From 8a85debc1c0d154823461015a4de0f02874618b9 Mon Sep 17 00:00:00 2001 From: Antigravity Agent Date: Fri, 20 Nov 2026 13:45:01 -0600 Subject: [PATCH 1/2] fix(indexing): partial settings rollback, clone cancellation & 409 lock guard, minified input halving retry, and pipeline concurrency (#183, #184, #185, #186) --- REQUIREMENTS.md | 8 +- app/api/routers/repositories.py | 10 + app/models/schemas.py | 2 +- app/services/embedding_batcher.py | 43 ++- app/services/embeddings.py | 308 +++++++++--------- app/services/git_manager.py | 80 ++++- app/services/indexing/__init__.py | 2 + app/services/indexing/channel_pipeline.py | 16 +- app/services/indexing/git_syncer.py | 27 +- ...ding_input_limits_and_pipeline_recovery.py | 268 ++++++++++++++- 10 files changed, 570 insertions(+), 194 deletions(-) diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index 32e81ae..05a726f 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -2,7 +2,7 @@ > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`. -**Test Verification Baseline:** **1361 Automated Tests** (997 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E). +**Test Verification Baseline:** **1365 Automated Tests** (1001 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E). --- @@ -1136,10 +1136,14 @@ code results are enriched with signature/kind/ast_symbol_id from SQLite._ - `test_embedding_cache_invalidation` - `test_embedding_cache_empty_inputs` -#### `tests/test_embedding_input_limits_and_pipeline_recovery.py` (3 tests) +#### `tests/test_embedding_input_limits_and_pipeline_recovery.py` (7 tests) - `test_per_input_token_limit_truncates_oversized_chunk_before_batching` - _Issue #178: Verifies oversized inputs exceeding max_tokens_per_input are truncated before batching and configurable via EMBEDDING_MAX_TOKENS_PER_INPUT._ - `test_http_400_oversized_input_splits_batch_skips_bad_chunk_and_continues_sync` - _Issue #178: Verifies that on an HTTP 400 error for a batch, the batcher splits and retries halves, skips only the bad input, logs its file path, and completes the repo sync._ - `test_pipeline_producer_stops_when_persist_stage_raises` - _Issue #179: Verifies that when the persist stage raises, the producer stops before reading all remaining files and threads are joined before executor shutdown._ +- `test_partial_embedding_settings_update_preserves_provider_and_rolls_back_on_failure` - _Issue #183: Verifies partial update with only api_concurrency preserves provider=api, and a failed update rolls back without mutating saved settings or active client._ +- `test_cancel_during_clone_stops_subprocess_updates_repo_status_and_sync_returns_409_when_locked` - _Issue #184: Verifies cancelling during clone terminates the git subprocess, sets DB repo status to error, and POST /sync returns 409 while lock is held._ +- `test_minified_css_dense_truncation_and_single_input_400_halving_retry` - _Issue #185: Verifies minified CSS uses dense token truncation and single-input HTTP 400 errors truncate in half and retry until accepted instead of skipping._ +- `test_pipeline_maintains_api_concurrency_overlap_without_clamping_to_1` - _Issue #186: Verifies ChannelIngestionPipeline runs up to api_concurrency=4 overlapping embedding requests in parallel and reports active_concurrency=4._ #### `tests/test_embedding_model_transition.py` (6 tests) - `test_update_embedding_config_rejection_when_index_has_content` - _Verifies that changing dense model/dimension without rebuild_index=True raises ValueError if index has content._ diff --git a/app/api/routers/repositories.py b/app/api/routers/repositories.py index e817e9b..9db2047 100644 --- a/app/api/routers/repositories.py +++ b/app/api/routers/repositories.py @@ -171,6 +171,16 @@ async def api_sync_repo(repo_id: int): return JSONResponse(status_code=404, content={"error": "Repo not found"}) r = dict(row) + if getattr(idx_service, "is_repo_sync_running", lambda _: False)(r["id"]): + return JSONResponse( + status_code=409, + content={ + "status": "already_running", + "error": f"Sync already in progress for {r['name']}", + "message": f"Sync already in progress for {r['name']}", + }, + ) + sync_fn = _get_sync_fn() def _run_manual_sync(rid: int): diff --git a/app/models/schemas.py b/app/models/schemas.py index 5274d31..7f9277e 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -181,7 +181,7 @@ class AutoSyncSettingsRequest(BaseModel): global_webhook_secret: Optional[str] = None class EmbeddingSettingsRequest(BaseModel): - provider: Optional[str] = "local" + provider: Optional[str] = None dense_model: Optional[str] = None sparse_model: Optional[str] = None threads: Optional[int] = Field(default=None, ge=1, le=128) diff --git a/app/services/embedding_batcher.py b/app/services/embedding_batcher.py index 21d4e71..e7759c0 100644 --- a/app/services/embedding_batcher.py +++ b/app/services/embedding_batcher.py @@ -27,6 +27,21 @@ def _extract_filepath_from_embed_text(text: str) -> Optional[str]: return None +def _chars_per_token_for_text(text: str) -> int: + """Returns conservative chars-per-token ratio: 2 for dense/minified code or data, 4 for normal prose/code.""" + if not text: + return 4 + lines = text.splitlines() + body = "\n".join(lines[5:]) if len(lines) > 5 and lines[0].startswith("Repo: ") else text + sample = body[:8192] + if not sample: + return 4 + punct_count = sum(1 for ch in sample if not ch.isalnum() and not ch.isspace()) + if (punct_count / len(sample)) >= 0.10 or (max((len(ln) for ln in sample.splitlines()), default=0) > 500 and punct_count > 0): + return 2 + return 4 + + def truncate_text_to_token_limit( text: str, max_tokens_per_input: int = 8000, @@ -35,14 +50,16 @@ def truncate_text_to_token_limit( """Truncates a single input text so its estimated token count does not exceed max_tokens_per_input.""" if not text or not max_tokens_per_input or max_tokens_per_input <= 0: return text - max_chars = int(max_tokens_per_input) * 4 + chars_per_tok = _chars_per_token_for_text(text) + max_chars = int(max_tokens_per_input) * chars_per_tok if len(text) <= max_chars: return text - est_tokens = estimate_token_count(text) + est_tokens = max(1, len(text) // chars_per_tok) label = filepath or _extract_filepath_from_embed_text(text) or "" logger.warning( f"Truncating oversized embedding input from '{label}' " - f"(~{est_tokens} estimated tokens, {len(text)} chars) to max_tokens_per_input={max_tokens_per_input} ({max_chars} chars)." + f"(~{est_tokens} estimated tokens, {len(text)} chars, {chars_per_tok} chars/token) " + f"to max_tokens_per_input={max_tokens_per_input} ({max_chars} chars)." ) return text[:max_chars] @@ -183,12 +200,17 @@ def reset(self, target_concurrency: Optional[int] = None) -> None: self._window_start = time.monotonic() self._window_tokens = 0 + def set_target_concurrency(self, target_concurrency: int) -> None: + with self._lock: + self._current_concurrency = max(1, int(target_concurrency or 1)) + self._consecutive_successes = 0 + def get_effective_concurrency(self, max_concurrency: int) -> int: max_c = max(1, int(max_concurrency or 1)) with self._lock: - if self._current_concurrency is None or self._current_concurrency > max_c: + if self._current_concurrency is None: self._current_concurrency = max_c - return max(1, self._current_concurrency) + return max(1, min(self._current_concurrency, max_c)) def record_rate_limit(self, wait_seconds: float, max_concurrency: int) -> int: max_c = max(1, int(max_concurrency or 1)) @@ -335,6 +357,8 @@ def _embed_items_with_bisect( telemetry_tracker.pace_tokens(slice_tokens, tpm_limit, sleep_fn=sleep_fn) attempt = 0 + trunc_retries = 0 + max_trunc_retries = 3 backoff = 1.0 while True: try: @@ -384,6 +408,15 @@ def _embed_items_with_bisect( or _extract_filepath_from_embed_text(batch_slice[0]) or "" ) + if trunc_retries < max_trunc_retries and len(batch_slice[0]) > 64: + trunc_retries += 1 + new_len = max(32, len(batch_slice[0]) // 2) + batch_slice = [batch_slice[0][:new_len]] + logger.warning( + f"Single embedding input from '{bad_path}' rejected with HTTP 400 ({exc}). " + f"Truncating in half to {new_len} chars (retry {trunc_retries}/{max_trunc_retries})." + ) + continue logger.error( f"Skipping bad embedding input from '{bad_path}' (~{estimate_token_count(batch_slice[0])} estimated tokens) " f"after HTTP 400 error: {exc}" diff --git a/app/services/embeddings.py b/app/services/embeddings.py index 32e5e89..eb65c42 100644 --- a/app/services/embeddings.py +++ b/app/services/embeddings.py @@ -59,30 +59,27 @@ def _get_db_emb_config() -> Dict[str, Any]: def init_embeddings( - provider: Optional[str] = None, - dense_model: Optional[str] = None, - sparse_model: Optional[str] = None, - threads: Optional[int] = None, - batch_size: Optional[int] = None, - api_concurrency: Optional[int] = None, - max_tokens_per_request: Optional[int] = None, - max_tokens_per_input: Optional[int] = None, - tpm_limit: Optional[int] = None, - litellm_url: Optional[str] = None, - litellm_api_key: Optional[str] = None, - vision_ocr_model: Optional[str] = None, - chat_model: Optional[str] = None, - allow_fallback: Optional[bool] = None, - fallback_dense_model: Optional[str] = None, + provider: Optional[str] = None, dense_model: Optional[str] = None, sparse_model: Optional[str] = None, + threads: Optional[int] = None, batch_size: Optional[int] = None, api_concurrency: Optional[int] = None, + max_tokens_per_request: Optional[int] = None, max_tokens_per_input: Optional[int] = None, + tpm_limit: Optional[int] = None, litellm_url: Optional[str] = None, litellm_api_key: Optional[str] = None, + vision_ocr_model: Optional[str] = None, chat_model: Optional[str] = None, + allow_fallback: Optional[bool] = None, fallback_dense_model: Optional[str] = None, ): global _dense_model, _sparse_model, _openai_client - global REQUESTED_PROVIDER, EMBEDDING_PROVIDER - global REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, SPARSE_MODEL_NAME + global REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, SPARSE_MODEL_NAME global EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, LITELLM_URL, LITELLM_API_KEY global EMBEDDING_API_CONCURRENCY, EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT global VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL cfg = _get_db_emb_config() + _prev_state = ( + REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, + SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY, + EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT, + LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL, + _dense_model, _sparse_model, _openai_client, + ) REQUESTED_PROVIDER = (provider or cfg.get("provider") or "local").lower().strip() EMBEDDING_PROVIDER = REQUESTED_PROVIDER @@ -95,17 +92,10 @@ def init_embeddings( EMBEDDING_BATCH_SIZE = int(batch_size) else: cfg_bs = cfg.get("batch_size") - if REQUESTED_PROVIDER == "api" and not os.getenv("EMBEDDING_BATCH_SIZE") and cfg_bs in (None, 32): - EMBEDDING_BATCH_SIZE = 128 - else: - EMBEDDING_BATCH_SIZE = int(cfg_bs or default_bs) + EMBEDDING_BATCH_SIZE = 128 if (REQUESTED_PROVIDER == "api" and not os.getenv("EMBEDDING_BATCH_SIZE") and cfg_bs in (None, 32)) else int(cfg_bs or default_bs) EMBEDDING_API_CONCURRENCY = int(api_concurrency if api_concurrency is not None else cfg.get("api_concurrency", 4)) - EMBEDDING_MAX_TOKENS_PER_REQUEST = int( - max_tokens_per_request if max_tokens_per_request is not None else cfg.get("max_tokens_per_request", 100000) - ) - EMBEDDING_MAX_TOKENS_PER_INPUT = int( - max_tokens_per_input if max_tokens_per_input is not None else cfg.get("max_tokens_per_input", 8000) - ) + EMBEDDING_MAX_TOKENS_PER_REQUEST = int(max_tokens_per_request if max_tokens_per_request is not None else cfg.get("max_tokens_per_request", 100000)) + EMBEDDING_MAX_TOKENS_PER_INPUT = int(max_tokens_per_input if max_tokens_per_input is not None else cfg.get("max_tokens_per_input", 8000)) EMBEDDING_TPM_LIMIT = int(tpm_limit if tpm_limit is not None else cfg.get("tpm_limit", 0)) LITELLM_URL = (litellm_url or cfg.get("litellm_url") or "http://litellm:4000/v1").strip() LITELLM_API_KEY = (litellm_api_key or cfg.get("litellm_api_key") or "dummy").strip() @@ -114,81 +104,105 @@ def init_embeddings( ALLOW_FALLBACK = allow_fallback if allow_fallback is not None else cfg.get("allow_fallback", False) FALLBACK_DENSE_MODEL = (fallback_dense_model if fallback_dense_model is not None else cfg.get("fallback_dense_model", "")).strip() + def _restore_prev_state() -> None: + global REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME + global SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY + global EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT + global LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL + global _dense_model, _sparse_model, _openai_client + ( + REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, + SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY, + EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT, + LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL, + prev_dense, prev_sparse, prev_client, + ) = _prev_state + with _dense_lock: + _dense_model = prev_dense + with _sparse_lock: + _sparse_model = prev_sparse + _openai_client = prev_client + # Set underlying OpenMP / BLAS thread guard os.environ["OMP_NUM_THREADS"] = str(EMBEDDING_NUM_THREADS) - sparse_disabled = SPARSE_MODEL_NAME.lower() in ("", "none", "off", "disabled", "false") - if REQUESTED_PROVIDER == "local": - _openai_client = None - try: - from fastembed import TextEmbedding, SparseTextEmbedding - logger.info(f"Initializing FastEmbed dense model: {DENSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") - new_dense = TextEmbedding(model_name=DENSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) - with _dense_lock: - _dense_model = new_dense - if not sparse_disabled: - logger.info(f"Initializing FastEmbed sparse model: {SPARSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") - new_sparse = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) - with _sparse_lock: - _sparse_model = new_sparse - else: - with _sparse_lock: - _sparse_model = None - logger.info("FastEmbed dense & sparse models initialized successfully with resource limits.") - except Exception as e: - logger.error(f"Failed to initialize local FastEmbed models: {e}") - with _dense_lock: - _dense_model = None - with _sparse_lock: - _sparse_model = None - - if not ALLOW_FALLBACK: - EMBEDDING_PROVIDER = "local" - raise RuntimeError( - f"Local FastEmbed initialization failed for dense model '{DENSE_MODEL_NAME}': {e}. " - "Fallback to API is disabled (allow_fallback=False)." - ) from e - else: - if not FALLBACK_DENSE_MODEL: - EMBEDDING_PROVIDER = "local" - raise ValueError( - f"Local FastEmbed initialization failed for '{DENSE_MODEL_NAME}': {e}, " - "and allow_fallback=True but no valid fallback_dense_model was configured." - ) from e - logger.warning(f"Local FastEmbed initialization failed: {e}. Falling back to API with model '{FALLBACK_DENSE_MODEL}'.") - EMBEDDING_PROVIDER = "api" - DENSE_MODEL_NAME = FALLBACK_DENSE_MODEL - - if EMBEDDING_PROVIDER == "api": - with _dense_lock: - _dense_model = None - try: - from openai import OpenAI - from app.services.llm_client import get_llm_extra_headers - logger.info(f"Using OpenAI/LiteLLM API embeddings at {LITELLM_URL} model {DENSE_MODEL_NAME}") - new_client = OpenAI( - base_url=LITELLM_URL, - api_key=LITELLM_API_KEY, - default_headers=get_llm_extra_headers(), - ) - _openai_client = new_client - if not sparse_disabled: - try: - from fastembed import SparseTextEmbedding + try: + if REQUESTED_PROVIDER == "local": + _openai_client = None + try: + from fastembed import TextEmbedding, SparseTextEmbedding + logger.info(f"Initializing FastEmbed dense model: {DENSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") + new_dense = TextEmbedding(model_name=DENSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) + with _dense_lock: + _dense_model = new_dense + if not sparse_disabled: + logger.info(f"Initializing FastEmbed sparse model: {SPARSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") new_sparse = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) with _sparse_lock: _sparse_model = new_sparse - except Exception: + else: with _sparse_lock: _sparse_model = None - else: + logger.info("FastEmbed dense & sparse models initialized successfully with resource limits.") + except Exception as e: + logger.error(f"Failed to initialize local FastEmbed models: {e}") + with _dense_lock: + _dense_model = None with _sparse_lock: _sparse_model = None - except Exception as oe: - logger.error(f"Failed to initialize API client: {oe}") - if REQUESTED_PROVIDER == "local" and ALLOW_FALLBACK: - raise RuntimeError(f"Fallback to API failed: {oe}") from oe + + if not ALLOW_FALLBACK: + EMBEDDING_PROVIDER = "local" + raise RuntimeError( + f"Local FastEmbed initialization failed for dense model '{DENSE_MODEL_NAME}': {e}. " + "Fallback to API is disabled (allow_fallback=False)." + ) from e + else: + if not FALLBACK_DENSE_MODEL: + EMBEDDING_PROVIDER = "local" + raise ValueError( + f"Local FastEmbed initialization failed for '{DENSE_MODEL_NAME}': {e}, " + "and allow_fallback=True but no valid fallback_dense_model was configured." + ) from e + logger.warning(f"Local FastEmbed initialization failed: {e}. Falling back to API with model '{FALLBACK_DENSE_MODEL}'.") + EMBEDDING_PROVIDER = "api" + DENSE_MODEL_NAME = FALLBACK_DENSE_MODEL + + if EMBEDDING_PROVIDER == "api": + with _dense_lock: + _dense_model = None + try: + from openai import OpenAI + from app.services.llm_client import get_llm_extra_headers + logger.info(f"Using OpenAI/LiteLLM API embeddings at {LITELLM_URL} model {DENSE_MODEL_NAME}") + new_client = OpenAI( + base_url=LITELLM_URL, + api_key=LITELLM_API_KEY, + default_headers=get_llm_extra_headers(), + ) + _openai_client = new_client + if not sparse_disabled: + try: + from fastembed import SparseTextEmbedding + new_sparse = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) + with _sparse_lock: + _sparse_model = new_sparse + except Exception: + with _sparse_lock: + _sparse_model = None + else: + with _sparse_lock: + _sparse_model = None + except Exception as oe: + logger.error(f"Failed to initialize API client: {oe}") + if REQUESTED_PROVIDER == "local" and ALLOW_FALLBACK: + raise RuntimeError(f"Fallback to API failed: {oe}") from oe + raise + telemetry_tracker.set_target_concurrency(EMBEDDING_API_CONCURRENCY) + except Exception: + _restore_prev_state() + raise # Auto-initialize on import safely @@ -232,29 +246,14 @@ def get_embedding_config() -> Dict[str, Any]: telemetry = telemetry_tracker.get_telemetry_snapshot(EMBEDDING_API_CONCURRENCY) return { - "provider": EMBEDDING_PROVIDER, - "requested_provider": REQUESTED_PROVIDER, - "effective_provider": EMBEDDING_PROVIDER, - "dense_model": DENSE_MODEL_NAME, - "requested_dense_model": REQUESTED_DENSE_MODEL, - "effective_dense_model": DENSE_MODEL_NAME, - "sparse_model": reported_sparse, - "requested_sparse_model": SPARSE_MODEL_NAME, - "effective_sparse_model": effective_sparse, - "threads": EMBEDDING_NUM_THREADS, - "batch_size": EMBEDDING_BATCH_SIZE, - "api_concurrency": EMBEDDING_API_CONCURRENCY, - "max_tokens_per_request": EMBEDDING_MAX_TOKENS_PER_REQUEST, - "max_tokens_per_input": EMBEDDING_MAX_TOKENS_PER_INPUT, - "tpm_limit": EMBEDDING_TPM_LIMIT, - "litellm_url": LITELLM_URL, - "vision_ocr_model": VISION_OCR_MODEL, - "chat_model": CHAT_MODEL, - "allow_fallback": ALLOW_FALLBACK, - "fallback_dense_model": FALLBACK_DENSE_MODEL, - "system_cpus": sys_res["cpus"], - "system_memory_gb": sys_res["memory_gb"], - **telemetry, + "provider": EMBEDDING_PROVIDER, "requested_provider": REQUESTED_PROVIDER, "effective_provider": EMBEDDING_PROVIDER, + "dense_model": DENSE_MODEL_NAME, "requested_dense_model": REQUESTED_DENSE_MODEL, "effective_dense_model": DENSE_MODEL_NAME, + "sparse_model": reported_sparse, "requested_sparse_model": SPARSE_MODEL_NAME, "effective_sparse_model": effective_sparse, + "threads": EMBEDDING_NUM_THREADS, "batch_size": EMBEDDING_BATCH_SIZE, "api_concurrency": EMBEDDING_API_CONCURRENCY, + "max_tokens_per_request": EMBEDDING_MAX_TOKENS_PER_REQUEST, "max_tokens_per_input": EMBEDDING_MAX_TOKENS_PER_INPUT, + "tpm_limit": EMBEDDING_TPM_LIMIT, "litellm_url": LITELLM_URL, "vision_ocr_model": VISION_OCR_MODEL, + "chat_model": CHAT_MODEL, "allow_fallback": ALLOW_FALLBACK, "fallback_dense_model": FALLBACK_DENSE_MODEL, + "system_cpus": sys_res["cpus"], "system_memory_gb": sys_res["memory_gb"], **telemetry, } @@ -300,29 +299,23 @@ def get_dense_dim_for_config( def update_embedding_config( - provider: Optional[str] = None, - dense_model: Optional[str] = None, - sparse_model: Optional[str] = None, - threads: Optional[int] = None, - batch_size: Optional[int] = None, - api_concurrency: Optional[int] = None, - max_tokens_per_request: Optional[int] = None, - max_tokens_per_input: Optional[int] = None, - tpm_limit: Optional[int] = None, - litellm_url: Optional[str] = None, - litellm_api_key: Optional[str] = None, - vision_ocr_model: Optional[str] = None, - chat_model: Optional[str] = None, - allow_fallback: Optional[bool] = None, - fallback_dense_model: Optional[str] = None, + provider: Optional[str] = None, dense_model: Optional[str] = None, sparse_model: Optional[str] = None, + threads: Optional[int] = None, batch_size: Optional[int] = None, api_concurrency: Optional[int] = None, + max_tokens_per_request: Optional[int] = None, max_tokens_per_input: Optional[int] = None, + tpm_limit: Optional[int] = None, litellm_url: Optional[str] = None, litellm_api_key: Optional[str] = None, + vision_ocr_model: Optional[str] = None, chat_model: Optional[str] = None, + allow_fallback: Optional[bool] = None, fallback_dense_model: Optional[str] = None, rebuild_index: bool = False, ) -> Dict[str, Any]: """Updates embedding configuration in SQLite and hot-reloads models in memory.""" from app.services.database import set_embedding_db_config, clear_indexing_metadata, get_db_connection from app.services.vector_store import get_vector_store - target_provider = (provider or EMBEDDING_PROVIDER).lower().strip() - target_dense = (dense_model or DENSE_MODEL_NAME).strip() + saved_cfg = _get_db_emb_config() + cur_provider = (saved_cfg.get("provider") or EMBEDDING_PROVIDER or "local").lower().strip() + cur_dense = (saved_cfg.get("dense_model") or DENSE_MODEL_NAME).strip() + target_provider = (provider if provider is not None else cur_provider).lower().strip() + target_dense = (dense_model if dense_model is not None else cur_dense).strip() has_indexed_content = False try: @@ -336,26 +329,24 @@ def update_embedding_config( except Exception as e: logger.debug(f"Error checking active index count: {e}") + model_changed = (target_dense != DENSE_MODEL_NAME or target_provider != EMBEDDING_PROVIDER) new_dim = None - try: - new_dim = get_dense_dim_for_config( - provider=target_provider, - dense_model=target_dense, - litellm_url=litellm_url, - litellm_api_key=litellm_api_key, - threads=threads, - ) - except Exception as probe_err: - logger.warning(f"Failed to probe candidate embedding dimension: {probe_err}") - current_dim = None - try: - current_dim = get_dense_dim() - except Exception: - pass + if model_changed or has_indexed_content: + if model_changed: + try: + new_dim = get_dense_dim_for_config( + provider=target_provider, dense_model=target_dense, + litellm_url=litellm_url, litellm_api_key=litellm_api_key, threads=threads, + ) + except Exception as probe_err: + logger.warning(f"Failed to probe candidate embedding dimension: {probe_err}") + try: + current_dim = get_dense_dim() + except Exception: + pass dim_changed = (new_dim is not None and current_dim is not None and new_dim != current_dim) - model_changed = (target_dense != DENSE_MODEL_NAME or target_provider != EMBEDDING_PROVIDER) if has_indexed_content and (dim_changed or model_changed) and not rebuild_index: msg = ( @@ -366,24 +357,15 @@ def update_embedding_config( raise ValueError(msg) cfg_kwargs = { - "provider": provider, - "dense_model": dense_model, - "sparse_model": sparse_model, - "threads": threads, - "batch_size": batch_size, - "api_concurrency": api_concurrency, - "max_tokens_per_request": max_tokens_per_request, - "max_tokens_per_input": max_tokens_per_input, - "tpm_limit": tpm_limit, - "litellm_url": litellm_url, - "litellm_api_key": litellm_api_key, - "vision_ocr_model": vision_ocr_model, - "chat_model": chat_model, - "allow_fallback": allow_fallback, - "fallback_dense_model": fallback_dense_model, + "provider": provider, "dense_model": dense_model, "sparse_model": sparse_model, + "threads": threads, "batch_size": batch_size, "api_concurrency": api_concurrency, + "max_tokens_per_request": max_tokens_per_request, "max_tokens_per_input": max_tokens_per_input, + "tpm_limit": tpm_limit, "litellm_url": litellm_url, "litellm_api_key": litellm_api_key, + "vision_ocr_model": vision_ocr_model, "chat_model": chat_model, + "allow_fallback": allow_fallback, "fallback_dense_model": fallback_dense_model, } - set_embedding_db_config(**cfg_kwargs) init_embeddings(**cfg_kwargs) + set_embedding_db_config(**cfg_kwargs) if (dim_changed or model_changed or rebuild_index) and has_indexed_content: logger.info("Coordinating safe index rebuild for embedding model change...") diff --git a/app/services/git_manager.py b/app/services/git_manager.py index 89f9628..149ef80 100644 --- a/app/services/git_manager.py +++ b/app/services/git_manager.py @@ -4,7 +4,7 @@ import subprocess import tempfile import time -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional import requests from app.models.schemas import CloneResult @@ -140,17 +140,67 @@ def get_remote_head_sha( return None -def _clone_fresh_into_dir(auth_url: str, branch: str, depth_str: str, dest_dir: str, git_env: Dict[str, str]) -> subprocess.CompletedProcess: +_ORIG_SUBPROCESS_RUN = subprocess.run + + +def _run_cancellable_git_cmd( + cmd: list[str], + *, + timeout: float, + env: Dict[str, str], + cwd: Optional[str] = None, + cancel_check: Optional[Callable[[], bool]] = None, +) -> subprocess.CompletedProcess: + """Runs a git subprocess while polling cancel_check so a cancelled sync stops cloning immediately.""" + if cancel_check is not None and cancel_check(): + return subprocess.CompletedProcess(cmd, returncode=-15, stdout="", stderr="Sync cancelled by user") + if subprocess.run is not _ORIG_SUBPROCESS_RUN or cancel_check is None: + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=timeout, env=env) + + proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) + deadline = time.monotonic() + float(timeout) + while True: + if cancel_check(): + proc.terminate() + try: + proc.wait(timeout=3.0) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=2.0) + return subprocess.CompletedProcess(cmd, returncode=-15, stdout="", stderr="Sync cancelled by user") + + remaining = deadline - time.monotonic() + if remaining <= 0: + proc.kill() + stdout, stderr = proc.communicate() + raise subprocess.TimeoutExpired(cmd, timeout, output=stdout, stderr=stderr) + try: + stdout, stderr = proc.communicate(timeout=min(0.2, remaining)) + return subprocess.CompletedProcess(cmd, returncode=proc.returncode, stdout=stdout or "", stderr=stderr or "") + except subprocess.TimeoutExpired: + continue + + +def _clone_fresh_into_dir( + auth_url: str, + branch: str, + depth_str: str, + dest_dir: str, + git_env: Dict[str, str], + cancel_check: Optional[Callable[[], bool]] = None, +) -> subprocess.CompletedProcess: """Clones a repository shallowly into dest_dir, falling back to default branch if --branch fails.""" if os.path.exists(dest_dir) and os.listdir(dest_dir): cleanup_repo_dir(dest_dir) cmd = ["git", "clone", "--depth", depth_str, "--branch", branch, "--single-branch", auth_url, dest_dir] - res = subprocess.run(cmd, capture_output=True, text=True, timeout=120, env=git_env) + res = _run_cancellable_git_cmd(cmd, timeout=120, env=git_env, cancel_check=cancel_check) if res.returncode != 0: + if res.returncode == -15 or (cancel_check is not None and cancel_check()): + return res if os.path.exists(dest_dir) and os.listdir(dest_dir): cleanup_repo_dir(dest_dir) cmd_fallback = ["git", "clone", "--depth", depth_str, auth_url, dest_dir] - res = subprocess.run(cmd_fallback, capture_output=True, text=True, timeout=120, env=git_env) + res = _run_cancellable_git_cmd(cmd_fallback, timeout=120, env=git_env, cancel_check=cancel_check) return res @@ -163,10 +213,16 @@ def shallow_clone_repo( repo_id: Optional[str] = None, target_dir: Optional[str] = None, depth: int = 1, + cancel_check: Optional[Callable[[], bool]] = None, ) -> CloneResult: """Shallow clones a git repository to a temporary or persistent directory.""" from app.services.database import get_effective_git_token + if cancel_check is None and repo_id is not None and str(repo_id).isdigit(): + from app.services.indexing.git_progress import progress_tracker + rid_int = int(str(repo_id)) + cancel_check = lambda: progress_tracker.is_cancelled(rid_int) + norm_url = normalize_git_url(git_url) use_ssh_native = is_ssh_git_url(git_url) and not token and not username depth_str = str(max(1, depth)) @@ -198,9 +254,13 @@ def shallow_clone_repo( git_base = ["git", "-C", repo_dir, "-c", f"safe.directory={safe_dir}"] logger.info(f"Existing shallow clone found in {repo_dir}; fetching latest refs (depth: {depth_str})...") subprocess.run(git_base + ["remote", "set-url", "origin", auth_url], cwd=repo_dir, capture_output=True, text=True, timeout=15, env=git_env) - f_res = subprocess.run(git_base + ["fetch", "--depth", depth_str, "origin", branch], cwd=repo_dir, capture_output=True, text=True, timeout=60, env=git_env) + f_res = _run_cancellable_git_cmd(git_base + ["fetch", "--depth", depth_str, "origin", branch], cwd=repo_dir, timeout=60, env=git_env, cancel_check=cancel_check) + if f_res.returncode == -15 or (cancel_check is not None and cancel_check()): + return CloneResult(temp_dir=None, commit_sha=None, error="Sync cancelled by user") if f_res.returncode != 0: - f_res = subprocess.run(git_base + ["fetch", "--depth", depth_str, "origin"], cwd=repo_dir, capture_output=True, text=True, timeout=60, env=git_env) + f_res = _run_cancellable_git_cmd(git_base + ["fetch", "--depth", depth_str, "origin"], cwd=repo_dir, timeout=60, env=git_env, cancel_check=cancel_check) + if f_res.returncode == -15 or (cancel_check is not None and cancel_check()): + return CloneResult(temp_dir=None, commit_sha=None, error="Sync cancelled by user") if f_res.returncode == 0: r_res = subprocess.run(git_base + ["reset", "--hard", "FETCH_HEAD"], cwd=repo_dir, capture_output=True, text=True, timeout=30, env=git_env) @@ -215,9 +275,9 @@ def shallow_clone_repo( logger.warning(f"Incremental fetch failed in {repo_dir} (rc={f_res.returncode}); staging fresh clone...") staging_dir = f"{repo_dir}.staging.{int(time.time() * 1000)}" - res = _clone_fresh_into_dir(auth_url, branch, depth_str, staging_dir, git_env) + res = _clone_fresh_into_dir(auth_url, branch, depth_str, staging_dir, git_env, cancel_check=cancel_check) if res.returncode != 0: - err_msg = res.stderr.strip() or res.stdout.strip() or "Git clone failed" + err_msg = "Sync cancelled by user" if res.returncode == -15 else (res.stderr.strip() or res.stdout.strip() or "Git clone failed") cleanup_repo_dir(staging_dir) return CloneResult(temp_dir=None, commit_sha=None, error=f"Clone failed: {err_msg}") @@ -228,9 +288,9 @@ def shallow_clone_repo( logger.warning(f"Target directory {repo_dir} exists but is not a valid git repository; cleaning up for fresh clone...") cleanup_repo_dir(repo_dir) - res = _clone_fresh_into_dir(auth_url, branch, depth_str, repo_dir, git_env) + res = _clone_fresh_into_dir(auth_url, branch, depth_str, repo_dir, git_env, cancel_check=cancel_check) if res.returncode != 0: - err_msg = res.stderr.strip() or res.stdout.strip() or "Git clone failed" + err_msg = "Sync cancelled by user" if res.returncode == -15 else (res.stderr.strip() or res.stdout.strip() or "Git clone failed") cleanup_repo_dir(repo_dir) return CloneResult(temp_dir=None, commit_sha=None, error=f"Clone failed: {err_msg}") diff --git a/app/services/indexing/__init__.py b/app/services/indexing/__init__.py index 9b4f002..b57788b 100644 --- a/app/services/indexing/__init__.py +++ b/app/services/indexing/__init__.py @@ -24,6 +24,7 @@ from app.services.indexing.local_syncer import sync_local_paths, sync_local_repo from app.services.indexing.git_syncer import ( compute_git_repo_delta, + is_repo_sync_running, sync_single_git_repo, run_full_indexing, ) @@ -56,6 +57,7 @@ "sync_local_paths", "sync_local_repo", "compute_git_repo_delta", + "is_repo_sync_running", "sync_single_git_repo", "run_full_indexing", "GitSyncJob", diff --git a/app/services/indexing/channel_pipeline.py b/app/services/indexing/channel_pipeline.py index fdfa575..e19261b 100644 --- a/app/services/indexing/channel_pipeline.py +++ b/app/services/indexing/channel_pipeline.py @@ -212,6 +212,7 @@ def _stage1_producer() -> None: def _stage2_coalescer() -> None: try: batch_items: List[ParsedFileItem] = [] + active_futs: List[Future] = [] for item in self.parsed_channel.read_all(): if self.persist_channel.is_closed: return @@ -226,6 +227,7 @@ def _stage2_coalescer() -> None: proc_service, get_cached_embeddings_batch, execute_api_embeddings_batch, + active_futs, ): return batch_items = [] @@ -239,6 +241,7 @@ def _stage2_coalescer() -> None: proc_service, get_cached_embeddings_batch, execute_api_embeddings_batch, + active_futs, ): return self.persist_channel.complete() @@ -282,6 +285,7 @@ def _dispatch_batch( proc_service: Any, get_cached_embeddings_batch: Callable, execute_api_embeddings_batch: Callable, + active_futs: Optional[List[Future]] = None, ) -> bool: if self.persist_channel.is_closed: return False @@ -369,17 +373,17 @@ def _dispatch_batch( if cur_t: slices_triples.append((cur_h, cur_t, cur_fp)) - active_futs: List[Future] = [] + tracked_futs = active_futs if active_futs is not None else [] for s_hashes, s_texts, s_fps in slices_triples: if self.persist_channel.is_closed or self.parsed_channel.is_closed: return False - active_futs = [f for f in active_futs if not f.done()] + tracked_futs[:] = [f for f in tracked_futs if not f.done()] eff_conc = telemetry_tracker.get_effective_concurrency(max_conc) - while len(active_futs) >= eff_conc: + while len(tracked_futs) >= eff_conc: if self.persist_channel.is_closed or self.parsed_channel.is_closed: return False time.sleep(0.005) - active_futs = [f for f in active_futs if not f.done()] + tracked_futs[:] = [f for f in tracked_futs if not f.done()] eff_conc = telemetry_tracker.get_effective_concurrency(max_conc) fut = embed_executor.submit( @@ -388,13 +392,13 @@ def _dispatch_batch( texts=s_texts, model_name=model_name, batch_size=len(s_texts), - max_concurrency=1, + max_concurrency=max_conc, max_tokens_per_request=max_tok, max_tokens_per_input=max_tok_input, tpm_limit=tpm, filepaths=s_fps, ) - active_futs.append(fut) + tracked_futs.append(fut) slice_futures.append((s_hashes, fut)) else: if self.persist_channel.is_closed or self.parsed_channel.is_closed: diff --git a/app/services/indexing/git_syncer.py b/app/services/indexing/git_syncer.py index a5dd172..45bb036 100644 --- a/app/services/indexing/git_syncer.py +++ b/app/services/indexing/git_syncer.py @@ -21,6 +21,23 @@ def _get_repo_lock(repo_id: int) -> threading.Lock: return _repo_sync_locks[key] +def is_repo_sync_running(repo_id: int) -> bool: + """Returns True if the per-repository sync lock is currently held.""" + return _get_repo_lock(repo_id).locked() + + +def _mark_repo_cancelled(repo_id: int, repo_name: str, stage_msg: str = "Sync cancelled by user") -> None: + logger.info(f"Sync cancelled for repo '{repo_name}': {stage_msg}") + progress_tracker.log(repo_id, "WARN", stage_msg) + try: + with db_service.get_db_connection() as conn: + conn.execute("UPDATE git_repositories SET status = 'error', last_error = ? WHERE id = ?", ("Sync cancelled by user", repo_id)) + conn.commit() + except Exception: + pass + progress_tracker.finish_job(repo_id, "error", "Sync cancelled by user") + + import app.services.database as db_service import app.services.git_manager as gm_service import app.services.indexing.index_format as index_format @@ -121,8 +138,7 @@ def sync_single_git_repo(repo_id: int, force: bool = False): return True if progress_tracker.is_cancelled(repo_id): - logger.info(f"Sync cancelled for repo '{repo_name}'") - progress_tracker.log(repo_id, "WARN", "Sync cancelled by user") + _mark_repo_cancelled(repo_id, repo_name) return # Stage 2: Shallow Cloning @@ -164,8 +180,7 @@ def sync_single_git_repo(repo_id: int, force: bool = False): progress_tracker.log(repo_id, "INFO", f"Cloned successfully in {clone_duration}s (@ {commit_sha[:8] if commit_sha else 'head'})") if progress_tracker.is_cancelled(repo_id): - logger.info(f"Sync cancelled for repo '{repo_name}' after clone") - progress_tracker.log(repo_id, "WARN", "Sync cancelled by user") + _mark_repo_cancelled(repo_id, repo_name, "Sync cancelled by user") return try: @@ -207,8 +222,7 @@ def sync_single_git_repo(repo_id: int, force: bool = False): return True if progress_tracker.is_cancelled(repo_id): - logger.info(f"Sync cancelled for repo '{repo_name}' after delta scan") - progress_tracker.log(repo_id, "WARN", "Sync cancelled by user") + _mark_repo_cancelled(repo_id, repo_name, "Sync cancelled by user") return store = vs_service.get_vector_store() @@ -394,6 +408,7 @@ def _write_relational_batch() -> int: ) pipeline.run(_produce_parsed_files) if was_cancelled or progress_tracker.is_cancelled(repo_id): + _mark_repo_cancelled(repo_id, repo_name, "Sync cancelled by user") return # Stage 5: Vector Indexing & Cleanup diff --git a/tests/test_embedding_input_limits_and_pipeline_recovery.py b/tests/test_embedding_input_limits_and_pipeline_recovery.py index 9cecda7..d7aad6d 100644 --- a/tests/test_embedding_input_limits_and_pipeline_recovery.py +++ b/tests/test_embedding_input_limits_and_pipeline_recovery.py @@ -106,7 +106,7 @@ def __init__(self, idx: int): def fake_create(input, model): for idx, text in enumerate(input): - if "POISON_CHUNK_OVER_8192_TOKENS" in text: + if "POISON_CHUNK_OVER_8192_TOKENS" in text or "file_3.py" in text: raise FakeBadRequestError(idx) return _make_fake_embedding_response(input) @@ -184,3 +184,269 @@ def _failing_persist(items: list[ParsedFileItem]) -> None: assert produced_count < 25 assert not any("cannot schedule new futures after shutdown" in rec.message for rec in caplog.records) + + +def test_partial_embedding_settings_update_preserves_provider_and_rolls_back_on_failure(tmp_path, monkeypatch): + """Issue #183: Verifies partial update with only api_concurrency preserves provider=api, and a failed update rolls back without mutating saved settings or active client.""" + import builtins + from app.models.schemas import EmbeddingSettingsRequest + from app.services.database.connection import get_embedding_db_config + + db_file = str(tmp_path / "test_settings_183.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + # 1. Configure provider=api with an API model + emb_service.update_embedding_config( + provider="api", + dense_model="openai/text-embedding-3-small", + api_concurrency=4, + allow_fallback=False, + rebuild_index=True, + ) + orig_client = emb_service._openai_client + assert emb_service.get_embedding_config()["provider"] == "api" + + # 2. Partial request payload with ONLY api_concurrency=8 + req = EmbeddingSettingsRequest(api_concurrency=8) + assert req.provider is None + + updated = emb_service.update_embedding_config( + provider=req.provider, + dense_model=req.dense_model, + api_concurrency=req.api_concurrency, + ) + assert updated["provider"] == "api" + assert updated["effective_provider"] == "api" + assert updated["dense_model"] == "openai/text-embedding-3-small" + assert updated["api_concurrency"] == 8 + assert get_embedding_db_config()["provider"] == "api" + assert get_embedding_db_config()["api_concurrency"] == 8 + client_before_failed_update = emb_service._openai_client + assert client_before_failed_update is not None + + # 3. Now simulate a failed update (switching to local when fastembed is missing and allow_fallback=False) + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "fastembed": + raise ImportError("No module named 'fastembed'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + + with pytest.raises(RuntimeError, match="Local FastEmbed initialization failed"): + emb_service.update_embedding_config( + provider="local", + dense_model="openai/text-embedding-3-small", + api_concurrency=16, + allow_fallback=False, + rebuild_index=True, + ) + + # Saved settings and in-memory active config must remain untouched + cfg_after = emb_service.get_embedding_config() + db_cfg_after = get_embedding_db_config() + assert cfg_after["provider"] == "api" + assert cfg_after["effective_provider"] == "api" + assert cfg_after["dense_model"] == "openai/text-embedding-3-small" + assert cfg_after["api_concurrency"] == 8 + assert db_cfg_after["provider"] == "api" + assert db_cfg_after["api_concurrency"] == 8 + assert emb_service._openai_client is client_before_failed_update + + +@pytest.mark.asyncio +async def test_cancel_during_clone_stops_subprocess_updates_repo_status_and_sync_returns_409_when_locked( + tmp_path, monkeypatch +): + """Issue #184: Verifies cancelling during clone terminates the git subprocess, sets DB repo status to error, and POST /sync returns 409 while lock is held.""" + import subprocess + from app.api.routers.repositories import api_sync_repo + from app.services.git_manager import shallow_clone_repo + from app.services.indexing.git_progress import progress_tracker + from app.services.indexing.git_syncer import _get_repo_lock + + db_file = str(tmp_path / "test_cancel_clone_184.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + with get_db_connection() as conn: + conn.execute( + "INSERT INTO git_repositories (id, name, url, branch, commit_sha, status) VALUES (184, 'clone-cancel-repo', 'https://github.com/org/large.git', 'main', 'old', 'pending')" + ) + conn.commit() + + # 1. Verify POST /admin/api/repos/184/sync returns 409 when repo sync lock is held + lock = _get_repo_lock(184) + assert lock.acquire(blocking=False) + try: + resp = await api_sync_repo(184) + assert getattr(resp, "status_code", 200) == 409 + finally: + lock.release() + + # 2. Verify shallow_clone_repo terminates the subprocess when job is cancelled during clone + progress_tracker.get_or_create_job(184, "clone-cancel-repo") + terminated = [] + + class FakePopen: + def __init__(self, cmd, **kwargs): + self.cmd = cmd + self.returncode = None + + def communicate(self, timeout=None): + # Simulate cancel arriving during the first poll window + progress_tracker.cancel_job(184) + raise subprocess.TimeoutExpired(self.cmd, timeout or 0.2) + + def terminate(self): + terminated.append(True) + self.returncode = -15 + + def wait(self, timeout=None): + return self.returncode + + def kill(self): + self.returncode = -9 + + with patch("subprocess.Popen", side_effect=FakePopen): + clone_res = shallow_clone_repo("https://github.com/org/large.git", "main", repo_id="184") + + assert terminated == [True] + assert clone_res.temp_dir is None + assert "cancelled" in (clone_res.error or "").lower() + + # 3. Verify sync_single_git_repo updates git_repositories.status in DB from 'syncing' to 'error' when cancelled after clone + def clone_then_cancel(*args, **kwargs): + progress_tracker.cancel_job(184) + return CloneResult(temp_dir=str(tmp_path), commit_sha="new184", error=None) + + with patch("app.services.git_manager.get_remote_head_sha", return_value="new184"), \ + patch("app.services.git_manager.shallow_clone_repo", side_effect=clone_then_cancel), \ + patch("app.services.git_manager.cleanup_repo_dir"): + sync_single_git_repo(184) + + with get_db_connection() as conn: + row = conn.execute("SELECT status, last_error FROM git_repositories WHERE id = 184").fetchone() + snap = progress_tracker.get_snapshot(184) + assert row["status"] == "error" + assert snap["status"] == "error" + + +def test_minified_css_dense_truncation_and_single_input_400_halving_retry(): + """Issue #185: Verifies minified CSS uses dense token truncation and single-input HTTP 400 errors truncate in half and retry until accepted instead of skipping.""" + minified_css = "Repo: web\nFile: dist/Mdm_To_Pub.css\nLanguage: css\nSymbol: Mdm_To_Pub.css\nCode:\n" + ( + ".btn-primary{color:#fff;background:#0055aa;padding:4px 8px;margin:0;border:1px solid #003366;}" * 1600 + ) + assert len(minified_css) > 140000 + + first_truncated = truncate_text_to_token_limit( + minified_css, max_tokens_per_input=8000, filepath="dist/Mdm_To_Pub.css" + ) + # Dense punctuation ratio in minified CSS should truncate to <= 16000 chars (2 chars/token), not 32000 + assert len(first_truncated) <= 16000 + + # Even if the upstream model still rejects inputs > 6000 chars with HTTP 400, single-input retry halves it until accepted + class FakeLength400Error(Exception): + def __init__(self): + super().__init__("Error code: 400 - Invalid 'input[0]': maximum input length is 8192 tokens.") + self.status_code = 400 + + attempts = [] + + def fake_create(input, model): + for t in input: + attempts.append(len(t)) + if len(t) > 6000: + raise FakeLength400Error() + return _make_fake_embedding_response(input) + + mock_client = MagicMock() + mock_client.embeddings.create.side_effect = fake_create + + vecs = execute_api_embeddings_batch( + client=mock_client, + texts=["normal-chunk-0", minified_css], + model_name="text-embedding-3-small", + batch_size=128, + max_concurrency=2, + max_tokens_per_request=100000, + max_tokens_per_input=8000, + filepaths=["src/index.js", "dist/Mdm_To_Pub.css"], + ) + # Both inputs must be embedded (neither is None / skipped) + assert len(vecs) == 2 + assert vecs[0] is not None + assert vecs[1] is not None + assert any(length <= 6000 for length in attempts) + + +def test_pipeline_maintains_api_concurrency_overlap_without_clamping_to_1(tmp_path, monkeypatch): + """Issue #186: Verifies ChannelIngestionPipeline runs up to api_concurrency=4 overlapping embedding requests in parallel and reports active_concurrency=4.""" + import threading + import time + from app.services.embedding_batcher import telemetry_tracker + + db_file = str(tmp_path / "test_conc_186.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + telemetry_tracker.reset(target_concurrency=4) + active = 0 + max_overlap = 0 + lock = threading.Lock() + barrier = threading.Barrier(4, timeout=2.0) + + def fake_create(input, model): + nonlocal active, max_overlap + with lock: + active += 1 + if active > max_overlap: + max_overlap = active + try: + barrier.wait() + except threading.BrokenBarrierError: + pass + time.sleep(0.02) + with lock: + active -= 1 + return _make_fake_embedding_response(input) + + mock_client = MagicMock() + mock_client.embeddings.create.side_effect = fake_create + + monkeypatch.setattr(emb_service, "EMBEDDING_PROVIDER", "api") + monkeypatch.setattr(emb_service, "_openai_client", mock_client) + monkeypatch.setattr(emb_service, "EMBEDDING_BATCH_SIZE", 2) + monkeypatch.setattr(emb_service, "EMBEDDING_API_CONCURRENCY", 4) + + def _produce(channel: BoundedChannel[ParsedFileItem]) -> None: + for i in range(8): + pt = VectorDocument( + id=f"doc-{i}", + dense_vector=None, + sparse_indices=None, + sparse_values=None, + text=f"def fn_{i}(): return {i}", + metadata={"_embed_text": f"chunk-{i}"}, + ) + channel.write(ParsedFileItem(filepath=f"repo://f{i}.py", rel_path=f"f{i}.py", repo="repo", points=[pt])) + + persisted = [] + pipeline = ChannelIngestionPipeline( + persist_batch_fn=lambda items: persisted.extend(items), + file_batch_size=8, + ) + pipeline.run(_produce) + + assert len(persisted) == 8 + assert max_overlap == 4 + assert emb_service.get_embedding_config()["active_concurrency"] == 4 + From 0bfe105cc69b2ec9afec3d381bc8e053363e9e93 Mon Sep 17 00:00:00 2001 From: Antigravity Agent Date: Fri, 20 Nov 2026 14:00:35 -0600 Subject: [PATCH 2/2] fix(embeddings): scope rollback to update_embedding_config to preserve init_embeddings contract --- app/services/embeddings.py | 188 ++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 95 deletions(-) diff --git a/app/services/embeddings.py b/app/services/embeddings.py index eb65c42..d77ccde 100644 --- a/app/services/embeddings.py +++ b/app/services/embeddings.py @@ -73,13 +73,6 @@ def init_embeddings( global VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL cfg = _get_db_emb_config() - _prev_state = ( - REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, - SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY, - EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT, - LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL, - _dense_model, _sparse_model, _openai_client, - ) REQUESTED_PROVIDER = (provider or cfg.get("provider") or "local").lower().strip() EMBEDDING_PROVIDER = REQUESTED_PROVIDER @@ -104,105 +97,82 @@ def init_embeddings( ALLOW_FALLBACK = allow_fallback if allow_fallback is not None else cfg.get("allow_fallback", False) FALLBACK_DENSE_MODEL = (fallback_dense_model if fallback_dense_model is not None else cfg.get("fallback_dense_model", "")).strip() - def _restore_prev_state() -> None: - global REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME - global SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY - global EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT - global LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL - global _dense_model, _sparse_model, _openai_client - ( - REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, - SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY, - EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT, - LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL, - prev_dense, prev_sparse, prev_client, - ) = _prev_state - with _dense_lock: - _dense_model = prev_dense - with _sparse_lock: - _sparse_model = prev_sparse - _openai_client = prev_client - # Set underlying OpenMP / BLAS thread guard os.environ["OMP_NUM_THREADS"] = str(EMBEDDING_NUM_THREADS) sparse_disabled = SPARSE_MODEL_NAME.lower() in ("", "none", "off", "disabled", "false") - try: - if REQUESTED_PROVIDER == "local": - _openai_client = None - try: - from fastembed import TextEmbedding, SparseTextEmbedding - logger.info(f"Initializing FastEmbed dense model: {DENSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") - new_dense = TextEmbedding(model_name=DENSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) - with _dense_lock: - _dense_model = new_dense - if not sparse_disabled: - logger.info(f"Initializing FastEmbed sparse model: {SPARSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") + if REQUESTED_PROVIDER == "local": + _openai_client = None + try: + from fastembed import TextEmbedding, SparseTextEmbedding + logger.info(f"Initializing FastEmbed dense model: {DENSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") + new_dense = TextEmbedding(model_name=DENSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) + with _dense_lock: + _dense_model = new_dense + if not sparse_disabled: + logger.info(f"Initializing FastEmbed sparse model: {SPARSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") + new_sparse = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) + with _sparse_lock: + _sparse_model = new_sparse + else: + with _sparse_lock: + _sparse_model = None + logger.info("FastEmbed dense & sparse models initialized successfully with resource limits.") + except Exception as e: + logger.error(f"Failed to initialize local FastEmbed models: {e}") + with _dense_lock: + _dense_model = None + with _sparse_lock: + _sparse_model = None + + if not ALLOW_FALLBACK: + EMBEDDING_PROVIDER = "local" + raise RuntimeError( + f"Local FastEmbed initialization failed for dense model '{DENSE_MODEL_NAME}': {e}. " + "Fallback to API is disabled (allow_fallback=False)." + ) from e + else: + if not FALLBACK_DENSE_MODEL: + EMBEDDING_PROVIDER = "local" + raise ValueError( + f"Local FastEmbed initialization failed for '{DENSE_MODEL_NAME}': {e}, " + "and allow_fallback=True but no valid fallback_dense_model was configured." + ) from e + logger.warning(f"Local FastEmbed initialization failed: {e}. Falling back to API with model '{FALLBACK_DENSE_MODEL}'.") + EMBEDDING_PROVIDER = "api" + DENSE_MODEL_NAME = FALLBACK_DENSE_MODEL + + if EMBEDDING_PROVIDER == "api": + with _dense_lock: + _dense_model = None + try: + from openai import OpenAI + from app.services.llm_client import get_llm_extra_headers + logger.info(f"Using OpenAI/LiteLLM API embeddings at {LITELLM_URL} model {DENSE_MODEL_NAME}") + new_client = OpenAI( + base_url=LITELLM_URL, + api_key=LITELLM_API_KEY, + default_headers=get_llm_extra_headers(), + ) + _openai_client = new_client + if not sparse_disabled: + try: + from fastembed import SparseTextEmbedding new_sparse = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) with _sparse_lock: _sparse_model = new_sparse - else: + except Exception: with _sparse_lock: _sparse_model = None - logger.info("FastEmbed dense & sparse models initialized successfully with resource limits.") - except Exception as e: - logger.error(f"Failed to initialize local FastEmbed models: {e}") - with _dense_lock: - _dense_model = None + else: with _sparse_lock: _sparse_model = None + except Exception as oe: + logger.error(f"Failed to initialize API client: {oe}") + if REQUESTED_PROVIDER == "local" and ALLOW_FALLBACK: + raise RuntimeError(f"Fallback to API failed: {oe}") from oe - if not ALLOW_FALLBACK: - EMBEDDING_PROVIDER = "local" - raise RuntimeError( - f"Local FastEmbed initialization failed for dense model '{DENSE_MODEL_NAME}': {e}. " - "Fallback to API is disabled (allow_fallback=False)." - ) from e - else: - if not FALLBACK_DENSE_MODEL: - EMBEDDING_PROVIDER = "local" - raise ValueError( - f"Local FastEmbed initialization failed for '{DENSE_MODEL_NAME}': {e}, " - "and allow_fallback=True but no valid fallback_dense_model was configured." - ) from e - logger.warning(f"Local FastEmbed initialization failed: {e}. Falling back to API with model '{FALLBACK_DENSE_MODEL}'.") - EMBEDDING_PROVIDER = "api" - DENSE_MODEL_NAME = FALLBACK_DENSE_MODEL - - if EMBEDDING_PROVIDER == "api": - with _dense_lock: - _dense_model = None - try: - from openai import OpenAI - from app.services.llm_client import get_llm_extra_headers - logger.info(f"Using OpenAI/LiteLLM API embeddings at {LITELLM_URL} model {DENSE_MODEL_NAME}") - new_client = OpenAI( - base_url=LITELLM_URL, - api_key=LITELLM_API_KEY, - default_headers=get_llm_extra_headers(), - ) - _openai_client = new_client - if not sparse_disabled: - try: - from fastembed import SparseTextEmbedding - new_sparse = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) - with _sparse_lock: - _sparse_model = new_sparse - except Exception: - with _sparse_lock: - _sparse_model = None - else: - with _sparse_lock: - _sparse_model = None - except Exception as oe: - logger.error(f"Failed to initialize API client: {oe}") - if REQUESTED_PROVIDER == "local" and ALLOW_FALLBACK: - raise RuntimeError(f"Fallback to API failed: {oe}") from oe - raise - telemetry_tracker.set_target_concurrency(EMBEDDING_API_CONCURRENCY) - except Exception: - _restore_prev_state() - raise + telemetry_tracker.set_target_concurrency(EMBEDDING_API_CONCURRENCY) # Auto-initialize on import safely @@ -308,6 +278,11 @@ def update_embedding_config( rebuild_index: bool = False, ) -> Dict[str, Any]: """Updates embedding configuration in SQLite and hot-reloads models in memory.""" + global REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME + global SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY + global EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT + global LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL + global _dense_model, _sparse_model, _openai_client from app.services.database import set_embedding_db_config, clear_indexing_metadata, get_db_connection from app.services.vector_store import get_vector_store @@ -364,7 +339,30 @@ def update_embedding_config( "vision_ocr_model": vision_ocr_model, "chat_model": chat_model, "allow_fallback": allow_fallback, "fallback_dense_model": fallback_dense_model, } - init_embeddings(**cfg_kwargs) + _prev_state = ( + REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, + SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY, + EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT, + LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL, + _dense_model, _sparse_model, _openai_client, + ) + try: + init_embeddings(**cfg_kwargs) + except Exception: + ( + REQUESTED_PROVIDER, EMBEDDING_PROVIDER, REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, + SPARSE_MODEL_NAME, EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, EMBEDDING_API_CONCURRENCY, + EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT, + LITELLM_URL, LITELLM_API_KEY, VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL, + prev_dense, prev_sparse, prev_client, + ) = _prev_state + with _dense_lock: + _dense_model = prev_dense + with _sparse_lock: + _sparse_model = prev_sparse + _openai_client = prev_client + telemetry_tracker.set_target_concurrency(EMBEDDING_API_CONCURRENCY) + raise set_embedding_db_config(**cfg_kwargs) if (dim_changed or model_changed or rebuild_index) and has_indexed_content: