Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions REQUIREMENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

> **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`.

**Test Verification Baseline:** **1361 Automated Tests** (997 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E).
**Test Verification Baseline:** **1365 Automated Tests** (1001 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E).

---

Expand Down Expand Up @@ -1136,10 +1136,14 @@ code results are enriched with signature/kind/ast_symbol_id from SQLite._
- `test_embedding_cache_invalidation`
- `test_embedding_cache_empty_inputs`

#### `tests/test_embedding_input_limits_and_pipeline_recovery.py` (3 tests)
#### `tests/test_embedding_input_limits_and_pipeline_recovery.py` (7 tests)
- `test_per_input_token_limit_truncates_oversized_chunk_before_batching` - _Issue #178: Verifies oversized inputs exceeding max_tokens_per_input are truncated before batching and configurable via EMBEDDING_MAX_TOKENS_PER_INPUT._
- `test_http_400_oversized_input_splits_batch_skips_bad_chunk_and_continues_sync` - _Issue #178: Verifies that on an HTTP 400 error for a batch, the batcher splits and retries halves, skips only the bad input, logs its file path, and completes the repo sync._
- `test_pipeline_producer_stops_when_persist_stage_raises` - _Issue #179: Verifies that when the persist stage raises, the producer stops before reading all remaining files and threads are joined before executor shutdown._
- `test_partial_embedding_settings_update_preserves_provider_and_rolls_back_on_failure` - _Issue #183: Verifies partial update with only api_concurrency preserves provider=api, and a failed update rolls back without mutating saved settings or active client._
- `test_cancel_during_clone_stops_subprocess_updates_repo_status_and_sync_returns_409_when_locked` - _Issue #184: Verifies cancelling during clone terminates the git subprocess, sets DB repo status to error, and POST /sync returns 409 while lock is held._
- `test_minified_css_dense_truncation_and_single_input_400_halving_retry` - _Issue #185: Verifies minified CSS uses dense token truncation and single-input HTTP 400 errors truncate in half and retry until accepted instead of skipping._
- `test_pipeline_maintains_api_concurrency_overlap_without_clamping_to_1` - _Issue #186: Verifies ChannelIngestionPipeline runs up to api_concurrency=4 overlapping embedding requests in parallel and reports active_concurrency=4._

#### `tests/test_embedding_model_transition.py` (6 tests)
- `test_update_embedding_config_rejection_when_index_has_content` - _Verifies that changing dense model/dimension without rebuild_index=True raises ValueError if index has content._
Expand Down
10 changes: 10 additions & 0 deletions app/api/routers/repositories.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,16 @@ async def api_sync_repo(repo_id: int):
return JSONResponse(status_code=404, content={"error": "Repo not found"})
r = dict(row)

if getattr(idx_service, "is_repo_sync_running", lambda _: False)(r["id"]):
return JSONResponse(
status_code=409,
content={
"status": "already_running",
"error": f"Sync already in progress for {r['name']}",
"message": f"Sync already in progress for {r['name']}",
},
)

sync_fn = _get_sync_fn()

def _run_manual_sync(rid: int):
Expand Down
2 changes: 1 addition & 1 deletion app/models/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ class AutoSyncSettingsRequest(BaseModel):
global_webhook_secret: Optional[str] = None

class EmbeddingSettingsRequest(BaseModel):
provider: Optional[str] = "local"
provider: Optional[str] = None
dense_model: Optional[str] = None
sparse_model: Optional[str] = None
threads: Optional[int] = Field(default=None, ge=1, le=128)
Expand Down
43 changes: 38 additions & 5 deletions app/services/embedding_batcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,21 @@ def _extract_filepath_from_embed_text(text: str) -> Optional[str]:
return None


def _chars_per_token_for_text(text: str) -> int:
"""Returns conservative chars-per-token ratio: 2 for dense/minified code or data, 4 for normal prose/code."""
if not text:
return 4
lines = text.splitlines()
body = "\n".join(lines[5:]) if len(lines) > 5 and lines[0].startswith("Repo: ") else text
sample = body[:8192]
if not sample:
return 4
punct_count = sum(1 for ch in sample if not ch.isalnum() and not ch.isspace())
if (punct_count / len(sample)) >= 0.10 or (max((len(ln) for ln in sample.splitlines()), default=0) > 500 and punct_count > 0):
return 2
return 4


def truncate_text_to_token_limit(
text: str,
max_tokens_per_input: int = 8000,
Expand All @@ -35,14 +50,16 @@ def truncate_text_to_token_limit(
"""Truncates a single input text so its estimated token count does not exceed max_tokens_per_input."""
if not text or not max_tokens_per_input or max_tokens_per_input <= 0:
return text
max_chars = int(max_tokens_per_input) * 4
chars_per_tok = _chars_per_token_for_text(text)
max_chars = int(max_tokens_per_input) * chars_per_tok
if len(text) <= max_chars:
return text
est_tokens = estimate_token_count(text)
est_tokens = max(1, len(text) // chars_per_tok)
label = filepath or _extract_filepath_from_embed_text(text) or "<unknown>"
logger.warning(
f"Truncating oversized embedding input from '{label}' "
f"(~{est_tokens} estimated tokens, {len(text)} chars) to max_tokens_per_input={max_tokens_per_input} ({max_chars} chars)."
f"(~{est_tokens} estimated tokens, {len(text)} chars, {chars_per_tok} chars/token) "
f"to max_tokens_per_input={max_tokens_per_input} ({max_chars} chars)."
)
return text[:max_chars]

Expand Down Expand Up @@ -183,12 +200,17 @@ def reset(self, target_concurrency: Optional[int] = None) -> None:
self._window_start = time.monotonic()
self._window_tokens = 0

def set_target_concurrency(self, target_concurrency: int) -> None:
with self._lock:
self._current_concurrency = max(1, int(target_concurrency or 1))
self._consecutive_successes = 0

def get_effective_concurrency(self, max_concurrency: int) -> int:
max_c = max(1, int(max_concurrency or 1))
with self._lock:
if self._current_concurrency is None or self._current_concurrency > max_c:
if self._current_concurrency is None:
self._current_concurrency = max_c
return max(1, self._current_concurrency)
return max(1, min(self._current_concurrency, max_c))

def record_rate_limit(self, wait_seconds: float, max_concurrency: int) -> int:
max_c = max(1, int(max_concurrency or 1))
Expand Down Expand Up @@ -335,6 +357,8 @@ def _embed_items_with_bisect(
telemetry_tracker.pace_tokens(slice_tokens, tpm_limit, sleep_fn=sleep_fn)

attempt = 0
trunc_retries = 0
max_trunc_retries = 3
backoff = 1.0
while True:
try:
Expand Down Expand Up @@ -384,6 +408,15 @@ def _embed_items_with_bisect(
or _extract_filepath_from_embed_text(batch_slice[0])
or "<unknown>"
)
if trunc_retries < max_trunc_retries and len(batch_slice[0]) > 64:
trunc_retries += 1
new_len = max(32, len(batch_slice[0]) // 2)
batch_slice = [batch_slice[0][:new_len]]
logger.warning(
f"Single embedding input from '{bad_path}' rejected with HTTP 400 ({exc}). "
f"Truncating in half to {new_len} chars (retry {trunc_retries}/{max_trunc_retries})."
)
continue
logger.error(
f"Skipping bad embedding input from '{bad_path}' (~{estimate_token_count(batch_slice[0])} estimated tokens) "
f"after HTTP 400 error: {exc}"
Expand Down
Loading
Loading