Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion REQUIREMENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

> **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`.

**Test Verification Baseline:** **1340 Automated Tests** (976 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E).
**Test Verification Baseline:** **1344 Automated Tests** (980 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E).

---

Expand Down Expand Up @@ -1022,6 +1022,12 @@ code results are enriched with signature/kind/ast_symbol_id from SQLite._
- `test_wait_for_db_retries_and_success`
- `test_wait_for_db_exhausts_retries`

#### `tests/test_delete_local_path.py` (4 tests)
- `test_delete_local_path_cleans_all_relational_tables_and_vectors` - _Verify DELETE /admin/api/paths/{id} removes rows from all 6 file-scoped tables and the vector store._
- `test_delete_local_path_with_shared_repo_only_purges_deleted_path_prefix` - _Verify deleting one local path that shares a repo name ('local') removes only its own files and vectors._
- `test_delete_git_repo_cleans_file_summaries_and_api_client_calls` - _Verify DELETE /admin/api/repos/{id} deletes file_summaries, api_client_calls, and all repo-scoped tables._
- `test_pgvector_upsert_batches_missing_embeddings_and_deletes_by_folder_prefix` - _Verify PgVectorStore.upsert_documents batches missing dense vectors and delete_by_path supports prefixes._

#### `tests/test_diagnose_evidence_chain.py` (8 tests)
- `test_chain_follows_the_call_path_to_the_query_that_drops_the_transaction`
- `test_chain_skips_loggers_interfaces_with_many_implementations_and_test_doubles`
Expand Down
135 changes: 67 additions & 68 deletions app/api/routers/repositories.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,10 +255,18 @@ async def api_cancel_repo_sync(repo_id: int):
return JSONResponse(status_code=400, content={"error": f"Repo {repo_id} is not currently syncing"})
return {"status": "cancelled", "repo_id": repo_id}

@router.delete(
"/admin/api/repos/{repo_id}",
dependencies=[Depends(require_role(Role.ADMIN))],
_FILE_SCOPED_TABLES = (
("indexed_files", "filepath"),
("file_summaries", "filepath"),
("ast_symbols", "filepath"),
("ast_relationships", "source_filepath"),
("api_routes", "filepath"),
("api_client_calls", "filepath"),
)
_REPO_SCOPED_TABLES = tuple(t for t, _ in _FILE_SCOPED_TABLES) + ("git_commits",)


@router.delete("/admin/api/repos/{repo_id}", dependencies=[Depends(require_role(Role.ADMIN))])
async def api_delete_repo(repo_id: int):
try:
with db_service.get_db_connection() as conn:
Expand All @@ -268,7 +276,7 @@ async def api_delete_repo(repo_id: int):
name = row["name"]

conn.execute("DELETE FROM git_repositories WHERE id = ?", (repo_id,))
for table in ("indexed_files", "ast_symbols", "ast_relationships", "api_routes", "git_commits"):
for table in _REPO_SCOPED_TABLES:
try:
conn.execute(f"DELETE FROM {table} WHERE repo = ?", (name,))
except Exception:
Expand All @@ -291,10 +299,7 @@ async def api_delete_repo(repo_id: int):
logger.error(f"Error deleting repo {repo_id}: {e}")
return JSONResponse(status_code=500, content={"error": "Failed to delete repository."})

@router.get(
"/admin/api/paths",
dependencies=[Depends(require_role(Role.VIEWER))],
)
@router.get("/admin/api/paths", dependencies=[Depends(require_role(Role.VIEWER))])
async def api_get_paths():
try:
with db_service.get_db_connection() as conn:
Expand All @@ -304,10 +309,7 @@ async def api_get_paths():
logger.error(f"Error getting paths: {e}")
return JSONResponse(status_code=500, content={"error": "Failed to get paths."})

@router.post(
"/admin/api/paths",
dependencies=[Depends(require_role(Role.ADMIN))],
)
@router.post("/admin/api/paths", dependencies=[Depends(require_role(Role.ADMIN))])
async def api_add_path(config: LocalPathConfig):
try:
raw_path = config.path.strip() if config.path else ""
Expand Down Expand Up @@ -343,25 +345,51 @@ async def api_add_path(config: LocalPathConfig):
logger.error(f"Error adding path: {e}")
return JSONResponse(status_code=500, content={"error": "Failed to add path."})

@router.delete(
"/admin/api/paths/{path_id}",
dependencies=[Depends(require_role(Role.ADMIN))],
)
@router.delete("/admin/api/paths/{path_id}", dependencies=[Depends(require_role(Role.ADMIN))])
async def api_delete_path(path_id: int):
try:
with db_service.get_db_connection() as conn:
row = conn.execute("SELECT path FROM indexed_paths WHERE id = ?", (path_id,)).fetchone()
row = conn.execute("SELECT path, repo FROM indexed_paths WHERE id = ?", (path_id,)).fetchone()
if not row:
return JSONResponse(status_code=404, content={"error": "Path not found"})
path_val = row["path"]
repo_val = (row["repo"] or "").strip() if "repo" in row.keys() else ""
norm_path = (path_val or "").replace("\\", "/").rstrip("/")

conn.execute("DELETE FROM indexed_paths WHERE id = ?", (path_id,))
conn.execute("DELETE FROM indexed_files WHERE filepath LIKE ?", (f"{path_val}%",))
repo_shared = False
if repo_val:
rem_paths = conn.execute("SELECT COUNT(*) FROM indexed_paths WHERE repo = ?", (repo_val,)).fetchone()[0]
try:
rem_repos = conn.execute("SELECT COUNT(*) FROM git_repositories WHERE name = ?", (repo_val,)).fetchone()[0]
except Exception:
rem_repos = 0
repo_shared = (rem_paths + rem_repos) > 0
if not repo_shared:
for table, _ in _FILE_SCOPED_TABLES:
try:
conn.execute(f"DELETE FROM {table} WHERE repo = ?", (repo_val,))
except Exception:
pass

for table, col in _FILE_SCOPED_TABLES:
try:
if norm_path:
conn.execute(
f"DELETE FROM {table} WHERE {col} = ? OR REPLACE({col}, '\\', '/') = ? OR REPLACE({col}, '\\', '/') LIKE ?",
(path_val, norm_path, f"{norm_path}/%"),
)
else:
conn.execute(f"DELETE FROM {table} WHERE {col} = ?", (path_val,))
except Exception:
pass
conn.commit()

try:
store = vs_service.get_vector_store()
store.delete_by_path(path_val)
if repo_val and not repo_shared:
store.delete_by_repo(repo_val)
store.delete_by_path(norm_path or path_val)
except Exception as e:
logger.error(f"Error removing points from vector DB for path {path_val}: {e}")

Expand All @@ -370,14 +398,8 @@ async def api_delete_path(path_id: int):
logger.error(f"Error deleting path {path_id}: {e}")
return JSONResponse(status_code=500, content={"error": "Failed to delete path."})

@router.post(
"/admin/api/sync",
dependencies=[Depends(require_role(Role.EDITOR))],
)
@router.post(
"/admin/api/reindex",
dependencies=[Depends(require_role(Role.EDITOR))],
)
@router.post("/admin/api/sync", dependencies=[Depends(require_role(Role.EDITOR))])
@router.post("/admin/api/reindex", dependencies=[Depends(require_role(Role.EDITOR))])
async def api_trigger_sync(force: bool = False):
if idx_service.is_indexing():
return JSONResponse(status_code=409, content={"status": "error", "error": "Indexing in progress", "message": "Indexing is already in progress."})
Expand All @@ -386,10 +408,7 @@ async def api_trigger_sync(force: bool = False):
threading.Thread(target=idx_service.run_full_indexing, kwargs=kwargs, daemon=True).start()
return {"status": "success", "force": force, "message": "Background ingestion pipeline dispatched."}

@router.post(
"/admin/api/search/test",
dependencies=[Depends(require_role(Role.VIEWER))],
)
@router.post("/admin/api/search/test", dependencies=[Depends(require_role(Role.VIEWER))])
async def api_test_search(payload: SearchRequest):
try:
query = payload.query.strip() if payload.query else ""
Expand All @@ -398,44 +417,27 @@ async def api_test_search(payload: SearchRequest):

search_mode = (payload.search_mode or "hybrid").lower().strip()
hits = search_service.execute_hybrid_search(
query_text=query,
doc_type=payload.type,
repo=payload.repo,
language=payload.language,
category=payload.category,
tag=payload.tag,
limit=payload.limit or 6,
dense_weight=payload.dense_weight,
search_mode=search_mode
query_text=query, doc_type=payload.type, repo=payload.repo,
language=payload.language, category=payload.category, tag=payload.tag,
limit=payload.limit or 6, dense_weight=payload.dense_weight, search_mode=search_mode,
)
results = []
for h in hits:
results.append({
"score": round(getattr(h, "score", 0.0), 4),
"dense_score": getattr(h, "dense_score", None),
"sparse_score": getattr(h, "sparse_score", None),
"dense_rank": getattr(h, "dense_rank", None),
"sparse_rank": getattr(h, "sparse_rank", None),
"payload": getattr(h, "payload", {})
})
return {
"query": query,
"type": payload.type,
"search_mode": search_mode,
"dense_weight": payload.dense_weight,
"results": results
}
results = [{
"score": round(getattr(h, "score", 0.0), 4),
"dense_score": getattr(h, "dense_score", None),
"sparse_score": getattr(h, "sparse_score", None),
"dense_rank": getattr(h, "dense_rank", None),
"sparse_rank": getattr(h, "sparse_rank", None),
"payload": getattr(h, "payload", {}),
} for h in hits]
return {"query": query, "type": payload.type, "search_mode": search_mode, "dense_weight": payload.dense_weight, "results": results}
except ValueError as ve:
logger.warning(f"Vector search rejected due to dimension or configuration error: {ve}")
return JSONResponse(status_code=400, content={"status": "error", "error": str(ve), "message": str(ve)})
except Exception as e:
logger.error(f"Error testing search: {e}")
return JSONResponse(status_code=500, content={"error": "Failed to execute search test.", "message": str(e)})

@router.get(
"/admin/api/browse",
dependencies=[Depends(require_role(Role.VIEWER))],
)
@router.get("/admin/api/browse", dependencies=[Depends(require_role(Role.VIEWER))])
async def api_browse_dir(path: str = "/"):
cleaned = path.strip() if path else "/"
if "\x00" in cleaned or any(part in ("..", ".") for part in cleaned.replace("\\", "/").split("/") if part):
Expand All @@ -454,22 +456,19 @@ async def api_browse_dir(path: str = "/"):
resolved = root_dir
try:
entries = os.scandir(resolved)
dirs = []
files = []
dirs, files = [], []
for e in entries:
if e.name.startswith("."):
continue
if e.is_dir():
dirs.append({"name": e.name, "path": os.path.abspath(e.path)})
else:
files.append({"name": e.name, "path": os.path.abspath(e.path)})
item = {"name": e.name, "path": os.path.abspath(e.path)}
(dirs if e.is_dir() else files).append(item)
dirs.sort(key=lambda x: x["name"].lower())
files.sort(key=lambda x: x["name"].lower())
return {
"current_path": resolved,
"parent_path": os.path.dirname(resolved) if resolved != "/" else "",
"directories": dirs,
"files": files
"files": files,
}
except Exception as e:
logger.error(f"Error browsing dir {path}: {e}")
Expand Down
25 changes: 19 additions & 6 deletions app/services/vector_store/pgvector_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from sqlalchemy import Engine, text

from app.services.database.engine import get_db_engine, is_postgres
from app.services.embeddings import get_dense_dim, get_dense_embedding
from app.services.embeddings import get_dense_dim, get_dense_embedding, get_dense_embeddings_batch
from app.services.text_decoding import strip_nul
from app.services.vector_store.base import VectorDocument, VectorSearchResult, VectorStore
from app.services.vector_store.pgvector_queries import (
Expand Down Expand Up @@ -185,6 +185,7 @@ def upsert_documents(

is_sqlite = getattr(self.engine.dialect, "name", "") == "sqlite"
records = []
missing_embed: List[Tuple[int, str]] = []
for doc in documents:
if isinstance(doc, VectorDocument):
doc_id = str(doc.id)
Expand Down Expand Up @@ -226,7 +227,7 @@ def upsert_documents(
payload[pk] = payload[pk].replace("\x00", "").replace("\\", "/")

if dense_vec is None and text_content:
dense_vec = get_dense_embedding(text_content)
missing_embed.append((len(records), text_content))

if "content" not in payload and text_content:
payload["content"] = text_content
Expand Down Expand Up @@ -258,6 +259,14 @@ def upsert_documents(
"payload": json.dumps(payload),
}))

if missing_embed:
try:
batch_vecs = get_dense_embeddings_batch([t for _, t in missing_embed])
except Exception:
batch_vecs = [get_dense_embedding(t) for _, t in missing_embed]
for (rec_idx, _), vec in zip(missing_embed, batch_vecs):
records[rec_idx]["embedding"] = (json.dumps(vec) if is_sqlite else str(vec)) if vec is not None else None

upsert_sql = text(f"""
INSERT INTO {self.table_name} (
id, collection_name, repo, doc_type, path, rel_path, title, folder,
Expand Down Expand Up @@ -323,13 +332,17 @@ def _upsert_records_individually(self, conn, upsert_sql, batch: List[Dict[str, A
return failed

def delete_by_path(self, filepath: str) -> bool:
"""Purges vectors associated with a specific file path."""
norm_fp = (filepath or "").replace("\\", "/")
"""Purges vectors associated with a specific file or directory path."""
norm_fp = (filepath or "").replace("\\", "/").rstrip("/")
prefix = f"{norm_fp}/%" if norm_fp else ""
try:
with self.engine.connect() as conn:
conn.execute(
text(f"DELETE FROM {self.table_name} WHERE collection_name = :collection_name AND (path = :path OR path = :norm_path)"),
{"collection_name": self.collection_name, "path": filepath, "norm_path": norm_fp},
text(
f"DELETE FROM {self.table_name} WHERE collection_name = :collection_name "
"AND (path = :path OR path = :norm_path OR (:prefix != '' AND path LIKE :prefix))"
),
{"collection_name": self.collection_name, "path": filepath, "norm_path": norm_fp, "prefix": prefix},
)
conn.commit()
return True
Expand Down
Loading
Loading