From 754dcafa6b315123e8dbd1459fc8258576df99c3 Mon Sep 17 00:00:00 2001 From: Antigravity Agent Date: Wed, 18 Nov 2026 14:08:45 -0600 Subject: [PATCH] feat(mcp): make permalinks opt-in, add tool parameter aliases, and resolve custom C#/TS/Python routes --- REQUIREMENTS.md | 9 +- app/mcp/handlers/diagnostic_handlers.py | 7 +- app/mcp/handlers/file_handlers.py | 15 +- app/mcp/handlers/grep_handlers.py | 21 +- app/mcp/handlers/route_handlers.py | 55 ++- app/mcp/handlers/script_handlers.py | 29 +- app/mcp/handlers/search_handlers.py | 4 +- app/mcp/handlers/symbol_handlers.py | 26 +- app/mcp/mcp_server.py | 43 +- app/mcp/tools.py | 39 +- app/services/chunking/api_route_extractor.py | 133 ++++-- .../chunking/client_url_resolution.py | 2 +- app/services/chunking/csharp_client_calls.py | 30 +- .../chunking/csharp_route_attributes.py | 401 +++++++++++++++++ app/services/chunking/symbol_extractor.py | 2 +- app/services/chunking/text_chunker.py | 26 +- app/services/compact_results.py | 13 +- app/services/database/connection.py | 4 +- app/services/database/engine.py | 4 +- app/services/indexing/channel_pipeline.py | 4 +- app/services/indexing/db_write_channel.py | 114 +++++ app/services/indexing/git_syncer.py | 64 +-- app/services/indexing/local_syncer.py | 224 +++++----- app/services/script_sdk_types.py | 6 +- main.py | 6 + tests/test_code_mode_result_types.py | 7 +- tests/test_compact_grep_and_read_output.py | 9 +- ...test_db_write_channel_and_custom_routes.py | 414 ++++++++++++++++++ tests/test_mcp_real_e2e_workflows.py | 16 +- 29 files changed, 1455 insertions(+), 272 deletions(-) create mode 100644 app/services/chunking/csharp_route_attributes.py create mode 100644 app/services/indexing/db_write_channel.py create mode 100644 tests/test_db_write_channel_and_custom_routes.py diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index c75ddd9..5ba65ee 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -2,7 +2,7 @@ > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`. -**Test Verification Baseline:** **1352 Automated Tests** (988 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E). +**Test Verification Baseline:** **1357 Automated Tests** (993 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E). --- @@ -1022,6 +1022,13 @@ code results are enriched with signature/kind/ast_symbol_id from SQLite._ - `test_wait_for_db_retries_and_success` - `test_wait_for_db_exhausts_retries` +#### `tests/test_db_write_channel_and_custom_routes.py` (5 tests) +- `test_database_write_channel_serializes_concurrent_sqlite_writers` - _Verifies DatabaseWriteChannel queues concurrent producers through a single writer thread without SQLite lock errors or interleaving._ +- `test_general_csharp_custom_route_and_verb_attributes` - _Verifies custom C# route and HTTP verb attributes are resolved generically from class declarations and inheritance chains._ +- `test_typescript_class_field_arrow_methods_and_axios_api_prefix_matching` - _Verifies TS class field arrow functions are extracted as Class.method symbols and their relative axios calls match /api/v{n} routes._ +- `test_permalinks_are_opt_in_and_direct_tool_aliases_work` - _Verifies find_routes and search omit permalinks by default (until include_permalinks=True) and direct tool calls accept common parameter aliases._ +- `test_code_mode_disabled_omits_execute_script_and_sse_post_routes_to_mcp` - _Verifies CODE_MODE_ENABLED=false omits execute_script from get_tool_definitions() and McpPingMiddleware routes POST /sse to /mcp._ + #### `tests/test_delete_local_path.py` (4 tests) - `test_delete_local_path_cleans_all_relational_tables_and_vectors` - _Verify DELETE /admin/api/paths/{id} removes rows from all 6 file-scoped tables and the vector store._ - `test_delete_local_path_with_shared_repo_only_purges_deleted_path_prefix` - _Verify deleting one local path that shares a repo name ('local') removes only its own files and vectors._ diff --git a/app/mcp/handlers/diagnostic_handlers.py b/app/mcp/handlers/diagnostic_handlers.py index 823f046..7c2bfb1 100644 --- a/app/mcp/handlers/diagnostic_handlers.py +++ b/app/mcp/handlers/diagnostic_handlers.py @@ -16,17 +16,20 @@ def _get_tools_attr(name, default): async def handle_diagnose_issue( - issue_text: Annotated[str, Field(description="Raw stack trace, exception log, HTTP failure payload, or bug symptom report across Python, TypeScript/JS/TSX, C#, COBOL, Go, Rust, Java, and C/C++.")], + issue_text: Annotated[Optional[str], Field(description="Raw stack trace, exception log, HTTP failure payload, or bug symptom report across Python, TypeScript/JS/TSX, C#, COBOL, Go, Rust, Java, and C/C++.")] = None, repo: Annotated[Optional[str], Field(description="Optional repository name to scope stack frame, route, and semantic candidate resolution.")] = None, context_lines: Annotated[int, Field(description="Surrounding source lines returned per resolved stack frame, clamped to 1..15 (default 5).")] = 5, limit: Annotated[int, Field(description="Maximum semantic code candidates returned alongside resolved stack frames, clamped to 1..20 (default 5).")] = 5, detail: Annotated[bool, Field(description="Set True to add full method bodies to call_chain steps and code, permalinks, callers, and callees to semantic_candidates (default False keeps the answer compact).")] = False, + query: Annotated[Optional[str], Field(description="Alias for `issue_text`. `issue_text` wins when both are given.")] = None, + error: Annotated[Optional[str], Field(description="Alias for `issue_text`. `issue_text` wins when both are given.")] = None, ) -> str: """Forensic triage: a ranked call chain with evidence lines, plus stack frames, routes, symbols, and semantic candidates.""" try: enforce_tool_permission(Role.VIEWER) - text = (issue_text or "").strip() + raw_text = issue_text if (issue_text is not None and issue_text.strip()) else (query or error or "") + text = raw_text.strip() if not text: return json.dumps({ "call_chain": {"entry": None, "terms": [], "steps": [], "other_hits": [], "nodes_read": 0}, diff --git a/app/mcp/handlers/file_handlers.py b/app/mcp/handlers/file_handlers.py index 7c2eaa9..e459e19 100644 --- a/app/mcp/handlers/file_handlers.py +++ b/app/mcp/handlers/file_handlers.py @@ -144,26 +144,29 @@ def _query_outline_rows(target_clean: str): async def handle_read_file( - path: Annotated[str, Field(description="Relative or absolute file path within an indexed repository, watched directory, or local storage (supports Python, TypeScript/JS/TSX, C#, COBOL, Go, Rust, Java, C/C++, Markdown, and config files).")], + path: Annotated[Optional[str], Field(description="Relative or absolute file path within an indexed repository, watched directory, or local storage (supports Python, TypeScript/JS/TSX, C#, COBOL, Go, Rust, Java, C/C++, Markdown, and config files).")] = None, repo: Annotated[Optional[str], Field(description="Optional repository name or 'local_storage' namespace. Recommended for relative paths or reconstructing ephemeral Git files from vector chunks.")] = None, start_line: Annotated[Optional[int], Field(description="1-based inclusive starting line number (default 1). Pass next_start_line from a prior call to paginate long files.")] = None, end_line: Annotated[Optional[int], Field(description="1-based inclusive ending line number (default end of file, bounded by read_file_max_lines=2000 per call).")] = None, outline_only: Annotated[bool, Field(description="Set True to return the file's AST symbol outline (classes, methods, functions, signatures, line ranges) instead of full source text.")] = False, limit: Annotated[int, Field(description="Maximum AST symbols returned per page when outline_only=True, clamped to 1..500 (default 150).")] = 150, offset: Annotated[int, Field(description="0-based AST symbol offset when outline_only=True (default 0). Advance via next_offset when has_more is True.")] = 0, + filepath: Annotated[Optional[str], Field(description="Alias for `path`. `path` wins when both are given.")] = None, + file: Annotated[Optional[str], Field(description="Alias for `path`. `path` wins when both are given.")] = None, ) -> str: """Read bounded 1-based line slices or inspect the AST symbol outline (outline_only=True) for an indexed file.""" + eff_path = path if (path is not None and path.strip()) else (filepath or file or "") try: enforce_tool_permission(Role.VIEWER) if outline_only: - return json.dumps(_read_file_outline_payload(path=path, repo=repo, limit=limit, offset=offset)) + return json.dumps(_read_file_outline_payload(path=eff_path, repo=repo, limit=limit, offset=offset)) reader_fn = _get_tools_attr("get_file_reader_service", get_file_reader_service) reader = reader_fn() result = reader.read_file( - path=path, + path=eff_path, repo=repo, start_line=start_line, end_line=end_line, @@ -182,7 +185,7 @@ async def handle_read_file( next_start_line = (e_line + 1) if (has_more and e_line is not None) else None payload: Dict[str, Any] = { - "file": _strip_repo_prefix(result.get("filepath") or path), + "file": _strip_repo_prefix(result.get("filepath") or eff_path), "lines": [s_line, e_line], "total_lines": t_lines, } @@ -196,14 +199,14 @@ async def handle_read_file( return json.dumps(payload, separators=(",", ":")) except ForbiddenError as fe: - logger.warning(f"Forbidden error in handle_read_file ({path}): {fe}") + logger.warning(f"Forbidden error in handle_read_file ({eff_path}): {fe}") return json.dumps({"error": f"Forbidden: {str(fe)}", "forbidden": True}) except FileNotFoundError as fne: return json.dumps({"error": f"File not found: {str(fne)}"}) except ValueError as ve: return json.dumps({"error": str(ve)}) except Exception as e: - logger.error(f"Error reading file '{path}': {e}") + logger.error(f"Error reading file '{eff_path}': {e}") return json.dumps({"error": f"Error reading file: {str(e)}"}) diff --git a/app/mcp/handlers/grep_handlers.py b/app/mcp/handlers/grep_handlers.py index 44b7bfb..00ba18b 100644 --- a/app/mcp/handlers/grep_handlers.py +++ b/app/mcp/handlers/grep_handlers.py @@ -69,7 +69,7 @@ def _sanitize_match_lines(match: Dict[str, Any]) -> Dict[str, Any]: async def handle_grep_code( - pattern: Annotated[str, Field(description="Exact literal string or regular expression to match across source files (Python, TypeScript/JS/TSX, C#, COBOL, Go, Rust, Java, C/C++, Markdown, configs).")], + pattern: Annotated[Optional[str], Field(description="Exact literal string or regular expression to match across source files (Python, TypeScript/JS/TSX, C#, COBOL, Go, Rust, Java, C/C++, Markdown, configs).")] = None, repo: Annotated[Optional[str], Field(description="Optional repository name or 'local_storage' to scope search. Omit to scan all indexed sources.")] = None, path_glob: Annotated[Optional[str], Field(description="Optional path filter. With wildcards it is an fnmatch glob ('*.cs', 'app/services/*'); without wildcards it is a file or folder ('Services_V2', 'src/app/main.py').")] = None, is_regex: Annotated[bool, Field(description="Set True to evaluate pattern as a Python regular expression; False (default) matches literal text. If a literal search finds nothing and the pattern looks like a regex (for example 'A|B'), the search runs again as a regex and pattern_mode says so.")] = False, @@ -78,10 +78,15 @@ async def handle_grep_code( limit: Annotated[int, Field(description="Maximum matched lines per page, clamped to 1..100 (default 30).")] = 30, offset: Annotated[int, Field(description="0-based match offset for pagination (default 0). Advance by next_offset when has_more is True.")] = 0, path: Annotated[Optional[str], Field(description="Alias for path_glob (a file, folder, or glob). path_glob wins when both are given.")] = None, + include_permalinks: Annotated[bool, Field(description="Include file-level Git web permalinks on matched file groups (default False keeps output compact).")] = False, + query: Annotated[Optional[str], Field(description="Alias for `pattern`. `pattern` wins when both are given.")] = None, + filepath: Annotated[Optional[str], Field(description="Alias for `path_glob` (a file, folder, or glob). `path_glob` wins when both are given.")] = None, + file: Annotated[Optional[str], Field(description="Alias for `path_glob` (a file, folder, or glob). `path_glob` wins when both are given.")] = None, ) -> str: """Deterministic literal and regex pattern search across indexed repositories with bounded line lengths and pagination.""" try: enforce_tool_permission(Role.VIEWER) + eff_pattern = pattern if (pattern is not None and pattern != "") else (query if query is not None else "") reader_fn = _resolve_reader_factory() reader = reader_fn() @@ -89,12 +94,12 @@ async def handle_grep_code( safe_offset = max(0, int(offset if offset is not None else 0)) safe_context = max(0, min(int(context_lines if context_lines is not None else 2), 10)) needed = safe_offset + safe_limit + 1 - path_filter = path_glob or path + path_filter = path_glob or path or filepath or file def _search(as_regex: bool) -> List[Dict[str, Any]]: if needed <= 100: raw_result = reader.grep_code( - pattern=pattern, + pattern=eff_pattern, repo=repo, path_glob=path_filter, is_regex=as_regex, @@ -104,7 +109,7 @@ def _search(as_regex: bool) -> List[Dict[str, Any]]: ) return raw_result.get("matches", []) raw_result = execute_code_grep( - pattern=pattern, + pattern=eff_pattern, storage_root=reader.storage_root, indexed_paths=reader._get_authorized_indexed_paths(), persistent_repos=reader._get_authorized_persistent_repos(), @@ -122,7 +127,7 @@ def _search(as_regex: bool) -> List[Dict[str, Any]]: pattern_mode = "regex" if is_regex else "literal" all_matches = _search(is_regex) - if not all_matches and not is_regex and looks_like_regex(pattern): + if not all_matches and not is_regex and looks_like_regex(eff_pattern): all_matches = _search(True) pattern_mode = "regex (the literal text had no matches, so the pattern ran as a regex)" @@ -131,7 +136,11 @@ def _search(as_regex: bool) -> List[Dict[str, Any]]: sanitized_matches = [_sanitize_match_lines(m) for m in page_slice] next_offset = (safe_offset + safe_limit) if has_more else None - shared_repo, files = group_grep_matches(sanitized_matches, block_separators=safe_context > 0) + shared_repo, files = group_grep_matches( + sanitized_matches, + block_separators=safe_context > 0, + include_permalinks=bool(include_permalinks), + ) payload: Dict[str, Any] = {"total_matches": len(all_matches)} if pattern_mode != ("regex" if is_regex else "literal"): payload["pattern_mode"] = pattern_mode diff --git a/app/mcp/handlers/route_handlers.py b/app/mcp/handlers/route_handlers.py index a3c1210..4aa8c26 100644 --- a/app/mcp/handlers/route_handlers.py +++ b/app/mcp/handlers/route_handlers.py @@ -46,7 +46,14 @@ def belongs(call_url: str, call_method: str) -> bool: return belongs -def _query_route_handlers(conn, local_roots: LocalSourceRoots, clean_q: Optional[str], method_clean: Optional[str], repo_clean: Optional[str]) -> List[Dict[str, Any]]: +def _query_route_handlers( + conn, + local_roots: LocalSourceRoots, + clean_q: Optional[str], + method_clean: Optional[str], + repo_clean: Optional[str], + include_permalinks: bool = False, +) -> List[Dict[str, Any]]: sql = ( "SELECT r.id, r.repo, r.filepath, r.framework, r.http_method, r.path_pattern, " "r.handler_symbol, r.start_line, r.end_line, g.url as git_url, g.commit_sha, g.provider " @@ -75,10 +82,7 @@ def _query_route_handlers(conn, local_roots: LocalSourceRoots, clean_q: Optional if dedup_key in seen_routes: continue seen_routes.add(dedup_key) - permalink = format_git_permalink( - r["git_url"], r["commit_sha"], rel_path, r["start_line"], r["end_line"], provider=r["provider"] - ) - all_routes.append({ + item: Dict[str, Any] = { "repo": r["repo"], "method": r["http_method"], "path": r["path_pattern"], @@ -86,12 +90,23 @@ def _query_route_handlers(conn, local_roots: LocalSourceRoots, clean_q: Optional "handler": r["handler_symbol"], "file": rel_path, "lines": [r["start_line"], r["end_line"]], - "permalink": permalink, - }) + } + if include_permalinks: + item["permalink"] = format_git_permalink( + r["git_url"], r["commit_sha"], rel_path, r["start_line"], r["end_line"], provider=r["provider"] + ) + all_routes.append(item) return all_routes -def _query_route_callers(conn, local_roots: LocalSourceRoots, clean_q: Optional[str], method_clean: Optional[str], repo_clean: Optional[str]) -> List[Dict[str, Any]]: +def _query_route_callers( + conn, + local_roots: LocalSourceRoots, + clean_q: Optional[str], + method_clean: Optional[str], + repo_clean: Optional[str], + include_permalinks: bool = False, +) -> List[Dict[str, Any]]: sql = ( "SELECT c.id, c.repo, c.filepath, c.http_method, c.url_pattern, c.caller_symbol, " "c.line_number, g.url as git_url, g.commit_sha, g.provider " @@ -135,18 +150,19 @@ def _query_route_callers(conn, local_roots: LocalSourceRoots, clean_q: Optional[ continue seen_callers.add(dedup_key) - permalink = format_git_permalink( - c["git_url"], c["commit_sha"], rel_path, c["line_number"], c["line_number"], provider=c["provider"] - ) - all_callers.append({ + item: Dict[str, Any] = { "repo": c["repo"], "method": c["http_method"], "url": url_pat, "caller": caller_sym, "file": rel_path, "line": c["line_number"], - "permalink": permalink, - }) + } + if include_permalinks: + item["permalink"] = format_git_permalink( + c["git_url"], c["commit_sha"], rel_path, c["line_number"], c["line_number"], provider=c["provider"] + ) + all_callers.append(item) return all_callers @@ -157,6 +173,10 @@ async def handle_find_routes( repo: Annotated[Optional[str], Field(description="Optional repository name filter.")] = None, limit: Annotated[int, Field(description="Maximum results per section per page, clamped to 1..200 (default 50).")] = 50, offset: Annotated[int, Field(description="0-based offset for pagination (default 0). Advance via next_offset when has_more is True.")] = 0, + include_permalinks: Annotated[bool, Field(description="Include Git web permalinks on returned routes and callers (default False keeps answers compact).")] = False, + path: Annotated[Optional[str], Field(description="Alias for `query` (route path or URL pattern filter). `query` wins when both are given.")] = None, + path_query: Annotated[Optional[str], Field(description="Alias for `query` (route path or URL pattern filter). `query` wins when both are given.")] = None, + route: Annotated[Optional[str], Field(description="Alias for `query` (route path or URL pattern filter). `query` wins when both are given.")] = None, ) -> str: """Discover HTTP/API route endpoints and/or client HTTP callers across indexed repositories with bounded pagination.""" try: @@ -167,7 +187,8 @@ async def handle_find_routes( if clean_kind not in ("handlers", "callers", "both"): return json.dumps({"error": f"Unsupported kind '{kind}'. Valid kinds are 'handlers', 'callers', or 'both'."}) - clean_q = query.strip() if query and query.strip() else None + raw_q = query if (query and query.strip()) else (path or path_query or route or "") + clean_q = raw_q.strip() if raw_q and raw_q.strip() else None method_clean = method.strip().upper() if method and method.strip() else None repo_clean = repo.strip() if repo and repo.strip() else None safe_limit = max(1, min(int(limit if limit is not None else 50), 200)) @@ -176,12 +197,12 @@ async def handle_find_routes( with _get_tools_attr("get_db_connection", get_db_connection)() as conn: local_roots = LocalSourceRoots.load(conn) all_routes = ( - _query_route_handlers(conn, local_roots, clean_q, method_clean, repo_clean) + _query_route_handlers(conn, local_roots, clean_q, method_clean, repo_clean, include_permalinks=bool(include_permalinks)) if clean_kind in ("handlers", "both") else [] ) all_callers = ( - _query_route_callers(conn, local_roots, clean_q, method_clean, repo_clean) + _query_route_callers(conn, local_roots, clean_q, method_clean, repo_clean, include_permalinks=bool(include_permalinks)) if clean_kind in ("callers", "both") else [] ) diff --git a/app/mcp/handlers/script_handlers.py b/app/mcp/handlers/script_handlers.py index f437846..403c871 100644 --- a/app/mcp/handlers/script_handlers.py +++ b/app/mcp/handlers/script_handlers.py @@ -57,6 +57,17 @@ def _tool_handlers() -> Dict[str, Any]: return {name: handler for name, handler, _desc in get_tool_definitions() if name != TOOL_NAME} +_SCRIPT_ARG_ALIASES: Dict[str, Dict[str, str]] = { + "find_routes": {"path": "query", "path_query": "query", "route": "query", "top_k": "limit"}, + "find_symbol": {"query": "symbol", "top_k": "limit"}, + "trace_path": {"name": "symbol", "query": "symbol", "top_k": "limit"}, + "read_file": {"filepath": "path", "file": "path"}, + "grep_code": {"query": "pattern", "filepath": "path_glob", "file": "path_glob", "top_k": "limit"}, + "diagnose_issue": {"query": "issue_text", "error": "issue_text", "top_k": "limit"}, + "search": {"top_k": "limit"}, +} + + async def call_tool_from_script(name: str, tool_args: Dict[str, Any]) -> Any: """Runs one tool for a script: checks the name and arguments, calls the handler, and unboxes the JSON result.""" if name == TOOL_NAME: @@ -65,11 +76,27 @@ async def call_tool_from_script(name: str, tool_args: Dict[str, Any]) -> Any: handler = handlers.get(name) if handler is None: raise ScriptToolError(f"Unknown tool '{name}'. Tools: {', '.join(sorted(handlers))}") + alias_map = _SCRIPT_ARG_ALIASES.get(name) + if alias_map and isinstance(tool_args, dict): + normalized_args: Dict[str, Any] = {} + for k, v in tool_args.items(): + target = alias_map.get(k) + if target and target not in tool_args and target not in normalized_args: + normalized_args[target] = v + else: + normalized_args[k] = v + tool_args = normalized_args params = inspect.signature(handler).parameters unknown = sorted(set(tool_args) - set(params)) if unknown: raise ScriptToolError(f"Unknown arguments for {name}: {', '.join(unknown)}. Valid: {', '.join(params)}") - missing = [p for p, spec in params.items() if spec.default is inspect.Parameter.empty and p not in tool_args] + from app.mcp.tools import _SCHEMA_REQUIRED_OVERRIDES + + schema_req = _SCHEMA_REQUIRED_OVERRIDES.get(name, []) if name != "find_symbol" else [] + missing = [ + p for p in params + if (params[p].default is inspect.Parameter.empty or p in schema_req) and p not in tool_args + ] if missing: raise ScriptToolError(f"Missing arguments for {name}: {', '.join(missing)}") raw = await handler(**tool_args) diff --git a/app/mcp/handlers/search_handlers.py b/app/mcp/handlers/search_handlers.py index 071089b..57abb45 100644 --- a/app/mcp/handlers/search_handlers.py +++ b/app/mcp/handlers/search_handlers.py @@ -50,6 +50,7 @@ async def handle_search( limit: Annotated[int, Field(description="Maximum chunks per page, clamped to 1..25 (default 5).")] = 5, offset: Annotated[int, Field(description="0-based result offset for pagination (default 0). Advance using next_offset when has_more is True.")] = 0, max_snippet_chars: Annotated[int, Field(description="Maximum characters per returned snippet, clamped to 200..6000 (default 2000).")] = 2000, + include_permalinks: Annotated[bool, Field(description="Include Git web permalinks on search hits (default False keeps results compact).")] = False, ) -> str: """Hybrid semantic + BM25 search across AST-chunked source code and documentation with bounded snippets and pagination.""" clean_query = query.strip() if query else "" @@ -113,10 +114,11 @@ async def handle_search( "score": score_val, "dense_score": d_val, "sparse_score": s_val, - "permalink": permalink, "content_truncated": was_truncated, "content": snippet_text, } + if include_permalinks: + item["permalink"] = permalink results.append(item) return json.dumps({ diff --git a/app/mcp/handlers/symbol_handlers.py b/app/mcp/handlers/symbol_handlers.py index efb53d9..5dbf25a 100644 --- a/app/mcp/handlers/symbol_handlers.py +++ b/app/mcp/handlers/symbol_handlers.py @@ -77,11 +77,12 @@ async def handle_find_symbol( limit: Annotated[int, Field(description="Maximum AST symbols per page, clamped to 1..100 (default 30).")] = 30, offset: Annotated[int, Field(description="0-based symbol offset for pagination (default 0). Advance via next_offset when has_more is True.")] = 0, name: Annotated[Optional[str], Field(description="Deprecated alias of `symbol`, kept for older callers. Use `symbol`; it wins when both are given.")] = None, + query: Annotated[Optional[str], Field(description="Alias for `symbol`. `symbol` wins when both are given.")] = None, ) -> str: """Instant deterministic AST symbol lookup (functions, classes, methods, interfaces) without token-heavy full-file scans.""" - # `symbol` is optional here only so that a call with the deprecated `name` alias still validates. + # `symbol` is optional here only so that a call with the deprecated `name` or `query` alias still validates. # The published schema marks `symbol` as required (see app/mcp/tools.py `_SCHEMA_REQUIRED_OVERRIDES`). - sym_name = (symbol or name or "").strip() + sym_name = (symbol or name or query or "").strip() if not sym_name: return json.dumps({"error": "symbol name cannot be empty: pass 'symbol' (the alias 'name' is also accepted)."}) @@ -93,24 +94,24 @@ async def handle_find_symbol( safe_offset = max(0, int(offset if offset is not None else 0)) with _get_tools_attr("get_db_connection", get_db_connection)() as conn: - query = ( + sql_query = ( "SELECT repo, filepath, name, full_symbol, kind, start_line, end_line, signature, language " "FROM ast_symbols WHERE " ) params: List[Any] = [] if exact: - query += "(name = ? OR full_symbol = ?)" + sql_query += "(name = ? OR full_symbol = ?)" params.extend([sym_name, sym_name]) else: - query += "(name LIKE ? OR full_symbol LIKE ?)" + sql_query += "(name LIKE ? OR full_symbol LIKE ?)" params.extend([f"%{sym_name}%", f"%{sym_name}%"]) if repo: - query += " AND repo = ?" + sql_query += " AND repo = ?" params.append(repo) - query += " ORDER BY repo ASC, filepath ASC, start_line ASC, name ASC" - rows = conn.execute(query, params).fetchall() + sql_query += " ORDER BY repo ASC, filepath ASC, start_line ASC, name ASC" + rows = conn.execute(sql_query, params).fetchall() local_roots = LocalSourceRoots.load(conn) all_symbols = [] @@ -157,7 +158,7 @@ async def handle_find_symbol( async def handle_trace_path( - symbol: Annotated[str, Field(description="Target function, method, or qualified class member (e.g. 'OrderService.create_order') across Python, TypeScript/JS/TSX, C#, COBOL, Go, Rust, Java, and C/C++.")], + symbol: Annotated[Optional[str], Field(description="Target function, method, or qualified class member (e.g. 'OrderService.create_order') across Python, TypeScript/JS/TSX, C#, COBOL, Go, Rust, Java, and C/C++.")] = None, repo: Annotated[Optional[str], Field(description="Optional repository name to scope intra-repo AST traversal (cross-repo HTTP boundaries still resolve automatically).")] = None, direction: Annotated[str, Field(description="Traversal direction: 'callers' (inbound), 'callees' (outbound), or 'both' (default 'both').")] = "both", depth: Annotated[int, Field(description="Maximum BFS hop depth, clamped to 1..5 (default 2).")] = 2, @@ -165,8 +166,11 @@ async def handle_trace_path( offset: Annotated[int, Field(description="0-based relationship offset per direction for pagination (default 0). Advance via next_offset when has_more is True.")] = 0, include_permalinks: Annotated[bool, Field(description="Include a Git permalink on every caller/callee node (default False keeps answers compact; the target always has its permalink).")] = False, include_unresolved: Annotated[bool, Field(description="Show library calls (no indexed symbol) and calls into test code as tree nodes. Default False lists unresolved call names in a compact 'unresolved' list per node and drops test code for a production target.")] = False, + name: Annotated[Optional[str], Field(description="Alias for `symbol`. `symbol` wins when both are given.")] = None, + query: Annotated[Optional[str], Field(description="Alias for `symbol`. `symbol` wins when both are given.")] = None, ) -> str: """Traverse AST call graphs, imports, inheritance, and cross-repo HTTP route/client boundaries via bounded BFS.""" + eff_symbol = (symbol if (symbol is not None and symbol.strip()) else (name or query or "")) try: from app.services.auth import enforce_tool_permission, Role, ForbiddenError enforce_tool_permission(Role.VIEWER) @@ -177,7 +181,7 @@ async def handle_trace_path( trace_fn = _get_tools_attr("trace_symbol_path", trace_symbol_path) raw_res = trace_fn( - symbol=symbol, + symbol=eff_symbol, repo=repo, direction=direction, depth=depth, @@ -198,7 +202,7 @@ async def handle_trace_path( return json.dumps({"error": stripped}) else: return json.dumps({ - "symbol": (symbol or "").strip(), + "symbol": (eff_symbol or "").strip(), "repo": repo, "direction": direction, "depth": depth, diff --git a/app/mcp/mcp_server.py b/app/mcp/mcp_server.py index 3ed8634..b60cf03 100644 --- a/app/mcp/mcp_server.py +++ b/app/mcp/mcp_server.py @@ -7,24 +7,35 @@ from mcp.server.fastmcp import FastMCP from mcp.server.transport_security import TransportSecuritySettings +from app.mcp.handlers.script_handlers import code_mode_enabled + MCP_SERVER_VERSION = "2.27.4" # x-release-please-version -MCP_SERVER_INSTRUCTIONS = ( - "ContextCortex: code and documentation search across indexed repositories " - "(AST symbols, call graphs, HTTP routes, hybrid search).\n" - "Select the first tool for the task:\n" - "- Bug report, stack trace, error log, or failing HTTP route: diagnose_issue. " - "One call gives the call chain from the route or frame to the likely fault, " - "with call-site lines and evidence. Then use read_file only for the lines " - "that you must confirm.\n" - "- Known symbol name: find_symbol. Exact text or regex: grep_code. " - "Concept or documentation: search.\n" - "- Callers or callees of a known symbol: trace_path. " - "HTTP handlers and client calls: find_routes.\n" - "- New codebase: get_architecture. Repository names: list_repositories.\n" - "- Several lookups in one round trip: execute_script.\n" - "Pass repo= when you know the repository." -) + +def build_mcp_server_instructions(include_code_mode: bool | None = None) -> str: + """Builds MCP server instructions, including `execute_script` guidance only when Code Mode is enabled.""" + if include_code_mode is None: + include_code_mode = code_mode_enabled() + code_mode_line = "- Several lookups in one round trip: execute_script.\n" if include_code_mode else "" + return ( + "ContextCortex: code and documentation search across indexed repositories " + "(AST symbols, call graphs, HTTP routes, hybrid search).\n" + "Select the first tool for the task:\n" + "- Bug report, stack trace, error log, or failing HTTP route: diagnose_issue. " + "One call gives the call chain from the route or frame to the likely fault, " + "with call-site lines and evidence. Then use read_file only for the lines " + "that you must confirm.\n" + "- Known symbol name: find_symbol. Exact text or regex: grep_code. " + "Concept or documentation: search.\n" + "- Callers or callees of a known symbol: trace_path. " + "HTTP handlers and client calls: find_routes.\n" + "- New codebase: get_architecture. Repository names: list_repositories.\n" + f"{code_mode_line}" + "Pass repo= when you know the repository." + ) + + +MCP_SERVER_INSTRUCTIONS = build_mcp_server_instructions() mcp_server = FastMCP( "ContextCortex", diff --git a/app/mcp/tools.py b/app/mcp/tools.py index 56ad411..aef61ff 100644 --- a/app/mcp/tools.py +++ b/app/mcp/tools.py @@ -147,9 +147,15 @@ def write_tools_enabled() -> bool: def get_tool_definitions() -> List[Tuple[str, Any, str]]: """Returns active MCP tool definitions (10 default read/search/code-mode tools, plus `manage` if enabled).""" + from app.mcp.handlers.script_handlers import code_mode_enabled + + defs = [ + item for item in _DEFAULT_TOOL_DEFINITIONS + if item[0] != "execute_script" or code_mode_enabled() + ] if write_tools_enabled(): - return list(_DEFAULT_TOOL_DEFINITIONS) + list(_WRITE_TOOL_DEFINITIONS) - return list(_DEFAULT_TOOL_DEFINITIONS) + return defs + list(_WRITE_TOOL_DEFINITIONS) + return defs _RESOURCE_TEMPLATE_DEFINITIONS = [ @@ -187,18 +193,37 @@ def get_tool_definitions() -> List[Tuple[str, Any, str]]: # Parameters that the published inputSchema marks as required, although the handler signature keeps them optional -# so that a deprecated alias (find_symbol `name`) still validates. FastMCP validates calls with the handler's -# pydantic model, not with this schema, so the alias keeps working on the server. -_SCHEMA_REQUIRED_OVERRIDES = {"find_symbol": ["symbol"]} +# so that deprecated or convenience aliases (e.g. find_symbol `name`/`query`, read_file `filepath`) still validate. +# FastMCP validates calls with the handler's pydantic model, not with this schema, so aliases keep working on the server. +_SCHEMA_REQUIRED_OVERRIDES = { + "find_symbol": ["symbol"], + "trace_path": ["symbol"], + "read_file": ["path"], + "grep_code": ["pattern"], + "diagnose_issue": ["issue_text"], +} + +# Convenience aliases accepted by handler pydantic models on direct calls, but omitted from the published +# JSON Schema / tools.help() so tool schemas stay compact (`find_symbol.name` is intentionally kept published). +_SCHEMA_HIDDEN_ALIAS_PROPERTIES = { + "find_routes": ("path", "path_query", "route"), + "find_symbol": ("query",), + "trace_path": ("name", "query"), + "read_file": ("filepath", "file"), + "grep_code": ("query", "filepath", "file"), + "diagnose_issue": ("query", "error"), +} def _apply_schema_required_overrides(server) -> None: for tool in server._tool_manager.list_tools(): + schema = tool.parameters + props = schema.get("properties", {}) + for hidden_prop in _SCHEMA_HIDDEN_ALIAS_PROPERTIES.get(tool.name, ()): + props.pop(hidden_prop, None) required = _SCHEMA_REQUIRED_OVERRIDES.get(tool.name) if not required: continue - schema = tool.parameters - props = schema.get("properties", {}) for param in required: prop = props.get(param) if prop is None: diff --git a/app/services/chunking/api_route_extractor.py b/app/services/chunking/api_route_extractor.py index 4876436..9ca5db8 100644 --- a/app/services/chunking/api_route_extractor.py +++ b/app/services/chunking/api_route_extractor.py @@ -55,23 +55,43 @@ def _segment_regex(segment: str) -> re.Pattern: return re.compile('^' + ''.join('[^/]+' if p.startswith('{') else re.escape(p) for p in parts) + '$', re.IGNORECASE) +_API_PREFIX_SEGMENTS = re.compile(r"^(?:api|v\d+|v\{[^{}]+\})$", re.IGNORECASE) + + def _match_base_relative_call(norm_route: str, norm_call: str) -> bool: """ A client URL that starts with a base it could not resolve (`{baseUrl}/api/v2/items/{id}`, `{API_HOST}/v2/items`) matches when the rest of its path matches the end of the route, segment by segment, with at least two literal segments (so `{base}/{id}` alone matches nothing). + Also matches an axios/SDK client path whose baseURL already includes `/api` or `/api/v{n}` (e.g. `/movie/{id}` + matching `/api/v3/movie/{id}`) when the route's leading segments are `api`/`v*` and the remaining suffix has at + least one literal segment. """ call_segs = [s for s in norm_call.split("?", 1)[0].strip("/").split("/") if s] + route_segs = [s for s in norm_route.strip("/").split("/") if s] lead = 0 while lead < len(call_segs) and _PARAM_SEGMENT.match(call_segs[lead]): lead += 1 - tail = call_segs[lead:] - if lead == 0 or sum(1 for s in tail if not _PARAM_SEGMENT.match(s)) < _MIN_LITERAL_SEGMENTS_FOR_SUFFIX: - return False - route_segs = [s for s in norm_route.strip("/").split("/") if s] - if len(tail) > len(route_segs): - return False - for call_seg, route_seg in zip(tail, route_segs[len(route_segs) - len(tail):]): + if lead > 0: + tail = call_segs[lead:] + if sum(1 for s in tail if not _PARAM_SEGMENT.match(s)) < _MIN_LITERAL_SEGMENTS_FOR_SUFFIX: + return False + if len(tail) > len(route_segs): + return False + target_route_segs = route_segs[len(route_segs) - len(tail):] + else: + # Check if route starts with /api(/vN) and call_segs matches the remaining route segments + api_lead = 0 + while api_lead < len(route_segs) and _API_PREFIX_SEGMENTS.match(route_segs[api_lead]): + api_lead += 1 + if api_lead == 0 or len(call_segs) != len(route_segs) - api_lead: + return False + if sum(1 for s in call_segs if not _PARAM_SEGMENT.match(s)) < 1: + return False + tail = call_segs + target_route_segs = route_segs[api_lead:] + + for call_seg, route_seg in zip(tail, target_route_segs): if _PARAM_SEGMENT.match(route_seg): continue # A client parameter ({eid}) only fills a route parameter, never a literal segment such as `publish`. @@ -201,6 +221,20 @@ def extract_api_routes_and_calls( )) continue + # Starlette / custom route decorator: @secured_expose(route="/api", methods=["GET", "POST"]) + m_py_route_dec = re.search(r'@\w+\s*\([^()]*\broute\s*=\s*["\']([^"\']+)["\']', window_text) if line.lstrip().startswith("@") else None + if m_py_route_dec: + path = normalize_path_pattern(m_py_route_dec.group(1)) + m_methods = re.search(r'\bmethods\s*=\s*[\[\(]([^\]\)]+)[\]\)]', window_text) + methods = [m.strip(" \"'").upper() for m in m_methods.group(1).split(",") if m.strip(" \"'")] if m_methods else ["GET", "POST"] + handler = find_enclosing_symbol(i + 1, symbols) or find_enclosing_symbol(i + 2, symbols) or find_enclosing_symbol(i, symbols) + for m in methods: + routes.append(ApiRouteRecord( + repo=repo, filepath=filepath, framework="starlette", + http_method=m, path_pattern=path, handler_symbol=handler, start_line=i, end_line=i + )) + continue + # Django m_django = re.search(r'(?:path|re_path)\s*\(\s*["\']([^"\']+)["\']\s*,\s*([\w\.]+)', line) if m_django: @@ -214,18 +248,28 @@ def extract_api_routes_and_calls( # JavaScript / TypeScript / TSX elif language in ("javascript", "typescript", "tsx"): - # Express.js + express_routers = {"app", "router", "server", "express"} + for m_er in re.finditer(r'\b(?:const|let|var)\s+(\w+)\s*=\s*(?:express\.)?Router\s*\(', code): + express_routers.add(m_er.group(1)) + # Express.js (including custom Router() vars and TS generic type parameters across lines) for i, line in enumerate(lines, start=1): - m_express = re.search(r'(?:app|router|server|express)\.(get|post|put|delete|patch|all|options|head)\s*\(\s*["\'`]([^"\'`]+)["\'`]', line, re.IGNORECASE) - if m_express: - method = m_express.group(1).upper() - path = normalize_path_pattern(m_express.group(2)) - handler = find_enclosing_symbol(i, symbols) - routes.append(ApiRouteRecord( - repo=repo, filepath=filepath, framework="express", - http_method=method, path_pattern=path, handler_symbol=handler, - start_line=i, end_line=i - )) + m_exp_head = re.search(r'\b(\w+)\.(get|post|put|delete|patch|all|options|head)\s*(?:<|\()', line, re.IGNORECASE) + if m_exp_head and (m_exp_head.group(1) in express_routers or m_exp_head.group(1).lower().endswith(("router", "routes"))): + win = " ".join(l.strip() for l in lines[i - 1:min(len(lines), i + 5)]) + m_express = re.search( + rf'\b{re.escape(m_exp_head.group(1))}\.(get|post|put|delete|patch|all|options|head)\s*(?:<[\s\S]*?>)?\s*\(\s*["\'`]([^"\'`]+)["\'`]', + win, + re.IGNORECASE, + ) + if m_express: + method = m_express.group(1).upper() + path = normalize_path_pattern(m_express.group(2)) + handler = find_enclosing_symbol(i, symbols) + routes.append(ApiRouteRecord( + repo=repo, filepath=filepath, framework="express", + http_method=method, path_pattern=path, handler_symbol=handler, + start_line=i, end_line=i + )) # NestJS: Controller prefix + decorators nest_prefix = "" @@ -310,32 +354,51 @@ def extract_api_routes_and_calls( # C# / ASP.NET Core Controllers & Minimal APIs elif language in ("c_sharp", "csharp"): + from app.services.chunking.csharp_route_attributes import ( + get_cached_csharp_attributes, + resolve_controller_route_from_attributes, + resolve_method_route_from_attribute_line, + ) + + cs_attr_specs = get_cached_csharp_attributes(repo, filepath, code) controller_route = "" controller_name = "" + is_abstract_controller = False group_prefixes: Dict[str, str] = {} for i, line in enumerate(lines, start=1): - m_class = re.search(r'class\s+([a-zA-Z0-9_]+Controller)\b', line) - if m_class: - c_full = m_class.group(1) - controller_name = c_full[:-10].lower() if c_full.endswith("Controller") else c_full.lower() - - m_croute = re.search(r'\[Route\s*\(\s*["\']([^"\']+)["\']\s*\)\]', line, re.IGNORECASE) - if m_croute and not controller_route: - controller_route = m_croute.group(1) - m_grp = re.search(r'\b(\w+)\s*=\s*(\w+)\.MapGroup\s*\(\s*["\']([^"\']*)["\']', line, re.IGNORECASE) if m_grp: grp_var, parent_var, grp_path = m_grp.group(1), m_grp.group(2), m_grp.group(3) parent_prefix = group_prefixes.get(parent_var, "") group_prefixes[grp_var] = _join_route_prefix(parent_prefix, grp_path) + pending_route = "" for i, line in enumerate(lines, start=1): - # Attribute routes [HttpGet("save")], [HttpGet("{id:guid}")], [HttpPost], etc. + m_croute = re.search(r'\[Route\s*\(\s*["\']([^"\']+)["\']\s*\)\]', line, re.IGNORECASE) + if m_croute: + pending_route = m_croute.group(1) + + m_class = re.search(r'class\s+([a-zA-Z0-9_]*Controller[a-zA-Z0-9_]*)\b', line) + if m_class: + c_full = m_class.group(1) + controller_name = c_full[:-10].lower() if c_full.endswith("Controller") else c_full.lower() + resolved_croute = resolve_controller_route_from_attributes(lines, i - 1, cs_attr_specs) + controller_route = resolved_croute or pending_route + is_abstract_controller = bool(re.search(r'\babstract\s+class\b', line)) + pending_route = "" + + # Attribute routes [HttpGet("save")], [HttpGet("{id:guid}")], [HttpPost], or custom HTTP verb attributes m_asp = re.search(r'\[(HttpGet|HttpPost|HttpPut|HttpDelete|HttpPatch)\s*(?:\(\s*["\']([^"\']*)["\']\s*\))?\]', line, re.IGNORECASE) - if m_asp: - method = m_asp.group(1)[4:].upper() - sub_path = m_asp.group(2) or "" - c_route_resolved = controller_route.replace("[controller]", controller_name) if controller_route else "" + custom_method_route = ( + (m_asp.group(1)[4:].upper(), m_asp.group(2) or "") + if m_asp + else resolve_method_route_from_attribute_line(line, cs_attr_specs) + ) + if custom_method_route: + method, sub_path = custom_method_route + if is_abstract_controller and not controller_route and not sub_path.startswith("/"): + continue + c_route_resolved = re.sub(r"\[controller\]", controller_name, controller_route, flags=re.IGNORECASE) if controller_route else "" if sub_path.startswith("/"): full_p = sub_path elif c_route_resolved: @@ -355,6 +418,12 @@ def extract_api_routes_and_calls( http_method=method, path_pattern=path, handler_symbol=handler, start_line=i, end_line=i )) + if c_route_resolved and re.fullmatch(r"\{[a-zA-Z_]\w*(?::[^{}]*)?\?\}", sub_path.strip("/")): + routes.append(ApiRouteRecord( + repo=repo, filepath=filepath, framework="aspnet", + http_method=method, path_pattern=normalize_path_pattern(c_route_resolved), handler_symbol=handler, + start_line=i, end_line=i + )) continue # Chained Minimal API: app.MapGroup("/api/v1").MapGet("/{id:guid}", ...) diff --git a/app/services/chunking/client_url_resolution.py b/app/services/chunking/client_url_resolution.py index ad42bf4..f2e2e4f 100644 --- a/app/services/chunking/client_url_resolution.py +++ b/app/services/chunking/client_url_resolution.py @@ -27,7 +27,7 @@ r'^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*(?::\s*string\s*)?=\s*(["\'`])([^"\'`\n]*)\2\s*;?\s*$' ) _CS_CONST = re.compile( - r'\b(?:const\s+string|static\s+readonly\s+string|readonly\s+static\s+string)\s+([A-Za-z_]\w*)\s*=\s*@?"([^"\n]*)"\s*;' + r'\b(?:const\s+string|static\s+readonly\s+string|readonly\s+static\s+string)\s+([A-Za-z_]\w*)\s*=\s*(?:\$@|@\$|\$|@)?"([^"\n]*)"\s*;' ) _JS_PLACEHOLDER = re.compile(r'\$\{\s*([A-Za-z_$][\w$]*)\s*\}') _CS_PLACEHOLDER = re.compile(r'\{([A-Za-z_]\w*)\}') diff --git a/app/services/chunking/csharp_client_calls.py b/app/services/chunking/csharp_client_calls.py index 0e4b9a1..5276ce5 100644 --- a/app/services/chunking/csharp_client_calls.py +++ b/app/services/chunking/csharp_client_calls.py @@ -29,6 +29,12 @@ r'\b\w+\s*(?:<[^()]*>)?\s*\(\s*(?:HttpMethod\.(\w+)|new\s+HttpMethod\s*\(\s*"(\w+)"\s*\))\s*,\s*' + _ARGUMENT, ) +# Wrapper where URL precedes HttpMethod: `BuildRequest(settings, $"{AppIndexerApiRoute}", HttpMethod.Post)` +_REQUEST_URL_THEN_METHOD = re.compile( + r'\b\w+\s*(?:<[^()]*>)?\s*\((?:[^()"]*,\s*)?' + + _ARGUMENT + + r'\s*,\s*(?:HttpMethod\.(\w+)|new\s+HttpMethod\s*\(\s*"(\w+)"\s*\))', +) _CLIENT_METHOD = re.compile( r'\b\w*client\.(Get|Post|Put|Delete|Patch)\w*Async\s*(?:<[^<>()]*>)?\(\s*' + _ARGUMENT + r'\s*[,)]', re.IGNORECASE, @@ -39,11 +45,13 @@ def _looks_like_path(text: str) -> bool: return "/" in text and " " not in text.strip() and not text.startswith("//") -def _literal_urls(statement: str) -> List[str]: +def _literal_urls(statement: str, consts: Optional[Dict[str, str]] = None) -> List[str]: """URL-like string literals of a statement; `"api/items/" + id` becomes `api/items/{id}`.""" + from app.services.chunking.client_url_resolution import expand_constants + urls = [] for m in _LITERAL_RE.finditer(statement): - text = m.group(1) + text = expand_constants(m.group(1), consts or {}, "csharp") if m.group(2): text = text.rstrip("/") + "/{" + m.group(2).split("+", 1)[1].strip().split(".")[0].rstrip("()") + "}" if _looks_like_path(text): @@ -51,7 +59,7 @@ def _literal_urls(statement: str) -> List[str]: return urls -def _resolve_variable(name: str, lines: List[str], line_no: int) -> List[str]: +def _resolve_variable(name: str, lines: List[str], line_no: int, consts: Optional[Dict[str, str]] = None) -> List[str]: """URL literals assigned to `name` in the nearest assignment above line `line_no` (1-based).""" short = name.rsplit(".", 1)[-1] assign = re.compile(r'\b' + re.escape(short) + r'\s*=(?![=>])') @@ -62,16 +70,19 @@ def _resolve_variable(name: str, lines: List[str], line_no: int) -> List[str]: continue statement = "\n".join(lines[idx:idx + _STATEMENT_LINES])[m.end():] statement = statement.split(";", 1)[0] - return _literal_urls(statement) + return _literal_urls(statement, consts) return [] def _urls_for_argument(arg: str, lines: List[str], line_no: int, consts: Optional[Dict[str, str]] = None) -> List[str]: + from app.services.chunking.client_url_resolution import expand_constants + if '"' in arg: - return _literal_urls(arg) + return _literal_urls(arg, consts) if consts and arg in consts: - return [consts[arg]] if _looks_like_path(consts[arg]) else [] - return _resolve_variable(arg, lines, line_no) + expanded = expand_constants(consts[arg], consts, "csharp") + return [expanded] if _looks_like_path(expanded) else [] + return _resolve_variable(arg, lines, line_no, consts) def extract_csharp_client_calls( @@ -83,6 +94,11 @@ def extract_csharp_client_calls( verb = (m_req.group(1) or m_req.group(2) or "").upper() method = verb if verb in _VERBS else None return [(method, url) for url in _urls_for_argument(m_req.group(3), lines, line_no, consts)] + m_rev = _REQUEST_URL_THEN_METHOD.search(line) + if m_rev: + verb = (m_rev.group(3) or m_rev.group(4) or "").upper() + method = verb if verb in _VERBS else None + return [(method, url) for url in _urls_for_argument(m_rev.group(1), lines, line_no, consts)] m_client = _CLIENT_METHOD.search(line) if m_client: method = m_client.group(1).upper() diff --git a/app/services/chunking/csharp_route_attributes.py b/app/services/chunking/csharp_route_attributes.py new file mode 100644 index 0000000..fa148a6 --- /dev/null +++ b/app/services/chunking/csharp_route_attributes.py @@ -0,0 +1,401 @@ +"""General resolution of custom C# route and HTTP-method attributes from source definitions. + +Instead of hardcoding framework- or project-specific attribute names (such as `[V3ApiController]` +or `[RestGetById]`), this module parses C# attribute class declarations (`class Attribute : ...`) +to determine: +1. Whether a class-level attribute implements `IRouteTemplateProvider` / `RouteAttribute` (or inherits + from a custom attribute that does) and what its route `Template` string is (resolving constructor + parameters, default values, constants, and `: base(...)` argument forwarding). +2. Whether a method-level attribute implements `IActionHttpMethodProvider` / `IRouteTemplateProvider` + or inherits from `HttpGetAttribute`, `HttpPostAttribute`, `HttpPutAttribute`, `HttpDeleteAttribute`, + `HttpPatchAttribute`, or `HttpMethodAttribute`, and what its HTTP verb and sub-route template are. +""" + +import os +import re +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + +_BUILTIN_METHOD_ATTRS: Dict[str, str] = { + "HttpGet": "GET", + "HttpPost": "POST", + "HttpPut": "PUT", + "HttpDelete": "DELETE", + "HttpPatch": "PATCH", + "HttpOptions": "OPTIONS", + "HttpHead": "HEAD", +} + +_ROUTE_INTERFACES = frozenset({ + "IRouteTemplateProvider", + "RouteAttribute", + "Route", +}) + +_METHOD_INTERFACES = frozenset({ + "IActionHttpMethodProvider", + "HttpMethodAttribute", +}) + +_CLASS_DECL_RE = re.compile( + r"\bclass\s+([A-Za-z_]\w*Attribute)\s*(?::\s*([^{]+))?\{" +) +_CONST_STRING_RE = re.compile( + r'\bconst\s+string\s+([A-Za-z_]\w*)\s*=\s*"([^"]*)"\s*;' +) +_TEMPLATE_ASSIGN_RE = re.compile( + r'\bTemplate\s*(?:=>|=)\s*(\$?"[^"]*")\s*;' +) +_HTTP_METHODS_RE = re.compile( + r'\bHttpMethods\b[^{;]*(?:\{\s*get[^;]*\}|=>)?[^{;]*\{\s*"(GET|POST|PUT|DELETE|PATCH|OPTIONS|HEAD)"', + re.IGNORECASE, +) + + +@dataclass +class CSharpAttributeSpec: + """Resolved or raw declaration metadata for a custom C# attribute class.""" + + short_name: str + class_name: str + base_types: List[str] = field(default_factory=list) + constants: Dict[str, str] = field(default_factory=dict) + ctor_params: List[Tuple[str, Optional[str]]] = field(default_factory=list) + base_call_args: List[str] = field(default_factory=list) + raw_template: Optional[str] = None + http_method: Optional[str] = None + is_route_provider: bool = False + + +def _strip_attr_suffix(name: str) -> str: + clean = name.strip().split(".")[-1] + return clean[:-9] if clean.endswith("Attribute") and len(clean) > 9 else clean + + +def _find_matching_brace(text: str, open_idx: int) -> int: + depth = 0 + for idx in range(open_idx, len(text)): + if text[idx] == "{": + depth += 1 + elif text[idx] == "}": + depth -= 1 + if depth == 0: + return idx + return len(text) + + +def _split_top_level_commas(text: str) -> List[str]: + parts: List[str] = [] + cur: List[str] = [] + depth = 0 + in_str = False + for ch in text: + if ch == '"': + in_str = not in_str + cur.append(ch) + elif not in_str: + if ch in "(<[{": + depth += 1 + cur.append(ch) + elif ch in ")>]}": + depth = max(0, depth - 1) + cur.append(ch) + elif ch == "," and depth == 0: + parts.append("".join(cur).strip()) + cur = [] + else: + cur.append(ch) + else: + cur.append(ch) + if cur: + parts.append("".join(cur).strip()) + return [p for p in parts if p] + + +def _resolve_value_token(token: str, constants: Dict[str, str], env: Dict[str, str]) -> str: + t = token.strip() + if t.startswith('$"') and t.endswith('"'): + inner = t[2:-1] + return re.sub( + r"\{([A-Za-z_]\w*)\}", + lambda m: env.get(m.group(1), constants.get(m.group(1), m.group(0))), + inner, + ) + if t.startswith('"') and t.endswith('"'): + return t[1:-1] + if t in env: + return env[t] + if t in constants: + return constants[t] + return t + + +def parse_csharp_attribute_specs(code: str) -> Dict[str, CSharpAttributeSpec]: + """Extracts C# `*Attribute` class definitions from a source string.""" + specs: Dict[str, CSharpAttributeSpec] = {} + if "Attribute" not in code: + return specs + + for m_cls in _CLASS_DECL_RE.finditer(code): + cls_name = m_cls.group(1) + short = _strip_attr_suffix(cls_name) + bases_raw = m_cls.group(2) or "" + bases = [ + re.sub(r"<.*>", "", b).strip().split(".")[-1] + for b in _split_top_level_commas(bases_raw) + if b.strip() + ] + body_start = m_cls.end() - 1 + body_end = _find_matching_brace(code, body_start) + body = code[body_start + 1 : body_end] + + consts: Dict[str, str] = { + m_c.group(1): m_c.group(2) for m_c in _CONST_STRING_RE.finditer(body) + } + + ctor_params: List[Tuple[str, Optional[str]]] = [] + base_args: List[str] = [] + ctor_re = re.compile( + r"\bpublic\s+" + + re.escape(cls_name) + + r"\s*\(([^)]*)\)\s*(?::\s*base\s*\(([^)]*)\))?" + ) + m_ctor = ctor_re.search(body) + if m_ctor: + raw_params = _split_top_level_commas(m_ctor.group(1)) + for rp in raw_params: + eq_parts = rp.split("=", 1) + lhs = eq_parts[0].strip() + p_name = lhs.split()[-1].strip() if lhs else "" + default_val: Optional[str] = None + if len(eq_parts) == 2: + default_val = _resolve_value_token(eq_parts[1].strip(), consts, {}) + if p_name: + ctor_params.append((p_name, default_val)) + if m_ctor.group(2) is not None: + base_args = _split_top_level_commas(m_ctor.group(2)) + + raw_template: Optional[str] = None + m_tpl = _TEMPLATE_ASSIGN_RE.search(body) + if m_tpl: + raw_template = m_tpl.group(1) + + http_method: Optional[str] = None + m_verb = _HTTP_METHODS_RE.search(body) + if m_verb: + http_method = m_verb.group(1).upper() + + is_route = any(b in _ROUTE_INTERFACES for b in bases) or raw_template is not None + for b in bases: + b_short = _strip_attr_suffix(b) + if b_short in _BUILTIN_METHOD_ATTRS: + http_method = _BUILTIN_METHOD_ATTRS[b_short] + is_route = True + elif b in _METHOD_INTERFACES: + is_route = True + + spec = CSharpAttributeSpec( + short_name=short, + class_name=cls_name, + base_types=bases, + constants=consts, + ctor_params=ctor_params, + base_call_args=base_args, + raw_template=raw_template, + http_method=http_method, + is_route_provider=is_route, + ) + specs[short] = spec + specs[cls_name] = spec + + return specs + + +def evaluate_attribute_usage( + attr_name: str, + call_args_raw: Optional[str], + specs: Dict[str, CSharpAttributeSpec], + depth: int = 0, +) -> Optional[Tuple[Optional[str], str]]: + """Resolves `[AttrName(args)]` against parsed `CSharpAttributeSpec` definitions. + + Returns `(http_method_or_none, route_template)` if the attribute is a route or + HTTP action attribute, or `None` if it is an unrelated attribute (e.g., `[SkipValidation]`). + """ + if depth > 6: + return None + + short = _strip_attr_suffix(attr_name) + call_args = _split_top_level_commas(call_args_raw or "") + + if short == "Route": + tpl = _resolve_value_token(call_args[0], {}, {}) if call_args else "" + return None, tpl + + if short in _BUILTIN_METHOD_ATTRS: + tpl = _resolve_value_token(call_args[0], {}, {}) if call_args else "" + return _BUILTIN_METHOD_ATTRS[short], tpl + + spec = specs.get(short) or specs.get(attr_name) + if spec is None: + return None + + # Bind constructor parameters using call_args or defaults + env: Dict[str, str] = {} + for idx, (p_name, p_default) in enumerate(spec.ctor_params): + if idx < len(call_args): + env[p_name] = _resolve_value_token(call_args[idx], spec.constants, env) + elif p_default is not None: + env[p_name] = p_default + # Also bind capitalized property alias (e.g., `version` -> `Version` when `Version = version;`) + if p_name in env and p_name[0].islower(): + env.setdefault(p_name[0].upper() + p_name[1:], env[p_name]) + + # Check if this attribute delegates to a base attribute class + for b in spec.base_types: + b_short = _strip_attr_suffix(b) + if b_short in _BUILTIN_METHOD_ATTRS or b_short == "Route" or b_short in specs or b in specs: + resolved_base_args = [ + f'"{_resolve_value_token(arg, spec.constants, env)}"' + for arg in spec.base_call_args + ] + base_res = evaluate_attribute_usage( + b_short, + ", ".join(resolved_base_args), + specs, + depth=depth + 1, + ) + if base_res is not None: + base_method, base_tpl = base_res + return spec.http_method or base_method, base_tpl + + if not spec.is_route_provider and spec.http_method is None: + return None + + tpl_out = "" + if spec.raw_template is not None: + tpl_out = _resolve_value_token(spec.raw_template, spec.constants, env) + elif call_args: + tpl_out = _resolve_value_token(call_args[0], spec.constants, env) + + return spec.http_method, tpl_out + + +_REPO_ATTR_CACHE: Dict[str, Dict[str, CSharpAttributeSpec]] = {} + + +def load_repo_csharp_attributes(filepath: str, code: str) -> Dict[str, CSharpAttributeSpec]: + """Loads C# custom attribute specs from the current file and any `*Attribute*.cs` files in the same repo tree.""" + local_specs = parse_csharp_attribute_specs(code) + norm_fp = (filepath or "").replace("\\", "/") + if "://" in norm_fp or not os.path.isabs(norm_fp): + return local_specs + + # Walk up to find the repository/solution root directory + cur_dir = os.path.dirname(norm_fp) + repo_root: Optional[str] = None + for _ in range(8): + if not cur_dir or cur_dir == "/": + break + if os.path.exists(os.path.join(cur_dir, ".git")) or any( + f.endswith(".sln") for f in (os.listdir(cur_dir) if os.path.isdir(cur_dir) else ()) + ): + repo_root = cur_dir + break + parent = os.path.dirname(cur_dir) + if parent == cur_dir: + break + cur_dir = parent + + if not repo_root: + return local_specs + + if repo_root not in _REPO_ATTR_CACHE: + scanned: Dict[str, CSharpAttributeSpec] = {} + try: + for root, dirs, files in os.walk(repo_root): + dirs[:] = [d for d in dirs if not d.startswith(".") and d not in ("bin", "obj", "node_modules", ".git")] + for fname in files: + if fname.endswith(".cs") and "Attribute" in fname: + fpath = os.path.join(root, fname) + try: + with open(fpath, "r", encoding="utf-8", errors="ignore") as f: + scanned.update(parse_csharp_attribute_specs(f.read())) + except Exception: + pass + except Exception: + pass + _REPO_ATTR_CACHE[repo_root] = scanned + + merged = dict(_REPO_ATTR_CACHE[repo_root]) + merged.update(local_specs) + return merged + + +def cache_repo_csharp_attributes(repo: str, root_dir: str) -> Dict[str, CSharpAttributeSpec]: + """Scans a cloned repository directory for `*Attribute*.cs` files and caches specs under `repo`.""" + scanned: Dict[str, CSharpAttributeSpec] = {} + if os.path.isdir(root_dir): + try: + for root, dirs, files in os.walk(root_dir): + dirs[:] = [d for d in dirs if not d.startswith(".") and d not in ("bin", "obj", "node_modules", ".git")] + for fname in files: + if fname.endswith(".cs") and "Attribute" in fname: + fpath = os.path.join(root, fname) + try: + with open(fpath, "r", encoding="utf-8", errors="ignore") as f: + scanned.update(parse_csharp_attribute_specs(f.read())) + except Exception: + pass + except Exception: + pass + _REPO_ATTR_CACHE[f"repo:{repo}"] = scanned + _REPO_ATTR_CACHE[root_dir.replace("\\", "/")] = scanned + return scanned + + +def get_cached_csharp_attributes(repo: str, filepath: str, code: str) -> Dict[str, CSharpAttributeSpec]: + """Returns merged custom attribute specs from repo cache, disk tree (if local), and current file.""" + specs: Dict[str, CSharpAttributeSpec] = {} + repo_key = f"repo:{repo}" + if repo_key in _REPO_ATTR_CACHE: + specs.update(_REPO_ATTR_CACHE[repo_key]) + specs.update(load_repo_csharp_attributes(filepath, code)) + return specs + + +_ATTR_USAGE_RE = re.compile(r"\[\s*([A-Za-z_]\w*)\s*(?:\(([^)\]]*)\))?\s*\]") + + +def resolve_controller_route_from_attributes( + lines: List[str], + class_line_idx: int, + specs: Dict[str, CSharpAttributeSpec], +) -> str: + """Resolves custom or `[Route(...)]` class-level route template above a C# controller class declaration.""" + start_idx = max(0, class_line_idx - 10) + for idx in range(start_idx, class_line_idx + 1): + for m_attr in _ATTR_USAGE_RE.finditer(lines[idx]): + attr_name = m_attr.group(1) + attr_args = m_attr.group(2) + res = evaluate_attribute_usage(attr_name, attr_args, specs) + if res is not None: + method, tpl = res + if method is None and tpl: + return tpl + return "" + + +def resolve_method_route_from_attribute_line( + line: str, + specs: Dict[str, CSharpAttributeSpec], +) -> Optional[Tuple[str, str]]: + """Resolves a method-level HTTP verb and sub-route template from a line with `[SomeAttr(...)]`.""" + for m_attr in _ATTR_USAGE_RE.finditer(line): + attr_name = m_attr.group(1) + attr_args = m_attr.group(2) + res = evaluate_attribute_usage(attr_name, attr_args, specs) + if res is not None and res[0] is not None: + return res[0], res[1] + return None + diff --git a/app/services/chunking/symbol_extractor.py b/app/services/chunking/symbol_extractor.py index 7ff4452..74db587 100644 --- a/app/services/chunking/symbol_extractor.py +++ b/app/services/chunking/symbol_extractor.py @@ -97,7 +97,7 @@ def extract_symbols_and_chunks( def _resolve_container_info(n) -> Optional[Tuple[str, str]]: if n.type not in target_types: return None - if n.type in ("variable_declarator", "pair"): + if n.type in ("variable_declarator", "pair", "public_field_definition", "field_definition"): return extract_callable_variable_or_pair(n, source_bytes) n_name = extract_node_name(n, source_bytes) if not n_name: diff --git a/app/services/chunking/text_chunker.py b/app/services/chunking/text_chunker.py index 56b67a9..47eb7f8 100644 --- a/app/services/chunking/text_chunker.py +++ b/app/services/chunking/text_chunker.py @@ -177,9 +177,9 @@ def extract_markdown_doc_links(content: str, filepath: str, repo: str) -> List[C # Tree-sitter node type queries per language for AST chunking & outline CODE_CONTAINER_TYPES = { "python": {"class_definition", "function_definition", "async_function_definition"}, - "javascript": {"class_declaration", "function_declaration", "method_definition", "variable_declarator", "pair"}, - "typescript": {"class_declaration", "interface_declaration", "function_declaration", "method_definition", "type_alias_declaration", "enum_declaration", "variable_declarator", "pair"}, - "tsx": {"class_declaration", "interface_declaration", "function_declaration", "method_definition", "type_alias_declaration", "enum_declaration", "variable_declarator", "pair"}, + "javascript": {"class_declaration", "function_declaration", "method_definition", "field_definition", "public_field_definition", "variable_declarator", "pair"}, + "typescript": {"class_declaration", "interface_declaration", "function_declaration", "method_definition", "public_field_definition", "field_definition", "type_alias_declaration", "enum_declaration", "variable_declarator", "pair"}, + "tsx": {"class_declaration", "interface_declaration", "function_declaration", "method_definition", "public_field_definition", "field_definition", "type_alias_declaration", "enum_declaration", "variable_declarator", "pair"}, "go": {"function_declaration", "method_declaration", "type_declaration"}, "rust": {"function_item", "impl_item", "struct_item", "trait_item", "enum_item"}, "c_sharp": {"class_declaration", "interface_declaration", "method_declaration", "struct_declaration", "enum_declaration", "record_declaration", "record_struct_declaration", "constructor_declaration"}, @@ -199,10 +199,10 @@ def extract_markdown_doc_links(content: str, filepath: str, repo: str) -> List[C def extract_callable_variable_or_pair(node, source_bytes: bytes) -> Optional[Tuple[str, str]]: """ - For JS/TS/TSX `variable_declarator` or `pair` nodes, returns `(name, kind)` - only when the assigned value is an arrow function, function expression, or + For JS/TS/TSX `variable_declarator`, `pair`, or `public_field_definition`/`field_definition` nodes, + returns `(name, kind)` only when the assigned value is an arrow function, function expression, or a recognized React/store higher-order wrapper (memo, forwardRef, create, etc.). - Returns None for plain data variables or object literal properties. + Returns None for plain data variables, class fields, or object literal properties. """ if node.type == "variable_declarator": name_node = node.child_by_field_name("name") @@ -222,6 +222,20 @@ def extract_callable_variable_or_pair(node, source_bytes: bytes) -> Optional[Tup return (name, "function_declaration") return None + if node.type in ("public_field_definition", "field_definition"): + name_node = node.child_by_field_name("name") or node.child_by_field_name("property") + val_node = node.child_by_field_name("value") + if not name_node or not val_node: + return None + if val_node.type not in ("arrow_function", "function_expression", "function", "generator_function"): + return None + if name_node.type not in ("property_identifier", "identifier", "private_property_identifier"): + return None + name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="ignore").strip() + if not name: + return None + return (name, "method_definition") + if node.type == "pair": key_node = node.child_by_field_name("key") val_node = node.child_by_field_name("value") diff --git a/app/services/compact_results.py b/app/services/compact_results.py index 7c326d8..3387887 100644 --- a/app/services/compact_results.py +++ b/app/services/compact_results.py @@ -38,12 +38,14 @@ def _file_lines(matches: List[Dict[str, Any]], block_separators: bool) -> List[s def group_grep_matches( - matches: Iterable[Dict[str, Any]], block_separators: bool = True + matches: Iterable[Dict[str, Any]], + block_separators: bool = True, + include_permalinks: bool = False, ) -> Tuple[Optional[str], List[Dict[str, Any]]]: """ Groups grep matches by (repo, file) in first-seen order. Returns the repo that all groups share (or None) and the file groups. A group has `file`, `repo` (only when the repos are different), `lines`, and, when known, - `symbols` ({line: enclosing symbol}) and `permalink` (the file permalink; add `#L`). + `symbols` ({line: enclosing symbol}) and `permalink` (the file permalink when include_permalinks=True; add `#L`). Set block_separators=False when there are no context lines (ripgrep writes no `--` then). """ groups: Dict[Tuple[Any, Any], List[Dict[str, Any]]] = {} @@ -57,9 +59,10 @@ def group_grep_matches( group: Dict[str, Any] = {"file": file_path} if shared_repo is None and repo: group["repo"] = repo - permalink = next((_permalink_base(m.get("permalink")) for m in file_matches if m.get("permalink")), None) - if permalink: - group["permalink"] = permalink + if include_permalinks: + permalink = next((_permalink_base(m.get("permalink")) for m in file_matches if m.get("permalink")), None) + if permalink: + group["permalink"] = permalink symbols = {str(m["line"]): m["symbol"] for m in file_matches if m.get("symbol")} if symbols: group["symbols"] = symbols diff --git a/app/services/database/connection.py b/app/services/database/connection.py index eaf9d3d..23911d8 100644 --- a/app/services/database/connection.py +++ b/app/services/database/connection.py @@ -52,9 +52,9 @@ def get_db_connection() -> Any: except Exception: pass - conn = sqlite3.connect(db_path, timeout=15.0) + conn = sqlite3.connect(db_path, timeout=60.0) conn.row_factory = sqlite3.Row - conn.execute("PRAGMA busy_timeout=15000;") + conn.execute("PRAGMA busy_timeout=60000;") conn.execute("PRAGMA journal_mode=WAL;") conn.execute("PRAGMA foreign_keys=ON;") return conn diff --git a/app/services/database/engine.py b/app/services/database/engine.py index 0872166..6b5e624 100644 --- a/app/services/database/engine.py +++ b/app/services/database/engine.py @@ -103,7 +103,7 @@ def get_db_engine(database_url: Optional[str] = None, reset: bool = False, **kwa if url.startswith("sqlite"): engine = create_engine( url, - connect_args={"check_same_thread": False, "timeout": 15.0}, + connect_args={"check_same_thread": False, "timeout": 60.0}, pool_pre_ping=True, **kwargs, ) @@ -113,7 +113,7 @@ def _set_sqlite_pragmas(dbapi_connection, connection_record): if isinstance(dbapi_connection, sqlite3.Connection): cursor = dbapi_connection.cursor() try: - cursor.execute("PRAGMA busy_timeout=15000;") + cursor.execute("PRAGMA busy_timeout=60000;") except Exception as e: logger.debug(f"Failed to set busy_timeout: {e}") try: diff --git a/app/services/indexing/channel_pipeline.py b/app/services/indexing/channel_pipeline.py index e81cfdd..5cf75f5 100644 --- a/app/services/indexing/channel_pipeline.py +++ b/app/services/indexing/channel_pipeline.py @@ -428,7 +428,9 @@ def _finalize_and_persist_batch( }) if items_to_cache: - set_cached_embeddings_batch(items_to_cache) + from app.services.indexing.db_write_channel import submit_db_write + + submit_db_write(lambda: set_cached_embeddings_batch(items_to_cache)) trim_memory() for pt, h in batch.point_hashes: diff --git a/app/services/indexing/db_write_channel.py b/app/services/indexing/db_write_channel.py new file mode 100644 index 0000000..5a1dbcc --- /dev/null +++ b/app/services/indexing/db_write_channel.py @@ -0,0 +1,114 @@ +"""Dedicated bounded database write channel (actor queue) in front of the relational database. + +When multiple git repositories or local paths are synced concurrently, each sync pipeline +produces embedding cache batches and relational AST/route batches. Instead of allowing concurrent +pipeline consumers to contend on SQLite/database locks directly, all write batches are enqueued +into a shared BoundedChannel and executed sequentially by a single dedicated writer thread, +providing natural backpressure to upstream pipeline stages. +""" + +import logging +import threading +from concurrent.futures import Future +from dataclasses import dataclass +from typing import Any, Callable, Optional, TypeVar + +from app.services.indexing.channel_pipeline import BoundedChannel + +logger = logging.getLogger("contextcortex.indexer.db_write_channel") + +R = TypeVar("R") + + +@dataclass +class DatabaseWriteCommand: + """Unit of database write work submitted to the DatabaseWriteChannel.""" + + action: Callable[[], Any] + future: Future + + +class DatabaseWriteChannel: + """Single-consumer bounded channel that serializes database write transactions + with backpressure across all concurrent ingestion pipelines.""" + + def __init__(self, capacity: int = 32) -> None: + self.capacity = max(1, int(capacity)) + self._channel: BoundedChannel[DatabaseWriteCommand] = BoundedChannel(capacity=self.capacity) + self._worker_lock = threading.Lock() + self._worker_thread: Optional[threading.Thread] = None + self._stopped = False + + def _ensure_worker_started(self) -> None: + with self._worker_lock: + if self._stopped: + self._channel = BoundedChannel(capacity=self.capacity) + self._stopped = False + if self._worker_thread is None or not self._worker_thread.is_alive(): + self._worker_thread = threading.Thread( + target=self._consume_loop, + name="cc-db-write-channel-worker", + daemon=True, + ) + self._worker_thread.start() + + def _consume_loop(self) -> None: + for cmd in self._channel.read_all(): + if cmd.future.cancelled(): + continue + try: + result = cmd.action() + cmd.future.set_result(result) + except BaseException as exc: + cmd.future.set_exception(exc) + + def submit_async(self, action: Callable[[], R]) -> Future: + """Enqueues a database write callable, blocking if the bounded channel is full, + and returns a Future that resolves when the write completes.""" + self._ensure_worker_started() + fut: Future = Future() + cmd = DatabaseWriteCommand(action=action, future=fut) + written = self._channel.write(cmd) + if not written: + fut.set_exception(RuntimeError("DatabaseWriteChannel is closed or cancelled.")) + return fut + + def submit(self, action: Callable[[], R]) -> R: + """Enqueues a database write callable with backpressure and waits for its result.""" + if threading.current_thread() is self._worker_thread: + # Re-entrant call from within the writer thread itself executes inline + return action() + fut = self.submit_async(action) + return fut.result() + + @property + def pending_count(self) -> int: + """Approximate number of queued write commands waiting for the DB writer.""" + return self._channel._queue.qsize() + + def shutdown(self, wait: bool = True) -> None: + """Completes the channel and optionally waits for pending writes to drain.""" + with self._worker_lock: + self._stopped = True + worker = self._worker_thread + self._channel.complete() + if wait and worker is not None and worker.is_alive() and threading.current_thread() is not worker: + worker.join(timeout=10.0) + + +_global_db_write_channel: Optional[DatabaseWriteChannel] = None +_global_db_write_lock = threading.Lock() + + +def get_db_write_channel() -> DatabaseWriteChannel: + """Returns the process-wide DatabaseWriteChannel singleton.""" + global _global_db_write_channel + with _global_db_write_lock: + if _global_db_write_channel is None: + _global_db_write_channel = DatabaseWriteChannel(capacity=32) + return _global_db_write_channel + + +def submit_db_write(action: Callable[[], R]) -> R: + """Submits a database write transaction through the global DatabaseWriteChannel.""" + return get_db_write_channel().submit(action) diff --git a/app/services/indexing/git_syncer.py b/app/services/indexing/git_syncer.py index 83d2b78..3c605cf 100644 --- a/app/services/indexing/git_syncer.py +++ b/app/services/indexing/git_syncer.py @@ -235,28 +235,39 @@ def sync_single_git_repo(repo_id: int, force: bool = False): except Exception as qe: logger.error(f"Error deleting vector for {fpath}: {qe}") + from app.services.indexing.channel_pipeline import ( + BoundedChannel, + ChannelIngestionPipeline, + ParsedFileItem, + ) + from app.services.indexing.db_write_channel import submit_db_write + if db_paths_to_clean: - with db_service.get_db_connection() as conn: - for fpath in db_paths_to_clean: - conn.execute("DELETE FROM ast_symbols WHERE filepath = ?", (fpath,)) - conn.execute("DELETE FROM ast_relationships WHERE source_filepath = ?", (fpath,)) - conn.execute("DELETE FROM api_routes WHERE filepath = ?", (fpath,)) - conn.execute("DELETE FROM api_client_calls WHERE filepath = ?", (fpath,)) - conn.execute("DELETE FROM file_summaries WHERE filepath = ?", (fpath,)) - for fpath in deleted_filepaths: - conn.execute("DELETE FROM indexed_files WHERE filepath = ?", (fpath,)) - conn.commit() + def _clean_stale_paths() -> None: + with db_service.get_db_connection() as conn: + for fpath in db_paths_to_clean: + conn.execute("DELETE FROM ast_symbols WHERE filepath = ?", (fpath,)) + conn.execute("DELETE FROM ast_relationships WHERE source_filepath = ?", (fpath,)) + conn.execute("DELETE FROM api_routes WHERE filepath = ?", (fpath,)) + conn.execute("DELETE FROM api_client_calls WHERE filepath = ?", (fpath,)) + conn.execute("DELETE FROM file_summaries WHERE filepath = ?", (fpath,)) + for fpath in deleted_filepaths: + conn.execute("DELETE FROM indexed_files WHERE filepath = ?", (fpath,)) + conn.commit() + + submit_db_write(_clean_stale_paths) files_to_sync = added_files + modified_files total_files = len(files_to_sync) logger.info(f"Ingesting {total_files} added/modified files for repo '{repo_name}'...") progress_tracker.log(repo_id, "INFO", f"Ingesting {total_files} added/modified files...") - from app.services.indexing.channel_pipeline import ( - BoundedChannel, - ChannelIngestionPipeline, - ParsedFileItem, - ) + try: + from app.services.chunking.csharp_route_attributes import cache_repo_csharp_attributes + + cache_repo_csharp_attributes(repo_name, temp_dir) + except Exception: + pass BATCH_SIZE = 25 total_vectors_count = 0 @@ -361,16 +372,19 @@ def _persist_parsed_batch(batch_items: list[ParsedFileItem]) -> None: raise RuntimeError(f"Vector store upsert failed during git indexing for repo '{repo_name}'") total_vectors_count += len(batch_points) - with db_service.get_db_connection() as conn: - total_symbols_count += persist_sync_batch( - conn, - batch_indexed_files, - batch_summaries, - batch_symbols, - batch_relationships, - batch_routes, - batch_calls, - ) + def _write_relational_batch() -> int: + with db_service.get_db_connection() as conn: + return persist_sync_batch( + conn, + batch_indexed_files, + batch_summaries, + batch_symbols, + batch_relationships, + batch_routes, + batch_calls, + ) + + total_symbols_count += submit_db_write(_write_relational_batch) trim_memory() pipeline = ChannelIngestionPipeline( diff --git a/app/services/indexing/local_syncer.py b/app/services/indexing/local_syncer.py index 4a61a4c..48978d3 100644 --- a/app/services/indexing/local_syncer.py +++ b/app/services/indexing/local_syncer.py @@ -135,6 +135,15 @@ def _sync_local_paths_internal(force: bool = False, repo_filter: str = None): ParsedFileItem, ) + try: + from app.services.chunking.csharp_route_attributes import cache_repo_csharp_attributes + + for target in active_paths: + if target.get("type") != "file" and os.path.isdir(target["path"]): + cache_repo_csharp_attributes(target["repo"], os.path.abspath(target["path"])) + except Exception: + pass + all_points = [] all_symbols = [] all_routes = [] @@ -219,17 +228,23 @@ def _collect_embedded_local_batch(batch_items: list[ParsedFileItem]) -> None: logger.error(f"Failed to embed local files in pipeline: {e}") return False + from app.services.indexing.db_write_channel import submit_db_write + # Purge old points for modified files store = vs_service.get_vector_store() for (fpath, repo) in files_to_delete: try: store.delete_by_path(fpath) - with db_service.get_db_connection() as conn: - conn.execute("DELETE FROM ast_symbols WHERE filepath = ?", (fpath,)) - conn.execute("DELETE FROM ast_relationships WHERE source_filepath = ?", (fpath,)) - conn.execute("DELETE FROM api_routes WHERE filepath = ?", (fpath,)) - conn.execute("DELETE FROM api_client_calls WHERE filepath = ?", (fpath,)) - conn.commit() + + def _purge_local_file(target_fp: str = fpath) -> None: + with db_service.get_db_connection() as conn: + conn.execute("DELETE FROM ast_symbols WHERE filepath = ?", (target_fp,)) + conn.execute("DELETE FROM ast_relationships WHERE source_filepath = ?", (target_fp,)) + conn.execute("DELETE FROM api_routes WHERE filepath = ?", (target_fp,)) + conn.execute("DELETE FROM api_client_calls WHERE filepath = ?", (target_fp,)) + conn.commit() + + submit_db_write(_purge_local_file) except Exception as e: logger.error(f"Failed to delete old points for {fpath}: {e}") @@ -247,104 +262,107 @@ def _collect_embedded_local_batch(batch_items: list[ParsedFileItem]) -> None: # Batch update database tables try: - with db_service.get_db_connection() as conn: - if files_to_update_cache: - conn.executemany( - """INSERT INTO indexed_files (filepath, repo, doc_type, language, commit_sha, mtime) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(filepath) DO UPDATE SET - repo = EXCLUDED.repo, - doc_type = EXCLUDED.doc_type, - language = EXCLUDED.language, - commit_sha = EXCLUDED.commit_sha, - mtime = EXCLUDED.mtime""", - files_to_update_cache - ) - if all_summaries: - conn.executemany( - """INSERT INTO file_summaries (filepath, repo, title, folder, category, tags, headings, keywords, summary_text, mtime) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(filepath) DO UPDATE SET - repo = EXCLUDED.repo, - title = EXCLUDED.title, - folder = EXCLUDED.folder, - category = EXCLUDED.category, - tags = EXCLUDED.tags, - headings = EXCLUDED.headings, - keywords = EXCLUDED.keywords, - summary_text = EXCLUDED.summary_text, - mtime = EXCLUDED.mtime""", - all_summaries - ) - if all_symbols: - for s in all_symbols: - cursor = conn.execute( - "INSERT INTO ast_symbols (repo, filepath, name, full_symbol, kind, start_line, end_line, signature, language) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - (s["repo"], s["filepath"], s["name"], s["full_symbol"], s["kind"], s["start_line"], s["end_line"], s["signature"], s["language"]) + def _persist_local_batch() -> None: + with db_service.get_db_connection() as conn: + if files_to_update_cache: + conn.executemany( + """INSERT INTO indexed_files (filepath, repo, doc_type, language, commit_sha, mtime) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(filepath) DO UPDATE SET + repo = EXCLUDED.repo, + doc_type = EXCLUDED.doc_type, + language = EXCLUDED.language, + commit_sha = EXCLUDED.commit_sha, + mtime = EXCLUDED.mtime""", + files_to_update_cache ) - s["inserted_id"] = cursor.lastrowid - - if all_relationships: - # Map symbol (repo, filepath, name) to inserted_id - sym_map = {} - syms_by_file = {} - for s in all_symbols: - if "inserted_id" in s: - sym_map[(s["repo"], s["filepath"], s["name"])] = s["inserted_id"] - if s.get("full_symbol"): - sym_map[(s["repo"], s["filepath"], s["full_symbol"])] = s["inserted_id"] - syms_by_file.setdefault((s["repo"], s["filepath"]), []).append(s) - - rel_tuples = [] - for r in all_relationships: - src_id = None - r_line = r.get("line_number") - if r_line is not None: - candidates = [ - s for s in syms_by_file.get((r["repo"], r["source_filepath"]), []) - if (s.get("start_line") or 0) <= r_line <= (s.get("end_line") or 0) - and (s.get("name") == r["source_symbol"] or s.get("full_symbol") == r["source_symbol"]) - ] - if candidates: - candidates.sort(key=lambda s: (s.get("end_line") or 0) - (s.get("start_line") or 0)) - src_id = candidates[0].get("inserted_id") - if src_id is None: - src_id = sym_map.get((r["repo"], r["source_filepath"], r["source_symbol"])) - rel_tuples.append(( - r["repo"], - src_id, - r["source_filepath"], - r["source_symbol"], - r["target_symbol"], - r["relationship_type"], - r["line_number"] - )) - - conn.executemany( - """INSERT INTO ast_relationships - (repo, source_symbol_id, source_filepath, source_symbol, target_symbol, relationship_type, line_number) - VALUES (?, ?, ?, ?, ?, ?, ?)""", - rel_tuples - ) - if all_routes: - route_tuples = [ - (r["repo"], r["filepath"], r["framework"], r["http_method"], r["path_pattern"], r.get("handler_symbol"), r["start_line"], r["end_line"]) - for r in all_routes - ] - conn.executemany( - "INSERT INTO api_routes (repo, filepath, framework, http_method, path_pattern, handler_symbol, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - route_tuples - ) - if all_calls: - call_tuples = [ - (c["repo"], c["filepath"], c.get("http_method"), c["url_pattern"], c.get("caller_symbol"), c["line_number"]) - for c in all_calls - ] - conn.executemany( - "INSERT INTO api_client_calls (repo, filepath, http_method, url_pattern, caller_symbol, line_number) VALUES (?, ?, ?, ?, ?, ?)", - call_tuples - ) - conn.commit() + if all_summaries: + conn.executemany( + """INSERT INTO file_summaries (filepath, repo, title, folder, category, tags, headings, keywords, summary_text, mtime) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(filepath) DO UPDATE SET + repo = EXCLUDED.repo, + title = EXCLUDED.title, + folder = EXCLUDED.folder, + category = EXCLUDED.category, + tags = EXCLUDED.tags, + headings = EXCLUDED.headings, + keywords = EXCLUDED.keywords, + summary_text = EXCLUDED.summary_text, + mtime = EXCLUDED.mtime""", + all_summaries + ) + if all_symbols: + for s in all_symbols: + cursor = conn.execute( + "INSERT INTO ast_symbols (repo, filepath, name, full_symbol, kind, start_line, end_line, signature, language) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (s["repo"], s["filepath"], s["name"], s["full_symbol"], s["kind"], s["start_line"], s["end_line"], s["signature"], s["language"]) + ) + s["inserted_id"] = cursor.lastrowid + + if all_relationships: + # Map symbol (repo, filepath, name) to inserted_id + sym_map = {} + syms_by_file = {} + for s in all_symbols: + if "inserted_id" in s: + sym_map[(s["repo"], s["filepath"], s["name"])] = s["inserted_id"] + if s.get("full_symbol"): + sym_map[(s["repo"], s["filepath"], s["full_symbol"])] = s["inserted_id"] + syms_by_file.setdefault((s["repo"], s["filepath"]), []).append(s) + + rel_tuples = [] + for r in all_relationships: + src_id = None + r_line = r.get("line_number") + if r_line is not None: + candidates = [ + s for s in syms_by_file.get((r["repo"], r["source_filepath"]), []) + if (s.get("start_line") or 0) <= r_line <= (s.get("end_line") or 0) + and (s.get("name") == r["source_symbol"] or s.get("full_symbol") == r["source_symbol"]) + ] + if candidates: + candidates.sort(key=lambda s: (s.get("end_line") or 0) - (s.get("start_line") or 0)) + src_id = candidates[0].get("inserted_id") + if src_id is None: + src_id = sym_map.get((r["repo"], r["source_filepath"], r["source_symbol"])) + rel_tuples.append(( + r["repo"], + src_id, + r["source_filepath"], + r["source_symbol"], + r["target_symbol"], + r["relationship_type"], + r["line_number"] + )) + + conn.executemany( + """INSERT INTO ast_relationships + (repo, source_symbol_id, source_filepath, source_symbol, target_symbol, relationship_type, line_number) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + rel_tuples + ) + if all_routes: + route_tuples = [ + (r["repo"], r["filepath"], r["framework"], r["http_method"], r["path_pattern"], r.get("handler_symbol"), r["start_line"], r["end_line"]) + for r in all_routes + ] + conn.executemany( + "INSERT INTO api_routes (repo, filepath, framework, http_method, path_pattern, handler_symbol, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + route_tuples + ) + if all_calls: + call_tuples = [ + (c["repo"], c["filepath"], c.get("http_method"), c["url_pattern"], c.get("caller_symbol"), c["line_number"]) + for c in all_calls + ] + conn.executemany( + "INSERT INTO api_client_calls (repo, filepath, http_method, url_pattern, caller_symbol, line_number) VALUES (?, ?, ?, ?, ?, ?)", + call_tuples + ) + conn.commit() + + submit_db_write(_persist_local_batch) except Exception as e: logger.error(f"Failed writing local index updates to SQLite: {e}") return False diff --git a/app/services/script_sdk_types.py b/app/services/script_sdk_types.py index fdac29b..505c098 100644 --- a/app/services/script_sdk_types.py +++ b/app/services/script_sdk_types.py @@ -49,8 +49,8 @@ "resolved_frames": "any[]", "matched_routes": "any[]", "matched_symbols": "any[]", "semantic_candidates": "any[]"}, "Route": {"repo": _S, "method": _S, "path": _S, "framework": _SN, "handler": _S, "file": _S, "lines": _LINES, - "permalink": _SN}, - "RouteCaller": {"repo": _S, "method": _SN, "url": _S, "caller": _S, "file": _S, "line": _N, "permalink": _SN}, + "permalink?": _SN}, + "RouteCaller": {"repo": _S, "method": _SN, "url": _S, "caller": _S, "file": _S, "line": _N, "permalink?": _SN}, "FindRoutesResult": {"query": _SN, "kind": _S, "method": _SN, "repo": _SN, "total": _N, "total_routes?": _N, "total_callers?": _N, **_PAGE, "routes?": "Route[]", "callers?": "RouteCaller[]"}, "TraceTarget": {"symbol": _S, "name": _S, "kind": _SN, "repo": _S, "file": _S, "lines": _LINES, "permalink": _SN, @@ -67,7 +67,7 @@ "FindSymbolResult": {"query": _S, "exact": "boolean", "total": _N, **_PAGE, "symbols": "SymbolHit[]"}, "SearchHit": {"repo": _SN, "file": _S, "doc_type": _S, "lines": "[number, number] | null", "symbol": _SN, "kind": _SN, "signature": _SN, "language": _SN, "heading": _SN, "tags": "string[]", "score": _N, - "dense_score": _NN, "sparse_score": _NN, "permalink": _SN, "content_truncated": "boolean", + "dense_score": _NN, "sparse_score": _NN, "permalink?": _SN, "content_truncated": "boolean", "content": _S}, "SearchResult": {"query": _S, "scope": _S, "mode": _S, "total": _N, **_PAGE, "max_snippet_chars": _N, "results": "SearchHit[]"}, diff --git a/main.py b/main.py index 5ee167d..48c9e96 100644 --- a/main.py +++ b/main.py @@ -180,6 +180,12 @@ async def __call__(self, scope, receive, send): path = scope.get("path", "").rstrip("/") or "/" has_session_id = b"session_id=" in scope.get("query_string", b"") + if path == "/sse" and method in ("POST", "DELETE") and not has_session_id: + scope = dict(scope) + scope["path"] = "/mcp" + scope["raw_path"] = b"/mcp" + path = "/mcp" + # Check if request targets an MCP endpoint with POST (standalone probe without active SSE session) if method == "POST" and not has_session_id and path in ("/mcp", "/sse", "/messages", ""): body = b"" diff --git a/tests/test_code_mode_result_types.py b/tests/test_code_mode_result_types.py index fbebb0d..632b41d 100644 --- a/tests/test_code_mode_result_types.py +++ b/tests/test_code_mode_result_types.py @@ -60,7 +60,9 @@ def test_result_types_agree_with_real_tool_results(chain_db): # noqa: F811 _check(result, "DiagnoseResult", "diagnose_issue", seen) assert result["call_chain"]["steps"] and result["working_path_comparison"]["steps"] for kind in ("both", "handlers", "callers"): - _check(_call("find_routes", query="publish", kind=kind, repo=REPO), "FindRoutesResult", f"find_routes({kind})", seen) + for permalinks in (False, True): + res_routes = _call("find_routes", query="publish", kind=kind, repo=REPO, include_permalinks=permalinks) + _check(res_routes, "FindRoutesResult", f"find_routes({kind}, permalinks={permalinks})", seen) for permalinks in (False, True): trace = _call("trace_path", symbol="ToolService.PublishInstance", repo=REPO, depth=3, include_permalinks=permalinks) _check(trace, "TracePathResult", "trace_path", seen) @@ -68,7 +70,8 @@ def test_result_types_agree_with_real_tool_results(chain_db): # noqa: F811 _check(_call("trace_path", symbol="Nope.Missing", repo=REPO), "TracePathResult", "trace_path(missing)", seen) _check(_call("find_symbol", symbol="GetByInstanceId", repo=REPO, exact=False), "FindSymbolResult", "find_symbol", seen) with patch("app.mcp.tools.execute_hybrid_search", _fake_search): - _check(_call("search", query="publish", repo=REPO), "SearchResult", "search", seen) + for permalinks in (False, True): + _check(_call("search", query="publish", repo=REPO, include_permalinks=permalinks), "SearchResult", "search", seen) assert seen == set(RESULT_INTERFACES), f"not checked: {sorted(set(RESULT_INTERFACES) - seen)}" diff --git a/tests/test_compact_grep_and_read_output.py b/tests/test_compact_grep_and_read_output.py index 8e3f36c..99c30f5 100644 --- a/tests/test_compact_grep_and_read_output.py +++ b/tests/test_compact_grep_and_read_output.py @@ -39,13 +39,16 @@ def _match(repo, file, line, symbol=None, permalink=None, context=1, lines=SOURC def test_matches_group_by_file_with_merged_context_and_block_separators(): link = "https://git.example/org/hub/blob/abc/Reader.cs" - shared_repo, files = group_grep_matches([ + matches = [ _match("hub", "Reader.cs", 3, "Reader.GetAccessPreflightAsync", f"{link}#L3"), _match("hub", "Reader.cs", 6, "Reader.GetByInstanceId", f"{link}#L6"), _match("hub", "Reader.cs", 12), - ]) + ] + shared_repo, files = group_grep_matches(matches) assert shared_repo == "hub" - assert files == [{ + assert "permalink" not in files[0] + shared_repo, files_with_link = group_grep_matches(matches, include_permalinks=True) + assert files_with_link == [{ "file": "Reader.cs", "permalink": link, "symbols": {"3": "Reader.GetAccessPreflightAsync", "6": "Reader.GetByInstanceId"}, diff --git a/tests/test_db_write_channel_and_custom_routes.py b/tests/test_db_write_channel_and_custom_routes.py new file mode 100644 index 0000000..fe47cd1 --- /dev/null +++ b/tests/test_db_write_channel_and_custom_routes.py @@ -0,0 +1,414 @@ +"""Tests for DatabaseWriteChannel bounded queue backpressure, general C# custom route attribute +resolution (IRouteTemplateProvider, IActionHttpMethodProvider, base constructor forwarding), +and TypeScript/JavaScript class field arrow function extraction.""" + +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +from app.services.chunking.api_route_extractor import ( + extract_api_routes_and_calls, + match_route_and_call, +) +from app.services.chunking.csharp_route_attributes import ( + evaluate_attribute_usage, + parse_csharp_attribute_specs, +) +from app.services.chunking.symbol_extractor import extract_symbols_and_chunks +from app.services.database import get_db_connection, init_db +from app.services.indexing.db_write_channel import DatabaseWriteChannel, submit_db_write + + +def test_database_write_channel_serializes_concurrent_sqlite_writers(tmp_path, monkeypatch): + """Verifies DatabaseWriteChannel queues concurrent producers through a single writer thread without SQLite lock errors or interleaving.""" + db_file = str(tmp_path / "db_write_channel.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + channel = DatabaseWriteChannel(capacity=4) + active_writers = 0 + max_concurrent_writers = 0 + guard = threading.Lock() + writer_thread_ids = set() + + def _make_write_op(idx: int): + def _op() -> int: + nonlocal active_writers, max_concurrent_writers + with guard: + active_writers += 1 + if active_writers > max_concurrent_writers: + max_concurrent_writers = active_writers + writer_thread_ids.add(threading.get_ident()) + try: + time.sleep(0.005) + with get_db_connection() as conn: + conn.execute( + "INSERT INTO system_metadata (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = EXCLUDED.value", + (f"k_{idx}", f"v_{idx}"), + ) + conn.commit() + return idx + finally: + with guard: + active_writers -= 1 + + return _op + + try: + with ThreadPoolExecutor(max_workers=8) as pool: + futs = [pool.submit(channel.submit, _make_write_op(i)) for i in range(24)] + results = [f.result(timeout=10.0) for f in futs] + + assert sorted(results) == list(range(24)) + assert max_concurrent_writers == 1 + assert len(writer_thread_ids) == 1 + + # Also verify global submit_db_write helper + val = submit_db_write(lambda: "ok") + assert val == "ok" + finally: + channel.shutdown(wait=True) + + +def test_general_csharp_custom_route_and_verb_attributes(): + """Verifies custom C# route and HTTP verb attributes are resolved generically from class declarations and inheritance chains.""" + csharp_code = """ +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Routing; + +namespace SampleApp.Http +{ + public class VersionedApiControllerAttribute : Attribute, IRouteTemplateProvider + { + public const string CONTROLLER_RESOURCE = "[controller]"; + + public VersionedApiControllerAttribute(int version, string resource = CONTROLLER_RESOURCE) + { + Template = $"api/v{version}/{resource}"; + } + + public string Template { get; } + } + + public class V3ApiControllerAttribute : VersionedApiControllerAttribute + { + public V3ApiControllerAttribute(string resource = "[controller]") + : base(3, resource) + { + } + } + + public class RestGetByIdAttribute : Attribute, IActionHttpMethodProvider, IRouteTemplateProvider + { + public IEnumerable HttpMethods => new[] { "GET" }; + public string Template => "{id:int}"; + } + + public class RestPostByIdAttribute : HttpPostAttribute + { + } + + public class RestPutByIdAttribute : HttpPutAttribute + { + public RestPutByIdAttribute() + : base("{id:int?}") + { + } + } + + public class RestDeleteByIdAttribute : HttpDeleteAttribute + { + public RestDeleteByIdAttribute() + : base("{id:int}") + { + } + } + + public abstract class RestController : Controller + { + [RestGetById] + public virtual TResource GetResourceById(int id) => default; + } + + public abstract class ProviderControllerBase : RestController + { + [RestPutById] + public object UpdateProvider(TResource body) => null; + } + + [V3ApiController] + public class MovieController : RestController + { + [HttpGet] + public object AllMovies() => null; + + [RestGetById] + public object GetById(int id) => null; + + [RestPostById] + public object AddMovie(object body) => null; + + [RestPutById] + public object UpdateMovie(object body) => null; + + [RestDeleteById] + public void DeleteMovie(int id) { } + + [HttpGet("lookup")] + public object Lookup(string term) => null; + } + + [V3ApiController("system/status")] + public class SystemController : Controller + { + [HttpGet] + public object GetStatus() => null; + } +} +""" + specs = parse_csharp_attribute_specs(csharp_code) + assert evaluate_attribute_usage("V3ApiController", None, specs) == (None, "api/v3/[controller]") + assert evaluate_attribute_usage("V3ApiController", '"system/status"', specs) == (None, "api/v3/system/status") + assert evaluate_attribute_usage("RestGetById", None, specs) == ("GET", "{id:int}") + assert evaluate_attribute_usage("RestPostById", None, specs) == ("POST", "") + assert evaluate_attribute_usage("RestPutById", None, specs) == ("PUT", "{id:int?}") + assert evaluate_attribute_usage("RestDeleteById", None, specs) == ("DELETE", "{id:int}") + + res = extract_symbols_and_chunks(csharp_code, "src/MovieController.cs", repo="radarr") + route_set = {(r.http_method, r.path_pattern, r.handler_symbol) for r in res.api_routes} + + assert ("GET", "/api/v3/movie", "MovieController.AllMovies") in route_set + assert ("GET", "/api/v3/movie/{id}", "MovieController.GetById") in route_set + assert ("POST", "/api/v3/movie", "MovieController.AddMovie") in route_set + assert ("PUT", "/api/v3/movie/{id}", "MovieController.UpdateMovie") in route_set + assert ("DELETE", "/api/v3/movie/{id}", "MovieController.DeleteMovie") in route_set + assert ("GET", "/api/v3/movie/lookup", "MovieController.Lookup") in route_set + assert ("GET", "/api/v3/system/status", "SystemController.GetStatus") in route_set + # Abstract base controllers without a class-level route template should not emit bare "/{id}" routes + assert all(r.path_pattern != "/{id}" for r in res.api_routes) + + +def test_typescript_class_field_arrow_methods_and_axios_api_prefix_matching(): + """Verifies TS class field arrow functions are extracted as Class.method symbols and their relative axios calls match /api/v{n} routes.""" + ts_code = """ +class RadarrAPI { + public getMovies = async () => { + const response = await this.axios.get('/movie'); + return response.data; + }; + + public getMovie = async ({ id }: { id: number }) => { + const response = await this.axios.get(`/movie/${id}`); + return response.data; + }; + + public addMovie = async (options: Record) => { + const response = await this.axios.post(`/movie`, options); + return response.data; + }; +} +""" + res = extract_symbols_and_chunks(ts_code, "server/api/servarr/radarr.ts", repo="overseerr") + by_full = {s.full_symbol: s for s in res.symbols} + + assert "RadarrAPI" in by_full + assert "RadarrAPI.getMovies" in by_full + assert by_full["RadarrAPI.getMovies"].kind == "method_definition" + assert "RadarrAPI.getMovie" in by_full + assert "RadarrAPI.addMovie" in by_full + + # Ensure class chunk does not duplicate method bodies + class_chunks = [c for c in res.chunks if c.symbol == "RadarrAPI"] + assert all("this.axios.get('/movie')" not in c.content for c in class_chunks) + + call_tuples = [(c.http_method, c.url_pattern, c.caller_symbol) for c in res.api_client_calls] + assert ("GET", "/movie", "RadarrAPI.getMovies") in call_tuples + assert ("GET", "/movie/{id}", "RadarrAPI.getMovie") in call_tuples + assert ("POST", "/movie", "RadarrAPI.addMovie") in call_tuples + + # Relative axios client paths match versioned /api/v3/* server routes + assert match_route_and_call("/api/v3/movie", "/movie") + assert match_route_and_call("/api/v3/movie/{id}", "/movie/{id}") + assert match_route_and_call("/api/v3/movie/lookup", "/movie/lookup") + assert not match_route_and_call("/api/v3/series", "/movie") + + +def test_permalinks_are_opt_in_and_direct_tool_aliases_work(tmp_path, monkeypatch): + """Verifies find_routes and search omit permalinks by default (until include_permalinks=True) and direct tool calls accept common parameter aliases.""" + import asyncio + import json + from types import SimpleNamespace + from unittest.mock import patch + + from app.mcp.handlers.diagnostic_handlers import handle_diagnose_issue + from app.mcp.handlers.file_handlers import handle_read_file + from app.mcp.handlers.grep_handlers import handle_grep_code + from app.mcp.handlers.route_handlers import handle_find_routes + from app.mcp.handlers.search_handlers import handle_search + from app.mcp.handlers.symbol_handlers import handle_find_symbol, handle_trace_path + + db_file = str(tmp_path / "opt_in_permalinks.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + with get_db_connection() as conn: + conn.execute( + "INSERT INTO git_repositories (name, url, branch, commit_sha, provider) VALUES (?, ?, ?, ?, ?)", + ("radarr", "https://github.com/Radarr/Radarr", "develop", "1234567890abcdef", "github"), + ) + conn.execute( + "INSERT INTO api_routes (repo, filepath, framework, http_method, path_pattern, handler_symbol, start_line, end_line) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ("radarr", "src/Radarr.Api.V3/Movies/MovieImportController.cs", "aspnet", "POST", "/api/v3/movie/import", "MovieImportController.Import", 18, 25), + ) + conn.execute( + "INSERT INTO api_client_calls (repo, filepath, http_method, url_pattern, caller_symbol, line_number) " + "VALUES (?, ?, ?, ?, ?, ?)", + ("radarr", "frontend/src/ImportMovie.ts", "POST", "/api/v3/movie/import", "importMovies", 42), + ) + conn.execute( + "INSERT INTO ast_symbols (repo, filepath, name, full_symbol, kind, start_line, end_line, signature, language) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ("radarr", "src/Radarr.Api.V3/Movies/MovieImportController.cs", "Import", "MovieImportController.Import", "method_declaration", 18, 25, "public List Import()", "c_sharp"), + ) + conn.commit() + + # 1. find_routes omits permalink by default and accepts `path` / `route` aliases + routes_default = json.loads(asyncio.run(handle_find_routes(path="/api/v3/movie/import", repo="radarr"))) + assert routes_default["total_routes"] == 1 + assert "permalink" not in routes_default["routes"][0] + assert "permalink" not in routes_default["callers"][0] + + routes_with_links = json.loads( + asyncio.run(handle_find_routes(route="/api/v3/movie/import", repo="radarr", include_permalinks=True)) + ) + assert "github.com/Radarr/Radarr/blob/1234567890abcdef" in (routes_with_links["routes"][0].get("permalink") or "") + assert "github.com/Radarr/Radarr/blob/1234567890abcdef" in (routes_with_links["callers"][0].get("permalink") or "") + + # 2. search omits permalink by default and includes it when include_permalinks=True + fake_hit = SimpleNamespace( + payload={ + "repo": "radarr", + "rel_path": "src/Radarr.Api.V3/Movies/MovieImportController.cs", + "start_line": 18, + "end_line": 25, + "full_symbol": "MovieImportController.Import", + "kind": "method_declaration", + "language": "c_sharp", + "permalink_url": "https://github.com/Radarr/Radarr/blob/1234567890abcdef/src/MovieImportController.cs#L18-L25", + "content": "public List Import()", + }, + score=0.95, + dense_score=0.9, + sparse_score=0.8, + ) + with patch("app.mcp.tools.execute_hybrid_search", return_value=[fake_hit]): + search_default = json.loads(asyncio.run(handle_search(query="Import", repo="radarr"))) + assert "permalink" not in search_default["results"][0] + search_with_link = json.loads( + asyncio.run(handle_search(query="Import", repo="radarr", include_permalinks=True)) + ) + assert search_with_link["results"][0]["permalink"].startswith("https://github.com/Radarr/Radarr/blob/") + + # 3. find_symbol accepts `query` alias, trace_path accepts `name`/`query`, read_file accepts `filepath`, + # grep_code accepts `query`/`filepath`, and diagnose_issue accepts `query`/`error`. + sym_res = json.loads(asyncio.run(handle_find_symbol(query="MovieImportController.Import", repo="radarr"))) + assert sym_res["total"] == 1 and sym_res["symbols"][0]["symbol"] == "MovieImportController.Import" + + with patch("app.mcp.tools.trace_symbol_path", return_value={"target": {"symbol": "MovieImportController.Import"}, "callers": [], "callees": []}): + trace_res = json.loads(asyncio.run(handle_trace_path(name="MovieImportController.Import", repo="radarr"))) + assert trace_res["target"]["symbol"] == "MovieImportController.Import" + + outline_res = json.loads( + asyncio.run( + handle_read_file( + filepath="src/Radarr.Api.V3/Movies/MovieImportController.cs", + repo="radarr", + outline_only=True, + ) + ) + ) + assert outline_res["total_symbols"] == 1 + + with patch("app.mcp.handlers.diagnostic_handlers.diagnose_issue", return_value={"call_chain": {"entry": "Import", "terms": [], "steps": [], "other_hits": [], "nodes_read": 1}}): + diag_res = json.loads(asyncio.run(handle_diagnose_issue(error="NullReferenceException in Import", repo="radarr"))) + assert diag_res["call_chain"]["entry"] == "Import" + + fake_reader = SimpleNamespace( + storage_root=str(tmp_path), + grep_code=lambda **kw: { + "matches": [ + { + "repo": "radarr", + "file": kw.get("path_glob") or "MovieImportController.cs", + "line": 18, + "match_text": kw.get("pattern"), + "context": [], + "permalink": "https://github.com/Radarr/Radarr/blob/123/MovieImportController.cs#L18", + } + ] + }, + ) + with patch("app.mcp.handlers.grep_handlers._resolve_reader_factory", return_value=lambda: fake_reader): + grep_res = json.loads(asyncio.run(handle_grep_code(query="Import", filepath="MovieImportController.cs"))) + assert grep_res["total_matches"] == 1 + assert grep_res["path_filter"] == "MovieImportController.cs" + assert "permalink" not in grep_res["files"][0] + + +def test_code_mode_disabled_omits_execute_script_and_sse_post_routes_to_mcp(monkeypatch): + """Verifies CODE_MODE_ENABLED=false omits execute_script from get_tool_definitions() and McpPingMiddleware routes POST /sse to /mcp.""" + import asyncio + from app.mcp.mcp_server import build_mcp_server_instructions + from app.mcp.tools import get_tool_definitions + from main import McpPingMiddleware + + monkeypatch.setenv("CODE_MODE_ENABLED", "false") + names_disabled = {name for name, _fn, _desc in get_tool_definitions()} + assert "execute_script" not in names_disabled + assert "execute_script" not in build_mcp_server_instructions() + + monkeypatch.setenv("CODE_MODE_ENABLED", "true") + names_enabled = {name for name, _fn, _desc in get_tool_definitions()} + assert "execute_script" in names_enabled + assert "execute_script" in build_mcp_server_instructions() + + captured_scopes = [] + + async def _dummy_app(scope, receive, send): + captured_scopes.append(dict(scope)) + + mw = McpPingMiddleware(_dummy_app) + + async def _empty_receive(): + return {"type": "http.request", "body": b"{}", "more_body": False} + + async def _noop_send(_msg): + pass + + # POST /sse without session_id routes to /mcp for Streamable HTTP clients + asyncio.run( + mw( + {"type": "http", "method": "POST", "path": "/sse", "query_string": b""}, + _empty_receive, + _noop_send, + ) + ) + assert captured_scopes[-1]["path"] == "/mcp" + + # GET /sse stays on /sse for legacy SSE transport + asyncio.run( + mw( + {"type": "http", "method": "GET", "path": "/sse", "query_string": b""}, + _empty_receive, + _noop_send, + ) + ) + assert captured_scopes[-1]["path"] == "/sse" + diff --git a/tests/test_mcp_real_e2e_workflows.py b/tests/test_mcp_real_e2e_workflows.py index d144649..89cb4b6 100644 --- a/tests/test_mcp_real_e2e_workflows.py +++ b/tests/test_mcp_real_e2e_workflows.py @@ -214,21 +214,15 @@ def real_e2e_env(tmp_path, monkeypatch): @pytest.mark.asyncio async def test_e2e_grep_code_survives_broken_venv_symlink_and_enriches_ast(real_e2e_env): """Verifies `grep_code` does not crash on broken `.venv/lib64` symlinks and returns AST symbol metadata.""" - res_raw = await handle_grep_code( - pattern="Missing bearer token", - repo="platform-core", - context_lines=2, - ) - res = json.loads(res_raw) - assert "error" not in res, f"grep_code failed: {res}" - assert res["total_matches"] == 1 - assert res["repo"] == "platform-core" + res = json.loads(await handle_grep_code(pattern="Missing bearer token", repo="platform-core", context_lines=2)) + assert "error" not in res and res["total_matches"] == 1 and res["repo"] == "platform-core" [group] = res["files"] - assert group["file"].endswith("app/api/users.py") + assert group["file"].endswith("app/api/users.py") and "permalink" not in group [(match_line, symbol)] = group["symbols"].items() assert symbol == "validate_token" assert any(line.startswith(f"{match_line}:") and "Missing bearer token" in line for line in group["lines"]) - assert "blob/400ce382" in group.get("permalink", "") and "#L" not in group["permalink"] + res_link = json.loads(await handle_grep_code(pattern="Missing bearer token", repo="platform-core", include_permalinks=True)) + assert "blob/400ce382" in res_link["files"][0].get("permalink", "") and "#L" not in res_link["files"][0]["permalink"] @pytest.mark.asyncio