From 602f9918fbeff3d1542e8a9a22257615eabe3f4d Mon Sep 17 00:00:00 2001 From: Antigravity Agent Date: Tue, 17 Nov 2026 18:53:39 -0600 Subject: [PATCH] docs: update v2.27.4 settings, MCP tool counts, admin API payloads, and large repo guide (#172) --- .env.example | 79 +++++++++++--- DEVELOPER_DOCS.md | 127 ++++++++++++++--------- docs/architecture/mcp-protocol.md | 2 +- docs/guide/configuration.md | 85 ++++++++++++--- docs/guide/index.md | 2 +- docs/index.md | 6 +- docs/reference/rest-api.md | 155 +++++++++++++++++++++++----- docs/requirements/index.md | 2 +- docs/requirements/non-functional.md | 5 +- docs/requirements/verification.md | 2 +- scripts/portable/.env.example | 6 ++ 11 files changed, 361 insertions(+), 110 deletions(-) diff --git a/.env.example b/.env.example index f4ccee07..31f7b2ac 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,9 @@ POSTGRES_PORT=5432 # DATABASE_URL=sqlite:///app/data/index_cache.db DATABASE_URL=postgresql+psycopg://contextcortex:cortexsecret@postgres:5432/contextcortex +# Optional explicit SQLite database path when DATABASE_URL is unset. +# CACHE_DB_PATH=/app/data/index_cache.db + # ------------------------------------------------------------------------------ # 2. Server, Vault & Local Storage Paths # ------------------------------------------------------------------------------ @@ -31,6 +34,9 @@ PORT=3000 # LOG_LEVEL=INFO VAULT_PATH=/docs +# Base data directory for persistent application storage. +# DATA_DIR=/app/data + # Managed local storage directory for direct file uploads, replacements, and incremental indexing. # Defaults to data/storage or /app/data/storage if unset. LOCAL_STORAGE_PATH=/app/data/storage @@ -42,15 +48,21 @@ LOCAL_STORAGE_PATH=/app/data/storage # Default when DATABASE_URL points to PostgreSQL is pgvector; otherwise qdrant. VECTOR_STORE_PROVIDER=pgvector +# Vector store operating mode: embedded | remote +# VECTOR_STORE_MODE=remote + # Embedded storage path for file-based vector backends (e.g. local Qdrant) -VECTOR_STORE_STORAGE_PATH=/app/data/vectors +VECTOR_STORE_STORAGE_PATH=/app/data/vector_storage -# Vector collection name for documents and chunk embeddings +# Vector collection name for documents and chunk embeddings (alias: COLLECTION_NAME) VECTOR_STORE_COLLECTION=knowledge_rag_v1 # Optional remote connection endpoint for Qdrant if using an external Qdrant server: # QDRANT_URL=http://qdrant:6333 +# Default dense semantic weight (0.0 to 1.0) for hybrid search fusion (default: 0.5). +# HYBRID_DENSE_WEIGHT=0.5 + # ------------------------------------------------------------------------------ # 4. OpenAI-Compatible AI & Embedding Gateway # ------------------------------------------------------------------------------ @@ -60,10 +72,36 @@ VECTOR_STORE_COLLECTION=knowledge_rag_v1 OPENAI_API_URL=http://litellm:4000/v1 OPENAI_API_KEY=dummy -# Batch size and CPU thread limits for ingestion pipelines: +# Embedding provider: local (FastEmbed ONNX) | api (OpenAI-compatible gateway). +# Default is local in source/container installs and api in the Windows portable bundle. +# EMBEDDING_PROVIDER=local +# EMBEDDING_MODEL=BAAI/bge-small-en-v1.5 +# SPARSE_MODEL=Qdrant/bm25 + +# Optional API fallback when local FastEmbed model initialization fails: +# EMBEDDING_ALLOW_FALLBACK=false +# EMBEDDING_FALLBACK_DENSE_MODEL= + +# Optional default Vision OCR and Chat Summarization models: +# VISION_OCR_MODEL= +# CHAT_MODEL= + +# CPU thread limit for local FastEmbed ONNX models (EMBEDDING_PROVIDER=local only). +# Has no effect when EMBEDDING_PROVIDER=api. Default: min(2, system_cpus). EMBEDDING_NUM_THREADS=4 + +# Maximum chunks per embedding batch request (default: 32 for local, 128 for api, max 256 per API request). +# Chunks are batched across files during indexing. EMBEDDING_BATCH_SIZE=32 +# Parallel API embedding concurrency and rate-limit pacing (when EMBEDDING_PROVIDER=api): +# Number of parallel in-flight embedding API requests during indexing (default: 4). +# EMBEDDING_API_CONCURRENCY=4 +# Estimated token cap per API batch request (default: 100000). +# EMBEDDING_MAX_TOKENS_PER_REQUEST=100000 +# Optional tokens-per-minute pacing limit for strict quotas (default: 0 / unlimited). +# EMBEDDING_TPM_LIMIT=0 + # Extra HTTP headers sent on every gateway request, as a JSON object. # LLM_EXTRA_HEADERS={"X-User-Agent":"contextcortex","X-User-Name":"jdoe"} @@ -71,7 +109,19 @@ EMBEDDING_BATCH_SIZE=32 # EXTRA_CA_CERTS_DIR=/certs # ------------------------------------------------------------------------------ -# 5. MCP 2026-07-28 Authentication & RBAC Security Engine +# 5. MCP Tools & Code Mode Sandbox +# ------------------------------------------------------------------------------ +# Enable the optional `manage` MCP write tool (default: false; exposes 10 read/search/code-mode tools). +# MCP_ENABLE_WRITE_TOOLS=false + +# Code Mode (`execute_script`) sandbox limits: +# CODE_MODE_ENABLED=true +# CODE_MODE_TIMEOUT_SECONDS=30 +# CODE_MODE_MAX_TOOL_CALLS=50 +# CODE_MODE_MAX_MEMORY_MB=64 + +# ------------------------------------------------------------------------------ +# 6. MCP 2026-07-28 Authentication & RBAC Security Engine # ------------------------------------------------------------------------------ # Enable or disable Bearer token & API key authentication (true | false). # When false (default), requests operate in unauthenticated local developer bypass mode. @@ -94,20 +144,23 @@ AUTH_JWKS_URI= ADMIN_INITIAL_KEY= # ------------------------------------------------------------------------------ -# 6. Background Poller, Sync, and Git Integration +# 7. Webhooks, Sync, and Git Integration # ------------------------------------------------------------------------------ -# Interval in seconds for background poller daemon to scan repositories for remote updates. -AUTO_SYNC_INTERVAL=300 - # Global webhook secret for HMAC verification of push webhooks from GitHub / GitLab / Gitea. GLOBAL_WEBHOOK_SECRET= -# Optional GitHub Personal Access Token for higher rate limits during remote repository cloning. +# Optional Personal Access Tokens for higher rate limits and private repository cloning. +# For self-hosted instances, you can also add per-host credentials in the Admin UI. GITHUB_TOKEN= - -# Optional GitLab Personal Access Token (read_repository scope) for cloning GitLab repositories. -# For self-hosted GitLab, you can also add a per-host credential in the admin UI. GITLAB_TOKEN= +# GITEA_TOKEN= +# BITBUCKET_TOKEN= -# Directory used for temporary repository checkouts and shallow clones. +# Directory used for temporary repository checkouts and ephemeral shallow clones. TMP_REPOS_DIR=/tmp/rag_repos + +# Directory used for persistent shallow clones when keep_shallow is enabled. +# PERSISTENT_REPOS_DIR=/app/data/repos + +# Maximum timeout in seconds for git clone operations (default: 300). +# GIT_CLONE_TIMEOUT_SECONDS=300 diff --git a/DEVELOPER_DOCS.md b/DEVELOPER_DOCS.md index 50689539..58df5ac7 100644 --- a/DEVELOPER_DOCS.md +++ b/DEVELOPER_DOCS.md @@ -93,11 +93,13 @@ cd contextcortex 6. **Set Environment Variables (Optional):** ```bash export VECTOR_STORE_PROVIDER="pgvector" # "pgvector" or "qdrant" - export COLLECTION_NAME="knowledge_rag_v1" + export VECTOR_STORE_COLLECTION="knowledge_rag_v1" # Alias: COLLECTION_NAME export OPENAI_API_URL="http://litellm:4000/v1" # OpenAI-compatible gateway endpoint export OPENAI_API_KEY="sk-your-key" # Gateway API key / Bearer token - export EMBEDDING_NUM_THREADS=2 # CPU thread cap (defaults to min(2, system_cpus)) - export EMBEDDING_BATCH_SIZE=32 # Batch size for embedding requests (defaults to 32) + export EMBEDDING_PROVIDER="local" # "local" (FastEmbed ONNX) or "api" (OpenAI-compatible gateway) + export EMBEDDING_NUM_THREADS=2 # CPU thread cap for local FastEmbed ONNX only (defaults to min(2, system_cpus); no effect when EMBEDDING_PROVIDER=api) + export EMBEDDING_BATCH_SIZE=32 # Texts per embedding call (defaults to 32 for local, 128 for api, max 256 per API request) + export EMBEDDING_API_CONCURRENCY=4 # Max concurrent in-flight API embedding requests when EMBEDDING_PROVIDER=api export LLM_EXTRA_HEADERS='{"X-User-Agent":"contextcortex"}' # Optional JSON extra headers for LLM gateway requests export EXTRA_CA_CERTS_DIR="/certs" # Optional directory of private CA certificates (*.crt) export GITHUB_TOKEN="ghp_your_token" # Optional: For higher rate limits & private repos @@ -171,25 +173,54 @@ npm run docs:preview | Variable | Description | Default | | :--- | :--- | :--- | +| `HOST` | Listen address (`0.0.0.0` in Docker, `localhost` in Windows portable bundle) | `0.0.0.0` | +| `PORT` | HTTP server port for FastAPI, Admin UI, and MCP endpoints | `3000` | +| `LOG_LEVEL` | Application log verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) | `INFO` | | `DATABASE_URL` | SQLAlchemy connection string (e.g. `postgresql+psycopg://...` or `sqlite:///...`) | `sqlite:////app/data/index_cache.db` | +| `DATA_DIR` | Root data directory for persistent application state | `/app/data` | +| `CACHE_DB_PATH` | Path to persistent SQLite cache database | `/app/data/index_cache.db` | | `LOCAL_STORAGE_PATH` | Storage directory for managed local storage file uploads | `/app/data/storage` | -| `PERSISTENT_REPOS_DIR` | Storage directory for retained shallow git clones (`keep_shallow`) | `/app/data/repos` | -| `AUTH_ENABLED` | Enable MCP 2026-07-28 OAuth 2.1 & API Key RBAC | `false` | +| `VAULT_PATH` | Default path to the Markdown documentation directory | `/docs` | +| `TMP_REPOS_DIR` | Ephemeral workspace directory for shallow Git clones during sync | `/tmp/contextcortex_repos` | +| `PERSISTENT_REPOS_DIR` | Storage directory for retained shallow Git clones (`keep_shallow`) | `/app/data/repos` | +| `GIT_CLONE_TIMEOUT_SECONDS` | Timeout in seconds for `git clone` / `git fetch` operations | `300` | | `VECTOR_STORE_PROVIDER` | Vector database backend (`pgvector` or `qdrant`) | `pgvector` (Postgres) / `qdrant` (SQLite) | | `VECTOR_STORE_MODE` | Vector store mode (`embedded` or `remote`) | `embedded` | -| `COLLECTION_NAME` | Vector collection name | `knowledge_rag_v1` | +| `VECTOR_STORE_STORAGE_PATH` | Storage directory for embedded Qdrant (`QDRANT_STORAGE_PATH` alias) | `/app/data/qdrant_storage` | +| `VECTOR_STORE_COLLECTION` | Vector collection name (`COLLECTION_NAME` alias) | `knowledge_rag_v1` | | `QDRANT_URL` | URL to the remote Qdrant vector database | `http://localhost:6333` | -| `QDRANT_STORAGE_PATH` | Storage directory for embedded Qdrant | `/app/data/qdrant_storage` | +| `HYBRID_DENSE_WEIGHT` | Default dense semantic weight (`0.0` to `1.0`) for hybrid search fusion | `0.5` | | `OPENAI_API_URL` | OpenAI-compatible gateway base URL passed to the OpenAI SDK | `http://litellm:4000/v1` | | `OPENAI_API_KEY` | API key for the OpenAI-compatible gateway | `dummy` | | `LLM_EXTRA_HEADERS` | JSON object of extra HTTP headers sent on every gateway request | `None` | +| `EMBEDDING_PROVIDER` | Dense embedding execution backend (`local` FastEmbed ONNX or `api` gateway) | `local` | +| `EMBEDDING_MODEL` | Active dense embedding model identifier | `BAAI/bge-small-en-v1.5` (`local`) / `text-embedding-3-small` (`api`) | +| `SPARSE_MODEL` | Local FastEmbed sparse lexical model for BM25 token weights | `Qdrant/bm25` | +| `EMBEDDING_ALLOW_FALLBACK` | Allow fallback to local dense model if API embedding initialization fails | `false` | +| `EMBEDDING_FALLBACK_DENSE_MODEL` | Local dense model used when `EMBEDDING_ALLOW_FALLBACK=true` | `BAAI/bge-small-en-v1.5` | +| `EMBEDDING_NUM_THREADS` | ONNX Runtime CPU threads per local FastEmbed model (`EMBEDDING_PROVIDER=local` only; no effect when `EMBEDDING_PROVIDER=api`) | `min(2, system_cpus)` | +| `EMBEDDING_BATCH_SIZE` | Texts per embedding call across files (clamped to `256` per API request) | `32` (`local`) / `128` (`api`) | +| `EMBEDDING_API_CONCURRENCY` | Maximum concurrent in-flight API embedding requests during indexing (`EMBEDDING_PROVIDER=api`) | `4` | +| `EMBEDDING_MAX_TOKENS_PER_REQUEST` | Estimated token budget cap per API embedding request batch | `100000` | +| `EMBEDDING_TPM_LIMIT` | Client-side tokens-per-minute pacing limit (`0` = disabled, reactive `429` handling only) | `0` | +| `VISION_OCR_MODEL` | Multimodal vision model for scanned PDF / diagram OCR fallback | `gpt-4o-mini` | +| `CHAT_MODEL` | Default chat completion model for large-file executive summarization | `gpt-4o-mini` | | `EXTRA_CA_CERTS_DIR` | Directory of private CA certificates (`*.crt`) trusted at container start | `/certs` | +| `MCP_ENABLE_WRITE_TOOLS` | Expose the state-mutating `manage` MCP tool (`sync_repo`, `upload`, `replace`, `delete`) | `false` | +| `CODE_MODE_ENABLED` | Enable the sandboxed `execute_script` Code Mode MCP tool | `true` | +| `CODE_MODE_TIMEOUT_SECONDS` | Maximum wall-clock execution time in seconds for a single `execute_script` call | `10.0` | +| `CODE_MODE_MAX_TOOL_CALLS` | Maximum inner `tools.*()` invocations allowed per `execute_script` run | `50` | +| `CODE_MODE_MAX_MEMORY_MB` | Maximum QuickJS heap memory in megabytes per `execute_script` sandbox | `64` | +| `AUTH_ENABLED` | Enable MCP OAuth 2.1 (RFC 9728) & `cc_` API Key RBAC | `false` | +| `AUTH_OIDC_ISSUER` | OpenID Connect Identity Provider issuer URL | `None` | +| `AUTH_JWKS_URI` | Custom JSON Web Key Set URL override for JWT verification | `None` | +| `AUTH_RESOURCE_INDICATOR` | RFC 8707 / RFC 9728 OAuth 2.1 Resource Indicator URI | `https://contextcortex.local` | +| `ADMIN_INITIAL_KEY` | Initial Admin API key seeded on first startup | `None` | +| `GLOBAL_WEBHOOK_SECRET` | Fallback HMAC secret for `/api/webhooks/git` when a repo has no per-repo secret | `None` | | `GITHUB_TOKEN` | Optional GitHub Personal Access Token for rate limits & private repos | `None` | -| `VAULT_PATH` | Default path to the markdown documentation directory | `/docs` | -| `CACHE_DB_PATH` | Path to persistent SQLite cache database | `/app/data/index_cache.db` | -| `CHUNK_SIZE` | Maximum character length per Markdown/doc chunk | `1500` | -| `CODE_CHUNK_SIZE` | Maximum character length per AST code symbol/function chunk | `4000` | -| `CHUNK_OVERLAP` | Character overlap between consecutive chunks | `200` | +| `GITLAB_TOKEN` | Optional GitLab Personal Access Token | `None` | +| `GITEA_TOKEN` | Optional Gitea / Forgejo Access Token | `None` | +| `BITBUCKET_TOKEN` | Optional Bitbucket App Password / Token | `None` | --- @@ -311,33 +342,34 @@ Example configuration: } ``` -### Available MCP Tools (19 Tools): -- `search_code(query="JWT authentication handler", repo="backend-api")` [Role: `viewer`] -- `search_docs(query="caddy reverse proxy configuration")` [Role: `viewer`] -- `grep_code(pattern="enforce_tool_permission", repo="backend-api", path_glob="*.py")` [Role: `viewer`] -- `diagnose_issue(issue_text="Traceback (most recent call last): ...", repo="backend-api")` [Role: `viewer`] -- `find_symbol(symbol="extract_symbols_and_chunks", exact=true)` [Role: `viewer`] -- `trace_path(symbol="authenticate_user", repo="backend-api")` [Role: `viewer`] -- `find_routes(repo="backend-api")` [Role: `viewer`] -- `find_api_callers(path="/api/v1/auth/login")` [Role: `viewer`] -- `get_file_outline(filepath="app/services/search.py")` [Role: `viewer`] -- `list_repositories()` [Role: `viewer`] -- `get_recent_changes(repo="backend-api", filepath="app/services/search.py", limit=10)` [Role: `viewer`] -- `sync_repository(repo="backend-api")` [Role: `editor`] -- `index_status()` [Role: `viewer`] -- `get_architecture(repo="backend-api")` [Role: `viewer`] -- `manage_adr(action="list", repo="backend-api")` [Role: `editor` for mutations, `viewer` for list/get] -- `manage_local_file(action="upload", file_path="notes/spec.md", content="# Spec")` [Role: `editor` for mutations, `viewer` for read] -- `what_is_ingested(source_type="all", detail_level="summary")` [Role: `viewer`] -- `read_file(path="app/services/search.py", repo="backend-api", start_line=1, end_line=120)` [Role: `viewer`] -- `summarize_file(path="app/services/search.py", repo="backend-api")` [Role: `viewer`] - -### Available MCP Resources: -- `knowledge://catalog/summary` +### Available MCP Tools (10 Default Tools + Optional `manage` Write Tool): +- `diagnose_issue(issue_text="Traceback (most recent call last): ...", repo="backend-api")` [Role: `viewer`] — First tool to call on bug reports, stack traces, or regressions; correlates candidate symbols, recent commits, `call_chain`, and `working_path_comparison`. +- `search(query="JWT authentication handler", repo="backend-api", content_type="code", mode="hybrid", dense_weight=0.5)` [Role: `viewer`] — Calibrated hybrid semantic + BM25 search across code and documentation (`content_type`: `"all"`, `"code"`, or `"docs"`). +- `grep_code(pattern="enforce_tool_permission", repo="backend-api", path_glob="*.py")` [Role: `viewer`] — Exact or regex source code search with context lines. +- `find_symbol(symbol="extract_symbols_and_chunks", exact=true)` [Role: `viewer`] — Locate function, class, or method definitions via the AST index. +- `trace_path(symbol="authenticate_user", repo="backend-api", direction="both", depth=1)` [Role: `viewer`] — Multi-hop caller, callee, import, and inheritance graph traversal (`depth` `1` to `5`). +- `find_routes(repo="backend-api", direction="both")` [Role: `viewer`] — Discover declared HTTP/REST API routes and outbound client HTTP calls. +- `read_file(path="app/services/search.py", repo="backend-api", start_line=1, end_line=120, outline_only=false, summarize=false)` [Role: `viewer`] — Read line-bounded source files, AST symbol outlines (`outline_only=True`), or LLM executive summaries (`summarize=True`). +- `list_repositories(include_status=false, include_catalog=false, recent_commits=0)` [Role: `viewer`] — List indexed repositories alongside optional sync status, ingestion catalog counts, and recent commits. +- `get_architecture(repo="backend-api", include_adrs=true)` [Role: `viewer`] — Synthesize repository entry points, language distribution, directory hotspots, and Architectural Decision Records (ADRs). +- `execute_script(code="const h = await tools.help('search'); return h;")` [Role: `viewer`, enabled when `CODE_MODE_ENABLED=true`] — Execute sandboxed JavaScript in QuickJS with `await tools.(args)` and `await tools.help('')` to chain multiple read-only MCP calls in one round-trip. +- `manage(action="sync_repo", repo="backend-api")` [Role: `editor` for `sync_repo`, `upload`, `replace`, `delete`; `viewer` for `read`, `preview`; registered only when `MCP_ENABLE_WRITE_TOOLS=true`] — Unified administrative tool for repository sync and managed local storage mutations. + +### Available MCP Resources & Resource Templates: +- `knowledge://catalog/summary` — Global ingestion catalog and vector index summary (`application/json`). +- `contextcortex://code-mode/sdk.d.ts` — TypeScript ambient declarations (`declare namespace tools`) and return-shape contracts for the `execute_script` sandbox (`text/plain`). +- `knowledge://repo/{repo}/summary` — Repository metadata, branch, commit SHA, and file/symbol counts. +- `knowledge://repo/{repo}/architecture` — Synthesized repository architecture overview. +- `knowledge://repo/{repo}/files/{page}` — Paginated list of indexed files (`100` files per page). +- `knowledge://repo/{repo}/routes/{page}` — Paginated list of discovered API routes (`100` routes per page). +- `knowledge://repo/{repo}/adrs` — Architectural Decision Records for the specified repository. ### Available MCP Prompts: - `search_infrastructure_docs(topic="docker-compose network topology")` -- `find_implementation_symbol(symbol="execute_hybrid_search")` +- `find_implementation_symbol(symbol="execute_hybrid_search", repo="backend-api")` +- `onboard_repository(repo="backend-api")` +- `triage_error(issue_text="KeyError: 'user_id'", repo="backend-api")` +- `investigate_change_impact(symbol="authenticate_user", repo="backend-api")` --- @@ -348,18 +380,21 @@ Example configuration: | :--- | :--- | :--- | :---: | | List / Search API Keys | `GET /admin/api/auth/keys` | - | `Role.ADMIN` | | Create / Revoke API Keys | `POST/DELETE /admin/api/auth/keys` | - | `Role.ADMIN` | -| Upload Local Storage File | `POST /admin/api/storage/upload` | `manage_local_file (upload)` | `Role.EDITOR` | -| Replace Local Storage File | `PUT /admin/api/storage/file` | `manage_local_file (replace)` | `Role.EDITOR` | -| Delete Local Storage File | `DELETE /admin/api/storage/file` | `manage_local_file (delete)` | `Role.EDITOR` | -| Read Local Storage File | `GET /admin/api/storage/file` | `manage_local_file (read)` | `Role.VIEWER` | +| Add / Delete Git Repository | `POST/DELETE /admin/api/repos` | - | `Role.ADMIN` | +| Add / Delete Monitored Path | `POST/DELETE /admin/api/paths` | - | `Role.ADMIN` | +| Upload Local Storage File | `POST /admin/api/storage/upload` | `manage (action="upload")` | `Role.EDITOR` | +| Replace Local Storage File | `PUT /admin/api/storage/file` | `manage (action="replace")` | `Role.EDITOR` | +| Delete Local Storage File | `DELETE /admin/api/storage/file` | `manage (action="delete")` | `Role.EDITOR` | +| Read Local Storage File | `GET /admin/api/storage/file` | `manage (action="read")` | `Role.VIEWER` | | Browse Storage Directory Tree | `GET /admin/api/storage/tree` | - | `Role.VIEWER` | -| Query Ingestion Catalog | `GET /admin/api/ingestion/catalog` | `what_is_ingested` | `Role.VIEWER` | +| Query Ingestion Catalog | `GET /admin/api/ingestion/catalog` | `list_repositories(include_catalog=True)` | `Role.VIEWER` | | Repository Navigator Tree | `GET /admin/api/navigator/tree` | - | `Role.VIEWER` | -| File Symbol Outline | `GET /admin/api/navigator/file-outline` | - | `Role.VIEWER` | -| Symbol Impact Intelligence | `GET /admin/api/navigator/symbol-impact` | - | `Role.VIEWER` | -| Sync Git Repository | `POST /admin/api/repositories/{id}/sync`| `sync_repository` | `Role.EDITOR` | -| Manage ADRs | - | `manage_adr` (create/update) | `Role.EDITOR` | -| Code & Document Searches | `POST /admin/api/search` | `search_code`, `search_docs` | `Role.VIEWER` | +| File Symbol Outline | `GET /admin/api/navigator/file-outline` | `read_file(outline_only=True)` | `Role.VIEWER` | +| Symbol Impact Intelligence | `GET /admin/api/navigator/symbol-impact` | `trace_path` | `Role.VIEWER` | +| Sync Git Repository | `POST /admin/api/repos/{id}/sync` | `manage (action="sync_repo")` | `Role.EDITOR` | +| Read Architecture & ADRs | - | `get_architecture` | `Role.VIEWER` | +| Code, Doc & Issue Searches | `POST /admin/api/search/test` | `search`, `grep_code`, `diagnose_issue` | `Role.VIEWER` | +| Sandboxed Multi-Tool Script | - | `execute_script` | `Role.VIEWER` | ### Codebase Navigator Component Hierarchy The frontend architecture implements a modular 3-pane layout under `frontend/src/components/navigator/`: diff --git a/docs/architecture/mcp-protocol.md b/docs/architecture/mcp-protocol.md index 383b6a99..84cdb6dd 100644 --- a/docs/architecture/mcp-protocol.md +++ b/docs/architecture/mcp-protocol.md @@ -30,7 +30,7 @@ flowchart TD end subgraph FastMCPEngine["FastMCP 2.0 Engine"] - Tools["20 MCP Tools"] + Tools["10 Default MCP Tools (+ Optional manage)"] Resources["Dynamic Catalog Resources"] Prompts["Guided System Prompts"] end diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index ade5587d..fd03d2d4 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1,35 +1,90 @@ # Configuration and Environment Variables -This document defines all configuration variables and parameters for ContextCortex. +This document defines all configuration variables and runtime parameters for ContextCortex. ## Environment Variables Reference -Configure the application by setting these environment variables in your system environment or a `.env` file. +Configure the application by setting these environment variables in the system environment or in a `.env` file. | Variable Name | Description | Default Value | Example | | :--- | :--- | :--- | :--- | -| `HOST` | Listen address. `localhost` listens on both loopback addresses (`127.0.0.1` and `::1`), so clients that resolve `localhost` to `::1` first (Windows) connect at once. An IP address gives one listener on that address. | `0.0.0.0` (Docker) / `localhost` (Windows portable) | `127.0.0.1` | +| `HOST` | Listen address. `localhost` binds both loopback addresses (`127.0.0.1` and `::1`). | `0.0.0.0` (Docker) / `localhost` (Windows portable) | `127.0.0.1` | +| `PORT` | HTTP server port for FastAPI, Web Admin Dashboard, and MCP endpoints. | `3000` | `3000` | +| `LOG_LEVEL` | Application log verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`). | `INFO` | `INFO` | | `DATABASE_URL` | SQLAlchemy connection string for relational data. | `sqlite:////app/data/index_cache.db` | `postgresql+psycopg://user:pass@postgres:5432/contextcortex` | +| `DATA_DIR` | Root directory for persistent application data. | `/app/data` | `/app/data` | +| `CACHE_DB_PATH` | File path for the SQLite relational database when `DATABASE_URL` is unset. | `/app/data/index_cache.db` | `/app/data/index_cache.db` | +| `LOCAL_STORAGE_PATH` | Host directory for managed file and document uploads. | `/app/data/storage` | `/drives/storage` | +| `VAULT_PATH` | Default path to the Markdown documentation directory seeded on first startup. | `/docs` | `/docs` | +| `TMP_REPOS_DIR` | Temporary directory for ephemeral shallow Git clones (`keep_shallow=false`). | `/tmp/contextcortex_repos` | `/tmp/contextcortex_repos` | +| `PERSISTENT_REPOS_DIR` | Storage directory for retained shallow Git clones (`keep_shallow=true`). | `/app/data/repos` | `/app/data/repos` | +| `GIT_CLONE_TIMEOUT_SECONDS` | Timeout in seconds for `git clone` and `git fetch` commands. | `300` | `600` | | `VECTOR_STORE_PROVIDER` | Active vector search engine (`pgvector` or `qdrant`). | `pgvector` (PostgreSQL) / `qdrant` (SQLite) | `pgvector` | -| `QDRANT_URL` | URL of remote Qdrant service (when `VECTOR_STORE_PROVIDER=qdrant` in remote mode). | `http://localhost:6333` | `http://10.0.0.10:6333` | -| `COLLECTION_NAME` | Name of the primary vector collection in vector store. | `knowledge_rag_v1` | `knowledge_rag_v1` | +| `VECTOR_STORE_MODE` | Qdrant deployment mode (`embedded` local disk or `remote` server). | `embedded` | `remote` | +| `VECTOR_STORE_STORAGE_PATH` | On-disk storage directory for embedded Qdrant (`QDRANT_STORAGE_PATH` alias). | `/app/data/qdrant_storage` | `/app/data/qdrant_storage` | +| `VECTOR_STORE_COLLECTION` | Name of the primary vector collection (`COLLECTION_NAME` alias). | `knowledge_rag_v1` | `knowledge_rag_v1` | +| `QDRANT_URL` | URL of the remote Qdrant service when `VECTOR_STORE_MODE=remote`. | `http://localhost:6333` | `http://10.0.0.10:6333` | +| `HYBRID_DENSE_WEIGHT` | Default dense semantic weight (`0.0` to `1.0`) for hybrid search fusion. | `0.5` | `0.7` | | `OPENAI_API_URL` | OpenAI-compatible gateway base URL. The OpenAI SDK appends `/embeddings`, `/chat/completions`, and `/models`. | `http://litellm:4000/v1` | `https://gateway.example.com/api/v3` | -| `OPENAI_API_KEY` | API key / Bearer token for the OpenAI-compatible gateway. | `dummy` | `sk-xxxxxxxxxxxx` | +| `OPENAI_API_KEY` | API key or Bearer token for the OpenAI-compatible gateway. | `dummy` | `sk-xxxxxxxxxxxx` | | `LLM_EXTRA_HEADERS` | JSON object of extra HTTP headers sent on every gateway request. | None | `{"X-User-Agent":"contextcortex","X-User-Name":"jdoe"}` | -| `EMBEDDING_NUM_THREADS` | Maximum CPU worker threads allocated for ingestion pipelines. | `min(2, system_cpus)` | `4` | -| `EMBEDDING_BATCH_SIZE` | Maximum batch size processed during embedding generation. | `32` | `64` | -| `HYBRID_DENSE_WEIGHT` | Default dense semantic weight (`0.0` to `1.0`) for hybrid search fusion. | `0.5` | `0.7` | +| `EMBEDDING_PROVIDER` | Dense embedding execution backend: `local` (FastEmbed ONNX on CPU) or `api` (OpenAI-compatible `/embeddings` gateway). | `local` | `api` | +| `EMBEDDING_MODEL` | Active dense embedding model identifier. | `BAAI/bge-small-en-v1.5` (`local`) / `text-embedding-3-small` (`api`) | `text-embedding-3-small` | +| `SPARSE_MODEL` | Local FastEmbed sparse lexical model used for BM25 keyword weights. | `Qdrant/bm25` | `Qdrant/bm25` | +| `EMBEDDING_ALLOW_FALLBACK` | Allow fallback to a local FastEmbed dense model if API embedding initialization fails. | `false` | `true` | +| `EMBEDDING_FALLBACK_DENSE_MODEL` | Local dense model loaded when `EMBEDDING_ALLOW_FALLBACK=true`. | `BAAI/bge-small-en-v1.5` | `BAAI/bge-small-en-v1.5` | +| `EMBEDDING_NUM_THREADS` | Maximum CPU worker threads per local FastEmbed ONNX model (`EMBEDDING_PROVIDER=local` only). Has **no effect** on dense embeddings when `EMBEDDING_PROVIDER=api`. | `min(2, system_cpus)` | `4` | +| `EMBEDDING_BATCH_SIZE` | Number of text chunks sent per embedding call across files (clamped to `256` per API request). | `32` (`local`) / `128` (`api`) | `128` | +| `EMBEDDING_API_CONCURRENCY` | Maximum concurrent in-flight HTTP `/embeddings` requests during indexing when `EMBEDDING_PROVIDER=api`. | `4` | `8` | +| `EMBEDDING_MAX_TOKENS_PER_REQUEST` | Estimated token budget cap per API embedding request batch before splitting. | `100000` | `100000` | +| `EMBEDDING_TPM_LIMIT` | Optional client-side tokens-per-minute pacing limit (`0` disables proactive pacing and relies on automatic HTTP `429` backoff). | `0` | `1000000` | +| `VISION_OCR_MODEL` | Multimodal vision model used for scanned PDF and diagram OCR fallback. | `gpt-4o-mini` | `gpt-4o-mini` | +| `CHAT_MODEL` | Default chat completion model used for large-file executive summarization. | `gpt-4o-mini` | `gpt-4o-mini` | | `EXTRA_CA_CERTS_DIR` | Folder of private CA certificates (`*.crt`) added to the trust store at container start. | `/certs` | `/certs` | -| `LOCAL_STORAGE_PATH` | Host path for managed file and document uploads. | `/app/data/storage` | `/drives/storage` | -| `AUTH_ENABLED` | Enables MCP OAuth 2.1 authentication and API key validation. | `false` | `true` | +| `MCP_ENABLE_WRITE_TOOLS` | Expose the state-mutating `manage` MCP tool (`sync_repo`, `upload`, `replace`, `delete`). | `false` | `true` | +| `CODE_MODE_ENABLED` | Enable the sandboxed `execute_script` Code Mode MCP tool. | `true` | `true` | +| `CODE_MODE_TIMEOUT_SECONDS` | Maximum wall-clock execution time in seconds for a single `execute_script` call. | `10.0` | `15.0` | +| `CODE_MODE_MAX_TOOL_CALLS` | Maximum inner `tools.*()` invocations allowed per `execute_script` run. | `50` | `50` | +| `CODE_MODE_MAX_MEMORY_MB` | Maximum QuickJS heap memory in megabytes per `execute_script` sandbox. | `64` | `64` | +| `AUTH_ENABLED` | Enable MCP OAuth 2.1 authentication and `cc_` API key validation. | `false` | `true` | | `AUTH_OIDC_ISSUER` | OpenID Connect Identity Provider issuer URL. | None | `https://auth.company.com/realms/master` | | `AUTH_JWKS_URI` | Custom JSON Web Key Set URL override for token verification. | None | `https://auth.company.com/realms/master/protocol/openid-connect/certs` | -| `AUTH_RESOURCE_INDICATOR`| RFC 8707 / RFC 9728 Resource Indicator for ContextCortex. | `https://contextcortex.local` | `https://contextcortex.wileyriley.com` | -| `ADMIN_INITIAL_KEY` | Bootstrap API key seeded during container initialization. | None | `cc_admin_initial_secret` | -| `GITHUB_TOKEN` | Global GitHub personal access token for higher API limits. | None | `ghp_xxxxxxxxxxxx` | +| `AUTH_RESOURCE_INDICATOR` | RFC 8707 / RFC 9728 Resource Indicator URI for ContextCortex. | `https://contextcortex.local` | `https://contextcortex.example.com` | +| `ADMIN_INITIAL_KEY` | Bootstrap Admin API key seeded during container initialization. | None | `cc_admin_initial_secret` | +| `GLOBAL_WEBHOOK_SECRET` | Fallback HMAC secret for `/api/webhooks/git` when a repository has no per-repo secret. | None | `whsec_xxxxxxxxxxxx` | +| `GITHUB_TOKEN` | Global GitHub personal access token for higher API limits and private repositories. | None | `ghp_xxxxxxxxxxxx` | | `GITLAB_TOKEN` | Global GitLab personal access token. | None | `glpat-xxxxxxxxxxxx` | | `GITEA_TOKEN` | Global Gitea or Forgejo access token. | None | `xxxxxxxxxxxxxxxx` | -| `AUTO_SYNC_INTERVAL` | Default polling interval in minutes for tracked repositories. | `60` | `30` | +| `BITBUCKET_TOKEN` | Global Bitbucket App Password or repository access token. | None | `ATBBxxxxxxxxxxxxxxxx` | + +> **Note on Auto-Sync Polling Interval:** Background repository polling (`auto_sync_interval_mins`, default `15` minutes) is stored in `system_metadata` and configured at runtime via **Settings → Repository Auto-Sync** or `POST /admin/api/settings/auto-sync`. + +--- + +## Large Repositories + +Indexing a large repository (`10,000+` files or `100,000+` AST chunks) behaves differently depending on whether dense embeddings run locally on CPU (`EMBEDDING_PROVIDER=local`) or through an external gateway (`EMBEDDING_PROVIDER=api`). + +### 1. Why Adding CPU Cores or RAM Does Not Speed Up API Embedding +- When `EMBEDDING_PROVIDER=api`, dense vector generation is network-bound and rate-limit-bound on the upstream OpenAI-compatible gateway (`/v1/embeddings`), not CPU-bound on the ContextCortex host. +- `EMBEDDING_NUM_THREADS` controls ONNX Runtime threads **only** for local FastEmbed models (`EMBEDDING_PROVIDER=local` and local sparse BM25 tokenization). Changing `EMBEDDING_NUM_THREADS` has no effect on remote API embedding latency or concurrency. +- The Windows standalone portable bundle runs directly as native Win32 processes (`python.exe` and `postgres.exe`) without a hypervisor or container cgroup limit. It already has access to all host CPU cores and system memory. + +### 2. Cross-File Chunk Batching and Parallel Requests +- ContextCortex does not issue one HTTP `/embeddings` call per file. During repository synchronization, the indexer parses files in super-batches, collects all resulting AST and document chunks across files into a unified buffer, and packs them into large request batches. +- Each API request packs up to `EMBEDDING_BATCH_SIZE` chunks (default `128` for `api`, `32` for `local`, capped at `256` per request) while keeping estimated tokens below `EMBEDDING_MAX_TOKENS_PER_REQUEST` (default `100000` tokens). +- When `EMBEDDING_PROVIDER=api`, the embedding engine dispatches up to `EMBEDDING_API_CONCURRENCY` (default `4`) batches in parallel across a thread pool and reassembles vectors in deterministic chunk order before upserting to PostgreSQL `pgvector` or Qdrant. + +### 3. Automatic HTTP `429` Rate-Limit Handling and Sync Telemetry +- Enterprise gateways (such as Azure OpenAI standard tiers or LiteLLM proxies) enforce Tokens-Per-Minute (TPM) and Requests-Per-Minute (RPM) quotas and return HTTP `429 Too Many Requests` when saturated. +- ContextCortex handles HTTP `429` responses automatically: + 1. **Header-Aware Backoff**: Reads `Retry-After-ms`, `Retry-After`, or `x-ratelimit-reset-tokens` / `x-ratelimit-reset-requests` headers from the gateway response (falling back to exponential backoff with jitter when no header is present). + 2. **Adaptive Concurrency (AIMD)**: Halves the active in-flight request limit immediately upon a `429` response and gradually scales concurrency back up to `EMBEDDING_API_CONCURRENCY` after consecutive successful batches. + 3. **Optional Proactive Pacing**: Set `EMBEDDING_TPM_LIMIT` (for example `EMBEDDING_TPM_LIMIT=1000000`) to enable a client-side token bucket that paces requests before hitting gateway `429` responses. +- **Inspecting Live Throughput and `429` Wait Time**: Query `GET /admin/api/repos/{repo_id}/sync-status` (or watch the live progress banner in the **Git Repositories** tab) to inspect real-time indexing metrics: + - `chunks_embedded` and `chunks_per_sec`: Rolling embedding throughput in chunks per second. + - `embed_batches_completed` / `embed_batches_total` and `in_flight_requests`: Active batch pipeline progress and current concurrency window. + - `rate_limit_429_count` and `rate_limit_wait_seconds`: Total HTTP `429` responses encountered and cumulative seconds spent waiting on gateway rate limits during the sync job. --- diff --git a/docs/guide/index.md b/docs/guide/index.md index 56f1f29d..bf8db30e 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -55,4 +55,4 @@ ContextCortex natively supports corporate and restricted-network environments: - **[Restricted-Network Builds](/guide/enterprise-build)**: Configure private CA certificates, internal package mirrors (`APT_MIRROR`, `PIP_INDEX_URL`, `NPM_REGISTRY`), and verified local image overlays. - **[User Guide](/guide/user-guide)**: Explore all 9 Web Admin Dashboard modules with step-by-step instructions and screenshots. - **[System Architecture](/architecture/)**: Review component diagrams, AST ingestion pipelines, MCP protocol flows, and database ERDs. -- **[Reference Manual](/reference/mcp-tools)**: Inspect all 20 MCP tools, dynamic resources, guided prompts, and Admin REST API endpoints. +- **[Reference Manual](/reference/mcp-tools)**: Inspect all 10 default MCP tools (plus the optional `manage` write tool), dynamic resources, guided prompts, and Admin REST API endpoints. diff --git a/docs/index.md b/docs/index.md index f0c01370..3fceb52e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,8 +25,8 @@ hero: features: - title: Dual-Engine Persistence details: Run with PostgreSQL 16 and pgvector HNSW cosine indexing in production, or zero-dependency SQLite WAL caching for local development. - - title: FastMCP 2.0 & 20 Agent Tools - details: Expose 20 specialized MCP tools (including a sandboxed Code Mode) over SSE and Streamable HTTP transports with zero-allocation JSON-RPC ping and RFC 9728 OAuth 2.1 RBAC. + - title: FastMCP 2.0 & 10-Tool Surface + details: Expose 10 consolidated MCP tools (including sandboxed Code Mode, plus an optional manage write tool) over SSE and Streamable HTTP transports with zero-allocation JSON-RPC ping and RFC 9728 OAuth 2.1 RBAC. - title: Syntax-Aware AST Chunking details: Parse 10 programming languages with Tree-sitter to extract functions, classes, signatures, REST API routes, and caller-callee graphs. - title: Calibrated Hybrid Search & Navigator @@ -40,5 +40,5 @@ features: | **Getting Started & Configuration** | Installation, Docker Compose stack, environment variables, database profiles, LiteLLM proxy setup, and restricted-network mirror builds. | [Overview](/guide/) · [Getting Started](/guide/getting-started) · [Configuration](/guide/configuration) · [Restricted-Network Builds](/guide/enterprise-build) | | **User Guide (Web Dashboard)** | Overview metrics, Codebase Navigator & Omni-Search (`Ctrl+K`), Search & Inspector, Git Repositories, Local Paths, Storage & PDF OCR, Settings, Diagnostics, and Themes. | [User Guide Hub](/guide/user-guide) · [Navigator](/guide/user-guide/navigator) · [Search Inspector](/guide/user-guide/search) | | **System Architecture** | Component interactions, Tree-sitter AST chunking pipeline, calibrated hybrid search scoring, MCP transports, OAuth 2.1 RBAC, and unified database ERD. | [Architecture Overview](/architecture/) · [System Design](/architecture/system-design) · [Data Pipeline](/architecture/data-pipeline) | -| **Reference Manual** | Complete specification of all 20 MCP tools, dynamic resources, guided prompts, Admin REST API routes, and contributor workflows. | [MCP Tools Reference](/reference/mcp-tools) · [Admin REST API](/reference/rest-api) · [Developer Guide](/reference/developer) | +| **Reference Manual** | Complete specification of the 10 default MCP tools (plus optional `manage`), dynamic resources, guided prompts, Admin REST API routes, and contributor workflows. | [MCP Tools Reference](/reference/mcp-tools) · [Admin REST API](/reference/rest-api) · [Developer Guide](/reference/developer) | | **Software Requirements (SRS)** | Functional requirements, non-functional latency/maintainability targets, and automated verification traceability across 1,051 tests. | [SRS Overview](/requirements/) · [Functional](/requirements/functional) · [Verification Matrix](/requirements/verification) | diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 3b150f68..789dd038 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -48,45 +48,146 @@ All administrative routes are prefixed with `/admin/api` unless otherwise noted. ### 1. List Registered Repositories - **Method**: `GET` - **Path**: `/admin/api/repos` -- **Response**: Array of registered Git repositories, branches, commit SHAs, `keep_shallow` and `auto_sync_enabled` flags, and sync statuses. +- **Required Role**: `Role.VIEWER` +- **Response**: Array of registered Git repositories, branches, commit SHAs, `keep_shallow` and `auto_sync` flags, `retain_commits_count`, `file_count`, and sync statuses. -### 2. Register Repository +### 2. Register Repository (`POST /admin/api/repos`) - **Method**: `POST` - **Path**: `/admin/api/repos` -- **Request Body**: +- **Required Role**: `Role.ADMIN` +- **Request Fields (`RepoConfig`)**: + - `name` (string, required): Unique repository identifier used across search, navigator, and MCP tools. + - `url` (string, required): Remote HTTPS or SSH Git clone URL. + - `branch` (string, optional, default `"main"`): Target Git branch to track and index. + - `pat` or `auth_token` (string, optional): Personal access token or repository token override (`pat` is accepted as an alias for `auth_token`). + - `provider` (string, optional, default `"github"`): Git provider hint (`"github"`, `"gitlab"`, `"gitea"`, `"bitbucket"`, or `"generic"`). Auto-detected from `url` when omitted. + - `auth_user` (string, optional): Optional HTTP basic auth username when required by the Git host. + - `auto_sync` (boolean, optional, default `true`): Enable periodic background SHA polling for this repository. + - `keep_shallow` (boolean, optional, default `false`): Retain the shallow clone on disk under `PERSISTENT_REPOS_DIR` for instant full-file reads and fast incremental fetches. + - `retain_commits_count` (integer, optional, default `20`): Number of recent commit metadata records retained in `git_commits` (`0` disables commit history storage). + - `webhook_secret` (string, optional): Per-repository HMAC secret for `/api/webhooks/git`. +- **Scripted `curl` Example**: + ```bash + curl -X POST http://localhost:3000/admin/api/repos \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer cc_admin_your_key" \ + -d '{ + "name": "my-service", + "url": "https://github.com/org/my-service.git", + "branch": "main", + "pat": "ghp_optional_override_token", + "provider": "github", + "keep_shallow": true, + "retain_commits_count": 20 + }' + ``` +- **Response (`200 OK`)**: ```json { - "name": "my-service", - "url": "https://github.com/org/my-service.git", - "branch": "main", - "provider": "github", - "keep_shallow": true, - "auth_token": "ghp_optional_override_token" + "status": "success", + "id": 1 } ``` + Registration immediately spawns a background synchronization thread for the new `id`. -### 3. Toggle Shallow Retention & Auto-Sync +### 3. Toggle Shallow Retention, Auto-Sync & Commit Retention - **Methods**: - - `PATCH /admin/api/repos/{repo_id}/keep-shallow` - - `PATCH /admin/api/repos/{repo_id}/auto-sync` -- **Response**: Updates persistent shallow clone retention (`keep_shallow`) or periodic background auto-sync for the repository. + - `PATCH /admin/api/repos/{repo_id}/keep-shallow` (`{"keep_shallow": true}`) + - `PATCH /admin/api/repos/{repo_id}/auto-sync` (`{"auto_sync": true}`) + - `PATCH /admin/api/repos/{repo_id}/retain-commits` (`{"retain_commits_count": 20}`) +- **Required Role**: `Role.ADMIN` +- **Response**: Updates persistent shallow clone retention (`keep_shallow`), periodic background auto-sync (`auto_sync`), or stored commit count (`retain_commits_count`) for the repository. ### 4. Synchronize, Stream Progress & Cancel Sync - **Methods**: - - `POST /admin/api/repos/{repo_id}/sync`: Triggers background repository synchronization. - - `GET /admin/api/repos/sync/stream`: Server-Sent Events (SSE) progress stream for active repository syncs. - - `GET /admin/api/repos/{repo_id}/sync-status`: Current sync phase and progress counters. - - `POST /admin/api/repos/{repo_id}/cancel-sync`: Cancels an in-flight synchronization run. - - `DELETE /admin/api/repos/{repo_id}`: Purges repository metadata, shallow clone cache, and associated vector points. - -### 5. Monitored Local Paths & Global Indexing -- **Methods**: - - `GET /admin/api/paths`: Lists monitored local host directories and vaults. - - `POST /admin/api/paths`: Registers a new monitored host directory (`path`, `category`, `recursive`). - - `DELETE /admin/api/paths/{path_id}`: Removes a monitored local path. - - `POST /admin/api/reindex`: Triggers a full re-index across all enabled sources. - -### 6. Interactive Calibrated Hybrid Search Tester + - `POST /admin/api/repos/{repo_id}/sync` (or `POST /admin/api/repos/sync/{repo_id}`): Triggers background repository synchronization (`Role.EDITOR`). + - `GET /admin/api/repos/sync/stream`: Server-Sent Events (SSE) progress stream for active repository syncs (`Role.VIEWER`). + - `GET /admin/api/repos/sync-status`: Snapshot of all active repository sync jobs (`Role.VIEWER`). + - `GET /admin/api/repos/{repo_id}/sync-status`: Current sync phase, file/chunk counters, embedding throughput (`chunks_per_sec`), and HTTP `429` wait telemetry (`rate_limit_429_count`, `rate_limit_wait_seconds`) for a single repository (`Role.VIEWER`). + - `POST /admin/api/repos/{repo_id}/cancel-sync`: Cancels an in-flight synchronization run (`Role.EDITOR`). + +### 5. Delete Repository (`DELETE /admin/api/repos/{repo_id}`) +- **Method**: `DELETE` +- **Path**: `/admin/api/repos/{repo_id}` +- **Required Role**: `Role.ADMIN` +- **Scripted `curl` Example**: + ```bash + curl -X DELETE http://localhost:3000/admin/api/repos/1 \ + -H "Authorization: Bearer cc_admin_your_key" + ``` +- **Response (`200 OK`)**: + ```json + { + "status": "success", + "name": "my-service", + "deleted": "my-service" + } + ``` +- **What `DELETE /admin/api/repos/{repo_id}` Removes**: + 1. The repository registration row in `git_repositories`. + 2. All relational rows where `repo = ` across `indexed_files`, `file_summaries`, `ast_symbols`, `ast_relationships`, `api_routes`, `api_client_calls`, and `git_commits`. + 3. All dense and sparse vector points for `` in the active vector store (`pgvector` or `qdrant`). + 4. The retained shallow clone directory on disk under `PERSISTENT_REPOS_DIR` (if `keep_shallow` was enabled). + +### 6. Monitored Local Paths & Global Indexing (`GET` / `POST` / `DELETE /admin/api/paths`) +- **List Monitored Paths (`GET /admin/api/paths`)**: + - **Required Role**: `Role.VIEWER` + - **Response**: Array of rows from `indexed_paths` (`id`, `path`, `type`, `recursive`, `category`, `repo`, `enabled`). +- **Register Monitored Path (`POST /admin/api/paths`)**: + - **Required Role**: `Role.ADMIN` + - **Request Fields (`LocalPathConfig`)**: + - `path` (string, required): Existing absolute directory or file path on the host/container filesystem (validated against path traversal and null bytes). + - `type` (string, optional, default `"directory"`): Path entry type (`"directory"` or `"file"`). + - `recursive` (boolean, optional, default `true`): Recursively traverse subdirectories during indexing. + - `category` (string, optional, default `null`): Optional classification tag attached to indexed chunks. + - `repo` (string, optional, default `"local"`): Logical repository/source label assigned to files indexed from this path. + - **Scripted `curl` Example**: + ```bash + curl -X POST http://localhost:3000/admin/api/paths \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer cc_admin_your_key" \ + -d '{ + "path": "/docs/architecture", + "type": "directory", + "recursive": true, + "category": "runbooks", + "repo": "local" + }' + ``` + - **Response (`200 OK`)**: + ```json + { + "status": "success", + "id": 2, + "path": "/docs/architecture" + } + ``` + Registration immediately triggers a background `run_full_indexing()` pass. +- **Delete Monitored Path (`DELETE /admin/api/paths/{path_id}`)**: + - **Required Role**: `Role.ADMIN` + - **Scripted `curl` Example**: + ```bash + curl -X DELETE http://localhost:3000/admin/api/paths/2 \ + -H "Authorization: Bearer cc_admin_your_key" + ``` + - **Response (`200 OK`)**: + ```json + { + "status": "success", + "path": "/docs/architecture", + "deleted": "/docs/architecture" + } + ``` + - **What `DELETE /admin/api/paths/{path_id}` Removes**: + 1. The monitored path entry in `indexed_paths`. + 2. All relational rows matching the removed path (or its subpath prefix `/%`, plus `repo`-scoped rows when no other registered path or Git repository shares that `repo` label) across `indexed_files`, `file_summaries`, `ast_symbols`, `ast_relationships`, `api_routes`, and `api_client_calls`. + 3. All vector points matching the removed path (and unshared `repo` label) in the active vector store (`pgvector` or `qdrant`). Files on the underlying host disk are never deleted. +- **Trigger Global Re-Index (`POST /admin/api/reindex` or `POST /admin/api/sync`)**: + - **Required Role**: `Role.EDITOR` + - **Query Parameter**: `force` (boolean, optional, default `false`). + - **Response**: Dispatches a background indexing run across all enabled sources (returns `409 Conflict` if an indexing run is already active). + +### 7. Interactive Calibrated Hybrid Search Tester - **Method**: `POST` - **Path**: `/admin/api/search/test` - **Request Body**: diff --git a/docs/requirements/index.md b/docs/requirements/index.md index 3704c76f..09be69f6 100644 --- a/docs/requirements/index.md +++ b/docs/requirements/index.md @@ -28,7 +28,7 @@ ContextCortex provides: The requirements are organized into three primary sections: 1. **[Functional Requirements](/requirements/functional)**: - Specifies the functional behavior, data operations, all 20 MCP tools, and user interface capabilities. + Specifies the functional behavior, data operations, the 10 default MCP tools (plus the optional `manage` write tool), and user interface capabilities. 2. **[Non-Functional Requirements](/requirements/non-functional)**: Defines performance targets, security standards, maintainability limits (sub-500 LOC per file), and reliability constraints. diff --git a/docs/requirements/non-functional.md b/docs/requirements/non-functional.md index ebe67b01..fef8afaf 100644 --- a/docs/requirements/non-functional.md +++ b/docs/requirements/non-functional.md @@ -36,8 +36,9 @@ This section specifies the non-functional requirements and quality attributes fo --- ### NFR-06: Resource Boundaries and Batch Capping -- **Requirement**: Embedding generation and background indexing must not starve host CPU or memory resources. -- **Batch & Thread Cap**: Embedding requests must be batched according to `EMBEDDING_BATCH_SIZE` (default `32`), and any local worker thread count must default to `min(2, N_cpu)`. +- **Requirement**: Embedding generation and background indexing must not starve host CPU, memory, or upstream gateway rate limits. +- **Local Thread Cap**: `EMBEDDING_NUM_THREADS` (default `min(2, N_cpu)`) caps ONNX Runtime CPU worker threads per local FastEmbed model (`EMBEDDING_PROVIDER=local` and local sparse BM25 tokenization). It has no effect on remote dense embeddings when `EMBEDDING_PROVIDER=api`. +- **Cross-File Batching & API Concurrency**: Embedding requests batch chunks across files according to `EMBEDDING_BATCH_SIZE` (default `32` for `local`, `128` for `api`, clamped to `256` per API request) and `EMBEDDING_MAX_TOKENS_PER_REQUEST` (default `100000`). When `EMBEDDING_PROVIDER=api`, up to `EMBEDDING_API_CONCURRENCY` (default `4`) batches execute in parallel with automatic HTTP `429` `Retry-After` backoff, AIMD concurrency scaling, and optional `EMBEDDING_TPM_LIMIT` (default `0` / disabled) token-bucket pacing. --- diff --git a/docs/requirements/verification.md b/docs/requirements/verification.md index de46a496..88a4c951 100644 --- a/docs/requirements/verification.md +++ b/docs/requirements/verification.md @@ -18,7 +18,7 @@ ContextCortex is verified by an automated test suite comprising **1,051 automate | **FR-05** | Universal Git Ingestion & Shallow Retention | Automated Integration Test | `tests/backend/test_multi_git_providers.py`, `tests/backend/test_git_manager.py`, `frontend/src/tests/GitRepoManager.test.tsx` | | **FR-06** | Managed Local Storage, Read & Summarize | Automated Unit & Integration | `tests/backend/test_local_storage_service.py`, `tests/backend/test_file_reader_and_summarizer.py` | | **FR-07** | PDF Ingestion & Vision OCR | Automated Integration Test | `tests/test_pdf_extractor.py`, `tests/test_pdf_storage_api.py`, `frontend/src/tests/PdfPreviewModal.test.tsx` | -| **FR-08** | FastMCP 2.0, 20 Tools & JSON-RPC Ping | Automated End-to-End Test | `tests/backend/test_mcp_v2.py`, `tests/test_mcp_ping.py`, `tests/backend/test_tools.py`, `tests/backend/test_grep_code.py`, `tests/backend/test_diagnostics.py` | +| **FR-08** | FastMCP 2.0, 10-Tool Surface & JSON-RPC Ping | Automated End-to-End Test | `tests/backend/test_mcp_v2.py`, `tests/test_mcp_ping.py`, `tests/backend/test_tools.py`, `tests/backend/test_grep_code.py`, `tests/backend/test_diagnostics.py` | | **FR-09** | RFC 9728 OAuth 2.1 & RBAC | Automated Unit & Security | `tests/backend/test_auth_jwt_and_keys.py`, `tests/backend/test_auth_middleware_and_rbac.py` | | **FR-10** | Hero Split Codebase Navigator & Omni-Search | Automated Component & E2E | `frontend/src/tests/CodeNavigator.test.tsx`, `NavigatorOmniSearch.test.tsx`, `NavigatorCodeViewer.test.tsx`, `e2e/navigator.spec.ts` | | **FR-11** | AI Gateway & Dynamic Model Discovery | Automated Integration Test | `tests/backend/test_ai_gateway_settings.py`, `tests/test_litellm_service.py`, `tests/test_llm_client.py`, `tests/test_model_discovery_api.py`, `frontend/src/tests/AIGatewaySettings.test.tsx` | diff --git a/scripts/portable/.env.example b/scripts/portable/.env.example index c1792780..a3271e73 100644 --- a/scripts/portable/.env.example +++ b/scripts/portable/.env.example @@ -32,6 +32,12 @@ OPENAI_API_KEY=your-corporate-api-key EMBEDDING_PROVIDER=api EMBEDDING_MODEL=text-embedding-3-small +# Batching, parallel API concurrency, and rate-limit pacing for large repositories: +# EMBEDDING_BATCH_SIZE=128 +# EMBEDDING_API_CONCURRENCY=4 +# EMBEDDING_MAX_TOKENS_PER_REQUEST=100000 +# EMBEDDING_TPM_LIMIT=0 + # Optional chat and vision OCR models (you can also select them in the Admin UI) # CHAT_MODEL=gpt-4o-mini # VISION_OCR_MODEL=gpt-4o-mini