diff --git a/.codecityignore b/.codecityignore index cb96790c2..84f4076f7 100644 --- a/.codecityignore +++ b/.codecityignore @@ -1,2 +1,2 @@ -api/tests/fixtures/large-repo -api/tests/fixtures/sample-repo +packages/api/api/tests/fixtures/large-repo +packages/api/api/tests/fixtures/sample-repo diff --git a/.dockerignore b/.dockerignore index 6d937248b..58b9da986 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,34 +1,29 @@ -# Version control + CI scaffolding +# At the build-context root, because that is what `docker build .` sends. +# Globs, not package paths: a new package inherits them. + .git .github -# Python local -.venv -.mypy_cache -.pytest_cache -.ruff_cache -api/__pycache__ +# Python +**/.venv **/__pycache__ **/*.pyc +**/.mypy_cache +**/.pytest_cache +**/.ruff_cache + +# Node +**/node_modules +**/dist +**/.vite -# Node local -node_modules -app/node_modules -app/dist -app/.vite -# Codecity local artifacts +# Local-run scratch and tool artifacts .local -.codecity .codecityignore - -# Tool artifacts .claude -.superpowers docs - -# Stale static dir (legacy) -api/static +TODO.md # OS / editors **/.DS_Store @@ -38,9 +33,3 @@ api/static *~ .idea .vscode - -# Repo-level scaffolding not needed at build time -TODO.md -# NOTE: README.md and LICENSE are NOT ignored — pyproject.toml references -# them (`readme = "README.md"`, `license = { file = "LICENSE" }`), and -# hatchling validates their existence during the wheel build. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1606f361e..a477119d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,8 +49,8 @@ jobs: # pyproject.toml's pytest addopts already include # `--cov=api --cov-report=term --cov-fail-under=80`. The extra # `--cov-report=xml:/srv/api/coverage.xml` lands the XML report - # inside the mounted `./api` dir so it's visible on the host for - # artifact upload (the compose service only mounts ./api, uv.lock, + # inside the mounted `./packages/api/api` dir so it's visible on the host for + # artifact upload (the compose service only mounts ./packages/api/api, uv.lock, # and pyproject.toml — not the repo root). run: | docker compose -f docker-compose.test.yml run --rm pytest \ @@ -61,7 +61,7 @@ jobs: # Stdlib-only, so it runs on the runner rather than paying for a container. - name: Python comment cap - run: python3 bin/check-comments.py api bin scripts + run: python3 bin/check-comments.py packages/api/api packages/api/scripts bin # Reads PYRIGHT_VERSION from the repo-root .env, which compose loads # automatically — no `env:` mirror needed here, unlike NPM_VERSION @@ -72,10 +72,15 @@ jobs: - name: Run vitest with coverage # Override the default compose command (which runs `npm test`) to # run `npm run coverage` instead. Same apt-get / npm bootstrap as - # the compose service. Output lands in app/coverage/ on the host - # via the ./app:/app bind mount. + # the compose service. Output lands in packages/app/coverage/ on the host + # via the ./packages/app:/app bind mount. run: | docker compose -f docker-compose.test.yml run --rm vitest \ + sh -c "apt-get update && apt-get install -y --no-install-recommends libexpat1 fontconfig fonts-dejavu-core && npm install -g npm@$NPM_VERSION && (cd /city && npm ci) && npm ci && npm run coverage" + + - name: Run the city's vitest with coverage + run: | + docker compose -f docker-compose.test.yml run --rm city-vitest \ sh -c "apt-get update && apt-get install -y --no-install-recommends libexpat1 fontconfig fonts-dejavu-core && npm install -g npm@$NPM_VERSION && npm ci && npm run coverage" - name: Upload pytest coverage report @@ -83,7 +88,7 @@ jobs: uses: actions/upload-artifact@v7 with: name: pytest-coverage - path: api/coverage.xml + path: packages/api/api/coverage.xml if-no-files-found: warn - name: Upload vitest coverage report @@ -91,17 +96,27 @@ jobs: uses: actions/upload-artifact@v7 with: name: vitest-coverage - path: app/coverage/ + path: packages/app/coverage/ + if-no-files-found: warn + + - name: Upload city vitest coverage report + if: always() + uses: actions/upload-artifact@v7 + with: + name: city-vitest-coverage + path: packages/city/coverage/ if-no-files-found: warn - name: Lint + typecheck run: | docker compose -f docker-compose.test.yml run --rm vitest \ - sh -c "apt-get update && apt-get install -y --no-install-recommends libexpat1 fontconfig fonts-dejavu-core && npm install -g npm@$NPM_VERSION && npm ci && npm run lint && npm run typecheck" + sh -c "apt-get update && apt-get install -y --no-install-recommends libexpat1 fontconfig fonts-dejavu-core && npm install -g npm@$NPM_VERSION && (cd /city && npm ci) && npm ci && npm run lint && npm run typecheck && npm run format:check" - # Its own service: the app-scoped vitest service can't see the root config. - - name: Format check (prettier) - run: docker compose -f docker-compose.test.yml run --rm prettier + # Prettier lives inside each package, so there is no repo-wide format + # check — README, AGENTS.md, compose and these workflows belong to no + # package and are hand-formatted. + - name: city/ typecheck + format check + run: docker compose -f docker-compose.test.yml run --rm packages # Only the schema is written out here; the container owns everything node. # npm on the runner cannot work: compose mounts anonymous volumes over @@ -110,7 +125,7 @@ jobs: run: | curl -LsSf https://astral.sh/uv/install.sh | sh mkdir -p .local - "$HOME/.local/bin/uv" run python scripts/gen_openapi.py > .local/openapi.generated.json + "$HOME/.local/bin/uv" run --project packages/api python packages/api/scripts/gen_openapi.py > .local/openapi.generated.json docker compose -f docker-compose.test.yml run --rm gentypes - name: Trivy scan diff --git a/.gitignore b/.gitignore index 6e79221f6..5b82fd4c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -# Output -.codecity/ +# Repo-level only. Anything a single package generates is ignored by that +# package's own .gitignore, so lifting one out takes its ignores with it. # Your own settings and credentials, seeded from .env.local.example. # .env itself is tracked: it holds what everyone shares. @@ -9,49 +9,14 @@ # always safe to delete. .local/ -# Tool artifacts -.superpowers/ - -# Generated test fixtures -# run `bash api/tests/fixtures/setup.sh` to regenerate -api/tests/fixtures/sample-repo/ -api/tests/fixtures/.sample-repo-ready -api/tests/fixtures/.sample-repo-setup.lock -# run `bash api/tests/fixtures/large-repo-setup.sh` to regenerate -api/tests/fixtures/large-repo/ +# The root is not an npm project, but running a package's vitest from here +# leaves vite's cache behind. Nothing installs into it. +node_modules/ # OS .DS_Store Thumbs.db -# Dependencies -node_modules/ - -# Vite caches -.vite/ -app/.vite/ - -# vitest coverage report output -app/coverage/ - -# pytest-cov output -# .coverage = SQLite data file (default) -# coverage.xml = Cobertura XML (CI artifact, written into api/ via -# --cov-report=xml:/srv/api/coverage.xml in ci.yml) -.coverage -coverage.xml -api/coverage.xml - -# Python -__pycache__/ -*.pyc -.venv/ -dist/ -*.egg-info/ -.pytest_cache/ -.mypy_cache/ -.ruff_cache/ - # Editors *.swp *.swo @@ -59,8 +24,5 @@ dist/ .idea/ .vscode/ -# Visual regression diff output (generated by verify-references.ts) -app/tests/visual/references-diff/ - # Claude Code session scratch (per-dev handoff notes, local settings) .claude/ diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index 27406d644..000000000 --- a/.prettierignore +++ /dev/null @@ -1,27 +0,0 @@ -# At the root, not app/: prettier resolves this relative to its cwd (#165). - -node_modules -package-lock.json -coverage -dist -api/static - -# Python, tooling and local-run scratch. -.venv -.local -.codecity -.claude -.superpowers -.mypy_cache -.pytest_cache -.ruff_cache -uv.lock - -# Scanner fixtures: a nested git repo whose file contents, line counts and -# mtimes the scan tests assert on. Formatting them rewrites those and fails -# test_scan, while the outer `git status` stays clean because it is its own repo. -api/tests/fixtures - -# Auto-generated from the OpenAPI schema by `just gen-types` (openapi-typescript). -# Edit api/models/*.py and regenerate; never hand-format this file. -app/src/types/manifest.generated.ts diff --git a/.prettierrc.json b/.prettierrc.json deleted file mode 100644 index 6f49d1e83..000000000 --- a/.prettierrc.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "printWidth": 100, - "tabWidth": 2, - "useTabs": false, - "singleQuote": true, - "trailingComma": "es5", - "semi": true, - "endOfLine": "lf", - "arrowParens": "always" -} diff --git a/AGENTS.md b/AGENTS.md index 150c454d1..c8639b9a1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,20 +110,65 @@ leaving those containers pointing at a gone network (`just dev` then fails with ## Layout -- `app/` — Preact + TypeScript frontend. Two routes, one view each: `/` is the - landing (pick a project) and `/city?src=…` is a world. The URL is the source of - truth for both — `router/` owns it, and `?src`, `?mode`, `?commit` and `?sel` - all survive Back and Forward. - - `src/city/` — the 3D city, a signals-driven mini-app. Layout runs in a - worker under `src/city/layout/` (snapshot-tested — keep output identical). - - `src/state/` — seven stores under `stores/`, each named for the question it - answers. `settings/` is its own subsystem: schema, drafts, reactions and - indicators over the fields they operate on. +Everything the product is made of lives under `packages/`. `bin/` and +`.github/` are how it gets built and shipped; the split is what stops the two +kinds from interleaving alphabetically at the repo root. + +Each package is independent: its own manifest, its own lockfile, its own +installed dependencies. Lifting one into a repo of its own is a copy, not a +untangling. + +- `packages/app/` (`codecity`) and `packages/city/` (`@codecity/city`) — two + separate npm projects, each with its own `package.json`, `package-lock.json`, + `node_modules` and prettier. + There is no npm project at the repo root, so nothing formats `README.md`, + `AGENTS.md`, the compose files or the workflows: those belong to no package + and are hand-formatted on purpose. +- `packages/city/` (`@codecity/city`) — the 3D renderer and the client every + backend call goes through. `createCity(canvas)` is the whole entry point: hand + it a canvas and an api base, and it fetches, builds, and reports what it is + doing. It depends on `three`, `three-mesh-bvh` and `rbush`, and on nothing + else — no Preact, no signals, no reactive runtime. Everything is per instance, + so two cities on one page share no settings, selection, timeline or GPU + resources; the landing's wallpaper and the `/city` scene are two such cities. + - Values in, events out. The consumer owns settings values and pushes them + with `updateSettings`; the city reports with `on(kind, listener)`. Layout + runs in a worker under `src/layout/` (snapshot-tested — keep output + identical). + - `src/index.ts` is the public surface. `tests/index.ts` is a second one, + `@codecity/city/testing`: the wire fixtures and stubs a consumer needs to + test against a city. The renderer stubs sit behind + `@codecity/city/testing/three` — a `vi.mock('three')` factory that awaits + the main barrel deadlocks, because the barrel reaches source that imports + three. + - Every import inside this package is RELATIVE, and there is no path alias to + add one back. An alias in published source resolves only if the consumer + maps the same prefix; ours did, which hid the fact that nobody else could. + The app reaches this package through `@codecity/city` and nothing else — a + test may reach past that surface, by explicit path so it says so, but no + file under `src/` may. +- `packages/app/` — Preact + TypeScript frontend. Two routes, one view each: + `/` is the landing (pick a project) and `/city?src=…` is a world. The URL is + the source of truth for both — `router/` owns it, and `?src`, `?mode`, + `?commit` and `?sel` all survive Back and Forward. + - This is where signals live. `state/stores/city.ts` is the seam: it holds the + handle, mirrors the city's hover and selection onto app signals, and + attaches the app's half of each event family (`attachCityChrome`, + `attachBuildProgress`, `attachScanProgress`). + - `src/state/` — stores under `stores/`, each named for the question it + answers. `settings/` is its own subsystem: the city declares the fields and + what each one costs, the app owns their values, persistence and signals. - `src/components/` — grouped by what a component is, not where it appears. A component used by exactly one thing lives beside that thing instead. -- `api/` — FastAPI backend that walks the repo and serves the manifest. Layered, - and imports only ever point down: `routers/` → `scan/` → `git/` and `cache/` → - `models/`, `core/`, `utils/`. + - `tests/integration/` is the seam under test: the app driving a real city. + Everything testing the city itself lives in `packages/city/tests/`. +- `packages/api/` — the Python project: `pyproject.toml`, `uv.lock`, and its own + README and LICENSE (hatchling refuses paths outside the project directory). + The importable package is `packages/api/api/`, because Python resolves + `import api` by finding a directory named `api` and the manifest has to sit + above the directory it names. FastAPI backend that walks the repo and serves + the manifest. Layered, and imports only ever point down: `routers/` → + `scan/` → `git/` and `cache/` → `models/`, `core/`, `utils/`. - `routers/` — the whole HTTP surface, one module per route family. `sse.py` is the streaming plumbing the two SSE routes share, not a route. - `git/`, `scan/` and `cache/` each curate a barrel in `__init__`. diff --git a/Dockerfile b/Dockerfile index aa67b8c37..314da55ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,24 +2,23 @@ # ─────── Stage 1: build the frontend ─────── FROM node:24-bookworm-slim AS web-builder -# Pin npm to host version — container ships npm 11.13.0, which is stricter -# about lockfile shape and refuses npm ci with EUSAGE on the host-generated -# lockfile (missing @emnapi/core, @emnapi/runtime entries). -# -# Canonical version source: this ARG default. The repo-root `.env` file -# mirrors it for docker-compose + justfile; .github/workflows/ci.yml mirrors -# it as an `env:` block. Bump all three together. +# The container's own npm is stricter about lockfile shape and refuses npm ci on +# a host-generated lockfile. Mirrored in .env and ci.yml — bump all three. ARG NPM_VERSION=11.6.2 RUN npm install -g npm@${NPM_VERSION} -WORKDIR /build -# .npmrc carries legacy-peer-deps=true (openapi-typescript's stale peer range -# vs TS 6) — it MUST be copied before `npm ci` or resolution fails with ERESOLVE. -COPY app/package.json app/package-lock.json app/.npmrc ./ +WORKDIR /build/app +COPY packages/app/package.json packages/app/package-lock.json ./ +# The app links @codecity/city via `file:../city`, so npm needs that manifest +# first. .npmrc rides along: legacy-peer-deps for openapi-typescript's peer range. +COPY packages/city/package.json packages/city/package-lock.json packages/city/.npmrc /build/city/ +RUN --mount=type=cache,target=/root/.npm \ + cd /build/city && npm ci --no-audit --no-fund RUN --mount=type=cache,target=/root/.npm \ npm ci --no-audit --no-fund -COPY app/ ./ +COPY packages/city/ /build/city/ +COPY packages/app/ ./ RUN npm run build -# Output: /build/dist/ +# Output: /build/app/dist/ # ─────── Stage 2: runtime ─────── FROM python:3.13-slim AS runtime @@ -29,60 +28,46 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ CODECITY_CACHE_ROOT=/cache \ UV_LINK_MODE=copy -# System deps. The base image's apt snapshot can lag published security fixes, -# so apply available upgrades before installing — Trivy fails CI on FIXED -# HIGH/CRITICAL OS CVEs (e.g. libcurl, pulled in by git). Only useful if this -# layer actually re-runs: ci.yml excludes this stage from the build cache, -# because a cached copy pins the upgrade to the day it was first built. -# Note: PID 1 init duties are handled by Docker's --init flag (compose: init: true), -# so we don't install tini here. +# Upgrade before installing: the base image's apt snapshot lags, and Trivy fails +# CI on fixed HIGH/CRITICAL OS CVEs. ci.yml keeps this stage out of the cache, +# or a cached copy pins the upgrade to the day it was first built. RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ git git-lfs ca-certificates wget \ && rm -rf /var/lib/apt/lists/* -# uv is the package manager here, so pip is dead weight — and its vendored -# msgpack/setuptools are what Trivy fails us on. The find guards the glob: -# a base-image bump must not silently no-op and hand the CVEs back. +# uv is the package manager, so pip is dead weight whose vendored msgpack and +# setuptools Trivy fails on. The find guards the glob against a silent no-op. RUN rm -rf /usr/local/lib/python3.*/site-packages/pip \ /usr/local/lib/python3.*/site-packages/pip-*.dist-info \ && rm -f /usr/local/bin/pip /usr/local/bin/pip3 /usr/local/bin/pip3.* \ && ! find /usr/local/lib -name 'pip' -maxdepth 5 -print | grep -q . -# uv: bring in the static binary from the official image. COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ WORKDIR /srv -# Lockfile-first layering: a source change won't bust the dep install layer. -# README.md + LICENSE are referenced by pyproject.toml and validated at -# wheel-build time by hatchling — copy them alongside the manifests. -COPY pyproject.toml uv.lock README.md LICENSE ./ +# Lockfile first, so a source change doesn't bust the dep layer. README+LICENSE +# ride along: pyproject references them and hatchling validates them at build. +COPY packages/api/pyproject.toml packages/api/uv.lock packages/api/README.md packages/api/LICENSE ./ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-dev --no-install-project -# Python source -COPY api/ ./api/ +COPY packages/api/api/ ./api/ -# Built frontend → /srv/api/static (matches api/app.py DEFAULT_STATIC_DIR -# resolution: Path(__file__).resolve().parent / "static"). No env var needed. -COPY --from=web-builder /build/dist /srv/api/static +# Matches api/app.py's DEFAULT_STATIC_DIR (__file__.parent / "static"). +COPY --from=web-builder /build/app/dist /srv/api/static -# pyproject.toml uses hatch-vcs (`source = "vcs"`) for dynamic versioning, -# but .dockerignore excludes .git to keep the build context small. Feed the -# version through setuptools_scm's escape hatch so the project install below -# can compute its version without a git repo. Use the unscoped -# SETUPTOOLS_SCM_PRETEND_VERSION because hatch-vcs 0.5.0 doesn't forward -# dist_name to setuptools_scm, so the _FOR_ form never matches. +# hatch-vcs reads the version from git tags, and .dockerignore drops .git. The +# unscoped name is required: hatch-vcs 0.5.0 never forwards dist_name. ARG VERSION=0.0.0+dev ENV SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} -# Install the project (registers `api` as importable; metadata for version). RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-dev -# Non-root user. /cache is the mount point for the named volume. +# /cache is where the named volume lands. RUN useradd --create-home --uid 10001 codecity \ && mkdir -p /cache \ && chown codecity:codecity /cache /srv @@ -93,22 +78,14 @@ EXPOSE 8080 HEALTHCHECK --interval=10s --timeout=2s --start-period=3s --retries=3 \ CMD wget -qO- http://127.0.0.1:8080/api/health || exit 1 -# Invoke the venv's python directly. Bypassing `uv run` avoids a startup -# re-sync that re-downloads dev deps and tries to reinstall the console -# script into /srv/.venv/bin (read-only for the non-root runtime user). -# Zombie reaping + signal propagation are handled by Docker's --init. -# `python -m api` launches a single uvicorn process (api.app:app) — single -# process by design, see api/core/security.py (the allowed_roots trust set is -# in-memory; multi-worker would split it). +# The venv's python directly: `uv run` would re-sync at startup and try to +# rewrite /srv/.venv/bin, which the non-root user cannot. Single process by +# design — api/core/security.py holds the trust set in memory. ENTRYPOINT ["/srv/.venv/bin/python", "-m", "api"] -# --host is explicit because the CLI defaults to loopback (an unauthenticated -# API that serves any scanned root should not reach the network by default). -# In a container that default would make the port unreachable from the host: -# here 0.0.0.0 is the container's own namespace, and only published ports get -# out. Any `command:` override must repeat this — see docker-compose.dev.yml. +# The CLI defaults to loopback, which in a container means unreachable. Any +# `command:` override replaces this outright and must repeat it. CMD ["--port", "8080", "--host", "0.0.0.0"] -# Populated by CI via --build-arg. ARG GIT_SHA=dev ARG VERSION=0.0.0+dev LABEL org.opencontainers.image.title="codecity" \ diff --git a/README.md b/README.md index 6aea72ddb..c42fd9731 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

- codecity + codecity

codecity.io

@@ -221,7 +221,7 @@ The pre-push hook runs the full lint + tests before pushing; bypass with `git pu | `just setup` | one-time: pre-push hooks, npm packages, `.env.local` | | `just dev` | Vite HMR + API auto-reload at `http://.localhost:/` | | `just url` | print this worktree's dev URL (`open $(just url)`) | -| `just test` | pytest + vitest in containers | +| `just test` | pytest and both vitest suites, in containers | | `just lint` | ruff, pyright, eslint, prettier, and typecheck | | `just gen-types` | regenerate the frontend wire types from the OpenAPI schema | | `just clean` | tear down this worktree's containers and volumes | diff --git a/api/routers/manifest.py b/api/routers/manifest.py deleted file mode 100644 index 03a2de6be..000000000 --- a/api/routers/manifest.py +++ /dev/null @@ -1,320 +0,0 @@ -"""The manifest routes: GET /api/manifest (SSE stream), GET -/api/manifest/signature, GET /api/timeline (SSE stream). - -Source classification/resolution lives in api.git.source; these are the -thin HTTP handlers over it. A ResolveError carries a status + message, plus a -code where the UI answers the failure differently: the signature route turns it -into an HTTPException, while the manifest and timeline SSE routes turn it into -an `error` event (EventSource can't read 4xx bodies).""" - -from __future__ import annotations - -import asyncio -import logging -import threading -from pathlib import Path -from typing import Any, AsyncIterator - -from fastapi import APIRouter, HTTPException, Query, Request -from sse_starlette.sse import EventSourceResponse - -from api.core.constants import ErrorCode, ScanEvent -from api.routers import sse -from api.routers.sse import Put, stream -from api.utils.labels import label_from_source -from api.models.events import ScanStreamMessage -from api.models.manifest import Manifest, SignatureResponse -from api.cache import ( - cache_clear_timeline, - cache_load_manifest, - cache_load_newest_manifest, - cache_load_ref_manifest, - cache_save_manifest, - cache_save_ref_manifest, -) -from api.git import ( - BranchNotFoundError, - CloneProgress, - CloneError, - HostUnreachableError, - RepoNotFoundError, - ResolveError, - SourceKind, - SourceRef, - classify, - clone_dir_for, - ensure_clone, - resolve_local, - resolve_ref, - resolve_source, -) -from api.scan import ( - ScanCancelledError, - normalize_excludes, - reconstruct_manifest, - scan_tree, - signature_tree, -) - -router = APIRouter(prefix="/api", tags=["manifest"]) - - -logger = logging.getLogger("codecity.manifest") - - -@router.get("/manifest/signature", response_model=SignatureResponse) -def signature( - src: str = Query(...), - branch: str | None = Query(None), - no_cache: bool = Query(False), - exclude: list[str] = Query(default_factory=list), -) -> SignatureResponse: - try: - target = resolve_source(src, branch) - except ResolveError as e: - raise HTTPException(e.status, e.message) - try: - sig = signature_tree( - str(target), - use_cache=not no_cache, - extra_exclude_paths=normalize_excludes(exclude), - ) - except Exception as e: # noqa: BLE001 - raise HTTPException(500, f"signature failed: {e}") - return sig - - -@router.get( - "/manifest/cached", - response_model=Manifest, - responses={404: {"description": "Nothing cached for this source."}}, -) -def cached_manifest( - src: str = Query(...), - branch: str | None = Query(None), -) -> Any: - """The newest manifest already on disk for this source, or 404. Never - scans, never clones, never resolves a ref over the network. - - Backs the landing backdrop, which wants a city to show rather than a - current one. Everything else wants the truth and goes to /api/manifest.""" - if not src: - raise HTTPException(400, "missing 'src' query param") - kind = classify(src) - if kind is SourceKind.INVALID: - raise HTTPException(400, "unrecognized source: pass a local path or a git URL") - if kind is SourceKind.REMOTE: - # The clone dir keys on the branch AS PASSED, so a repo first opened - # without one lives elsewhere than the branch recorded for it later. - roots = [clone_dir_for(src, branch)] - if branch: - roots.append(clone_dir_for(src, None)) - else: - try: - roots = [resolve_local(src)] - except ResolveError: - raise HTTPException(404, "nothing cached for this source") - for root in roots: - manifest = cache_load_newest_manifest(root) - if manifest is not None: - return manifest - raise HTTPException(404, "nothing cached for this source") - - -@router.get( - "/manifest", - responses={ - 200: { - "description": ( - "Server-Sent Events stream (`text/event-stream`). Named events and " - "their JSON `data` payloads: `clone-progress` (CloneProgressEvent), " - "`scan-progress` (ScanProgressEvent), `manifest-partial` " - "(PartialManifestEvent), `manifest-complete` (CompleteManifestEvent), " - "`error` (ErrorEvent). The client closes the connection on " - "`manifest-complete`/`error`. When `ref` is set, the manifest is " - "reconstructed as of that commit instead of the working tree " - "(a remote source still emits `clone-progress` if it isn't cloned " - "yet, but never `scan-progress`/`manifest-partial` for the " - "reconstruction itself — the city is already drawn, so a skeleton " - "would flash placeholders)." - ), - "model": ScanStreamMessage, - }, - }, -) -async def manifest( - request: Request, - src: str = Query(""), - branch: str | None = Query(None), - no_cache: bool = Query(False), - exclude: list[str] = Query(default_factory=list), - ref: str | None = Query(None), -) -> EventSourceResponse: - use_cache = not no_cache - excludes = normalize_excludes(exclude) - - async def gen() -> AsyncIterator[dict[str, Any]]: - # Validate WITHOUT cloning: the clone runs on the worker below so its - # progress streams. Failures here are error EVENTS, not 4xx. - if not src: - yield sse.error("missing 'src' query param") - return - kind = classify(src) - if kind is SourceKind.INVALID: - yield sse.error("unrecognized source: pass a local path or a git URL") - return - # The PENDING label, and the only name derivation in this route: the - # scanner bakes the canonical tree.name later. See the README. - pending_label = label_from_source(src) - # Stamped onto every manifest this stream emits: a read sends it back to - # name the same root, so it has to be the branch AS PASSED. - source = SourceRef(src, branch) - local_path: Path | None = None - if kind is SourceKind.LOCAL: - try: - local_path = await asyncio.to_thread(resolve_local, src) - except ResolveError as e: - yield sse.error(e.message, e.code) - return - - built: dict[str, Any] = {"manifest": None, "sig": None, "path": None} - - def work(_put: Put, cancel: threading.Event) -> None: - def _on_clone(p: CloneProgress) -> None: - _put( - sse.event( - ScanEvent.CLONE_PROGRESS, - { - "label": pending_label, - "stage": p.stage, - "percent": p.percent, - "objects": p.objects, - "objects_total": p.objects_total, - "mib": p.mib, - }, - ) - ) - - def _on_clone_heartbeat(mb_on_disk: int | None) -> None: - # Silent promisor-fetch phase: no stage/percent, just the working - # tree growing on disk, so the UI shows activity not a freeze. - _put( - sse.event( - ScanEvent.CLONE_PROGRESS, - {"label": pending_label, "mb_on_disk": mb_on_disk}, - ) - ) - - def _on_scan(files_scanned: int) -> None: - _put( - sse.event( - ScanEvent.SCAN_PROGRESS, - {"label": pending_label, "files_scanned": files_scanned}, - ) - ) - - try: - # Clone phase (git only): emit `clone-progress` FIRST, then clone - # with live progress + cancel support. - if kind is SourceKind.REMOTE: - _put(sse.event(ScanEvent.CLONE_PROGRESS, {"label": pending_label})) - try: - path = ensure_clone( - src, - branch, - on_progress=_on_clone, - on_heartbeat=_on_clone_heartbeat, - cancel_event=cancel, - ) - except RepoNotFoundError as e: - _put(sse.error(str(e), ErrorCode.REPO_NOT_FOUND)) - return - except (BranchNotFoundError, HostUnreachableError) as e: - _put(sse.error(str(e))) - return - except CloneError as e: - _put(sse.error(str(e))) - return - else: - assert local_path is not None - path = local_path - - built["path"] = path - - # no_cache means rebuild EVERYTHING for this source, so the - # per-HEAD timeline bundle has to go too. - if not use_cache: - cache_clear_timeline(path.resolve()) - - # Resolve to a sha FIRST: it is both the cache key and the - # early "bad ref" error. No skeleton events — see the README. - if ref is not None: - sha = resolve_ref(path, ref) - if sha is None: - _put(sse.error(f"ref does not resolve to a commit: {ref}")) - return - if use_cache: - cached_ref = cache_load_ref_manifest(path.resolve(), sha) - if cached_ref is not None: - _put( - sse.event( - ScanEvent.MANIFEST_COMPLETE, - {"manifest": cached_ref}, - ) - ) - return - m = reconstruct_manifest( - str(path), source, sha, use_cache=use_cache - ) - cache_save_ref_manifest(path.resolve(), sha, m) - _put(sse.event(ScanEvent.MANIFEST_COMPLETE, {"manifest": m})) - return - - _put(sse.event(ScanEvent.SCAN_PROGRESS, {"label": pending_label})) - - # The signature costs a full stat-walk and scan_tree computes - # the same value anyway, so only pay when a cache exists. - if use_cache: - sig = signature_tree( - str(path), use_cache=use_cache, extra_exclude_paths=excludes - ).content_signature - built["sig"] = sig - cached = cache_load_manifest(path.resolve(), sig) - if cached is not None: - _put( - sse.event(ScanEvent.MANIFEST_COMPLETE, {"manifest": cached}) - ) - return - - # Cold scan: partial + complete manifests, with heartbeat progress. - for ev in scan_tree( - str(path), - source, - use_cache=use_cache, - cancel_event=cancel, - on_scan_progress=_on_scan, - extra_exclude_paths=excludes, - ): - if ev.phase is ScanEvent.MANIFEST_COMPLETE: - built["manifest"] = ev.manifest - built["sig"] = ev.manifest.content_signature - _put(sse.event(ev.phase, {"manifest": ev.manifest})) - except ScanCancelledError: - pass # client disconnected mid-clone/scan; nothing to report - except Exception as e: # noqa: BLE001 - logger.exception("manifest scan failed for src=%s", src) - _put(sse.error(f"scan failed: {e}")) - finally: - _put(None) # sentinel - - async def save() -> None: - """Write-through on a clean finish; the read is gated by no_cache, - the write never is. Nothing to save when the scan errored.""" - final, sig, path = built["manifest"], built["sig"], built["path"] - if final is not None and sig is not None and path is not None: - await asyncio.to_thread(cache_save_manifest, path.resolve(), sig, final) - - async for item in stream(request, work, on_complete=save): - yield item - - return EventSourceResponse(gen()) diff --git a/app/src/api/apiUrl.ts b/app/src/api/apiUrl.ts deleted file mode 100644 index eef82594c..000000000 --- a/app/src/api/apiUrl.ts +++ /dev/null @@ -1,28 +0,0 @@ -// api/apiUrl.ts — Shared builder for backend `/api` URLs. Centralizes the origin -// plus the app's deploy base (import.meta.env.BASE_URL) so every call works under -// a subpath, not only at the domain root, and sets query params uniformly -// (undefined values are skipped, so callers can pass optional params inline). - -/** - * Build a URL for the `/api/` endpoint. `path` has no leading slash. - * Array values emit one repeated query param per entry (e.g. `exclude`); scalar - * values overwrite, matching the backend's repeated-param contract. - */ -export function apiUrl( - path: string, - params?: Record -): string { - const base = import.meta.env.BASE_URL || '/'; - const url = new URL(`${base}api/${path}`, window.location.origin); - if (params) { - for (const [key, value] of Object.entries(params)) { - if (value == null) continue; - if (Array.isArray(value)) { - for (const v of value) url.searchParams.append(key, v); - } else { - url.searchParams.set(key, value); - } - } - } - return url.toString(); -} diff --git a/app/src/api/branches.ts b/app/src/api/branches.ts deleted file mode 100644 index 7ab54e3d4..000000000 --- a/app/src/api/branches.ts +++ /dev/null @@ -1,39 +0,0 @@ -// api/branches.ts — Client for GET /api/branches. Fetches the remote branch -// list for a git URL so the picker can offer a valid-for-this-repo dropdown -// instead of a free-text field. Remote URLs only (local sources have no branch). - -import { URL_PARAMS } from '@/constants/urlParams'; -import { apiUrl } from '@/api/apiUrl'; -import { ScanError, type ScanErrorCode } from '@/api/manifest'; - -export interface BranchList { - branches: string[]; - default: string | null; -} - -export async function fetchBranches(src: string): Promise { - const resp = await fetch(apiUrl('branches', { [URL_PARAMS.SRC]: src })); - if (!resp.ok) { - let message = `branch lookup failed (${resp.status})`; - let code: ScanErrorCode | undefined; - try { - // The API's error envelope is { error, code? }; fall back to FastAPI's - // { detail } for anything that bypasses the app's handler. - const body = (await resp.json()) as { - error?: string; - code?: ScanErrorCode; - detail?: string; - }; - if (body?.error) message = body.error; - else if (body?.detail) message = body.detail; - code = body?.code; - } catch (_) { - /* non-JSON error body: keep the status-based message */ - } - // Same carrier the manifest stream uses, so a caller keys its remedy on the - // code whichever request surfaced the failure. - throw new ScanError(message, code); - } - const body = (await resp.json()) as BranchList; - return { branches: body.branches ?? [], default: body.default ?? null }; -} diff --git a/app/src/api/commit.ts b/app/src/api/commit.ts deleted file mode 100644 index 8e634e38d..000000000 --- a/app/src/api/commit.ts +++ /dev/null @@ -1,25 +0,0 @@ -// api/commit.ts — lazy fetcher for the full commit -// message body. Called when the user clicks "Show full message" in -// the commit pane. Author + subject are already in the manifest; -// body comes from /api/commit on demand to keep the manifest small. - -import { apiUrl } from '@/api/apiUrl'; -import type { SourceRef } from '@/types'; - -export interface CommitDetail { - sha: string; - authors: string[]; - date: string; - subject: string; - body: string; -} - -export async function fetchCommitDetail(source: SourceRef, sha: string): Promise { - const resp = await fetch( - apiUrl('commit', { src: source.src, branch: source.branch ?? undefined, sha }) - ); - if (!resp.ok) { - throw new Error(`commit fetch failed: ${resp.status}`); - } - return (await resp.json()) as CommitDetail; -} diff --git a/app/src/api/config.ts b/app/src/api/config.ts deleted file mode 100644 index db8dc88ca..000000000 --- a/app/src/api/config.ts +++ /dev/null @@ -1,52 +0,0 @@ -// One-shot memoized fetch of /api/config. Failures fail closed: better the -// "local is disabled" UI than a path input the server will reject. - -import { apiUrl } from '@/api/apiUrl'; -import type { components } from '@/types/manifest.generated'; - -// Derived from the OpenAPI schema rather than re-declared, so a field added to -// the backend's ConfigResponse cannot drift from what this layer exposes. -export type ServerConfig = components['schemas']['ConfigResponse']; - -// Pre-boot defaults. `version` matches the backend's own metadata-lookup -// fallback. -export const DEFAULT_SERVER_CONFIG: ServerConfig = { - allowLocalRepos: false, - hosted: false, - featuredRepo: '', - version: '0.0.0+unknown', -}; - -let _cached: Promise | null = null; - -export async function fetchServerConfig(): Promise { - try { - const resp = await fetch(apiUrl('config')); - if (!resp.ok) return DEFAULT_SERVER_CONFIG; - const body = (await resp.json()) as Partial; - // Over the defaults, overriding only what the body actually carries, so a - // truncated or half-written response can't zero a field to its falsy value. - return { - ...DEFAULT_SERVER_CONFIG, - allowLocalRepos: !!body.allowLocalRepos, - hosted: !!body.hosted, - ...(typeof body.featuredRepo === 'string' ? { featuredRepo: body.featuredRepo } : {}), - ...(typeof body.version === 'string' && body.version ? { version: body.version } : {}), - }; - } catch (_) { - return DEFAULT_SERVER_CONFIG; - } -} - -/** Read config from here: one network call, then the cached promise. - * `fetchServerConfig` is exposed only for tests wanting a fresh roundtrip. */ -export function getServerConfig(): Promise { - if (_cached === null) _cached = fetchServerConfig(); - return _cached; -} - -/** Test-only: clear the memoized promise so successive tests can - * return different responses without leaking state. */ -export function _resetServerConfigForTests(): void { - _cached = null; -} diff --git a/app/src/api/discover.ts b/app/src/api/discover.ts deleted file mode 100644 index 2c6174fa0..000000000 --- a/app/src/api/discover.ts +++ /dev/null @@ -1,50 +0,0 @@ -// One-shot memoized fetch of /api/discover. Every failure resolves to an empty -// list rather than rejecting: Discover is one tab on the landing page, and the -// tab hides itself when the list is empty, so "couldn't fetch it" and "the -// server has it switched off" want the same handling. - -import { apiUrl } from '@/api/apiUrl'; -import type { components } from '@/types/manifest.generated'; - -// Derived from the OpenAPI schema rather than re-declared, so a field added to -// the backend's DiscoverEntry cannot drift from what this layer exposes. -export type DiscoverEntry = components['schemas']['DiscoverEntry']; -type DiscoverResponse = components['schemas']['DiscoverResponse']; - -const EMPTY: readonly DiscoverEntry[] = []; - -function usable(entry: DiscoverEntry): boolean { - return Boolean(entry?.url) && Boolean(entry?.label); -} - -export async function fetchDiscover(): Promise { - try { - const resp = await fetch(apiUrl('discover')); - if (!resp.ok) return EMPTY; - const body = (await resp.json()) as Partial; - if (!Array.isArray(body.repos)) return EMPTY; - // A row with no URL has nothing to open and a row with no label has - // nothing to click, so both are dropped rather than rendered blank. - return body.repos.filter(usable); - } catch (_) { - return EMPTY; - } -} - -let _cached: Promise | null = null; - -/** - * Memoized variant. First call hits the network; subsequent calls return the - * cached promise. `fetchDiscover` is exposed only for tests that want a fresh - * roundtrip. - */ -export function getDiscover(): Promise { - if (_cached === null) _cached = fetchDiscover(); - return _cached; -} - -/** Test-only: clear the memoized promise so successive tests can - * return different responses without leaking state. */ -export function _resetDiscoverForTests(): void { - _cached = null; -} diff --git a/app/src/api/file.ts b/app/src/api/file.ts deleted file mode 100644 index b6f4b1361..000000000 --- a/app/src/api/file.ts +++ /dev/null @@ -1,97 +0,0 @@ -// api/file.ts — endpoint helpers for /api/file (raw file content reads). - -import { apiUrl } from '@/api/apiUrl'; -import type { components } from '@/types/manifest.generated'; -import type { SourceRef } from '@/types'; - -/** URL for a file's bytes: a repo-relative path plus the source it is relative - * to. `sha` pins a git blob (Timeline), else `mtime` versions the working tree. */ -export function fileUrl( - source: SourceRef, - path: string, - mtime?: string, - sha?: string | null -): string { - const repo = { src: source.src, branch: source.branch ?? undefined }; - // A blob sha IS the version, so the mtime is redundant alongside it. - return apiUrl('file', sha ? { ...repo, path, sha } : { ...repo, path, mtime }); -} - -// The server knows the file and hasn't got its bytes yet: an unpulled Git LFS -// object, or history a blobless clone hasn't backfilled. A wait, not a failure. -const PENDING_STATUS = 202; - -const PENDING_FALLBACK = 'This file has not been downloaded yet.'; - -/** Thrown rather than a plain Error so a wait can be told from a failure, with - * the server's wording for WHICH fetch is outstanding. */ -export class ContentPendingError extends Error { - constructor(message: string) { - super(message); - this.name = 'ContentPendingError'; - } -} - -/** Content, or the reason there is none: ContentPendingError while the bytes - * are still being fetched, a plain Error on any other non-2xx. */ -export async function fetchContent( - url: string, - // 'high' for a pane the user is looking at, so it jumps the queue of - // background manifest and facade fetches in flight. - priority: RequestPriority = 'auto' -): Promise { - const resp = await fetch(url, { priority }); - if (resp.status === PENDING_STATUS) { - const pending = (await resp.json().catch(() => null)) as - components['schemas']['ContentPendingResponse'] | null; - throw new ContentPendingError(pending?.message || PENDING_FALLBACK); - } - if (!resp.ok) throw new Error(`HTTP ${resp.status}`); - return resp; -} - -/** Whether the bytes are still being fetched, for the loaders that can't see a - * status: an or