diff --git a/.devcontainer/PYTORCH_CUDA_INSTALL.md b/.devcontainer/PYTORCH_CUDA_INSTALL.md deleted file mode 100644 index 814a1269..00000000 --- a/.devcontainer/PYTORCH_CUDA_INSTALL.md +++ /dev/null @@ -1,47 +0,0 @@ -# Dev Container PyTorch CUDA Installation - -## Optimized Setup - -We have optimized the project configuration to handle PyTorch CUDA dependencies declaratively. - -## How it Works - -1. **`pyproject.toml` Configuration**: - We explicitly define the PyTorch CUDA index and map the relevant packages to it: - ```toml - [[tool.uv.index]] - name = "pytorch-cu124" - url = "https://download.pytorch.org/whl/cu124" - explicit = true - - [tool.uv.sources] - torch = { index = "pytorch-cu124" } - torchvision = { index = "pytorch-cu124" } - torchaudio = { index = "pytorch-cu124" } - ``` - -2. **`uv.lock`**: - The lockfile now pins these packages to the versions found in the CUDA index (e.g., `+cu124` versions). - -3. **Dockerfile Optimization**: - - `Dockerfile.gpu` simply runs `uv sync --frozen`. - - It copies `pyproject.toml` and `uv.lock` *before* the source code to leverage Docker layer caching. - - Dependencies are installed during the image build (because we set `SKIP_IMAGE_UV_SYNC=false` in `devcontainer.json`). - -4. **Dev Container**: - - The container starts with all dependencies pre-installed. - - `postCreateCommand` runs `uv sync` just to ensure everything is up-to-date, but it should be instant. - -## Verification - -To verify the installation inside the container: - -```bash -uv run python -c "import torch; print(f'PyTorch: {torch.__version__}, CUDA: {torch.cuda.is_available()}')" -``` - -## Troubleshooting - -If you see "No solution found" errors during build: -1. Ensure `uv.lock` is up-to-date with `pyproject.toml`. Run `uv lock` locally if needed. -2. Check if the PyTorch index URL is accessible. diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json index cdd61126..4d781550 100644 --- a/.devcontainer/devcontainer-lock.json +++ b/.devcontainer/devcontainer-lock.json @@ -4,6 +4,16 @@ "version": "1.1.0", "resolved": "ghcr.io/devcontainers/features/github-cli@sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671", "integrity": "sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671" + }, + "ghcr.io/devcontainers/features/node:1": { + "version": "1.7.1", + "resolved": "ghcr.io/devcontainers/features/node@sha256:8c0de46939b61958041700ee89e3493f3b2e4131a06dc46b4d9423427d06e5f6", + "integrity": "sha256:8c0de46939b61958041700ee89e3493f3b2e4131a06dc46b4d9423427d06e5f6" + }, + "ghcr.io/shyim/devcontainers-features/bun:0": { + "version": "0.0.1", + "resolved": "ghcr.io/shyim/devcontainers-features/bun@sha256:689eae681aa08981175829a59953ba67a7d311f6a05c15d1bbbcb2da2839827e", + "integrity": "sha256:689eae681aa08981175829a59953ba67a7d311f6a05c15d1bbbcb2da2839827e" } } } diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5a807401..5c0a54d7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,75 +1,79 @@ { - "name": "VideoAnnotator (GPU)", - "build": { - "dockerfile": "../Dockerfile.gpu", - "context": "..", - "args": { - "SKIP_IMAGE_UV_SYNC": "true", - "SKIP_TORCH_INSTALL": "true" - } - }, - "runArgs": [ - "--gpus", - "all" - ], - "features": { - "ghcr.io/devcontainers/features/github-cli:1": {}, - "ghcr.io/devcontainers/features/node:1": { - "version": "lts" - }, - "ghcr.io/shyim/devcontainers-features/bun:0": {} - }, - "forwardPorts": [ - 18011, - 19011 - ], - "portsAttributes": { - "18011": { - "label": "VideoAnnotator API", - "onAutoForward": "notify" - }, - "19011": { - "label": "Viewer dev server (Vite)", - "onAutoForward": "notify" - } - }, - "onCreateCommand": "grep -qF scripts/shell_aliases.sh ~/.bashrc || echo \"source ${containerWorkspaceFolder}/scripts/shell_aliases.sh\" >> ~/.bashrc", - "postCreateCommand": "uv sync --inexact --extra dev \u0026\u0026 HADOLINT_DEST_DIR=/usr/local/bin bash scripts/install_hadolint.sh \u0026\u0026 uv run pre-commit install \u0026\u0026 (cd viewer \u0026\u0026 bun install)", - "mounts": [ - "source=videoannotator-claude-home,target=/root/.claude,type=volume", - "source=videoannotator-viewer-node-modules,target=${containerWorkspaceFolder}/viewer/node_modules,type=volume" - ], - "containerEnv": { - "UV_LINK_MODE": "copy", - "UV_NO_SYNC": "1", - "HUGGINGFACE_TOKEN": "${localEnv:HUGGINGFACE_TOKEN}", - "HF_AUTH_TOKEN": "${localEnv:HF_AUTH_TOKEN}", - "OLLAMA_BASE_URL": "http://host.docker.internal:11434", - "HF_HOME": "${containerWorkspaceFolder}/models/huggingface", - "TORCH_HOME": "${containerWorkspaceFolder}/models/torch", - "PYANNOTE_CACHE": "${containerWorkspaceFolder}/models/pyannote", - "DEEPFACE_HOME": "${containerWorkspaceFolder}/models/deepface" - }, - "customizations": { - "vscode": { - "settings": { - "python.defaultInterpreterPath": "${containerWorkspaceFolder}/.venv/bin/python", - "python.formatting.provider": "none", - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true - } - } - }, - "extensions": [ - "charliermarsh.ruff", - "ms-python.python", - "ms-python.vscode-pylance", - "anthropic.claude-code" - ] - } - }, - "shutdownAction": "stopContainer" + "name": "VideoAnnotator (GPU)", + "build": { + "dockerfile": "../Dockerfile", + "context": "..", + "args": { + "SKIP_IMAGE_UV_SYNC": "true" + } + }, + "runArgs": [ + "--gpus", + "all", + "--memory=12g" + ], + "features": { + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/node:1": { + "version": "lts" + }, + "ghcr.io/shyim/devcontainers-features/bun:0": {} + }, + "forwardPorts": [ + 18011, + 19011 + ], + "portsAttributes": { + "18011": { + "label": "VideoAnnotator API", + "onAutoForward": "notify" + }, + "19011": { + "label": "Viewer dev server (Vite)", + "onAutoForward": "notify" + } + }, + "onCreateCommand": "grep -qF scripts/shell_aliases.sh ~/.bashrc || echo \"source ${containerWorkspaceFolder}/scripts/shell_aliases.sh\" >> ~/.bashrc", + "postCreateCommand": "uv sync --inexact --all-extras && HADOLINT_DEST_DIR=/usr/local/bin bash scripts/install_hadolint.sh && uv run pre-commit install && (cd viewer && bun install)", + "postStartCommand": "bash scripts/check_workspace_mount.sh", + "mounts": [ + "source=videoannotator-claude-home,target=/root/.claude,type=volume", + "source=videoannotator-gh-config,target=/root/.config/gh,type=volume", + "source=videoannotator-viewer-node-modules,target=${containerWorkspaceFolder}/viewer/node_modules,type=volume", + "source=videoannotator-venv,target=${containerWorkspaceFolder}/.venv,type=volume", + "source=videoannotator-models,target=/app/models,type=volume" + ], + "containerEnv": { + "UV_LINK_MODE": "copy", + "UV_NO_SYNC": "1", + "HUGGINGFACE_TOKEN": "${localEnv:HUGGINGFACE_TOKEN}", + "HF_AUTH_TOKEN": "${localEnv:HF_AUTH_TOKEN}", + "OLLAMA_BASE_URL": "http://host.docker.internal:11434", + "VIDEOANNOTATOR_MODELS_DIR": "/app/models", + "VIDEOANNOTATOR_LOG_DIR": "${containerWorkspaceFolder}/logs", + "VIDEOANNOTATOR_DB_PATH": "${containerWorkspaceFolder}/videoannotator.db", + "STORAGE_ROOT": "${containerWorkspaceFolder}/storage/jobs" + }, + "customizations": { + "vscode": { + "settings": { + "python.defaultInterpreterPath": "${containerWorkspaceFolder}/.venv/bin/python", + "python.formatting.provider": "none", + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + } + }, + "extensions": [ + "charliermarsh.ruff", + "ms-python.python", + "ms-python.vscode-pylance", + "anthropic.claude-code" + ] + } + }, + "shutdownAction": "stopContainer" } diff --git a/.devcontainer/devcontainer.json.bak-20260108-122537 b/.devcontainer/devcontainer.json.bak-20260108-122537 deleted file mode 100644 index b178ad83..00000000 --- a/.devcontainer/devcontainer.json.bak-20260108-122537 +++ /dev/null @@ -1,63 +0,0 @@ -{ - "name": "VideoAnnotator (GPU)", - "build": { - "dockerfile": "../Dockerfile.gpu", - "context": "..", - "args": { - "SKIP_IMAGE_UV_SYNC": "false", - "SKIP_TORCH_INSTALL": "false" - } - }, - "runArgs": [ - "--gpus", - "all" - ], - "features": {}, - "forwardPorts": [ - 18011, - 18012, - 18013, - 18014, - 18015, - 19011, - 19012, - 19013, - 19014, - 19015 - ], - "portsAttributes": { - "18011": { - "label": "VideoAnnotator API (default)", - "onAutoForward": "notify" - }, - "19011": { - "label": "Video Annotation Viewer", - "onAutoForward": "notify" - } - }, - "postCreateCommand": "uv sync && uv sync --extra dev && HADOLINT_DEST_DIR=/usr/local/bin bash scripts/install_hadolint.sh && uv run pre-commit install", - "containerEnv": { - "UV_LINK_MODE": "copy" - }, - "customizations": { - "vscode": { - "settings": { - "python.defaultInterpreterPath": ".venv/bin/python", - "python.formatting.provider": "none", - "[python]": { - "editor.defaultFormatter": "astral-sh.ruff", - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true - } - } - }, - "extensions": [ - "astral-sh.ruff", - "ms-python.python", - "ms-python.vscode-pylance", - "GitHub.copilot-chat" - ] - } - } -} diff --git a/.devcontainer/noop.txt b/.devcontainer/noop.txt deleted file mode 100644 index 6c7539b6..00000000 --- a/.devcontainer/noop.txt +++ /dev/null @@ -1,3 +0,0 @@ -This file is copied into the container along with environment.yml* from the -parent folder. This is done to prevent the Dockerfile COPY instruction from -failing if no environment.yml is found. diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..a6564a3f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +# Allowlist: the build context is only what the Dockerfile copies. Everything else +# (.git, .venv, models, databases, .env with tokens, viewer/node_modules) stays out. +* +!pyproject.toml +!uv.lock +!.python-version +!README.md +!src/ +!configs/ +# CI only: tests/launcher/Dockerfile.e2e adds these; the published image never copies them. +!tests/launcher/e2e/ +**/__pycache__ +**/*.pyc diff --git a/.gitattributes b/.gitattributes index 4a648003..3401eb0f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -27,3 +27,6 @@ # Shell scripts should always be LF *.sh text eol=lf + +# Windows double-click wrapper: cmd.exe wants CRLF +launcher/*.cmd text eol=crlf diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index c9771935..1f98ace9 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -7,11 +7,15 @@ on: branches: [master, main, develop] release: types: [published] + schedule: + - cron: "17 4 * * 1" # Mondays: extras-restore-e2e workflow_dispatch: jobs: test: runs-on: ${{ matrix.os }} + # Normally 4-5 minutes; a stuck apt mirror once held a job for an hour. + timeout-minutes: 45 # Windows has known SQLite file-locking issues during test teardown # (tracked separately); keep it informational so it doesn't block CI. continue-on-error: ${{ matrix.os == 'windows-latest' }} @@ -19,18 +23,21 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.12"] + python-version: ["3.12", "3.13"] + env: + # Otherwise uv follows .python-version (3.13) and the 3.12 jobs test 3.13. + UV_PYTHON: ${{ matrix.python-version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - name: Set up uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v10.2.0 # no moving major tags since v8 with: enable-cache: true @@ -56,15 +63,17 @@ jobs: uv sync --dev --all-extras - name: Lint with ruff + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' run: | uv run ruff check src/videoannotator tests - name: Check formatting with ruff + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' run: | uv run ruff format --check src/videoannotator tests - name: Type check with mypy - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' run: | uv run mypy src/videoannotator @@ -78,31 +87,208 @@ jobs: # note. No-op on Linux/Windows. KMP_DUPLICATE_LIB_OK: "TRUE" run: | - uv run pytest -q --cov=src/videoannotator --cov-report=xml --cov-report=term-missing --cov-fail-under=45 + uv run pytest -q -m "not real_models and not performance" --cov=src/videoannotator --cov-report=xml --cov-report=term-missing --cov-fail-under=45 - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v7 with: - file: ./coverage.xml + files: ./coverage.xml flags: unittests name: codecov-umbrella fail_ci_if_error: false + viewer: + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: viewer + steps: + - uses: actions/checkout@v7 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.x" + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Lint + run: bun run lint + + # Plain `tsc --noEmit` compiles nothing here (tsconfig.json only references). + - name: Typecheck + run: bun run typecheck + + - name: Unit tests + run: bun run test:run + + # Fails when src/videoannotator/viewer_static/ doesn't match a fresh build of viewer/. + - name: Bundle matches source + working-directory: . + run: bash scripts/build_viewer.sh --check + + # Every other job installs the dev group and every extra, which hid a core + # dependency missing from `pip install videoannotator` (httpx, 2026-10-02). + launcher: + # The launcher's sh script against tests/launcher/cases.json (spec 024). No + # engine needed; macOS catches BSD-userland differences (research R12). + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v7 + - name: Install bats, jq and shellcheck (Ubuntu) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y bats jq shellcheck + - name: Install bats, jq and shellcheck (macOS) + if: runner.os == 'macOS' + run: brew install bats-core jq shellcheck + - name: shellcheck + run: shellcheck launcher/videoannotator-start launcher/install.sh + - name: bats + run: bats tests/launcher + + launcher-windows: + # The PowerShell launcher against the same cases (spec 024). + runs-on: windows-latest + steps: + - uses: actions/checkout@v7 + - name: Install PSScriptAnalyzer and Pester 5 + shell: pwsh + run: | + Install-Module PSScriptAnalyzer -Force -Scope CurrentUser + Install-Module Pester -MinimumVersion 5.5 -MaximumVersion 5.99 -Force -SkipPublisherCheck -Scope CurrentUser + - name: PSScriptAnalyzer + shell: pwsh + run: Invoke-ScriptAnalyzer -Path launcher -Recurse -Severity Warning -EnableExit + - name: Pester (PowerShell 7) + shell: pwsh + run: Invoke-Pester tests/launcher -CI + - name: Pester (Windows PowerShell 5.1) + shell: powershell + run: | + Install-Module Pester -MinimumVersion 5.5 -MaximumVersion 5.99 -Force -SkipPublisherCheck -Scope CurrentUser + Invoke-Pester tests/launcher -CI + + e2e-images: + # The slim image, and the CI-only e2e image built from it (Dockerfile.e2e), + # saved for the launcher-e2e jobs to load into Docker and Podman. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Build the slim image + run: docker build -t localhost/videoannotator:ci . + - name: Build the e2e image + run: docker build -f tests/launcher/Dockerfile.e2e -t localhost/videoannotator:e2e . + - name: Save + run: docker save localhost/videoannotator:e2e | gzip > e2e-image.tar.gz + - uses: actions/upload-artifact@v4 + with: + name: e2e-image + path: e2e-image.tar.gz + retention-days: 2 + + launcher-e2e: + # The launcher end to end with a real engine (spec 024, research R12). + runs-on: ubuntu-latest + needs: [e2e-images] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + engine: [docker, podman] + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v4 + with: + name: e2e-image + - name: Load the image into ${{ matrix.engine }} + run: gunzip -c e2e-image.tar.gz | ${{ matrix.engine }} load + - name: End to end + run: tests/launcher/test_launcher_e2e.sh ${{ matrix.engine }} + + extras-restore-e2e: + # Weekly: an installed pipeline group is restored after the container is + # recreated, from the download cache (spec 024, research R7). Too heavy + # (torch) for every push. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + needs: [e2e-images] + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v4 + with: + name: e2e-image + - name: Load the image + run: gunzip -c e2e-image.tar.gz | docker load + - name: Install scene, recreate, restore + run: tests/launcher/test_extras_restore_e2e.sh docker + + core-install: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + + - name: Set up uv + uses: astral-sh/setup-uv@v10.2.0 # no moving major tags since v8 + with: + enable-cache: true + + - name: Install core only (no dev group, no extras) + run: | + uv sync --frozen --no-dev + + - name: Server starts on a fresh database and serves the API + run: | + export VIDEOANNOTATOR_DB_PATH="$RUNNER_TEMP/va.db" VIDEOANNOTATOR_LOG_DIR="$RUNNER_TEMP/logs" + uv run --no-sync videoannotator server --port 18011 > "$RUNNER_TEMP/server.log" 2>&1 & + for i in $(seq 1 60); do curl -sf localhost:18011/health && break; sleep 2; done + curl -sf localhost:18011/health + curl -sf "localhost:18011/api/v1/pipelines/?include_unavailable=true" > /dev/null + ! grep -E "ERROR|Traceback" "$RUNNER_TEMP/server.log" || { cat "$RUNNER_TEMP/server.log"; exit 1; } + + docs: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + + - name: Set up uv + uses: astral-sh/setup-uv@v10.2.0 # no moving major tags since v8 + with: + enable-cache: true + + - name: Relative links resolve + run: python3 scripts/validate_docs_links.py + + # The site Pages publishes (docs-site.yml); --strict fails on a broken link or anchor. + - name: Site builds + run: | + uv sync --frozen --only-group docs + uv run --no-sync mkdocs build --strict + integration-test: runs-on: ubuntu-latest needs: test - if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' + # On releases too: the image and PyPI jobs need it, and skip when it does. + if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || github.event_name == 'release' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: - python-version: "3.12" + python-version: "3.13" - name: Set up uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v10.2.0 # no moving major tags since v8 with: enable-cache: true @@ -118,7 +304,7 @@ jobs: - name: Run integration tests run: | - uv run pytest -q -m integration + uv run pytest -q -m "integration and not real_models" - name: Test CLI interface run: | @@ -131,15 +317,15 @@ jobs: if: github.event_name == 'workflow_dispatch' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: - python-version: "3.12" + python-version: "3.13" - name: Set up uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v10.2.0 # no moving major tags since v8 with: enable-cache: true @@ -170,16 +356,16 @@ jobs: runs-on: ubuntu-latest name: Draft JOSS paper steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build draft PDF - uses: openjournals/openjournals-draft-action@master + uses: openjournals/openjournals-draft-action@af22ceb4b8f9eff6b3a13c05747ccc2f573cf2e6 # master, 2026-10-01 with: journal: joss paper-path: paper/paper.md - name: Upload paper PDF - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: paper path: paper/paper.pdf @@ -192,10 +378,10 @@ jobs: security-events: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.35.0 + uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: "fs" scan-ref: "." @@ -203,22 +389,23 @@ jobs: output: "trivy-results.sarif" - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: "trivy-results.sarif" build-and-publish: runs-on: ubuntu-latest - needs: [test, integration-test] - if: github.event_name == 'release' && github.event.action == 'published' + needs: [test, viewer, integration-test] + # Not for release candidates (v1.6.0-rc1): a PyPI version can't be reused. + if: github.event_name == 'release' && github.event.action == 'published' && !contains(github.ref_name, '-') steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: - python-version: "3.12" + python-version: "3.13" - name: Install build dependencies run: | @@ -239,6 +426,79 @@ jobs: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + image: + # The slim image researchers run through the launcher (spec 024, + # contracts/image.md): ghcr.io/infantlab/videoannotator: on + # releases, :latest from the default branch. Docker Hub (docker-build + # below) stays a mirror where its secrets exist. + runs-on: ubuntu-latest + needs: [test, integration-test] + if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'release' && github.event.action == 'published') + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tags and labels + id: meta + uses: docker/metadata-action@v6 + with: + images: ghcr.io/infantlab/videoannotator + tags: | + type=semver,pattern={{version}} + type=raw,value=latest,enable={{is_default_branch}} + labels: | + org.opencontainers.image.title=VideoAnnotator + org.opencontainers.image.description=VideoAnnotator, slim: pipelines are installed from the viewer as needed + + - name: Build and push (slim) + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/amd64 + build-args: EXTRAS= + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=ghcr-slim + cache-to: type=gha,mode=max,scope=ghcr-slim + + release-launcher: + # The launcher and its installers as release assets, pinned to the + # release's version, which pins the image they run (spec 024, R16). + runs-on: ubuntu-latest + needs: [launcher, launcher-windows] + if: github.event_name == 'release' && github.event.action == 'published' + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + - name: Pin the version + run: | + version="${GITHUB_REF_NAME#v}" + sed -i "s/@VERSION@/$version/" launcher/videoannotator-start launcher/videoannotator-start.ps1 \ + launcher/install.sh launcher/install.ps1 + ! grep -l '@VERSION@' launcher/videoannotator-start launcher/videoannotator-start.ps1 \ + launcher/install.sh launcher/install.ps1 + - name: Attach to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "$GITHUB_REF_NAME" --clobber \ + launcher/videoannotator-start launcher/videoannotator-start.ps1 \ + launcher/videoannotator-start.cmd launcher/install.sh launcher/install.ps1 + docker-build: runs-on: ubuntu-latest needs: [test, integration-test] @@ -248,10 +508,10 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Check Docker Hub secrets run: | @@ -263,14 +523,14 @@ jobs: - name: Login to Docker Hub if: ${{ env.DOCKER_USERNAME != '' && env.DOCKER_PASSWORD != '' }} - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Extract metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: videoannotator/videoannotator tags: | @@ -282,7 +542,7 @@ jobs: - name: Build and push Docker image if: ${{ env.DOCKER_USERNAME != '' && env.DOCKER_PASSWORD != '' }} - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . push: true diff --git a/.github/workflows/docs-site.yml b/.github/workflows/docs-site.yml new file mode 100644 index 00000000..0b1041e9 --- /dev/null +++ b/.github/workflows/docs-site.yml @@ -0,0 +1,48 @@ +name: Docs site + +# Publishes docs/ (without the archive) to GitHub Pages from master. +# CI's `docs` job builds the same site on every push, so a broken build is caught before here. +on: + push: + branches: [master] + paths: [docs/**, mkdocs.yml, scripts/mkdocs_hooks.py] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up uv + uses: astral-sh/setup-uv@v10.2.0 # no moving major tags since v8 + with: + enable-cache: true + + - name: Build + run: | + uv sync --frozen --only-group docs + uv run --no-sync mkdocs build --strict + + - uses: actions/upload-pages-artifact@v5 + with: + path: site + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v5 diff --git a/.gitignore b/.gitignore index b6e797ec..d7cd036e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,8 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ +/lib/ +/lib64/ parts/ sdist/ var/ @@ -83,8 +83,7 @@ target/ profile_default/ ipython_config.py -# pyenv -.python-version +# pyenv: .python-version is tracked (uv reads it: the default Python for dev and images) # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -209,9 +208,6 @@ outputs/ .DS_Store Thumbs.db -# Docker -.dockerignore - # Test and benchmark data test_data/ sample_videos/ @@ -225,6 +221,7 @@ benchmark.json # Audio processing temp files *.rttm +!tests/fixtures/**/*.rttm *.lab *.TextGrid @@ -240,3 +237,4 @@ videoannotator.db test_*.db *.sqlite *.sqlite3 +site/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3849fbb1..ddb40618 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ exclude: ^(tokens/|src/videoannotator/viewer_static/|viewer/) repos: # Built-in hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -32,7 +32,7 @@ repos: # Python linting and formatting with Ruff (replaces Black, isort, flake8) - repo: https://github.com/astral-sh/ruff-pre-commit # Pinned to a known-good stable release for reproducibility. - rev: v0.14.0 + rev: v0.16.9 hooks: - id: ruff args: [--fix] @@ -40,17 +40,21 @@ repos: - id: ruff-format # Type checking - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + # Same command, version and config as CI (the mirrors-mypy hook checked a + # fraction of the package with an old mypy and missed errors CI caught). + - repo: local hooks: - id: mypy - args: [--config-file=pyproject.toml, --namespace-packages] - additional_dependencies: [types-PyYAML, types-requests] - exclude: ^(src/videoannotator/pipelines/|src/videoannotator/storage/|src/videoannotator/utils/|src/videoannotator/version\.py|src/videoannotator/exporters/|src/videoannotator/schemas/|examples/|scripts/) + name: mypy + entry: uv run --no-sync mypy src/videoannotator + language: system + types: [python] + files: ^src/videoannotator/ + pass_filenames: false # Security scanning - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.9.4 hooks: - id: bandit args: [-r, src, -f, json] @@ -59,48 +63,45 @@ repos: additional_dependencies: [pbr] stages: [manual] - # Documentation - - repo: https://github.com/pycqa/pydocstyle - rev: 6.3.0 - hooks: - - id: pydocstyle - args: [--convention=google] - files: ^src/api/ - stages: [manual] - - # YAML formatting - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 - hooks: - - id: prettier - files: \.(yaml|yml|json|md)$ - stages: [manual] - # Shell script linting - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.5 + rev: v0.11.0.1 hooks: - id: shellcheck files: ^scripts/ stages: [manual] + # The launcher runs on researchers' machines: checked on every commit. + - id: shellcheck + name: shellcheck (launcher) + files: ^launcher/(videoannotator-start|install\.sh)$ + + # Launcher behaviour, from tests/launcher/cases.json (needs bats and jq) + - repo: local + hooks: + - id: bats-launcher + name: bats (launcher) + entry: bats tests/launcher + language: system + pass_filenames: false + files: ^(launcher/|tests/launcher/) # Dockerfile linting - repo: https://github.com/hadolint/hadolint - rev: v2.12.0 + rev: v2.15.1 hooks: - id: hadolint args: [--ignore, DL3008, --ignore, DL3009] # Python dependency security - repo: https://github.com/Lucas-C/pre-commit-hooks-safety - rev: v1.3.2 + rev: v1.4.2 hooks: - id: python-safety-dependencies-check stages: [manual] # Git commit message linting - repo: https://github.com/commitizen-tools/commitizen - rev: v3.7.1 + rev: v4.19.0 hooks: - id: commitizen stages: [commit-msg] diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..24ee5b1b --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/.specify/feature.json b/.specify/feature.json index 3d830f78..a01f9d18 100644 --- a/.specify/feature.json +++ b/.specify/feature.json @@ -1,3 +1,3 @@ { - "feature_directory": "specs/005-pipeline-extras-install" + "feature_directory": "specs/024-container-feels-local" } diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md index 46d05efb..4bb78dc3 100644 --- a/.specify/memory/constitution.md +++ b/.specify/memory/constitution.md @@ -1,6 +1,64 @@ +`specs/024-container-feels-local/plan.md`: researchers start VideoAnnotator with `launcher/ +videoannotator-start` (sh, plus a PowerShell twin), which finds Docker or Podman, shares chosen +folders read-only at their real paths (remembered in a host `start.conf`), runs the slim GHCR +image on 127.0.0.1 and opens the viewer connected. App, installed pipelines (restored after +updates and restarts), model weights, activity and the researcher's files each persist on their +own. Inside a container the server never lists its own filesystem. After 024: +`specs/023-viewer-overhaul` (needs research first). + + ## Recent Changes +- 012-python-313-support: `requires-python` widens to `>=3.12,<3.14`; CI tests both; + `.python-version` makes 3.13 the dev/Docker default; ruff/mypy targets stay at 3.12 (oldest + supported). - Post-005 follow-up: added `GET /api/v1/auth/me` (any authenticated caller can check its own `is_admin` — closes the gap where a 403 from an admin-only endpoint was undiagnosable from the frontend) and made `generate-token` grant admin by default to a brand-new user while the diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e4d292f..d159b686 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,9 @@ This will: 2. Install dependencies: ```bash - pip install -e .[dev] + uv sync # installs the dev tools (dependency group `dev`) by default + # or, with pip >= 25.1: + pip install -e . --group dev ``` 3. Set up pre-commit hooks: @@ -95,7 +97,7 @@ docker-compose --profile dev up --build ## Testing Standards -Please follow our comprehensive testing standards outlined in [TESTING_STANDARDS.md](docs/TESTING_STANDARDS.md). +Please follow our comprehensive testing standards outlined in [testing standards](docs/testing/testing_standards.md). ### Running Tests diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..59b4c116 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,89 @@ +# VideoAnnotator image: one Dockerfile for CPU and GPU machines, and the dev container. +# +# Model weights are not in the image; they download on first use into /app/models +# (mount a volume there). Pipelines missing from the image can be installed from the +# viewer at runtime. +# +# No nvidia/cuda base: torch's Linux wheels bring their own CUDA runtime and cuDNN, and +# the host's driver arrives with `--gpus all`. The same image runs on CPU-only machines. +# A CPU-wheel variant would be ~3.6 GB smaller, but TensorFlow (DeepFace, face_analysis) +# and open_clip segfault together on CPU torch wheels (2026-10-02), so it waits for +# face_analysis's Phase 5 replacement. +# +# docker build -t videoannotator . # slim: core, no torch +# docker build --build-arg EXTRAS=all -t videoannotator:all . # every pipeline +# docker build --build-arg EXTRAS=scene,person -t videoannotator:scene-person . +# docker run --gpus all -p 18011:18011 -v videoannotator-models:/app/models videoannotator:all + +FROM ubuntu:24.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ARG DEBIAN_FRONTEND=noninteractive + +# ffmpeg: audio extraction and torchcodec's shared libraries (pyannote.audio 4). +# libgl1/libglib: OpenCV. git, git-lfs: the dev container. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl git git-lfs ffmpeg \ + libgl1 libglib2.0-0 libsm6 libxext6 libxrender1 libgomp1 locales \ + && locale-gen en_US.UTF-8 \ + && rm -rf /var/lib/apt/lists/* \ + && git lfs install --system + +COPY --from=ghcr.io/astral-sh/uv:0.12.22 /uv /uvx /usr/local/bin/ + +ENV LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/uv-python + +WORKDIR /app + +# true for the dev container, whose postCreateCommand syncs into its own .venv volume. +ARG SKIP_IMAGE_UV_SYNC=false +# Extras group(s) to install, e.g. "scene", "scene,person" or "all". Empty: slim image. +ARG EXTRAS="" + +COPY pyproject.toml uv.lock .python-version README.md ./ +# Ubuntu 24.04 ships Python 3.12; this installs the .python-version one (3.13). +RUN uv python install + +COPY src/ ./src/ +COPY configs/ ./configs/ + +# Non-editable, without the dev group. The cache mount keeps uv's download cache +# (several GB with extras) out of the image. +# hadolint ignore=SC2086 +RUN --mount=type=cache,target=/root/.cache/uv \ + if [ "$SKIP_IMAGE_UV_SYNC" = "true" ]; then \ + echo "[BUILD] Skipping uv sync (SKIP_IMAGE_UV_SYNC=true)"; \ + else \ + FLAGS=""; \ + for e in ${EXTRAS//,/ }; do FLAGS="$FLAGS --extra $e"; done; \ + uv sync --frozen --no-dev --no-editable $FLAGS; \ + fi + +# UV_NO_SYNC: the environment is built above; don't re-sync (and pull the dev group) +# at start. NVIDIA_*: what the nvidia/cuda base used to set; the NVIDIA runtime reads +# them with --gpus. +ENV PATH="/app/.venv/bin:${PATH}" \ + PYTHONUNBUFFERED=1 \ + UV_NO_SYNC=1 \ + NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility \ + VIDEOANNOTATOR_MODELS_DIR=/app/models \ + VIDEOANNOTATOR_LOG_DIR=/app/logs \ + VIDEOANNOTATOR_DB_PATH=/app/database/videoannotator.db \ + STORAGE_ROOT=/app/storage/jobs \ + UV_CACHE_DIR=/app/cache/uv + +# /app/cache: uv's download cache, for a named volume (videoannotator-cache), so +# pipelines installed from the viewer are restored quickly after the container is +# recreated (spec 024, R7). +# /app/launcher/requests: where videoannotator-start mounts its requests folder. +# No VOLUME: the dev container builds this file too, and would collect anonymous volumes. +RUN mkdir -p /app/data /app/output /app/logs /app/database /app/models \ + /app/cache/uv /app/launcher/requests + +EXPOSE 18011 + +CMD ["videoannotator", "server", "--host", "0.0.0.0", "--port", "18011"] diff --git a/Dockerfile.cpu b/Dockerfile.cpu deleted file mode 100644 index 1d5b8540..00000000 --- a/Dockerfile.cpu +++ /dev/null @@ -1,92 +0,0 @@ -# VideoAnnotator Production Docker Image - CPU Version -# This image does NOT include models/weights - they download automatically on first use -# -# By default this builds a SLIM image with no pipeline extras (no torch, no -# ML pipelines) — 004-extras-based-install / SC-002. Pass --build-arg -# EXTRAS= to include one or more pipeline families, or -# EXTRAS=all to reproduce the pre-v1.5.0 "everything installed" image. -# -# Usage: -# docker build -f Dockerfile.cpu -t videoannotator:cpu . # slim (no extras) -# docker build -f Dockerfile.cpu --build-arg EXTRAS=scene -t videoannotator:cpu-scene . -# docker build -f Dockerfile.cpu --build-arg EXTRAS=all -t videoannotator:cpu-all . -# docker run --rm -p 8000:8000 -v ${PWD}/data:/app/data videoannotator:cpu - -FROM ubuntu:24.04 - -# Use bash with pipefail so RUN commands that use a pipe fail when any stage does -SHELL ["/bin/bash","-o","pipefail","-lc"] - -# Prevent interactive prompts during package installation -ARG DEBIAN_FRONTEND=noninteractive - -# Install base packages and locales in a single RUN to reduce layers and avoid -# pulling recommended packages unnecessarily (DL3015, DL3059) -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl python3 python3-venv python3-pip python3-dev git git-lfs ffmpeg \ - libgl1-mesa-dri libglib2.0-0 libsm6 libxext6 libxrender1 libgomp1 locales \ - && locale-gen en_US.UTF-8 \ - && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ - && rm -rf /var/lib/apt/lists/* \ - && git lfs install - -# Export UTF-8 locale for all processes -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -ENV UV_LINK_MODE=copy - -# Install uv, copy sources, and install dependencies and CPU PyTorch in grouped RUNs -RUN curl -LsSf https://astral.sh/uv/install.sh | sh -ENV PATH="/root/.local/bin:${PATH}" - -WORKDIR /app - -# Extras group(s) to install, e.g. "scene" or "scene,person" or "all". -# Empty (default) = slim image, core dependencies only, no torch. -ARG EXTRAS="" - -# Copy source code explicitly (exclude models/weights) -COPY pyproject.toml uv.lock ./ -COPY api_server.py ./ -COPY src/ ./src/ -COPY configs/ ./configs/ -COPY scripts/ ./scripts/ - -# Install dependencies (extras opt-in via $EXTRAS), then, only if any extras -# were requested, override with CPU-only PyTorch wheels (matched versions). -# hadolint ignore=SC2086 -RUN set -eu; \ - if [ -n "$EXTRAS" ]; then \ - FLAGS=""; \ - for e in $(echo "$EXTRAS" | tr ',' ' '); do FLAGS="$FLAGS --extra $e"; done; \ - uv sync --frozen --no-editable $FLAGS; \ - else \ - uv sync --frozen --no-editable; \ - fi \ - && HADOLINT_DEST_DIR=/usr/local/bin bash scripts/install_hadolint.sh \ - && if [ -n "$EXTRAS" ]; then \ - uv pip install "torch==2.6.0+cpu" "torchvision==0.21.0+cpu" "torchaudio==2.6.0+cpu" --index-url https://download.pytorch.org/whl/cpu; \ - else \ - echo "[BUILD] EXTRAS not set — slim image, skipping CPU torch install"; \ - fi - -# Verify CPU setup (only meaningful when extras/torch were installed) -RUN if [ -n "$EXTRAS" ]; then \ - uv run python3 -c "\ -import torch; \ -print(f'[CPU BUILD] CUDA available: {torch.cuda.is_available()}'); \ -print(f'[CPU BUILD] PyTorch version: {torch.__version__}'); \ -print('[CPU BUILD] Production image ready - models will download on first use');"; \ - else \ - echo "[BUILD] Slim image (no extras) — skipping Torch verification"; \ - fi - -# Set environment for production -ENV PYTHONUNBUFFERED=1 - -# Create directories for mounted volumes -RUN mkdir -p /app/data /app/output /app/logs - -EXPOSE 18011 - -CMD ["uv", "run", "python3", "api_server.py", "--log-level", "info", "--port", "18011"] diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 2489e9a3..00000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,76 +0,0 @@ -# VideoAnnotator Development Docker Image - Simple Approach -# This image includes GPU support AND your local model cache for instant testing -# Copies your existing models/ and weights/ directories -# -# Usage: -# docker build -f dockerfile.dev -t videoannotator:dev . -# docker run --gpus all --rm -p 8000:8000 -v ${PWD}/data:/app/data videoannotator:dev - -FROM nvidia/cuda:12.6.0-runtime-ubuntu24.04 - -# Use bash with pipefail so RUN commands that use a pipe fail when any stage does -SHELL ["/bin/bash","-o","pipefail","-lc"] - -# Prevent interactive prompts during package installation -ARG DEBIAN_FRONTEND=noninteractive - -# Install base packages and locales in a single RUN to reduce layers and avoid -# pulling recommended packages unnecessarily (DL3015, DL3059) -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl python3 python3-venv python3-dev git git-lfs ffmpeg \ - libgl1-mesa-dri libglib2.0-0 libsm6 libxext6 libxrender1 libgomp1 locales \ - && locale-gen en_US.UTF-8 \ - && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ - && rm -rf /var/lib/apt/lists/* - -# Export UTF-8 locale for all processes -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -ENV UV_LINK_MODE=copy - -# Initialize Git LFS, install uv and set PATH, copy project files, and install -# dependencies in grouped RUNs to reduce layers. The uv installer is a piped -# command; using SHELL with pipefail above ensures failures are detected (DL4006). -RUN git lfs install \ - && curl -LsSf https://astral.sh/uv/install.sh | sh \ - && export PATH="/root/.local/bin:${PATH}" \ - && mkdir -p /app - -ENV PATH="/root/.local/bin:${PATH}" - -WORKDIR /app - -# Copy project files explicitly so production images can exclude models/weights, -# while the dev image intentionally includes them. -COPY pyproject.toml uv.lock ./ -COPY api_server.py ./ -COPY src/ ./src/ -COPY configs/ ./configs/ -COPY scripts/ ./scripts/ - -# Copy your local models and weights for fast offline-ish iteration -COPY models/ /app/models/ -COPY weights/ /app/weights/ - -# Install dependencies. As of 004-extras-based-install, torch/pipeline deps -# are opt-in extras rather than core dependencies, so the dev image (which -# is meant to exercise every pipeline against the mounted model cache) -# installs the `all` extras group explicitly. torch/torchvision/torchaudio -# come from the configured `pytorch-cu124` uv index in pyproject.toml. -RUN uv sync --frozen --no-editable --extra all \ - && HADOLINT_DEST_DIR=/usr/local/bin bash scripts/install_hadolint.sh \ - && uv tool install specify-cli --from git+https://github.com/github/spec-kit.git - -# Verify GPU access and model cache (no model downloading needed!) -RUN uv run python3 -c "import torch; from pathlib import Path; print(f'[DEV BUILD] CUDA available: {torch.cuda.is_available()}'); models_count = len(list(Path('/app/models').rglob('*'))) if Path('/app/models').exists() else 0; weights_count = len(list(Path('/app/weights').rglob('*'))) if Path('/app/weights').exists() else 0; print(f'[DEV BUILD] Models directory: {models_count} files'); print(f'[DEV BUILD] Weights directory: {weights_count} files'); print('[DEV BUILD] Development image ready with local model cache!');" - -# Set environment for development -ENV PYTHONUNBUFFERED=1 -ENV CUDA_VISIBLE_DEVICES=0 - -# Create directories for mounted volumes -RUN mkdir -p /app/data /app/output /app/logs - -EXPOSE 18011 - -CMD ["uv", "run", "python3", "api_server.py", "--log-level", "info", "--port", "18011"] diff --git a/Dockerfile.gpu b/Dockerfile.gpu deleted file mode 100644 index dc6afc5c..00000000 --- a/Dockerfile.gpu +++ /dev/null @@ -1,102 +0,0 @@ -# VideoAnnotator Production Docker Image - GPU Version -# This image does NOT include models/weights - they download automatically on first use -# -# By default this builds a SLIM image with no pipeline extras (no torch, no -# ML pipelines) — 004-extras-based-install / SC-002. Pass --build-arg -# EXTRAS= (with SKIP_IMAGE_UV_SYNC=false) to include one or -# more pipeline families, or EXTRAS=all to reproduce the pre-v1.5.0 image. -# torch for these groups resolves against the cu124 index configured in -# pyproject.toml's [tool.uv.sources] — no separate GPU torch install step -# needed here (unlike Dockerfile.cpu's CPU-wheel override). -# -# Usage: -# docker build -f Dockerfile.gpu --build-arg SKIP_IMAGE_UV_SYNC=false -t videoannotator:gpu . -# docker build -f Dockerfile.gpu --build-arg SKIP_IMAGE_UV_SYNC=false --build-arg EXTRAS=all -t videoannotator:gpu-all . -# docker run --gpus all --rm -p 8000:8000 -v ${PWD}/data:/app/data videoannotator:gpu - -FROM nvidia/cuda:12.6.0-runtime-ubuntu24.04 - -# Use bash with pipefail so RUN commands that use a pipe fail when any stage does -SHELL ["/bin/bash","-o","pipefail","-lc"] - -# Prevent interactive prompts during package installation -ARG DEBIAN_FRONTEND=noninteractive - -# Install base packages and locales in a single RUN to reduce layers and avoid -# pulling recommended packages unnecessarily (DL3015, DL3059) -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl python3 python3-venv python3-dev git git-lfs ffmpeg \ - libgl1-mesa-dri libglib2.0-0 libsm6 libxext6 libxrender1 libgomp1 locales \ - && locale-gen en_US.UTF-8 \ - && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ - && rm -rf /var/lib/apt/lists/* \ - && git lfs install - -# Export UTF-8 locale for all processes -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -ENV UV_LINK_MODE=copy - -# uv package manager -RUN curl -LsSf https://astral.sh/uv/install.sh | sh -ENV PATH="/root/.local/bin:${PATH}" - -WORKDIR /app - -# Build args to allow dev builds to skip network-heavy steps (defaults: true) -# For production image builds set these to "false" to perform installs in image. -ARG SKIP_IMAGE_UV_SYNC=true -ARG SKIP_TORCH_INSTALL=true -# Extras group(s) to install, e.g. "scene" or "scene,person" or "all". -# Empty (default) = slim image, core dependencies only, no torch. -ARG EXTRAS="" - -# Copy project definition and lock file first to leverage Docker cache -COPY pyproject.toml uv.lock ./ - -# Install dependencies (including torch, via extras opt-in with $EXTRAS) -# These steps can be skipped at build time for dev containers by setting -# build args SKIP_IMAGE_UV_SYNC=true. Post-create commands will run `uv sync` -# inside the running container where network is usually available. -# hadolint ignore=SC2086 -RUN set -eu; \ - if [ "${SKIP_IMAGE_UV_SYNC}" != "true" ]; then \ - if [ -n "$EXTRAS" ]; then \ - FLAGS=""; \ - for e in $(echo "$EXTRAS" | tr ',' ' '); do FLAGS="$FLAGS --extra $e"; done; \ - uv sync --frozen --no-install-project --no-editable $FLAGS; \ - else \ - uv sync --frozen --no-install-project --no-editable; \ - fi; \ - else \ - echo "[BUILD] Skipping uv sync at image build (SKIP_IMAGE_UV_SYNC=true)"; \ - fi - -# Copy the rest of the source code (explicitly exclude models/weights) -COPY api_server.py ./ -COPY src/ ./src/ -COPY configs/ ./configs/ -COPY scripts/ ./scripts/ - -# Install hadolint into a system location so pre-commit can find it. -RUN HADOLINT_DEST_DIR=/usr/local/bin bash scripts/install_hadolint.sh - -# Verify GPU access (optional) -# NOTE: Some builds may intentionally skip Torch installation/sync to speed up -# devcontainer image builds. In that case, skip the Torch import check. -RUN if [ "${SKIP_TORCH_INSTALL}" = "true" ]; then \ - echo "[BUILD] Skipping Torch verification (SKIP_TORCH_INSTALL=true)"; \ - else \ - uv run python3 -c "import torch; print(f'[GPU BUILD] CUDA available: {torch.cuda.is_available()}'); print(f'[GPU BUILD] PyTorch version: {torch.__version__}'); print('[GPU BUILD] Production image ready - models will download on first use')"; \ - fi - -# Set environment for production -ENV PYTHONUNBUFFERED=1 -ENV CUDA_VISIBLE_DEVICES=0 - -# Create directories for mounted volumes -RUN mkdir -p /app/data /app/output /app/logs - -EXPOSE 18011 - -CMD ["uv", "run", "python3", "api_server.py", "--log-level", "info", "--port", "18011"] diff --git a/Makefile b/Makefile index 1dd5af5b..47428266 100644 --- a/Makefile +++ b/Makefile @@ -144,16 +144,16 @@ sync: # Docker commands docker-build: - docker build -f Dockerfile.cpu -t videoannotator:cpu . + docker build -t videoannotator . docker-build-gpu: - docker build -f Dockerfile.gpu -t videoannotator:gpu . + docker build --build-arg EXTRAS=all -t videoannotator:all . docker-run: - docker run --rm -p 18011:18011 -v $(PWD)/data:/app/data -v $(PWD)/output:/app/output -v $(PWD)/database:/app/database -v $(PWD)/logs:/app/logs videoannotator:cpu + docker run --rm -p 18011:18011 -v $(PWD)/data:/app/data -v $(PWD)/output:/app/output -v $(PWD)/database:/app/database -v $(PWD)/logs:/app/logs videoannotator docker-run-gpu: - docker run --gpus all --rm -p 18011:18011 -v $(PWD)/data:/app/data -v $(PWD)/output:/app/output -v $(PWD)/database:/app/database -v $(PWD)/logs:/app/logs videoannotator:gpu + docker run --gpus all --rm -p 18011:18011 -v $(PWD)/data:/app/data -v $(PWD)/output:/app/output -v $(PWD)/database:/app/database -v $(PWD)/logs:/app/logs videoannotator:all docker-dev: docker compose up --build diff --git a/README.md b/README.md index 15cbda0b..68fb7f93 100644 --- a/README.md +++ b/README.md @@ -1,476 +1,137 @@ # VideoAnnotator -[![FastAPI](https://img.shields.io/badge/FastAPI-REST%20API-009688?logo=fastapi&logoColor=white)](http://localhost:18011/docs) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/) -[![uv](https://img.shields.io/badge/uv-package%20manager-FF4B4B?logo=uv&logoColor=white)](https://github.com/astral-sh/uv) -[![Docker](https://img.shields.io/badge/Docker-GPU%20Ready-2496ED?logo=docker&logoColor=white)](https://docs.docker.com/) [![CI](https://github.com/InfantLab/VideoAnnotator/actions/workflows/ci-cd.yml/badge.svg)](https://github.com/InfantLab/VideoAnnotator/actions/workflows/ci-cd.yml) -[![codecov](https://codecov.io/gh/InfantLab/VideoAnnotator/branch/master/graph/badge.svg)](https://codecov.io/gh/InfantLab/VideoAnnotator) -[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/InfantLab/VideoAnnotator) - -**Automated video analysis toolkit for human interaction research** - Extract comprehensive behavioral annotations from videos using AI pipelines, with an intuitive web interface for visualization and analysis. - -## 🎯 What is VideoAnnotator? - -VideoAnnotator automatically analyzes videos of human interactions and extracts rich behavioral data including: - -- **👥 Person tracking** - Multi-person detection and pose estimation with persistent IDs -- **😊 Facial analysis** - Emotions, expressions, gaze direction, and action units -- **🎬 Scene detection** - Environment classification and temporal segmentation -- **🎤 Audio analysis** - Speech recognition, speaker identification, and emotion detection - -**Perfect for researchers studying parent-child interactions, social behavior, developmental psychology, and human-computer interaction.** - -## 🖥️ Complete Solution: Processing + Visualization - -VideoAnnotator provides both **automated processing** and **interactive visualization** — and installing this repository gets you both, no separate setup required. - -### 📹 **VideoAnnotator** (this repository) - -**AI-powered video processing pipeline** - -- Processes videos to extract behavioral annotations -- REST API for integration with research workflows -- Supports batch processing and custom configurations -- Outputs standardized JSON data -- Bundles **[Video Annotation Viewer](https://github.com/InfantLab/video-annotation-viewer)**, served at `/viewer` - -### 🌐 **[Video Annotation Viewer](https://github.com/InfantLab/video-annotation-viewer)** (companion project, bundled here) - -**Interactive web-based visualization tool** - -- Load and visualize VideoAnnotator results -- Synchronized video playback with annotation overlays -- Timeline scrubbing with pose, face, and audio data -- Export tools for further analysis -- Also fully independent: usable standalone with output from any tool that produces COCO/WebVTT/RTTM/scene-JSON files, no VideoAnnotator install required - -
- Running Video Annotation Viewer standalone (without VideoAnnotator) - - ```bash - git clone https://github.com/InfantLab/video-annotation-viewer.git - cd video-annotation-viewer - npm install - npm run dev - ``` - - Note: Ensure Node and NPM are installed. On macOS with Homebrew: `brew install node`. Open - http://localhost:3000 and load your files, or connect it to a VideoAnnotator server via `.env` or - the in-app Settings page — see that repo's README for details. - -
- -**Complete workflow**: `Your Videos → [VideoAnnotator Processing] → Annotation Data → [Video Annotation Viewer, at /viewer] → Interactive Analysis` - -## 🚀 Get Started in 60 Seconds - -### 1. Quick Setup - -```bash -# Install modern Python package manager -curl -LsSf https://astral.sh/uv/install.sh | sh # Linux/Mac -# powershell -c "irm https://astral.sh/uv/install.ps1 | iex" # Windows - -# Clone -git clone https://github.com/InfantLab/VideoAnnotator.git -cd VideoAnnotator -``` - -### 2. Start the Server - -```bash -scripts/start_server.sh -``` - -This syncs dependencies, sets up the local database and an admin API key (prompting for an admin email -the first time), starts the API server, and — since [Video Annotation Viewer](https://github.com/InfantLab/video-annotation-viewer) -is bundled and served by this same process, not a separate frontend to start — prints a one-click viewer -login link once the server is actually up. Safe to re-run any time (e.g. after a container/VS Code -restart). Pass `--background` to keep it running after this terminal closes, or `--help` for all options -(custom port, non-interactive mode, extras to sync, etc). - -**Optional shorthands.** In the dev container, two aliases are already set up. Elsewhere, add -`source /path/to/VideoAnnotator/scripts/shell_aliases.sh` to your `~/.bashrc` or `~/.zshrc`: - -| Alias | Runs | -|------------|---------------------------| -| `va-start` | `scripts/start_server.sh` | -| `va` | `uv run videoannotator` | - -They're only shorthand, and they work from any directory. `va-start --background` and -`va job submit video.mp4 --pipelines scene_detection` work as you'd expect. This README always -spells out the full commands. Note that `va server` starts only the bare server, without the -setup steps; use `va-start` for the one-command start. - -The default (core) install runs the server and viewer with no pipeline installed yet — each pipeline -family lives behind a named extras group (e.g. `face`, `audio`, `scene`, `person`, `all`; see -[pyproject.toml](pyproject.toml)'s `[project.optional-dependencies]`), so the install stays small until -you ask for a specific pipeline. `GET /api/v1/pipelines?include_unavailable=true` lists every pipeline -with `available`/`install_hint` fields showing exactly what's missing. As an admin, you can trigger an -install directly through the API instead of dropping to a terminal: - -```bash -curl -X POST "http://localhost:18011/api/v1/pipelines/extras/face/install" \ - -H "Authorization: Bearer YOUR_ADMIN_API_KEY" -# poll for completion: -curl "http://localhost:18011/api/v1/pipelines/extras/install-jobs/" \ - -H "Authorization: Bearer YOUR_ADMIN_API_KEY" -``` - -A completed install needs a server restart to activate — `GET /api/v1/pipelines`'s top-level -`restart_required` field tells you when one's pending. Restarting via this script (or with -`--skip-sync`) preserves whatever you've installed; see `scripts/start_server.sh --help`. - -This action needs an *admin* API key specifically — not just any authenticated one. -`uv run videoannotator generate-token` grants admin automatically while your deployment is still -single-user (the common case: it's your own server), but if you've re-issued a key under a -different email than your original `setup-db` admin, or you're not sure, check with: - -```bash -curl "http://localhost:18011/api/v1/auth/me" -H "Authorization: Bearer YOUR_API_KEY" -# {"id": "...", "username": "...", "email": "...", "is_admin": false} -``` - -If `is_admin` is `false` and you expect it to be `true`, grant it explicitly: -`uv run videoannotator generate-token --user you@example.com --admin`. - -
-Prefer the manual, step-by-step equivalent? - -```bash -uv sync -uv run videoannotator setup-db --admin-email you@example.com --admin-username you -uv run videoannotator server --host 0.0.0.0 --port 18011 -``` - -
- -### 3. Process Your First Video - -**Recommended — the viewer's GUI:** open the viewer login link `scripts/start_server.sh` printed in -Step 2 (not `/viewer` directly — that link is what logs you in the first time). Then go to -**New Job** (`/jobs/new`), upload a video, pick pipelines, and submit — no `curl`, no API key to copy -by hand. - -
-Or process it manually, e.g. for scripting/automation - -```bash -curl -X POST "http://localhost:18011/api/v1/jobs/" \ - -H "Authorization: Bearer YOUR_API_KEY" \ - -F "video=@your_video.mp4" \ - -F "selected_pipelines=person,face,scene,audio" - -# Check results at http://localhost:18011/docs -```ca - -
- -### 4. Visualize Results - -If you submitted through the viewer's GUI in Step 3, you're already there — the job shows up in its list -as it processes. Otherwise, open the same viewer login link from Step 2 to get in. - -Script said "no fresh API key this run" (admin already existed), or you lost the link? Mint a fresh one: - -```bash -uv run videoannotator generate-token --port 18011 -``` - -If you never open a viewer-connect link, the viewer falls back to a built-in demo token that won't -authenticate against your server, and the job list will show 401s — open the viewer's **Settings** panel -and paste in the key by hand to fix that. - -Set `VIDEOANNOTATOR_ENABLE_VIEWER=false` if you'd rather not serve it. - -**🎉 That's it!** You now have both automated video processing and interactive visualization. - -(Want to run Video Annotation Viewer on its own, without VideoAnnotator? See the standalone option -in the "Complete Solution: Processing + Visualization" section above.) - -## 🧠 AI Pipelines & Capabilities - -Authoritative pipeline metadata (names, tasks, modalities, capabilities) is generated from the registry: - -- Pipeline specification table: [docs/pipelines_spec.md](docs/pipelines_spec.md) (auto-generated; do not edit by hand) -- Pipeline API endpoint: http://localhost:18011/api/v1/pipelines -- Emotion output format spec: [docs/development/emotion_output_format.md](docs/development/emotion_output_format.md) - -Additional Specs: - -- Output Naming Conventions: [docs/development/output_naming_conventions.md](docs/development/output_naming_conventions.md) (stable patterns for downstream tooling) -- Emotion Validator Utility: [src/videoannotator/validation/emotion_validator.py](src/videoannotator/validation/emotion_validator.py) (programmatic validation of `.emotion.json` files) -- CLI Validation: `uv run videoannotator validate-emotion path/to/file.emotion.json` returns non-zero exit on failure - Client tools (e.g. the Video Annotation Viewer) should rely on those sources or the `/api/v1/pipelines` endpoint rather than hard-coding pipeline assumptions. - -### Person Tracking (1 pipeline) - -| Pipeline | Technology | Outputs | Stability | -|----------|-----------|---------|-----------| -| **Person Tracking & Pose** | YOLO11 + ByteTrack | COCO bounding boxes, 17-point pose keypoints, persistent person IDs | beta | - -### Face Analysis (3 pipelines) - -| Pipeline | Technology | Outputs | Stability | -|----------|-----------|---------|-----------| -| **Face Analysis** | DeepFace (TensorFlow/OpenCV) | Emotion labels, age/gender, action units | stable | -| **LAION CLIP Face Embedding** | LAION CLIP-derived model | 512-D semantic embeddings, zero-shot attribute & emotion tagging | experimental | -| **OpenFace3 Face Embedding** | OpenFace 3.0 (ONNX/PyTorch) | 512-D face embeddings for recognition or clustering | experimental | - -### Scene Detection (1 pipeline) - -| Pipeline | Technology | Outputs | Stability | -|----------|-----------|---------|-----------| -| **Scene Detection** | PySceneDetect + CLIP | Scene boundaries, environment classification, temporal segmentation | beta | - -### Audio Processing (4 pipelines + 1 combined) - -| Pipeline | Technology | Outputs | Stability | -|----------|-----------|---------|-----------| -| **Speech Recognition** | OpenAI Whisper | WebVTT transcripts with word-level timestamps | stable | -| **Speaker Diarization** | pyannote.audio | RTTM speaker turns with timestamps | stable | -| **Audio Processing** | Whisper + pyannote (combined) | WebVTT transcripts + RTTM speaker turns | beta | -| **LAION Empathic Voice** | LAION Empathic Insight + Whisper embeddings | Emotion segments, empathic scores, emotion timeline | stable | -| **Voice Emotion Baseline** | Spectral CNN over Whisper embeddings | _(planned — not yet implemented)_ | experimental | - -## 💡 Why VideoAnnotator? - -### **🎯 Built for Researchers** - -- **No coding required** - Web interface and REST API -- **Standardized outputs** - JSON formats compatible with analysis tools -- **Reproducible results** - Version-controlled processing pipelines -- **Batch processing** - Handle multiple videos efficiently - -### **🔬 Research-Grade Accuracy** - -- **State-of-the-art models** - YOLO11, OpenFace 3.0, Whisper -- **Validated pipelines** - Tested on developmental psychology datasets -- **Comprehensive metrics** - Confidence scores, validation tools -- **Flexible configuration** - Adjust parameters for your research needs - -### **⚡ Production Ready** - -- **Fast processing** - GPU acceleration, optimized pipelines -- **Scalable architecture** - Docker containers, API-first design -- **Cross-platform** - Windows, macOS, Linux support -- **Enterprise features** - Authentication, logging, monitoring - -### **🔒 Privacy & Data Protection** - -- **100% Local Processing** - All analysis runs on your hardware, no cloud dependencies -- **No Data Transmission** - Videos and results never leave your infrastructure -- **GDPR Compliant** - Full control over sensitive research data -- **Foundation Model Free** - No external API calls to commercial AI services -- **Research Ethics Ready** - Designed for studies requiring strict data confidentiality - -## 📊 Example Output - -VideoAnnotator generates rich, structured data like this: - -```json -{ - "person_tracking": [ - { - "timestamp": 12.34, - "person_id": 1, - "bbox": [0.2, 0.3, 0.4, 0.5], - "pose_keypoints": [...], - "confidence": 0.87 - } - ], - "face_analysis": [ - { - "timestamp": 12.34, - "person_id": 1, - "emotion": "happy", - "confidence": 0.91, - "facial_landmarks": [...], - "gaze_direction": [0.1, -0.2] - } - ], - "scene_detection": [ - { - "start_time": 0.0, - "end_time": 45.6, - "scene_type": "living_room", - "confidence": 0.95 - } - ], - "audio_analysis": [ - { - "start_time": 1.2, - "end_time": 3.8, - "speaker": "adult", - "transcript": "Look at this toy!", - "emotion": "excited" - } - ] -} -``` - -## 🔗 Integration & Export - -VideoAnnotator produces machine-readable outputs (primarily JSON files and API responses) intended to be easy to consume from common data tools. - -- **Python**: Load JSON into pandas / numpy for analysis (see [examples/](examples/)) -- **R / MATLAB**: Not currently supported with official helper packages, but the JSON outputs can be consumed using standard JSON readers -- **Visualization**: Bundled [Video Annotation Viewer](https://github.com/InfantLab/video-annotation-viewer) at `/viewer` for interactive playback + overlays; also runs standalone against output from other tools - -## 🛠️ Installation Options - -The quickstart above covers the recommended local install via `uv`. For more detail, see the [installation guide](docs/installation/INSTALLATION.md). - -### **Docker (CPU/GPU)** - -```bash -# CPU version (lightweight) -docker build -f Dockerfile.cpu -t videoannotator:cpu . -docker run -p 18011:18011 videoannotator:cpu - -# GPU version (faster processing) -docker build -f Dockerfile.gpu -t videoannotator:gpu . -docker run -p 18011:18011 --gpus all videoannotator:gpu - -# Development version (pre-cached models) -docker build -f Dockerfile.dev -t videoannotator:dev . -docker run -p 18011:18011 --gpus all videoannotator:dev -``` - -## 📚 Documentation & Resources - -| Resource | Description | -| ------------------------------------------------------------------------ | -------------------------------------- | -| **[📖 Interactive Docs](https://deepwiki.com/InfantLab/VideoAnnotator)** | Complete documentation with examples | -| **[🎮 Live API Testing](http://localhost:18011/docs)** | Interactive API when server is running | -| **[🚀 Getting Started Guide](docs/usage/GETTING_STARTED.md)** | Step-by-step setup and first video | -| **[🔧 Installation Guide](docs/installation/INSTALLATION.md)** | Detailed installation instructions | -| **[⚙️ Pipeline Specifications](docs/pipelines_spec.md)** | Auto-generated pipeline spec table | -| **[🎯 Demo Commands](docs/usage/demo_commands.md)** | Example commands and workflows | +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) +[![Python 3.12 | 3.13](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue.svg)](https://www.python.org/downloads/) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.16961751.svg)](https://doi.org/10.5281/zenodo.16961751) -## 👥 Research Applications +**Automatic annotation of videos of people, for behavioural research.** VideoAnnotator finds the +people, faces, speech and scenes in your videos, writes the results in standard formats, and shows +them on the video so you can check them. Everything runs on your own computer: your videos are never +uploaded anywhere. -### **Developmental Psychology** +It was built for studies of parent–child interaction, and suits any research that codes behaviour +from video: developmental psychology, clinical observation, classroom and human–computer +interaction studies. -- **Parent-child interaction** studies with synchronized behavioral coding -- **Social development** research with multi-person tracking -- **Language acquisition** studies with audio-visual alignment +![The viewer showing pose, face boxes with emotion labels and a scene label over a video of a parent and infant, with a timeline of speech and speakers below](docs/imgs/viewer.png) -### **Clinical Research** +## What it does -- **Autism spectrum** behavioral analysis with facial expression tracking -- **Therapy session** analysis with emotion and engagement metrics -- **Developmental assessment** with standardized behavioral measures +You choose videos and pipelines in the browser; VideoAnnotator runs them and keeps the results. -### **Human-Computer Interaction** +| Pipeline | What you get | File format | +|---|---|---| +| Person tracking | Each person's box and 17-point pose, with an ID that follows them | COCO | +| Face analysis | Faces with emotion, age and gender estimates | COCO | +| Face embeddings (OpenFace 3) | A 512-number description of each face, for telling people apart | JSON | +| Scene detection | Where the scene cuts, and what kind of place each scene is | JSON | +| Speech recognition | A transcript with word timings (Whisper) | WebVTT | +| Speaker diarization | Who spoke when (pyannote) | RTTM | +| VLM annotation | Your own question, asked of video frames by a local vision-language model (Ollama) | JSON | -- **User experience** research with attention and emotion tracking -- **Interface evaluation** with gaze direction and facial feedback -- **Accessibility** studies with comprehensive behavioral data +The viewer then lets you: -## 🏗️ Architecture & Performance +- Play the video with every result drawn on it, and a timeline of speech, speakers and scenes. +- Run the same settings again, or change one and compare the results. +- Keep named lists of videos (datasets) to run studies on. +- Write and test prompts for the VLM pipeline, and compare two models' labels against your own + coding (ELAN `.eaf`). -### **Modern Technology Stack** +Every output file records what made it: the pipeline and VideoAnnotator version, the model and its +exact weights, and the settings. That is what you need to report the method, and to get the same +results again later. -- **FastAPI** - High-performance REST API with automatic documentation -- **YOLO11** - State-of-the-art object detection and pose estimation -- **DeepFace / OpenFace 3.0 / LAION CLIP** - Facial analysis, embeddings, and emotion recognition -- **Whisper** - Robust speech recognition and transcription -- **pyannote.audio** - Speaker diarization and segmentation -- **LAION Empathic Insight** - Voice emotion analysis from Whisper embeddings -- **PySceneDetect + CLIP** - Scene boundary detection and environment classification -- **PyTorch** - GPU-accelerated machine learning inference +## Get started -### **Performance Characteristics** +You need [Docker Desktop](https://www.docker.com/products/docker-desktop/) or +[Podman Desktop](https://podman-desktop.io/), installed and started once. No Python. -- **Processing speed**: ~2-4x real-time with GPU acceleration -- **Memory usage**: 4-8GB RAM for typical videos -- **Storage**: ~100MB output per hour of video -- **Accuracy**: 90%+ for person detection, 85%+ for emotion recognition +1. **Install the start-up program**, with one line: -### **Scalability** + ```powershell + # Windows (PowerShell) + powershell -ExecutionPolicy Bypass -c "irm https://github.com/InfantLab/VideoAnnotator/releases/latest/download/install.ps1 | iex" + ``` -- **Batch processing**: Handle multiple videos simultaneously -- **Container deployment**: Docker support for cloud platforms -- **Distributed processing**: API-first design for microservices -- **Resource optimization**: CPU and GPU variants available + ```bash + # macOS and Linux + curl -LsSf https://github.com/InfantLab/VideoAnnotator/releases/latest/download/install.sh | sh + ``` -## 🤝 Contributing & Community +2. **Double-click Start VideoAnnotator.** The first time, it asks which folder your videos are + in and shares it read-only. Then it opens VideoAnnotator in your browser. Next time it just + starts. -### **Getting Involved** +3. **Run a job.** Choose **New job**, tick some videos and the pipelines to run. The first time + you choose a pipeline, VideoAnnotator offers to install it. **Jobs** shows progress; open a + finished job to see the results on the video. Results also go to a `VideoAnnotator` folder in + your home folder. -- **🐛 Report issues**: [GitHub Issues](https://github.com/InfantLab/VideoAnnotator/issues) -- **💬 Discussions**: [GitHub Discussions](https://github.com/InfantLab/VideoAnnotator/discussions) -- **📧 Contact**: Caspar Addyman at infantologist@gmail.com -- **🔬 Collaborations**: Open to research partnerships +Allow about 1 GB of disk, plus up to 12 GB if you install every pipeline, and 16 GB of RAM. A +GPU is optional. The VLM pipeline needs [Ollama](https://ollama.com) running on the same +computer. -### **Development** +The [installation guide](docs/installation/INSTALLATION.md) covers sharing more folders, the +GPU, labs and servers ([Docker Compose](docs/installation/INSTALLATION.md#docker-installation-alternative)), +and installing from source in your own Python environment +([developers](docs/installation/INSTALLATION.md#install-with-uv-developers)). -- **Code quality**: Automated linting, typing checks, and tests (see the CI badge above) -- **Documentation**: Comprehensive guides and API documentation -- **CI/CD**: Automated testing and deployment pipelines -- **Standards**: Following research software engineering best practices +## Learn more -## 📄 Citation & License +- **[Documentation](docs/README.md)**: install options, every pipeline, output formats, + configuration and troubleshooting. +- **[Getting started](docs/usage/GETTING_STARTED.md)**: a first job, step by step. +- **[Command line and API](docs/usage/demo_commands.md)**: everything the viewer does can also be + scripted, via `videoannotator` commands or the REST API (documented at + http://127.0.0.1:18011/docs while the server runs). +- **[Docker](docs/deployment/Docker.md)**: run it in a container, with or without a GPU. +- **[Changelog](CHANGELOG.md)**: what changed in each release. -### **Citation** +## Citing VideoAnnotator -If you use VideoAnnotator in your research, please cite: +If you use VideoAnnotator in your research, please cite it (GitHub's "Cite this repository" gives +the details from [CITATION.cff](CITATION.cff)): ``` Addyman, C. (2025). VideoAnnotator: Automated video analysis toolkit for human interaction research. Zenodo. https://doi.org/10.5281/zenodo.16961751 ``` -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.16961751.svg)](https://doi.org/10.5281/zenodo.16961751) - -### **License** - -MIT License - Full terms in [LICENSE](LICENSE) - -### **Funding & Support** - -- **The Global Parenting Initiative** (Funded by The LEGO Foundation) - -## 🙏 Acknowledgments - -### **Research Team** - -- **Caspar Addyman** - Stellenbosch University, South Africa [![ORCID](https://img.shields.io/badge/ORCID-0000--0003--0001--9548-green)](https://orcid.org/0000-0003-0001-9548) — Lead Developer & Corresponding Author -- **Jeremiah Ishaya** - Stellenbosch University, South Africa [![ORCID](https://img.shields.io/badge/ORCID-0000--0002--9014--9372-green)](https://orcid.org/0000-0002-9014-9372) -- **Irene Uwerikowe** - Stellenbosch University, South Africa [![ORCID](https://img.shields.io/badge/ORCID-0000--0002--1293--7349-green)](https://orcid.org/0000-0002-1293-7349) -- **Daniel Stamate** - Department of Computing, Goldsmiths, University of London, UK [![ORCID](https://img.shields.io/badge/ORCID-0000--0001--8565--6890-green)](https://orcid.org/0000-0001-8565-6890) -- **Jamie Lachman** - Department of Social Policy and Intervention, University of Oxford, UK [![ORCID](https://img.shields.io/badge/ORCID-0000--0001--9475--9218-green)](https://orcid.org/0000-0001-9475-9218) -- **Mark Tomlinson** - Stellenbosch University, South Africa [![ORCID](https://img.shields.io/badge/ORCID-0000--0001--5846--3444-green)](https://orcid.org/0000-0001-5846-3444) - +Please also cite the models behind the pipelines you used; each output file names them. -### **Open Source Dependencies** +## Contributing and contact -Built with and grateful to: +- Report a problem or ask for a feature: [GitHub Issues](https://github.com/InfantLab/VideoAnnotator/issues) +- Questions and ideas: [GitHub Discussions](https://github.com/InfantLab/VideoAnnotator/discussions) +- Contributing code: [CONTRIBUTING.md](CONTRIBUTING.md) +- Research collaborations: Caspar Addyman, infantologist@gmail.com -- **[YOLO & Ultralytics](https://ultralytics.com/)** - Object detection, tracking, and pose estimation -- **[DeepFace](https://github.com/serengil/deepface)** - Face detection and emotion recognition -- **[OpenFace 3.0](https://github.com/CMU-MultiComp-Lab/OpenFace-3.0)** - Facial behavior analysis and embeddings -- **[LAION](https://laion.ai/)** - CLIP face embeddings and empathic voice emotion models -- **[OpenAI Whisper](https://github.com/openai/whisper)** - Speech recognition -- **[pyannote.audio](https://github.com/pyannote/pyannote-audio)** - Speaker diarization -- **[PySceneDetect](https://www.scenedetect.com/)** - Scene boundary detection -- **[FastAPI](https://github.com/tiangolo/fastapi)** - Modern web framework -- **[PyTorch](https://pytorch.org/)** - Machine learning infrastructure +## Team -### **Development Tools & AI Assistance** +- **Caspar Addyman**, Stellenbosch University, South Africa ([ORCID](https://orcid.org/0000-0003-0001-9548)): lead developer and corresponding author +- **Jeremiah Ishaya**, Stellenbosch University, South Africa ([ORCID](https://orcid.org/0000-0002-9014-9372)) +- **Irene Uwerikowe**, Stellenbosch University, South Africa ([ORCID](https://orcid.org/0000-0002-1293-7349)) +- **Daniel Stamate**, Department of Computing, Goldsmiths, University of London, UK ([ORCID](https://orcid.org/0000-0001-8565-6890)) +- **Jamie Lachman**, Department of Social Policy and Intervention, University of Oxford, UK ([ORCID](https://orcid.org/0000-0001-9475-9218)) +- **Mark Tomlinson**, Stellenbosch University, South Africa ([ORCID](https://orcid.org/0000-0001-5846-3444)) -Development was greatly helped by: +Funded by **The Global Parenting Initiative** (The LEGO Foundation). -- **[Visual Studio Code](https://code.visualstudio.com/)** - Primary development environment -- **[GitHub Copilot](https://github.com/features/copilot)** - AI pair programming assistance -- **[Claude Code](https://claude.ai/code)** - Architecture design and documentation -- **GPT-4 & Claude Models** - Code generation and debugging help +## Acknowledgements -_This project demonstrates how AI-assisted development can accelerate research software creation while maintaining code quality and comprehensive testing._ +VideoAnnotator stands on these open-source projects: +[Ultralytics YOLO](https://ultralytics.com/) (people and pose), +[DeepFace](https://github.com/serengil/deepface) (faces and emotion), +[OpenFace 3.0](https://github.com/CMU-MultiComp-Lab/OpenFace-3.0) (face embeddings), +[PySceneDetect](https://www.scenedetect.com/) and [OpenCLIP](https://github.com/mlfoundations/open_clip) +with weights trained on [LAION](https://laion.ai/)-2B (scenes), +[OpenAI Whisper](https://github.com/openai/whisper) (speech), +[pyannote.audio](https://github.com/pyannote/pyannote-audio) (speakers), +[Ollama](https://ollama.com) (vision-language models), +[PyTorch](https://pytorch.org/) and [FastAPI](https://fastapi.tiangolo.com/). ---- +Development was helped by [Visual Studio Code](https://code.visualstudio.com/), +[GitHub Copilot](https://github.com/features/copilot) and [Claude Code](https://claude.ai/code). -**🎥 Ready to start analyzing videos?** Follow the [60-second setup](#-get-started-in-60-seconds) above! +Released under the [MIT License](LICENSE). diff --git a/configs/README.md b/configs/README.md index 1d826a1d..410f5237 100644 --- a/configs/README.md +++ b/configs/README.md @@ -87,7 +87,7 @@ face_analysis: deepface: emotion_model: "VGG-Face" # Emotion recognition model age_gender_model: "VGG-Face" # Age/gender model - detector_backend: "opencv" # Face detector: opencv, mtcnn, retinaface, ssd + detector_backend: "retinaface" # Face detector: retinaface (default), mtcnn, yunet, ssd, opencv enforce_detection: false # Continue even if no face detected # Processing settings @@ -187,7 +187,7 @@ annotator = VideoAnnotator(config_path="config/full_pipeline.yaml") # Process single video results = annotator.process_video( video_path="path/to/video.mp4", - pipelines=["scene_detection", "person_tracking", "face_analysis"] + pipelines=["scene_detection", "person_tracking", "face_analysis"], ) # Save results @@ -200,10 +200,7 @@ annotator.save_results(results, "output/annotations.json") from videoannotator.pipelines import ScenePipeline # Configure and run scene detection -config = { - "threshold": 25.0, - "scene_labels": ["indoor", "outdoor", "office"] -} +config = {"threshold": 25.0, "scene_labels": ["indoor", "outdoor", "office"]} with ScenePipeline(config) as pipeline: annotations = pipeline.process("video.mp4") diff --git a/configs/default.yaml b/configs/default.yaml index b8a73e84..d76aadd2 100644 --- a/configs/default.yaml +++ b/configs/default.yaml @@ -4,7 +4,8 @@ # Storage Configuration storage: provider: "local" - root_path: "./storage/jobs" + # root_path: where job folders go. Default: `jobs` in the per-user data + # directory, beside the database. The STORAGE_ROOT environment variable wins. options: create_dirs: true @@ -73,22 +74,13 @@ face_analysis: deepface: emotion_model: "VGG-Face" age_gender_model: "VGG-Face" - detector_backend: "opencv" + detector_backend: "retinaface" # opencv is faster but flags lamps/objects as faces enforce_detection: false -# Audio Processing Pipeline -audio_processing: - speech_recognition: - model: "openai/whisper-base" - language: "auto" - - audio_classification: - confidence_threshold: 0.5 - - features: - sample_rate: 16000 - hop_length: 512 - n_mels: 128 +# Audio: configure speech_recognition and speaker_diarization as their own sections +# (keyed by pipeline name). The old `audio_processing` section here had no effect (its +# settings were ignored or equal to the defaults); that pipeline is deprecated in v1.6.0 +# and removed in v1.7.0. # Global Settings global: diff --git a/configs/high_performance.yaml b/configs/high_performance.yaml index 48ae9123..2dd9ae13 100644 --- a/configs/high_performance.yaml +++ b/configs/high_performance.yaml @@ -79,28 +79,10 @@ face_analysis: age_gender_model: "VGG-Face" detector_backend: "retinaface" # More accurate detector -# Audio Processing Pipeline -audio_processing: - speech_recognition: - model: "openai/whisper-large-v2" # Larger model - language: "auto" - - speaker_diarization: - model: "pyannote/speaker-diarization" - min_speakers: 1 - max_speakers: 15 - - audio_classification: - model: "yamnet" - confidence_threshold: 0.4 # Lower threshold - - features: - sample_rate: 16000 - hop_length: 256 # Higher resolution - n_mels: 256 # More mel bins - - segment_length: 15.0 # Shorter segments for precision - overlap: 0.25 +# Audio: configure speech_recognition and speaker_diarization as their own sections +# (keyed by pipeline name). The old `audio_processing` section here had no effect (its +# settings were ignored or equal to the defaults); that pipeline is deprecated in v1.6.0 +# and removed in v1.7.0. # Global Settings global: diff --git a/configs/laion_pipelines.yaml b/configs/laion_pipelines.yaml deleted file mode 100644 index 4a6cbd29..00000000 --- a/configs/laion_pipelines.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# LAION Pipelines Configuration for VideoAnnotator Batch Processing - -# Default LAION pipeline configurations -laion_face_analysis: - model_size: "small" # "small" or "large" - large is faster with GPU - confidence_threshold: 0.6 # Face detection confidence threshold - top_k_emotions: 5 # Number of top emotions to return - device: "auto" # "cpu", "cuda", or "auto" - face_detection_backend: "opencv" - min_face_size: 30 - max_faces: 10 - include_raw_scores: false - model_cache_dir: "./models/laion_face" - -laion_voice_analysis: - model_size: "small" # "small" or "large" - large is faster with GPU - whisper_model: "mkrausio/EmoWhisper-AnS-Small-v0.1" - cache_dir: "./models/laion_voice" - - # Audio processing - min_segment_duration: 1.0 # Minimum segment length in seconds - max_segment_duration: 15.0 # Maximum segment length in seconds - - # Segmentation strategy - segmentation_mode: "fixed_interval" # "fixed_interval", "diarization", "scene_based", "vad" - segment_overlap: 0.0 # Overlap between segments in seconds - - # Integration options - enable_diarization: false # Use speaker diarization for segmentation - enable_scene_alignment: false # Align with scene boundaries - - # Output configuration - include_raw_scores: false - include_transcription: true # Include speech transcription - top_k_emotions: 5 # Number of top emotions to return - -# Quality presets for different use cases -fast_preset: - laion_face_analysis: - model_size: "small" - confidence_threshold: 0.7 - top_k_emotions: 3 - max_faces: 5 - laion_voice_analysis: - model_size: "small" - min_segment_duration: 2.0 - max_segment_duration: 20.0 - include_transcription: false - top_k_emotions: 3 - -balanced_preset: - laion_face_analysis: - model_size: "small" - confidence_threshold: 0.6 - top_k_emotions: 5 - max_faces: 10 - laion_voice_analysis: - model_size: "small" - min_segment_duration: 1.0 - max_segment_duration: 15.0 - include_transcription: true - top_k_emotions: 5 - -high_quality_preset: - laion_face_analysis: - model_size: "large" - confidence_threshold: 0.5 - top_k_emotions: 5 - max_faces: 15 - laion_voice_analysis: - model_size: "large" - min_segment_duration: 1.0 - max_segment_duration: 10.0 - include_transcription: true - top_k_emotions: 5 - segmentation_mode: "diarization" # Use advanced segmentation if available diff --git a/configs/lightweight.yaml b/configs/lightweight.yaml index c6f86792..03b7e222 100644 --- a/configs/lightweight.yaml +++ b/configs/lightweight.yaml @@ -44,22 +44,10 @@ face_analysis: emotion_model: "VGG-Face" # Lightweight model detector_backend: "opencv" # Fastest detector -# Audio Processing Pipeline -audio_processing: - speech_recognition: - model: "openai/whisper-tiny" # Smallest model - language: "en" # Fixed language - - audio_classification: - confidence_threshold: 0.6 # Higher threshold - - features: - sample_rate: 8000 # Lower sample rate - hop_length: 1024 # Lower resolution - n_mels: 64 # Fewer mel bins - - segment_length: 60.0 # Longer segments - overlap: 0.1 # Minimal overlap +# Audio: configure speech_recognition and speaker_diarization as their own sections +# (keyed by pipeline name). The old `audio_processing` section here had no effect (its +# settings were ignored or equal to the defaults); that pipeline is deprecated in v1.6.0 +# and removed in v1.7.0. # Global Settings global: diff --git a/configs/openface3.yaml b/configs/openface3.yaml index b015812b..fd8887f7 100644 --- a/configs/openface3.yaml +++ b/configs/openface3.yaml @@ -62,10 +62,10 @@ face_analysis: save_face_crops: false save_visualizations: true -audio_processing: - enabled: true - whisper_model: "base" - sample_rate: 16000 +# Audio: configure speech_recognition and speaker_diarization as their own sections +# (keyed by pipeline name). The old `audio_processing` section here had no effect (its +# settings were ignored or equal to the defaults); that pipeline is deprecated in v1.6.0 +# and removed in v1.7.0. # Processing settings optimized for OpenFace 3.0 processing: diff --git a/docker-compose.yml b/docker-compose.yml index 54e0e660..9ae92f89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,14 @@ -version: "3.8" - services: # Development service videoannotator-dev: build: context: . - dockerfile: Dockerfile.cpu + dockerfile: Dockerfile volumes: - .:/app - ./data:/app/data - ./output:/app/output + - cache:/app/cache environment: - HUGGINGFACE_TOKEN # passed through from the host env / .env; needed by speaker diarization (pyannote) - HF_AUTH_TOKEN # pre-v1.5.0 name, still read as a fallback @@ -26,13 +25,31 @@ services: videoannotator-prod: build: context: . - dockerfile: Dockerfile.cpu + dockerfile: Dockerfile volumes: + - models:/app/models # model weights persist across rebuilds (spec 016) + - database:/app/database # jobs, users and API keys persist across rebuilds + - storage:/app/storage # each job's video and results, which the database points to + - cache:/app/cache # downloads, so installed pipelines are restored quickly (spec 024) - ./data:/app/data:ro - ./output:/app/output + # Spec 022: your videos, read where they are and never changed... + - ${VIDEOS_DIR:-./videos}:/videos:ro + # ...and your results, in a folder on your own machine. + - ${RESULTS_DIR:-~/VideoAnnotator}:/results + ports: + # This machine only: VIDEOANNOTATOR_PUBLISHED_LOCALLY below relies on it. + # Publishing more widely (e.g. "18011:18011") means removing that line. + - "127.0.0.1:18011:18011" environment: - HUGGINGFACE_TOKEN # passed through from the host env / .env; needed by speaker diarization (pyannote) - HF_AUTH_TOKEN # pre-v1.5.0 name, still read as a fallback + # /videos only when VIDEOS_DIR is set: otherwise Docker's empty ./videos would count + # as a shared folder and the wizard couldn't say how to share one (spec 024, R8). + - VIDEOANNOTATOR_INGEST_ROOTS=${VIDEOS_DIR:+/videos} + - VIDEOANNOTATOR_RESULTS_DIR=/results + - VIDEOANNOTATOR_PUBLISHED_LOCALLY=1 # every caller is this machine: the port is on 127.0.0.1 + - VIDEOANNOTATOR_HOST_PATHS=/videos=${VIDEOS_DIR:-./videos};/results=${RESULTS_DIR:-~/VideoAnnotator} - UV_NO_SYNC=1 # keep pipeline extras installed from the viewer across restarts (spec 011 FR-009) - PYTHONPATH=/app - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434} # Ollama on the host (vlm_annotation) @@ -46,13 +63,31 @@ services: videoannotator-gpu: build: context: . - dockerfile: Dockerfile.gpu + dockerfile: Dockerfile volumes: + - models:/app/models # model weights persist across rebuilds (spec 016) + - database:/app/database # jobs, users and API keys persist across rebuilds + - storage:/app/storage # each job's video and results, which the database points to + - cache:/app/cache # downloads, so installed pipelines are restored quickly (spec 024) - ./data:/app/data:ro - ./output:/app/output + # Spec 022: your videos, read where they are and never changed... + - ${VIDEOS_DIR:-./videos}:/videos:ro + # ...and your results, in a folder on your own machine. + - ${RESULTS_DIR:-~/VideoAnnotator}:/results + ports: + # This machine only: VIDEOANNOTATOR_PUBLISHED_LOCALLY below relies on it. + # Publishing more widely (e.g. "18011:18011") means removing that line. + - "127.0.0.1:18011:18011" environment: - HUGGINGFACE_TOKEN # passed through from the host env / .env; needed by speaker diarization (pyannote) - HF_AUTH_TOKEN # pre-v1.5.0 name, still read as a fallback + # /videos only when VIDEOS_DIR is set: otherwise Docker's empty ./videos would count + # as a shared folder and the wizard couldn't say how to share one (spec 024, R8). + - VIDEOANNOTATOR_INGEST_ROOTS=${VIDEOS_DIR:+/videos} + - VIDEOANNOTATOR_RESULTS_DIR=/results + - VIDEOANNOTATOR_PUBLISHED_LOCALLY=1 # every caller is this machine: the port is on 127.0.0.1 + - VIDEOANNOTATOR_HOST_PATHS=/videos=${VIDEOS_DIR:-./videos};/results=${RESULTS_DIR:-~/VideoAnnotator} - UV_NO_SYNC=1 # keep pipeline extras installed from the viewer across restarts (spec 011 FR-009) - PYTHONPATH=/app - CUDA_VISIBLE_DEVICES=0 @@ -71,44 +106,11 @@ services: profiles: - gpu - # Development service with GPU and pre-cached models - videoannotator-dev-gpu: - build: - context: . - dockerfile: Dockerfile.dev - volumes: - - ./data:/app/data - - ./output:/app/output - - ./logs:/app/logs - - ./database:/app/database # Persistent database for development - environment: - - HUGGINGFACE_TOKEN # passed through from the host env / .env; needed by speaker diarization (pyannote) - - HF_AUTH_TOKEN # pre-v1.5.0 name, still read as a fallback - - PYTHONPATH=/app - - CUDA_VISIBLE_DEVICES=0 - - VIDEOANNOTATOR_DB_PATH=/app/database/videoannotator_dev.db - - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434} # Ollama on the host (vlm_annotation) - extra_hosts: - - "host.docker.internal:host-gateway" # Linux Docker Engine; Docker Desktop provides it already - runtime: nvidia - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: 1 - capabilities: [gpu] - ports: - - "18011:18011" # FastAPI server (host 18011) - command: ["uv", "run", "videoannotator", "server", "--host", "0.0.0.0", "--port", "18011"] - profiles: - - dev-gpu - # Jupyter service for development jupyter: build: context: . - dockerfile: Dockerfile.cpu + dockerfile: Dockerfile volumes: - .:/app - ./data:/app/data @@ -119,7 +121,7 @@ services: - PYTHONPATH=/app # --inexact: .:/app shares the repo's .venv; an exact sync would uninstall # pipeline extras installed from the viewer (see scripts/start_server.sh). - command: ["bash", "-lc", "uv sync --inexact --extra dev && uv run jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root"] + command: ["bash", "-lc", "uv sync --inexact --group notebooks && uv run jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root"] profiles: - dev @@ -147,6 +149,16 @@ services: volumes: postgres_data: + models: + # Fixed name (not _models), shared with the dev container, so the + # weights download once per machine. + name: videoannotator-models + database: + name: videoannotator-database + storage: + name: videoannotator-storage + cache: + name: videoannotator-cache networks: default: diff --git a/docs/GETTING_STARTED_REVIEWERS.md b/docs/GETTING_STARTED_REVIEWERS.md index 1e5e0a4d..fe823a30 100644 --- a/docs/GETTING_STARTED_REVIEWERS.md +++ b/docs/GETTING_STARTED_REVIEWERS.md @@ -329,7 +329,7 @@ See [Pipeline Specifications](usage/pipeline_specs.md) ### Q: What about production deployment? **A**: Multiple options: -- Docker: `Dockerfile.cpu`, `Dockerfile.gpu` +- Docker: `Dockerfile` (one image for CPU and GPU) - Kubernetes: Manifests in `docs/deployment/` - Systemd: Service files provided - See [Deployment Guide](deployment/Docker.md) diff --git a/docs/README.md b/docs/README.md index 6767f6ff..f49073b3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,82 +1,53 @@ -# VideoAnnotator Documentation +# VideoAnnotator documentation -## Current Release: v1.4.1 +VideoAnnotator runs computer-vision and audio pipelines over videos of people (faces, bodies, +speech, scenes, and labels from a vision-language model), keeps a record of what made every result, +and shows the results on the video in its viewer. It runs on your own computer: videos never leave +it. -This documentation is organized into clear sections for different user types and development phases. +New here? Start with **[Getting started](usage/GETTING_STARTED.md)**. -## 📖 User Documentation +## Install -### Installation +- [Installation](installation/INSTALLATION.md): Windows, macOS and Linux, with or without a GPU, + and the dev container. +- [Docker](deployment/Docker.md): the CPU and GPU images, and Docker Compose. +- [Access tokens for gated models](installation/ENVIRONMENT_SETUP.md): the Hugging Face token + that speaker diarization needs. +- [Installation troubleshooting](installation/troubleshooting.md) -- [Installation Guide](installation/INSTALLATION.md) - **Modern uv-based setup** with CUDA support -- macOS specifics: see the macOS section inside the Installation Guide for libomp, Node, ffmpeg, and PATH fixes. -- [Environment Setup](installation/ENVIRONMENT_SETUP.md) - HuggingFace and configuration -- [Python Development 2025](installation/PythonDev2025.md) - **Modern development practices** with uv, Ruff, and Docker +## Use -### Usage +- [Getting started](usage/GETTING_STARTED.md): start the server, open the viewer, run a first job. +- [Command-line examples](usage/demo_commands.md) +- [Pipelines](usage/pipeline_specs.md): what each pipeline does and the files it writes. +- [Getting your results](usage/accessing_results.md): downloading and reading the output files. +- [Configuration](usage/configuration.md) and [environment variables](usage/environment_variables.md) +- [Connecting the viewer with an API key](usage/CLIENT_TOKEN_SETUP_GUIDE.md) +- [Scene detection and person tracking](usage/scene_detection.md) +- [Troubleshooting](usage/troubleshooting.md) -- [Getting Started](usage/GETTING_STARTED.md) - Quick start guide for new users -- [Accessing Results](usage/accessing_results.md) - **New**: Downloading annotations and artifacts -- [Configuration](usage/configuration.md) - Configuration guide -- [Pipeline Specifications](usage/pipeline_specs.md) - Detailed pipeline documentation -- [Scene Detection Guide](usage/scene_detection.md) - Scene detection usage -- [Demo Commands](usage/demo_commands.md) - Example commands and workflows -- [Troubleshooting](usage/troubleshooting.md) - Common issues and solutions +## Run it for a group -## Command Shortcuts (Containers) +- [Security overview](security/README.md): [authentication](security/authentication.md), + [CORS](security/cors.md) and the [production checklist](security/production_checklist.md). +- [Locales in the Docker images](locale.md) -If you are using the provided Docker/devcontainer images, a few convenience commands are available on `PATH`. -These are optional shortcuts; the canonical CLI remains `uv run videoannotator ...`. +## Contribute -| Action | Shortcut | Equivalent | -|---|---|---| -| Initialize the database + create an admin token | `setupdb --admin-email you@example.com --admin-username you` | `uv run videoannotator setup-db --admin-email you@example.com --admin-username you` | -| Run the VideoAnnotator CLI (any subcommand) | `va ...` | `uv run videoannotator ...` | -| Start the API server (recommended defaults) | `va` | `uv run videoannotator` | -| Start the API server (explicit subcommand) | `server ...` | `uv run videoannotator server ...` | -| Generate a new API token | `newtoken ...` | `uv run videoannotator generate-token ...` | -| List available pipelines (detailed) | `va pipelines --detailed` | `uv run videoannotator pipelines --detailed` | -| Validate a config file | `va config --validate configs/default.yaml` | `uv run videoannotator config --validate configs/default.yaml` | -| Submit a processing job | `va job submit video.mp4 --pipelines scene,person` | `uv run videoannotator job submit video.mp4 --pipelines scene,person` | -| Check job status | `va job status ` | `uv run videoannotator job status ` | -| List jobs | `va job list` | `uv run videoannotator job list` | -| Run all tests (quick/quiet) | `vatest` | `uv run pytest -q` | -| Run some tests (quick/quiet) | `vatest tests/unit/` | `uv run pytest -q tests/unit/` | +- [Contributing guide](../CONTRIBUTING.md) +- [Development notes](development/README.md): conventions, output formats, the + [pre-commit hooks](development/PRE_COMMIT_GUIDE.md). +- [Testing overview](testing/testing_overview.md), [testing standards](testing/testing_standards.md) + and [coverage](testing/coverage_report.md) +- [Roadmap for v1.6.0](development/roadmap_v1.6.0.md), then [v1.7.0](development/roadmap_v1.7.0.md) + and [v1.7 to v2.0](development/roadmap_v1.7_to_v2.0.md) +- [Changelog](../CHANGELOG.md) -For more copy-pasteable CLI workflows, see `usage/demo_commands.md`. +## Reviewing the software -### Deployment +- [Quick start for JOSS reviewers](GETTING_STARTED_REVIEWERS.md) +- [The JOSS paper](joss.md) -- [Docker Guide](deployment/Docker.md) - Container deployment (Docker Compose-first) - -## 🔧 Development Documentation - -### Active Development & Roadmap - -- **[Roadmap Overview (Archived)](archive/development/roadmap_overview.md)** - Historical release strategy notes -- [v1.4.0 Roadmap](development/roadmap_v1.4.0.md) - Roadmap for the v1.4.0 cycle -- [v1.5.0 Roadmap](development/roadmap_v1.5.0.md) - Modularity & integration release (JOSS resubmission response) -- [v1.6.0 Roadmap](development/roadmap_v1.6.0.md) - Public release: one repository, results out, agents, model refresh, corpus view, tutorials -- [v1.7.0 Roadmap](development/roadmap_v1.7.0.md) - Plugin ecosystem (planned as v1.6.0 until 2026-09-24) -- [v1.7 to v2.0 Roadmap](development/roadmap_v1.7_to_v2.0.md) - Remote pipelines, HPC dispatch, slim core -- [Modular Pipeline Architecture Spec](../specs/003-modular-pipeline-architecture/spec.md) - Full design spec underlying v1.5.0/v1.7.0 -- [Examples CLI Update Plan](development/EXAMPLES_CLI_UPDATE_CHECKLIST.md) - CLI modernization checklist - -## 🧪 Testing & QA - -### Current Testing - -- [Testing Overview](testing/testing_overview.md) - Complete testing strategy and results -- [Testing Standards](testing/testing_standards.md) - Quality assurance standards -- [Batch Testing Guide](testing/batch_testing_guide.md) - Batch processing test procedures - -## 📁 Archive - -Historical and superseded documentation lives under `archive/`: - -- [Archive Root](archive/) - Top-level archive index -- [Release Notes](archive/release-notes/) - Historical release notes -- [Development (Archived)](archive/development/) - Completed roadmaps, checklists, and completion summaries -- [Testing (Archived)](archive/testing/) - Historical QA checklists -- [Dated Updates](archive/2025/) - Dated update memos -- **v1.3.0** (Future Release) - Advanced features, security, and scalability enhancements +Older documents, kept for the record, are in [archive/](archive/README.md). They describe earlier +versions and may be wrong about this one. diff --git a/docs/Figure 1.docx b/docs/archive/Figure 1.docx similarity index 100% rename from docs/Figure 1.docx rename to docs/archive/Figure 1.docx diff --git a/docs/development/EXAMPLES_CLI_UPDATE_CHECKLIST.md b/docs/archive/development/EXAMPLES_CLI_UPDATE_CHECKLIST.md similarity index 97% rename from docs/development/EXAMPLES_CLI_UPDATE_CHECKLIST.md rename to docs/archive/development/EXAMPLES_CLI_UPDATE_CHECKLIST.md index 7b3057ac..a1be8d2c 100644 --- a/docs/development/EXAMPLES_CLI_UPDATE_CHECKLIST.md +++ b/docs/archive/development/EXAMPLES_CLI_UPDATE_CHECKLIST.md @@ -51,9 +51,7 @@ Ongoing cleanup to modernize all examples and documentation to use the `videoann - **Current**: Custom analysis workflow - **Update**: Integrate with API results retrieval -7. **examples/test_laion_voice_pipeline.py** - - **Current**: Direct pipeline testing - - **Update**: API-based LAION testing +7. ~~**examples/test_laion_voice_pipeline.py**~~: removed with the LAION pipelines (v1.6.0) ## 🔄 **Recommended Approach** diff --git a/docs/PARTIAL_SUCCESS_ADVISORY.md b/docs/archive/development/PARTIAL_SUCCESS_ADVISORY.md similarity index 100% rename from docs/PARTIAL_SUCCESS_ADVISORY.md rename to docs/archive/development/PARTIAL_SUCCESS_ADVISORY.md diff --git a/docs/installation/PythonDev2025.md b/docs/archive/development/PythonDev2025.md similarity index 95% rename from docs/installation/PythonDev2025.md rename to docs/archive/development/PythonDev2025.md index a705c2bd..e5b6f137 100644 --- a/docs/installation/PythonDev2025.md +++ b/docs/archive/development/PythonDev2025.md @@ -201,10 +201,11 @@ COPY . . CMD ["uv", "run", "python", "-m", "your_project"] ``` -### Production vs dev images (this repo) +### This repo -- `Dockerfile.cpu` and `Dockerfile.gpu` are production images and do NOT copy `models/` or `weights/` into the image; they download what they need at runtime. -- `Dockerfile.dev` is a development image and intentionally copies your local `models/` and `weights/` so repeated runs do not re-download. +This repo has one `Dockerfile` for CPU and GPU, on `ubuntu:24.04` rather than an `nvidia/cuda` base +(torch's wheels bring their own CUDA). Model weights are never copied in; they download into a +`/app/models` volume. See `docs/deployment/Docker.md`. **Host requirement:** Install **NVIDIA Container Toolkit** and run with `--gpus all`. @@ -223,7 +224,7 @@ Toolkit docs: install on your distro and Docker integrates GPU runtime. ([NVIDIA ```json { "name": "your-project (GPU)", - "build": { "dockerfile": "../Dockerfile.gpu", "context": ".." }, + "build": { "dockerfile": "../Dockerfile", "context": ".." }, "runArgs": ["--gpus", "all"], "features": {}, "postCreateCommand": "uv sync && pre-commit install", diff --git a/docs/performance/TOKEN_VALIDATION_FIX.md b/docs/archive/development/TOKEN_VALIDATION_FIX.md similarity index 100% rename from docs/performance/TOKEN_VALIDATION_FIX.md rename to docs/archive/development/TOKEN_VALIDATION_FIX.md diff --git a/docs/development/joss_testing_ci_polish.md b/docs/archive/development/joss_testing_ci_polish.md similarity index 100% rename from docs/development/joss_testing_ci_polish.md rename to docs/archive/development/joss_testing_ci_polish.md diff --git a/docs/development/scripts_inventory.md b/docs/archive/development/scripts_inventory.md similarity index 100% rename from docs/development/scripts_inventory.md rename to docs/archive/development/scripts_inventory.md diff --git a/docs/keypointsdf.png b/docs/archive/keypointsdf.png similarity index 100% rename from docs/keypointsdf.png rename to docs/archive/keypointsdf.png diff --git a/docs/testing/CLIENT_SERVER_DEBUG_TOOLS.md b/docs/archive/testing/CLIENT_SERVER_DEBUG_TOOLS.md similarity index 100% rename from docs/testing/CLIENT_SERVER_DEBUG_TOOLS.md rename to docs/archive/testing/CLIENT_SERVER_DEBUG_TOOLS.md diff --git a/docs/testing/Issues for Server Team.md b/docs/archive/testing/Issues for Server Team.md similarity index 100% rename from docs/testing/Issues for Server Team.md rename to docs/archive/testing/Issues for Server Team.md diff --git a/docs/testing/Jerry-issues.md b/docs/archive/testing/Jerry-issues.md similarity index 100% rename from docs/testing/Jerry-issues.md rename to docs/archive/testing/Jerry-issues.md diff --git a/docs/testing/SERVER_SIDE_IMPROVEMENTS.md b/docs/archive/testing/SERVER_SIDE_IMPROVEMENTS.md similarity index 100% rename from docs/testing/SERVER_SIDE_IMPROVEMENTS.md rename to docs/archive/testing/SERVER_SIDE_IMPROVEMENTS.md diff --git a/docs/testing/TEAM_HANDOFF_PACKAGE.md b/docs/archive/testing/TEAM_HANDOFF_PACKAGE.md similarity index 100% rename from docs/testing/TEAM_HANDOFF_PACKAGE.md rename to docs/archive/testing/TEAM_HANDOFF_PACKAGE.md diff --git a/docs/testing/TESTING_COLLABORATION_GUIDE.md b/docs/archive/testing/TESTING_COLLABORATION_GUIDE.md similarity index 100% rename from docs/testing/TESTING_COLLABORATION_GUIDE.md rename to docs/archive/testing/TESTING_COLLABORATION_GUIDE.md diff --git a/docs/testing/batch_testing_guide.md b/docs/archive/testing/batch_testing_guide.md similarity index 100% rename from docs/testing/batch_testing_guide.md rename to docs/archive/testing/batch_testing_guide.md diff --git a/docs/deployment/Docker.md b/docs/deployment/Docker.md index c7347b0c..b8920202 100644 --- a/docs/deployment/Docker.md +++ b/docs/deployment/Docker.md @@ -1,200 +1,129 @@ # VideoAnnotator Docker Deployment Guide -VideoAnnotator includes modern Docker containers using uv for fast, reliable dependency management. +One `Dockerfile` builds VideoAnnotator for CPU and GPU machines. It is slim by default; pipelines are +added at build time with `--build-arg EXTRAS=...` or installed later from the viewer. Model weights are +never in the image: they download on first use into `/app/models`, so mount a volume there. -## 🐳 Available Docker Images +There is no `nvidia/cuda` base image: torch's wheels bring their own CUDA runtime and cuDNN, and the +host's driver arrives with `--gpus all`. Without `--gpus all` the same image runs on the CPU. -### 1. **videoannotator:cpu** - Production CPU-only - -- **Size**: ~15-20GB (optimized) -- **Use case**: Production deployment without GPU -- **Features**: CPU-only PyTorch, models download on first use -- **Build time**: ~10-15 minutes +## Build ```bash -# Build CPU image -docker build -f Dockerfile.cpu -t videoannotator:cpu . +# Slim: core only (API, viewer, CLI), no torch +docker build -t videoannotator . + +# One or more pipeline families (see the extras in pyproject.toml) +docker build --build-arg EXTRAS=scene,person -t videoannotator:scene-person . -# Run CPU container -docker run -p 18011:18011 --rm -v "${PWD}/data:/app/data" -v "${PWD}/output:/app/output" videoannotator:cpu +# Every pipeline +docker build --build-arg EXTRAS=all -t videoannotator:all . ``` -### 2. **videoannotator:gpu** - Production GPU +With Podman on Windows, add `--network=host --format docker` (`podman build --network=host --format +docker -t videoannotator .`): the WSL kernel can't apply the nftables rules Podman's default build +network needs, and `--format docker` keeps the `SHELL` line. -- **Size**: ~25-30GB (optimized) -- **Use case**: Production deployment with GPU acceleration -- **Features**: CUDA PyTorch, models download on first use -- **Build time**: ~15-20 minutes +Measured on 2026-10-02 (Docker Desktop on Windows 11 with WSL 2, RTX 4060 Laptop GPU): -> These images bundle every pipeline's dependencies today. A slim, per-pipeline install (targeting -> an 80%+ size reduction for the default image) is planned in -> [`roadmap_v1.5.0.md`](../development/roadmap_v1.5.0.md). +| Image | On disk | Download (compressed) | Build time | +|---|---|---|---| +| Slim (default) | 1.35 GB | 347 MB | about 1.5 minutes | +| Every pipeline (`EXTRAS=all`) | 14.9 GB | 4.78 GB | about 12 minutes | -```bash -# Build GPU image -docker build -f Dockerfile.gpu -t videoannotator:gpu . - -# Run GPU container -docker run -p 18011:18011 --gpus all --rm -v "${PWD}/data:/app/data" -v "${PWD}/output:/app/output" videoannotator:gpu -``` +Most of the everything image is the GPU libraries that torch and TensorFlow bring (NVIDIA 3.6 GB, +TensorFlow 1.8 GB, torch 1.6 GB). The first job then downloads about 2.5 GB of model weights into +the models volume. -### 3. **videoannotator:dev** - Development with Local Model Cache +A job running all six default pipelines on the 14-second demo video used at most 5.7 GB of RAM and +2.4 GB of GPU memory, and took about 78 seconds on the GPU once the weights were downloaded (391 +seconds the first time). On the CPU alone it took 68 seconds with 5.0 GB of RAM: on a clip this short, +loading the models takes most of the time. -- **Size**: ~50GB (includes your local models/weights) -- **Use case**: Development and testing - instant model access -- **Features**: GPU support + your exact local model cache copied -- **Build time**: ~15-20 minutes (no model downloading, just copying) +## Run ```bash -# Build development image (includes models) -docker build -f Dockerfile.dev -t videoannotator:dev . - -# Run development container -docker run -p 18011:18011 --gpus all --rm \ +docker run --rm -p 18011:18011 --gpus all \ + -v videoannotator-models:/app/models \ + -v videoannotator-database:/app/database \ + -v videoannotator-storage:/app/storage \ -v "${PWD}/data:/app/data" \ -v "${PWD}/output:/app/output" \ - -v "${PWD}/database:/app/database" \ - -v "${PWD}/logs:/app/logs" \ - videoannotator:dev - -Notes: - -- Production images (`Dockerfile.cpu`, `Dockerfile.gpu`) do not copy `models/` or `weights/` into the image; they download what they need at runtime. -- The dev image (`Dockerfile.dev`) intentionally copies your local `models/` and `weights/` directories into the image so repeated runs do not re-download. -- If your repo uses Git LFS for models/weights, run `git lfs pull` before building the dev image. -``` - -## 📊 Image Comparison - -| Image | Size | Build Time | Model Access | GPU Support | Use Case | -| ------- | ----- | ---------- | -------------------- | ----------- | ------------------- | -| **cpu** | ~15GB | 10-15 min | Downloads at runtime | ❌ | Production CPU | -| **gpu** | ~25GB | 15-20 min | Downloads at runtime | ✅ | Production GPU | -| **dev** | ~50GB | 15-20 min | Local cache copied | ✅ | Development/Testing | - -## 🚀 Docker Compose Usage - -### Development with GPU (Recommended for Testing) - -```bash -# Start development service with pre-cached models -docker compose --profile dev-gpu up videoannotator-dev-gpu - -# Access API at http://localhost:18011 -# Interactive docs at http://localhost:18011/docs -``` - -### Production GPU - -```bash -# Start production GPU service -docker compose --profile gpu up videoannotator-gpu + videoannotator:all ``` -### CPU-only Production +Leave out `--gpus all` on a machine without an NVIDIA GPU. The API is at http://localhost:18011, the +viewer at http://localhost:18011/viewer. -```bash -# Start CPU service -docker compose --profile prod up videoannotator-prod -``` - -## Prerequisites +### Mounts -### NVIDIA Container Toolkit (for GPU) +| Path | Holds | Mount | +|---|---|---| +| `/app/models` | Model weights (`VIDEOANNOTATOR_MODELS_DIR`) | A named volume, so weights download once | +| `/app/database` | Jobs, users and API keys (`VIDEOANNOTATOR_DB_PATH`) | A named volume, so they survive a new container | +| `/app/storage` | Each job's video and results (`STORAGE_ROOT` is `/app/storage/jobs`) | A named volume, alongside the database | +| `/app/data` | Input videos | Your video folder | +| `/app/output` | Exported results | A host folder | +| `/app/logs` | Logs (`VIDEOANNOTATOR_LOG_DIR`) | Optional | -Install NVIDIA Container Toolkit for GPU support: +### Windows PowerShell -```bash -# Ubuntu/Debian -curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg -curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ - sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ - sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list -sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit -sudo systemctl restart docker +```powershell +docker run --rm -p 18011:18011 --gpus all -v videoannotator-models:/app/models -v videoannotator-database:/app/database -v videoannotator-storage:/app/storage -v "${PWD}\data:/app/data" videoannotator:all ``` -## 🗂️ Volume Mounts - -### Essential Mounts - -- **`./data:/app/data`** - Input videos and test data -- **`./output:/app/output`** - Processing results and annotations -- **`./logs:/app/logs`** - Application logs (dev only) -- **`./database:/app/database`** - Persistent SQLite database (dev only) - -### Example with All Mounts +## Docker Compose ```bash -docker run -p 18011:18011 --gpus all --rm \ - -v "${PWD}/data:/app/data" \ - -v "${PWD}/output:/app/output" \ - -v "${PWD}/logs:/app/logs" \ - -v "${PWD}/database:/app/database" \ - -e VIDEOANNOTATOR_DB_PATH=/app/database/videoannotator.db \ - videoannotator:dev +docker compose --profile gpu up videoannotator-gpu # GPU +docker compose --profile prod up videoannotator-prod # CPU ``` -## 🔧 Development Workflow +Both build the slim image and keep models and the database in named volumes +(`videoannotator-models`, `videoannotator-database`). -### For Fast Testing (Use dev image) +## Prerequisites for a GPU -1. **Build once**: `docker build -f Dockerfile.dev -t videoannotator:dev .` -2. **Run anytime**: `docker run -p 18011:18011 --gpus all videoannotator:dev` -3. **No wait**: Your local models copied, immediate testing +- **Linux**: the NVIDIA driver and the NVIDIA Container Toolkit: -### For Production (Use optimized images) + ```bash + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit + sudo systemctl restart docker + ``` -1. **Build**: `docker build -f Dockerfile.gpu -t videoannotator:gpu .` -2. **Deploy**: Models download on first use, smaller image size -3. **Scale**: Suitable for container orchestration +- **Windows**: Docker Desktop with the WSL 2 backend and a current NVIDIA driver. +- The driver must support CUDA 12.6: NVIDIA driver 560 or newer. -## 🎯 Quick Commands +Check GPU access: ```bash -# Build all images -docker build -f Dockerfile.cpu -t videoannotator:cpu . -docker build -f Dockerfile.gpu -t videoannotator:gpu . -docker build -f Dockerfile.dev -t videoannotator:dev . - -# Run development server (fastest for testing) -docker run -p 18011:18011 --gpus all --rm videoannotator:dev - -# Run production server -docker run -p 18011:18011 --gpus all --rm videoannotator:gpu - -# Check GPU access in container -docker run --gpus all --rm videoannotator:dev python -c "import torch; print(f'CUDA: {torch.cuda.is_available()}')" +docker run --gpus all --rm videoannotator:all python -c "import torch; print('CUDA:', torch.cuda.is_available())" ``` -## 🐛 Troubleshooting - -### Build Issues +## Troubleshooting -- **"NVIDIA files in CPU build"**: Fixed in latest Dockerfile.cpu with CPU-only PyTorch -- **"Dev image build context is large"**: `Dockerfile.dev` copies `models/` and `weights/` into the image; this is expected for fast iteration -- **"Build takes too long"**: Use dev image only when you need pre-cached models -- **"Package libgl1-mesa-glx not available"**: Fixed with correct Debian package names -- **"No solution found when resolving dependencies"**: Fixed with --frozen flag and Python <3.13 constraint +- **No GPU detected**: run with `--gpus all`; check the driver version (560+) and, on Linux, the + NVIDIA Container Toolkit. +- **Models download on every run**: mount a volume at `/app/models`. +- **Jobs or API keys gone after recreating the container**: mount a volume at `/app/database`. +- **Jobs listed but their videos or results missing after recreating the container**: mount a + volume at `/app/storage` too. Before v1.6.0 the image didn't say so, and job folders were lost + with the container. +- **Permission denied on a mounted folder**: check the host folder's permissions. -### Runtime Issues +## What's in the image -- **"No GPU detected"**: Ensure `--gpus all` flag and NVIDIA Docker runtime installed -- **"Models downloading slowly"**: Use dev image for testing, or cache models on host -- **"Permission denied"**: Check volume mount permissions and paths - -### Windows PowerShell - -```powershell -# Windows-specific command format -docker run -p 18011:18011 --gpus all --rm -v "${PWD}\data:/app/data" -v "${PWD}\output:/app/output" videoannotator:dev -``` +- Ubuntu 24.04, Python 3.13 (installed by uv), ffmpeg +- VideoAnnotator and the extras you chose, installed from `uv.lock` without the dev tools +- The server starts with `videoannotator server --host 0.0.0.0 --port 18011` -## Container Features +## The dev container -- **uv package manager** for fast, reliable dependency management -- **Python 3.12** runtime -- **CUDA 12.x** support (GPU containers; see the `FROM nvidia/cuda:...` tag in the Dockerfile) -- **Models download automatically** on first use (production) or pre-cached (dev) -- **FastAPI server** ready for deployment -- **Automatic dependency resolution** via uv.lock +`.devcontainer/devcontainer.json` builds the same `Dockerfile` with `SKIP_IMAGE_UV_SYNC=true`; its +`postCreateCommand` installs the environment into a `.venv` volume instead. The image itself is +1.13 GB. See [Dev Container (VS Code)](../installation/INSTALLATION.md#dev-container-vs-code) for +setup and memory. diff --git a/docs/development/dependency_audit_v1.6.0.md b/docs/development/dependency_audit_v1.6.0.md new file mode 100644 index 00000000..5df1efdd --- /dev/null +++ b/docs/development/dependency_audit_v1.6.0.md @@ -0,0 +1,238 @@ +# Dependency audit for v1.6.0 + +**Status**: draft, 2026-10-01. Part of v1.6.0 Phase 1 ([`roadmap_v1.6.0.md`](roadmap_v1.6.0.md)). +Scope follows the [pipeline review](pipeline_review_v1.6.0.md): where a recommendation depends on +a review decision still open, it says so. Each **Recommendation** is a proposal; the specs decide. + +Versions: *locked* is `uv.lock` on `1.6-dev` (2026-09-30); *latest* is PyPI on 2026-09-30. + +## 1. Summary + +**Update (spec 015, 2026-10-01)**: torch moved to **2.11**, not 2.14: pyannote.audio 4 imports +torchaudio, discontinued at 2.11 and compiled against torch 2.11. torch ≥ 2.12 waits for +pyannote.audio to drop torchaudio. pyannote.audio 4 also turned out to need the +`speaker-diarization-community-1` licence and to send telemetry by default (now off). + +The lockfile is about a year old. Most of it upgrades freely. Five things don't, and they're +connected: + +1. **torch is held at 2.6.0 by pyannote.audio 3.** torchaudio ≥ 2.9 removed `AudioMetaData`, + which pyannote.audio 3 reads audio through (`3ac2034`). Our own code doesn't use torchaudio. So + the torch upgrade *is* the pyannote.audio 4 migration (which reads audio through torchcodec and + FFmpeg's shared libraries instead). +2. **opencv-python is held below 5 by `face_analysis`**: its fallback detector and DeepFace's + default `opencv` detector use the Haar cascades that 5.0 removed. If the review replaces + `face_analysis`, the cap goes with it. +3. **TensorFlow** comes only from DeepFace (`face_analysis`). It's the one thing holding back + Python 3.14. +4. **transformers 5 / huggingface-hub 2**: the `<5` / `<1` caps were defensive (`d9c64e8`). They + only matter to `face_laion_clip` and `laion_voice`, which the review recommends dropping. + pyannote.audio 4 and the HF downloads in readiness still use huggingface-hub, so its upgrade + needs testing either way. +5. **`openface-test` stays `==0.1.13`** (`f4cdae7`): later releases pin Pillow 9.4, numpy 1.26 and + scipy 1.13, which must be built from source on Python ≥ 3.12. + +## 2. Python + +- **Now**: `>=3.12,<3.13` since v1.2.0 (`ca5c422`), with no recorded reason beyond what was + tested. 3.12 gets security fixes only, until October 2028. +- **3.13**: see §7 for the trial. With every other constraint unchanged, the lock resolves for 3.12 + and 3.13 together, adding only five backports of standard-library modules 3.13 removed + (`audioop-lts`, `standard-aifc`, `standard-chunk`, `standard-sunau`, plus `pyyaml-ft`), which the + audio stack needs. +- **3.14**: blocked by TensorFlow (no 3.14 release) through DeepFace, and `openai-whisper` and + `ultralytics` don't list 3.14 yet. Also `open-clip-torch` lists only up to 3.12 and + `tf-keras` up to 3.12, though both install on 3.13 (classifiers lag). +- **Also**: VTC 2 (review §3) requires Python ≥ 3.13. +- **Recommendation**: `requires-python = ">=3.12,<3.14"` in the first spec (keeps 3.12 users + working), CI on 3.12 and 3.13, the dev container and Docker images on 3.13. Add a non-blocking + 3.14 CI job once TensorFlow is gone or ships for 3.14. Move the floor to 3.13 at v1.7.0. + +## 3. Core dependencies + +**Done in spec 013** (2026-10-01): unused dependencies removed, numba moved to `audio`, core +upgraded; core-only install 73 → 40 packages, 746 → 135 MB. + +| Package | Constraint | Locked | Latest | Notes | Recommendation | +|---|---|---|---|---|---| +| fastapi | `>=0.115.0` | 0.116.1 | 0.142.2 | | Upgrade; raise floor | +| uvicorn[standard] | `>=0.30.0` | 0.35.0 | 0.54.0 | | Upgrade | +| python-multipart | `>=0.0.9` | 0.0.20 | 0.0.32 | Not imported, but FastAPI needs it for uploads | Keep, upgrade | +| pydantic | `>=2.0.0` | 2.11.7 | 2.13.5 | | Upgrade | +| sqlalchemy | `>=2.0.0` | 2.0.43 | 2.1.1 | 2.1 is a minor with deprecation removals; two `Job` models share one table (see CI fixes on `1.6-dev`), worth fixing first | Upgrade after the `Job` model cleanup | +| alembic | `>=1.12.0` | 1.16.4 | 1.20.0 | **Not imported anywhere.** Migrations are hand-written | Drop, or adopt for real migrations (spec decides) | +| numpy | `>=1.24.0` | 2.2.6 | 2.5.3 | Ceiling set by numba, by design | Upgrade; raise floor to 2.0 | +| pandas | `>=2.2.2` | 2.3.2 | 3.0.6 | **Major**: copy-on-write and a string dtype by default. Used in one module | Upgrade with a test of that module | +| numba | `>=0.60.0` | 0.61.2 | 0.68.0 | Used only by the audio pipeline (and librosa) | **Move to `audio`** | +| Pillow | `>=10.4.0` | 11.3.0 | 12.3.0 | | Upgrade | +| pycocotools | `>=2.0.10` | 2.0.10 | 2.0.11 | | Upgrade | +| webvtt-py | `>=0.5.1` | 0.5.1 | 0.5.1 | Last release 2024; classifiers stop at 3.12 | Keep; small, pure Python | +| praatio | `>=6.2.0` | 6.2.0 | 6.2.2 | | Upgrade | +| pyjwt | `>=2.10.1` | 2.10.1 | 2.15.1 | | Upgrade | +| cryptography | `>=45.0.6` | 45.0.6 | 50.0.2 | Not imported; pyjwt only needs it for RSA/EC keys, which we don't use | Drop unless an import turns up | +| requests, pyyaml, packaging, psutil, python-dotenv, typer | various | | | | Upgrade | +| moviepy | `>=1.0.3` | 2.2.1 | 2.2.1 | Only `visualization/video_utils.py` uses it, via `moviepy.editor`, **which moviepy 2 removed**, so that code is already broken; nothing imports the module | Drop with the dead module | +| matplotlib | `>=3.9.2` | 3.10.5 | 3.11.2 | Only listed in `version.py`'s version report | Drop | +| tqdm | `>=4.65.0` | 4.67.1 | 4.70.1 | Only in `version.py` | Drop (still arrives transitively where needed) | +| openpyxl, imageio, imageio-ffmpeg, av, rich, click, scikit-image | various | | | **Not imported anywhere in `src/`**; click comes with typer anyway | Drop, after checking `scripts/` and the docs' examples | + +Dropping the unused ones makes the core install smaller, which is v1.5.0's "slim by default" goal. + +## 4. Extras groups + +| Group | Pipelines | Key constraints | Recommendation | +|---|---|---|---| +| `face` | `face_analysis` | `deepface`, `tf-keras`, `opencv-python<5`, `imutils` (not imported: drop) | Depends on the review. If replaced: remove the group's TensorFlow stack and the opencv cap. If kept: keep `<5`, set the DeepFace detector default to `retinaface` | +| `face-laion` | `face_laion_clip` | `transformers<5`, `huggingface-hub<1`, torch | Remove if the pipeline is dropped | +| `face-openface3` | `face_openface3_embedding` | `openface-test==0.1.13`, scipy, opencv | Keep the pin. Show the non-commercial licence before install | +| `audio` | speech, diarization, `audio_processing` | torch/torchaudio `==2.6.0`, `pyannote.*=2.14,<2.15`) in every group, so combinations still +resolve to one torch and upgrades are deliberate. + +## 5. CUDA, drivers, containers + +- **Wheel index**: `pytorch-cu124` for Linux. torch 2.14 isn't built for cu124; its oldest CUDA + build is **cu126**, which matches the `nvidia/cuda:12.6.0-runtime-ubuntu24.04` Docker base. + Recommendation: move to `cu126` with the torch upgrade. That raises the minimum NVIDIA driver + from the CUDA 12.4 level to the 12.6 level (NVIDIA's release notes give the exact numbers; state + them in the install docs). +- **Docker**: `ubuntu:24.04` (CPU) and `nvidia/cuda:12.6.0-runtime-ubuntu24.04` (GPU, dev). Ubuntu + 24.04 ships Python 3.12, so 3.13 comes from uv (`uv python install`), not apt. Image size against + the v1.4.3 baseline is a separate roadmap item. +- **pyannote.audio 4 needs FFmpeg's shared libraries** at runtime (torchcodec). The images have the + `ffmpeg` CLI; check that the libraries torchcodec loads are there too. + +## 6. Tooling + +**Done in spec 013** (2026-10-01), except the viewer's packages and Bun (Phase 0). + +| Item | Now | Latest | Recommendation | +|---|---|---|---| +| pre-commit `mirrors-mypy` | v1.5.1 | v2.3.1 | **Upgrade, and match the project's mypy.** v1.5.1 is why pre-commit passed what CI's mypy failed (`6d6c347`) | +| `ruff-pre-commit` | v0.14.0 | v0.16.9 | Upgrade with the `ruff` dev dependency | +| `pre-commit-hooks` | v4.4.0 | v6.0.0 | Upgrade | +| `bandit` | 1.7.5 | 1.9.4 | Upgrade | +| `hadolint` | v2.12.0 | v2.15.1 | Upgrade | +| `shellcheck-py` | v0.9.0.5 | v0.11.0.1 | Upgrade | +| `commitizen` | v3.7.1 | v4.19.0 | Upgrade, or drop if unused | +| `pydocstyle` | 6.3.0 | 6.3.0 (2023, unmaintained) | Replace with ruff's `D` rules | +| `mirrors-prettier` | v3.0.3 | only v4 alphas | Drop, or use the viewer's own formatter for YAML/JSON/Markdown | +| `pre-commit-hooks-safety` | v1.3.2 | v1.4.2 | Upgrade, or replace with `uv`'s audit / pip-audit | +| `actions/checkout` | v4 | v7 | Upgrade: v4, `setup-python@v5` and `setup-uv@v5` run on Node 20, which CI now warns is deprecated | +| `actions/setup-python` | v5 | v7 | Upgrade (or drop: setup-uv can install Python) | +| `astral-sh/setup-uv` | v5 | v10 | Upgrade | +| `actions/upload-artifact`, `codecov-action`, docker actions | v4 / v4 / v3–v6 | v7 / v7 / v7 | Upgrade | +| `openjournals-draft-action` | `@master` | | Pin to a commit | +| JS: Bun | 1.x in CI, 1.4.2 in the dev container | | Keep Bun; drop `viewer/package-lock.json` (Phase 0 item) | +| Viewer packages | | | Separate audit of `viewer/package.json` (not done in this draft) | + +## 7. Python 3.13 trial + +Run 2026-10-01 in a separate worktree (branch `py313-trial`), CPython 3.13.15, Linux x86_64 with +CUDA. + +**A. Python 3.13, every other constraint unchanged** (`requires-python = ">=3.12,<3.14"`): +- `uv lock` resolves 311 packages for 3.12 and 3.13 together; the only additions are the five + standard-library backports in §2. +- `uv sync --all-extras` installs everything on 3.13, including the source builds + (`openai-whisper`) and `openface-test==0.1.13`: 641 packages, about 20 minutes, mostly CUDA wheel + downloads. +- All ten pipeline classes import. +- Test suite (`-m "not real_models"`, as CI runs it): **1335 passed, 33 skipped, 0 failed** in + 2 min 20 s. CI on 3.12 for the same commit: 1332 passed (fewer extras installed there). +- Real pipelines on the viewer's demo video (`2UWdXP…Peekaboo_h265.mp4`, run through the server's + `JobProcessor`): `speech_recognition`, `scene_detection`, `face_analysis`, + `face_openface3_embedding`, `person_tracking` and `speaker_diarization` all complete on 3.13. + `speaker_diarization`'s four segments are identical on 3.12 and 3.13 (start, end and speaker, + to every digit). + - `person_tracking` first failed with `No module named 'lap'`. ByteTrack imports `lap`; + ultralytics doesn't declare it and pip-installs it at runtime instead, which is how the 3.12 + dev environment got it. A fresh or offline install would fail. **Fixed on `1.6-dev`**: `lap` + is now in the `person` extra. Not a 3.13 issue. + - `face_analysis` finds 0 faces in this video on both 3.12 and 3.13 (same result, so not a + regression; noted in the pipeline review). + +So the Python upgrade doesn't depend on any library upgrade and can be its own small spec. + +**B. Python 3.13 with every upper bound and exact pin lifted, cu126 index** (resolution only): +resolves 318 packages to torch 2.14.1+cu126, torchaudio 2.11.0, torchvision 0.29.1, pyannote.audio +4.0.7, transformers 5.18.0, opencv-python 5.0.0.93, TensorFlow 2.21.0, numpy 2.5.3, pandas 3.0.6, +librosa 1.0.0, ultralytics 8.4.170, with `openface-test` still at 0.1.13. Two packages stay below +their latest, both for reasons outside our constraints: +- Pillow 11.3 (latest 12.3): `moviepy` 2.2.1 requires `pillow<12`. Dropping moviepy (§3) lifts it. +- huggingface-hub 1.33 (latest 2.0): `tokenizers` (via transformers) requires `<2.0`. Upstream; + 1.x is current. + +Resolution says nothing about behaviour: B's majors (pyannote.audio 4, opencv 5, pandas 3, +librosa 1.0, transformers 5) need the specs and their tests. + +## 8. Diarization and the torch cap (2026-10-02) + +Not an immediate priority; recorded so Phase 5 starts from it. + +**Why torch is held at 2.11.** pyannote.audio 4 imports torchaudio (audio I/O fallbacks and the +speaker-embedding code). torchaudio is discontinued: its last release, 2.11, is compiled against +torch 2.11 and fails to load on anything newer (`OSError: Could not load this library: +.../_torchaudio.abi3.so`). It declares no torch requirement, so a resolver pairs it with torch 2.14 +and the failure only shows at runtime. Every extras group pins the same torch so combinations such +as `[all]` resolve, so the whole stack waits. Our own code never imports torchaudio. Cost today: +three minor torch releases, nothing broken (outputs unchanged, see CHANGELOG). The cost grows when +a future model or library needs torch ≥ 2.12; whether torch 2.11 has builds for the newest GPUs +(RTX 50-series) is unchecked. + +**Upstream state.** pyannote.audio 4.0.7 (2026-06-30) still requires `torchaudio>=2.8`. No open +work to drop it: a PR relaxing the torch/torchaudio pins (#1977) was closed unmerged in February, +and there have been no commits since 2026-06-30. Open issues worth knowing: #2019 "Remote Code +Execution via Attacker-Controlled config.yaml Class Loading" (low risk for us: we load the +official models) and #1963 (4.x uses ~6× the VRAM of 3.3). The company's effort appears to be on +its hosted models. + +**Alternatives checked.** + +| Option | Removes torchaudio? | Notes | +|---|---|---| +| **VTC 2** (LAAC-LSCP): adult female, adult male, key child, other child | **No**: depends on `pyannote-audio>=3.4` | The tool the field wants (pipeline review: top Phase 5 priority). No LICENSE file in the repository. Installed by git clone + git-lfs, not from PyPI; Python ≥ 3.13; no Windows support | +| **NVIDIA Streaming Sortformer** (`diar_streaming_sortformer_4spk-v2.1`) via NeMo | **Yes** | NeMo 3.0.0 (2026-08, Apache-2.0); model not gated (no HF-token hurdle), NVIDIA Open Model License, updated 2026-09. But NeMo's `asr` extra brings 23 packages, among them wandb, hydra and `nv_one_logger_*` telemetry packages to vet against Local-First. It caps `lightning<=2.4.0` while pyannote needs `>=2.4`: both together only at exactly 2.4.0. Up to 4 speakers | +| **DiariZen** (BUT) | No | Vendors a pyannote fork pinned to torch 2.1.1, numpy 1.26.4, Python 3.10. Weights CC BY-NC. A step back | + +**Decision (2026-10-02): decide now, swap later.** +1. Keep pyannote through v1.6.0 Phase 1: nothing is broken, and no default changes without a + benchmark score. +2. Phase 5's diarization benchmark scores pyannote community-1, Sortformer and VTC 2, with + dependency health as a criterion alongside accuracy. +3. Before Phase 5 relies on it, a half-day spike: Sortformer in a scratch environment on torch + 2.14 without torchaudio; what the telemetry does and whether it can be switched off; install + size; output on the demo clip. +4. Settle VTC 2's licence with LAAC-LSCP early (maintainer to email). If it stays unlicensed it + can't ship, and the voice-type plan changes. + +Sources: https://github.com/pyannote/pyannote-audio/releases , +https://github.com/LAAC-LSCP/VTC , https://huggingface.co/coml/VTC-2.0 , +https://huggingface.co/nvidia/diar_streaming_sortformer_4spk-v2.1 , +https://github.com/BUTSpeechFIT/DiariZen , +https://neosophie.com/en/blog/20260223-diarization (2026 comparison: Sortformer v2-streaming and +DiariZen among the best open models). + +## Specs this suggests + +In order, each one small enough to review. The pipeline review's recommendations were all accepted +(2026-10-01). + +1. **Python 3.13**: `requires-python <3.14`, CI on 3.12 and 3.13, the dev container and Docker + images on 3.13. Nothing else: §7 shows it needs no library change. +2. **Core clean-up and tooling**: drop unused core dependencies, move numba to `audio`, upgrade the + core, pre-commit hooks and GitHub Actions. +3. **torch 2.14 + pyannote.audio 4 + cu126**: one change, since each forces the others. +4. **Drop pipelines**: `laion_voice`, `face_laion_clip`, `audio_processing`, with their extras. +5. **Face stack**: replace `face_analysis`; with it go DeepFace, TensorFlow and tf-keras, and the + opencv `<5` cap. This is the path to Python 3.14, so do it before rc1 if Phase 5's benchmark + allows, and add the 3.14 CI job with it. +6. **Viewer dependencies** (separate audit). diff --git a/docs/development/emotion_output_format.md b/docs/development/emotion_output_format.md index 2e85989b..5175c12b 100644 --- a/docs/development/emotion_output_format.md +++ b/docs/development/emotion_output_format.md @@ -22,7 +22,7 @@ Provide a lightweight, consistent JSON structure for emotion-related outputs acr ``` { "schema_version": 1, - "source_pipeline": "face_laion_clip", + "source_pipeline": "face_analysis", "media": { "type": "video", "path": "relative/or/logical/reference.mp4", @@ -42,7 +42,7 @@ Provide a lightweight, consistent JSON structure for emotion-related outputs acr } }, "metadata": { - "pipeline_variant": "laion-clip-face", + "pipeline_variant": "deepface", "generation_time": "2025-09-17T12:34:56Z" } } diff --git a/docs/development/handover_docker_images.md b/docs/development/handover_docker_images.md new file mode 100644 index 00000000..e14cb3ed --- /dev/null +++ b/docs/development/handover_docker_images.md @@ -0,0 +1,199 @@ +# Handover: build, check and measure the Docker image + +**For**: an agent with Docker on the host machine (outside the dev container). +**From**: the agent that rewrote the Docker setup on `1.6-dev`, 2026-10-02. It had no Docker. +**Status**: run once, 2026-10-02, at `ec5ab04`; results at the end. The commands below are corrected +from that run (`docker run -i`, `?include_unavailable=true`); rerun them after Docker changes. +**Ask Caspar before**: pushing anything, deleting images or volumes you didn't create, or changing +host settings (`.wslconfig`, Docker Desktop). + +## Why + +Three Dockerfiles (`Dockerfile.cpu`, `Dockerfile.gpu`, `Dockerfile.dev`) were replaced by one +`Dockerfile` on `ubuntu:24.04`. The changes: + +- No `nvidia/cuda` base. It was a 2024-08 snapshot never rebuilt. torch's wheels bring their own + CUDA 12.6 runtime and cuDNN, and `--gpus all` brings the driver. +- The CPU image used to replace torch with 2.6.0 CPU wheels after installing the extras. That was a + leftover from before the torch 2.11 upgrade, and it broke pyannote.audio 4. That step is gone. +- The GPU image used to skip installing VideoAnnotator itself (`--no-install-project`). It now + installs it. +- The dev tools (`dev` group) and uv's download cache used to ship in the images. Both are now left + out. +- The image starts with `videoannotator server --host 0.0.0.0 --port 18011`. +- One image serves CPU and GPU machines. A CPU-wheel variant would be about 3.6 GB smaller, but it + is blocked: TensorFlow (DeepFace) followed by open_clip segfaults on CPU torch wheels. + +The roadmap item this closes is v1.6.0 Phase 1, "Docker image size". v1.5.0 set a target of an 80% +smaller default image than v1.4.3 (about 30 GB) but never measured it. + +## Before you start + +1. **Machine**: Docker Desktop with the WSL 2 backend (Windows), or Docker Engine plus the NVIDIA + Container Toolkit (Linux). The NVIDIA driver must be 560 or newer (`nvidia-smi`). +2. **Where**: a terminal in a WSL distro or on Linux. Clone onto the Linux filesystem (`~/...`), not + under `/mnt/c`: the build context is slow over the Windows file bridge. +3. **Disk**: at least 60 GB free for Docker (`docker system df`). Add 30 GB if you do step 6. +4. **Memory**: this host froze once from memory exhaustion (see + `handover_windows_devcontainer_freeze.md`). Image builds aren't covered by the dev container's + 12 GB cap: installing the extras alone filled 10 GB with page cache. Ask Caspar to close heavy + apps (browsers, other containers). Run one build or container at a time. Stop the dev container + if it is running. +5. **Hugging Face token**: speaker diarization downloads a gated pyannote model. You need a token + whose account has accepted the model's licence. Ask Caspar for it and set + `export HUGGINGFACE_TOKEN=...`. Never write it into a file or a log you report. + +## Steps + +```bash +git clone -b 1.6-dev https://github.com/InfantLab/VideoAnnotator ~/va-images +cd ~/va-images && git log --oneline -1 # report this commit +``` + +### 1. Build + +Record the wall time of each build. + +```bash +time docker build -t va-check:slim . +time docker build --build-arg EXTRAS=all -t va-check:all . +time docker build --build-arg SKIP_IMAGE_UV_SYNC=true -t va-check:devcontainer . +docker images va-check --format '{{.Tag}}\t{{.Size}}' +``` + +If a build fails, stop and report the last 40 lines of its output. + +### 2. Slim image: the server starts and reports what's missing + +```bash +docker run -d --name va-slim -p 18011:18011 -e AUTH_REQUIRED=false va-check:slim +sleep 20; curl -s localhost:18011/health; echo +curl -s "localhost:18011/api/v1/pipelines/?include_unavailable=true" | python3 -c "import json,sys; [print(p['name'], p.get('available'), p.get('install_hint','')) for p in json.load(sys.stdin)['pipelines']]" +docker logs va-slim 2>&1 | grep -E "Logs:|Database:|ERROR" | head +docker rm -f va-slim +``` + +Expected: +- `/health` says healthy. +- Every pipeline shows `False`, with an install hint. +- The log shows `Logs: /app/logs` and `Database: /app/database/videoannotator.db`. + +### 3. Full image on the GPU: libraries and imports + +```bash +docker run -i --rm --gpus all va-check:all python - <<'EOF' +import torch +print("torch", torch.__version__, "cuda", torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else "") +import tensorflow as tf +print("tensorflow GPUs:", tf.config.list_physical_devices("GPU")) +with tf.device("/GPU:0"): + tf.nn.conv2d(tf.ones((1, 8, 8, 3)), tf.ones((3, 3, 3, 4)), 1, "SAME") +libs = sorted({l.split()[-1] for l in open("/proc/self/maps") if ".so" in l and any(k in l for k in ("cuda", "cudnn", "cublas", "cusolver", "cusparse", "nvJitLink"))}) +print("\n".join(libs)) +import open_clip, pyannote.audio, whisper, ultralytics +print("imports ok (tensorflow then open_clip in one process)") +EOF +``` + +Expected: +- torch reports `cuda True`. +- TensorFlow lists one GPU and the conv2d runs. +- Every CUDA library path is under `/app/.venv/.../nvidia/`. No library path comes from + `/usr/local/cuda`, because that folder no longer exists. +- The script reaches "imports ok". + +If TensorFlow finds no GPU, report it. DeepFace would then silently run on the CPU. That isn't fatal, +but it decides whether we need a fallback. + +### 4. Full image: a real job with every local pipeline + +```bash +docker volume create va-check-models +docker run -d --name va-all --gpus all -p 18011:18011 -e AUTH_REQUIRED=false \ + -e HUGGINGFACE_TOKEN -v va-check-models:/app/models va-check:all +sleep 30 +VIDEO=viewer/demo-assets/2UWdXP.joke1.rep3.take1.Peekaboo_h265.mp4 +JOB=$(curl -s -X POST localhost:18011/api/v1/jobs/ -F "video=@$VIDEO" \ + -F "selected_pipelines=face_analysis,face_openface3_embedding,person_tracking,scene_detection,speaker_diarization,speech_recognition" \ + | python3 -c "import json,sys; print(json.load(sys.stdin)['id'])") +while :; do S=$(curl -s localhost:18011/api/v1/jobs/$JOB | python3 -c "import json,sys; print(json.load(sys.stdin)['status'])"); echo "$S"; case $S in completed|failed|cancelled) break;; esac; sleep 15; done +curl -s localhost:18011/api/v1/jobs/$JOB/results | python3 -c "import json,sys; [print(k, v['status'], v.get('annotation_count'), v.get('error_message')) for k, v in json.load(sys.stdin)['pipeline_results'].items()]" +docker stats --no-stream va-all +docker exec va-all ls /app/database /app/logs +docker rm -f va-all +``` + +The first run downloads the weights (a few GB), so allow 10–20 minutes. Expected annotation +counts, from the same video in the dev container on 2026-10-02: + +| Pipeline | Count | +|---|---| +| face_analysis | 0 (DeepFace finds no faces in this clip; not an error) | +| face_openface3_embedding | 1 | +| person_tracking | 22 | +| scene_detection | 1 | +| speaker_diarization | 4 | +| speech_recognition | 1 | + +Every pipeline should say `completed`. Report the `docker stats` memory figure. The dev-container +peak for this job was about 6.5 GB, including VS Code. + +### 5. Full image without a GPU + +Repeat step 4 without `--gpus all`, keeping the same models volume so nothing downloads again. +Everything should still complete, more slowly. Report the wall time. + +### 6. Optional: the v1.4.3 baseline + +This measures the "~30 GB" image that v1.5.0's target was set against. It's long, and needs about +30 GB of disk. + +```bash +git -C ~/va-images worktree add ~/va-v143 v1.4.3 && cd ~/va-v143 +time docker build -f Dockerfile.gpu --build-arg SKIP_IMAGE_UV_SYNC=false --build-arg SKIP_TORCH_INSTALL=false -t va-check:v143 . +docker images va-check:v143 --format '{{.Size}}' +cd ~/va-images && git worktree remove --force ~/va-v143 +``` + +### 7. Compose file + +```bash +docker compose config -q && echo "compose ok" +``` + +### 8. Clean up + +```bash +docker rmi va-check:slim va-check:all va-check:devcontainer va-check:v143 2>/dev/null +docker volume rm va-check-models +docker builder prune -f # the build cache, several GB +``` + +## Results, 2026-10-02 (`ec5ab04`) + +Run on `proart`: Windows 11, Docker Desktop 29.8.1 / WSL 2, RTX 4060 Laptop 8 GB, driver 616.92, +16 GB for the Docker VM. "Disk" is Docker Desktop's containerd store (unpacked layers plus compressed +blobs); "compressed" is what a registry push or pull moves. + +| Image | Disk | Compressed | Build (wall) | +|---|---|---|---| +| `slim` | 1.35 GB | 347 MB | 79 s (cold) | +| `all` | 14.9 GB (≈10.1 GB unpacked; `.venv` 8.7 GB: nvidia 3.6, tensorflow 1.8, torch 1.6, triton 0.6) | 4.78 GB | 741 s | +| `devcontainer` | 1.13 GB | 296 MB | 41 s | +| v1.4.3 (`Dockerfile.gpu`) | 26.1 GB | 8.89 GB | 1188 s | + +Against v1.4.3: `slim` −95%, `all` −43% on disk (−46% compressed). v1.5.0's "80% smaller default +image" is met by the slim default; the everything image is not 80% smaller, and can't be while it +carries CUDA, TensorFlow and torch. + +| Check | Result | +|---|---| +| Slim server start | **Failed**: `No module named 'httpx'` (core dependency dropped by spec 013). Fixed: `httpx` back in core; a CI job now installs core only and starts the server | +| Slim pipelines | All 7 unavailable with install hints (listed only with `?include_unavailable=true`); `Logs: /app/logs`, `Database: /app/database/videoannotator.db` | +| First start on a fresh database | **Failed**: the first API key was generated before migrations (`no such table: users`), so a fresh install started with authentication on and no key. Fixed: security setup runs after migrations; test added | +| GPU libraries | torch 2.11.0+cu126 CUDA; TensorFlow sees the GPU, conv2d runs; every CUDA library from `/app/.venv/.../nvidia/` plus the driver's `libcuda` from `/usr/lib/wsl`; no `/usr/local/cuda`. The plain `ubuntu:24.04` base works | +| Job, GPU | All 6 completed, counts 0/1/22/1/4/1 as expected. 391 s cold (2.5 GB weight download), ~78 s warm; peak 5.7 GB RAM, 2.4 GB GPU | +| Job, CPU only | All 6 completed, same counts, 68 s, peak 5.0 GB (on a 14 s clip, model loading dominates) | +| Compose | Valid; obsolete `version:` key removed | +| Build context | No `.dockerignore` (sent `.git`, 190 MB). Fixed with an allowlist | +| Noise | TensorFlow logs `cuInit ... (303)` on CPU-only runs: harmless | diff --git a/docs/development/handover_first_time_user_run.md b/docs/development/handover_first_time_user_run.md new file mode 100644 index 00000000..bf664f9c --- /dev/null +++ b/docs/development/handover_first_time_user_run.md @@ -0,0 +1,74 @@ +# First-time user run, 2026-10-05 + +Phase 2, item E: follow the README as a new researcher would, on the one path +install → add videos → run → review → export, and note every point of friction. + +## How it was run + +- A fresh clone of `1.6-dev` at `6a86ba1` (the README rewrite), in a scratch folder. +- A fresh `HOME`, so the database, API key, model weights and logs all started empty. None of + the dev container's VideoAnnotator variables were set. uv's package cache was shared, which + only made `uv sync` faster. +- Exactly the README's Linux steps: `uv sync`, then `uv run videoannotator server`. The port + was `--port 18111`, because 18011 was taken by another server. +- The viewer was driven with Playwright (headless Chromium), from the link the server printed. +- The video was the 10-second demo clip as VP9 WebM, since headless Chromium can't play H.264. +- The pipeline was Scene Detection, installed from the viewer as the README says. + +## What happened, in order + +| Step | Result | +|---|---| +| `uv sync` | 4 s with a warm cache; 246 MB `.venv` | +| Server start | Database, first API key and one-click link printed. **The link named port 18011, not 18111.** Fixed | +| Open the link | Logs in and lands on Home: "0 of 7 pipelines ready" and the setup checklist. Good | +| New job → upload | Works. The native file input sits on its own line under "or pick files:" | +| Choose pipelines | **Every pipeline said "requires an administrator API key"** although the server's own first key asks for admin. Fixed | +| Install Scene Detection | **Failed in 1 s on Linux** (uv: no solution; torch 2.11 cu126 vs PyPI's cu13). Fixed. Then completed and activated without a restart; it took 25 min here, almost all downloading about 3 GB from the PyTorch index | +| Submit | Lands on the run (batch) page, which says the first run loads models and may download weights. Good | +| While the job ran | 344 polls of `GET /api/v1/jobs/{id}` over about 3 min: median 25 ms, slowest 349 ms. No stall | +| Job page | Clear: Run it again, Open in Viewer, Download Results | +| Review | "Choose a folder for your results" first, with "View without saving". Then the video plays with "Scene: clinic" over it | +| Export (zip) | **Held only the JSON: the WebM video was left out**, and the page promised a job log that is never written. Fixed | + +## Fixed in this pass (commit `084ba61` and the one after it) + +- The first-start key's user is now an admin (the key always asked for the `admin` scope). +- Extras installs add the project's CUDA index when the lock needs it (Linux torch). +- `server --port` reaches the first-start viewer link. +- Results zip: every accepted video format; no promise of a job log. +- The pipeline list no longer shows the variant as a version ("vpyscenedetect-clip"). +- No `pyannote.core` warning at start without the audio extra. +- The "lost this key" hint also gives the `uv run` form for a clone. + +## Not fixed: for a decision or a later pass + +1. *(Done 2026-10-05: per-user default, old jobs still open, Docker volume.)* + **Job files live in the folder the server was started from** (`./storage/jobs/...`, inside + the clone here). The database, models and logs live in the per-user data folder (spec 016). + Start the server from another folder and earlier jobs lose their videos and results. Suggest: + the same per-user default as the database, with `STORAGE_ROOT` still overriding it. That needs + a migration note for existing `./storage` folders. +2. **A new database logs four "[MIGRATION] Adding jobs... column" lines.** The users and + datasets layer (`database/`) creates `jobs` from its own model first, then the job storage + layer adds its columns. It's harmless, but it's the two-database-layers problem showing. +3. **Installing a pipeline can take a long time** (25 min for `scene` here, download-bound). + The viewer shows the size but no time hint, and nothing tells the user it's fine to leave the + page. +4. **Wizard wording and layout**: the subtitle "Process videos through the VideoAnnotator + pipeline (supports batch processing)"; the file input on its own line. +5. **Review asks for a results folder before showing anything.** "View without saving" is the + secondary button. A first-time user probably wants to see the result first and decide about + keeping it later. +6. **The viewer lists pipelines the job didn't run** (Person, Face, Emotion, Speech, Speaker, each + "(No data)") above the one it did. +7. **Scene label quality**: a home kitchen was labelled "clinic". This is the CLIP label set, a + Phase 5 benchmark question. +8. **The job page's Configuration shows `{}`** when a pipeline ran with its defaults; it could + show the defaults it used (provenance has them). +9. **The structural pass** ("anything off the path moves or goes") needs Caspar: candidates are + the Settings → Getting Help tab, the Getting Started page (overlaps Home's checklist), and + the jobs/batches double naming ("runs", "batch", "jobs"). +10. **Not covered by this run**: Windows and macOS installs, the GPU path, a long video (the + API-responsiveness item wants one), speaker diarization's Hugging Face token step, the VLM + pipeline with Ollama, and the datasets path. diff --git a/docs/development/handover_phase2.md b/docs/development/handover_phase2.md new file mode 100644 index 00000000..362e6900 --- /dev/null +++ b/docs/development/handover_phase2.md @@ -0,0 +1,152 @@ +# Handover: v1.6.0 Phase 2, "Clean First Contact" + +**For**: the agent taking over v1.6.0 Phase 2, starting cold. +**From**: the agent that finished Phase 1, 2026-10-02. +**Branch**: `1.6-dev`. The state at handover is the commit that adds this file. +**Maintainer**: Caspar Addyman. + +## Read first + +1. `docs/development/roadmap_v1.6.0.md`: Phase 2 is the plan, and its items are your task list. Also + read the Release Overview (why v1.6.0 exists: the JOSS scope check needs outside users) and the + Core Principles. +2. `CLAUDE.md` (commands, code style) and `AGENTS.md` (§23 for the viewer). +3. `.specify/memory/constitution.md` (v1.1.0). It is binding: Local-First, Stable Pipeline Contract, + Provenance, Modular, Backward Compatibility, and the viewer's Engineering Standards. +4. `CHANGELOG.md`, `[Unreleased]`: everything v1.6.0 has changed so far. + +## Where things stand + +- **Phase 0** (one repository, the viewer merged in): done. +- **Phase 1** (pipelines and dependencies): done. + - Python 3.13 support. + - torch 2.11 and pyannote.audio 4. + - The LAION pipelines dropped. + - One per-user folder each for models, logs and the database, used by both database layers. + - The output baseline re-captured against v1.5.0, checked by + `tests/integration/test_output_baseline.py` with real models. + - One `Dockerfile` for CPU and GPU, measured: slim 1.35 GB, every pipeline 14.9 GB, v1.4.3 26.1 GB. + - The Windows dev-container freeze diagnosed (memory exhaustion) and capped (`--memory=12g`). + + Still in Phase 1: "Download weights ahead of time". It's a feature, not housekeeping; move it to + Phase 2 or 3 with Caspar. +- **Phase 2**: not started. +- **Decided, not for now**: diarization alternatives and the torch 2.11 cap + (`dependency_audit_v1.6.0.md` §8). Phase 5 benchmarks Sortformer against pyannote. Caspar is to + ask LAAC-LSCP about VTC 2's licence. + +## Suggested order + +The README and docs describe behaviour, so fix the behaviour first. + +**A. Correctness (small, testable; one commit each)** +1. **A pipeline that fails must say so.** `speech_recognition` swallows transcription errors and + reports "completed" with no transcript. Make it raise, then audit the other pipelines for the + same pattern. +2. **`GET /api/v1/jobs/{id}/results/files/{pipeline}`** returns `OUTPUT_FILE_MISSING` for every + result stored in the database, and it's what every job advertises as `download_url`. The files + are in the job's storage folder (`_.json/.rttm/.vtt`). Serve them from + there, or stop advertising the URL. +3. **`videoannotator process