From 98d5257652af565c6d3ee08b0cae6e5d249765e4 Mon Sep 17 00:00:00 2001 From: Shawn Hwang Date: Tue, 10 Nov 2026 14:34:03 -0800 Subject: [PATCH 1/3] egocap: add station settings, task catalogs and camera roles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add protected deployment inspection, persistent operator setup and browser-uploaded task catalogs directly on main. Suspend recording keyboard and voice commands while Settings is open, serialize catalog/settings mutations, and keep configuration hidden from recording downloads. Use main’s available capture cameras for role assignments; LiveKit integration remains with its separate unmerged work. Signed-off-by: Shawn Hwang --- examples/egocap/AGENTS.md | 14 + examples/egocap/backend/README.md | 4 + examples/egocap/backend/egocap_web/app.py | 49 +- .../backend/egocap_web/operator_settings.py | 326 ++++++++ .../backend/egocap_web/preview_routes.py | 12 +- .../backend/egocap_web/recording_command.py | 4 +- .../egocap/backend/egocap_web/recordings.py | 4 +- .../backend/egocap_web/settings_routes.py | 117 +++ .../backend/egocap_web/task_file_routes.py | 141 +++- .../egocap/backend/egocap_web/task_files.py | 70 +- examples/egocap/data/taskfiles/README.md | 8 +- examples/egocap/frontend/src/App.test.tsx | 33 + examples/egocap/frontend/src/App.tsx | 46 +- .../egocap/frontend/src/CameraGrid.test.tsx | 31 + examples/egocap/frontend/src/CameraGrid.tsx | 14 +- .../src/RecordingPanel.taskCatalog.test.tsx | 113 +++ .../frontend/src/RecordingPanel.test.tsx | 18 + .../egocap/frontend/src/RecordingPanel.tsx | 62 +- .../src/RecordingPanel.voice.test.tsx | 31 + .../frontend/src/SettingsPanel.test.tsx | 307 +++++++ .../egocap/frontend/src/SettingsPanel.tsx | 754 ++++++++++++++++++ .../egocap/frontend/src/settingsApi.test.ts | 101 +++ examples/egocap/frontend/src/settingsApi.ts | 252 ++++++ .../frontend/src/settingsTestDoubles.ts | 88 ++ examples/egocap/frontend/src/styles.css | 409 ++++++++++ examples/egocap/frontend/src/taskCatalog.ts | 58 ++ .../frontend/src/useOperatorSettings.ts | 36 + .../tests/backend/test_operator_settings.py | 348 ++++++++ .../tests/backend/test_preview_routes.py | 6 +- .../egocap/tests/backend/test_recordings.py | 28 +- .../tests/backend/test_settings_routes.py | 173 ++++ .../tests/backend/test_task_file_import.py | 195 +++++ .../tests/backend/test_task_file_routes.py | 27 +- .../egocap/tests/backend/test_task_files.py | 12 +- 34 files changed, 3823 insertions(+), 68 deletions(-) create mode 100644 examples/egocap/backend/egocap_web/operator_settings.py create mode 100644 examples/egocap/backend/egocap_web/settings_routes.py create mode 100644 examples/egocap/frontend/src/RecordingPanel.taskCatalog.test.tsx create mode 100644 examples/egocap/frontend/src/SettingsPanel.test.tsx create mode 100644 examples/egocap/frontend/src/SettingsPanel.tsx create mode 100644 examples/egocap/frontend/src/settingsApi.test.ts create mode 100644 examples/egocap/frontend/src/settingsApi.ts create mode 100644 examples/egocap/frontend/src/settingsTestDoubles.ts create mode 100644 examples/egocap/frontend/src/taskCatalog.ts create mode 100644 examples/egocap/frontend/src/useOperatorSettings.ts create mode 100644 examples/egocap/tests/backend/test_operator_settings.py create mode 100644 examples/egocap/tests/backend/test_settings_routes.py create mode 100644 examples/egocap/tests/backend/test_task_file_import.py diff --git a/examples/egocap/AGENTS.md b/examples/egocap/AGENTS.md index bdcbb26d8f..6c6ee48e28 100644 --- a/examples/egocap/AGENTS.md +++ b/examples/egocap/AGENTS.md @@ -54,3 +54,17 @@ with its history can still work on it. Inspect changed paths against the intended MR base and audit path/install/dependency references. If an external configuration or CI edit is actually required, identify its exact path and purpose and obtain explicit user authorization before editing it. Do not silently broaden scope or duplicate files outside the allowed tree. Run all three test suites per "Run the tests yourself" above — check each package's own coverage target, not one combined number (see `pyproject.toml`'s `[tool.coverage.run]` comment for why). + +## Settings and catalog verification + +- When replaying settings changes, preserve the base branch's task-details field in fixtures and submit calls. +- Inject non-UTF-8 directory entries in catalog tests: macOS filesystems reject their creation. +- Keep generated backend test recordings out of the index before running the full pre-commit hook set. +- Modal ownership must suspend recording keyboard and voice commands regardless of the focused element. + +## Human testing before merge + +- Do not merge a PR or enable auto-merge until the user has tested the changes personally and explicitly approved the merge. Passing automated checks does not satisfy this requirement. +- Prepare and update the PR for testing, report the exact tested commit and remaining validation limits, then wait for the user's test results. + +- Settings on main must not import modules or describe active services that exist only in unmerged dependency branches. diff --git a/examples/egocap/backend/README.md b/examples/egocap/backend/README.md index 3d00987fec..ee27613890 100644 --- a/examples/egocap/backend/README.md +++ b/examples/egocap/backend/README.md @@ -29,6 +29,10 @@ or restarts it. | `WS /api/session` | The main live connection: capture-status snapshots, sampled binary tracking frames, and recording commands/replies/snapshots. One socket per browser tab; see [`frontend/README.md`](../frontend/README.md) for the full message shapes. | | `GET /stream{id}` | MJPEG camera preview, bound to the backend/session identity advertised in the `/api/session` snapshot. | | `GET /api/capture-status` | Thin HTTP inspection endpoint mirroring the latest snapshot; the frontend does not poll this during normal operation. | +| `GET /api/settings` | The backend's effective configuration (CaptureServer URL, sources, station file, recording root, 3D view, origin policy), read-only and gated by the `X-Control-Token` header; the page's Advanced settings tab shows it. Secrets are reported as present or absent, never by value. | +| `GET`/`PUT /api/operator-settings` | The operator's station setup: the camera type (`sensing_gmsl` or `oak_d`), camera roles (`ego`, `wrist_left`, `wrist_right`), the task catalog the recording panel's picker offers (`task_file`) and the task it preselects (`preselected_task`). Kept in `.egocap/operator_settings.json` under the recording output (hidden from the recordings browser) (`EGOCAP_OPERATOR_SETTINGS_FILE` overrides the path) so every browser sees the same values. `PUT` names only the fields it changes and carries `control_token` like a recording command; a wrong token is 403. | +| `POST /api/task-files/import`, `DELETE /api/task-files/{name}` | Import a task catalog the operator supplies — by `name` + `content` (text uploaded from the browser) — into the imported-catalog directory beside the operator settings, once it parses; and remove an imported one. `POST` carries `control_token` in JSON; `DELETE` uses the `X-Control-Token` header and refuses removal of the selected catalog (409); imports also refuse replacement of that catalog until it is deselected. Imports appear in `GET /api/task-files` (as `imported`) and shadow a shipped catalog of the same name. | +| `GET /api/tools` | The operator tools the Settings drawer offers (headset calibration, glove calibration, fisheye rectification for the SENSING cameras), each with whether this build can launch it. None can yet. | | `GET /api/task-files`, `GET /api/task-files/{name}` | Read-only task-catalog API; see [`../data/taskfiles/README.md`](../data/taskfiles/README.md). | | `GET /recordings/...` | The optional recordings browser's API and file routes, off unless `EGOCAP_RECORDINGS_ENABLED=1`; see [`../recordings_frontend/README.md`](../recordings_frontend/README.md). | diff --git a/examples/egocap/backend/egocap_web/app.py b/examples/egocap/backend/egocap_web/app.py index 609e95fa3b..a6b1f79c0c 100644 --- a/examples/egocap/backend/egocap_web/app.py +++ b/examples/egocap/backend/egocap_web/app.py @@ -35,6 +35,8 @@ from .capture_preview import MAX_CAMERA_JPEG_BYTES from .capture_status import CaptureStatusMonitor from .config import VisualizationConfig, config_from_env, visualization_from_env +from .operator_settings import OperatorSettingsStore, operator_settings_path +from .operator_settings import router as operator_settings_router from .preview_routes import router as preview_router from .recording_adapter import ( MANUS_GLOVE_SPEC, @@ -45,6 +47,8 @@ from .recording_command import RecordingContext from .recording_session import RecordingSessionCoordinator, TaskContext from .recordings import router as recordings_router +from .settings_routes import EffectiveSettings +from .settings_routes import router as settings_router from .station_config import ResolvedStation, StationConfigError, load_station_config, station_config_path_from_env from .task_file_routes import router as task_file_router from .viser_scene import VisualizationScene @@ -319,6 +323,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # UI must never end up pointed at two different roots mid-process. recording_output_root = _recording_output_root(resolved_station.data_dir if resolved_station else None) app.state.recording_output_root = recording_output_root + # Station roles and task choices stay under a hidden directory excluded from recording downloads. + operator_settings_file = operator_settings_path(recording_output_root) + app.state.operator_settings_store = OperatorSettingsStore(operator_settings_file) + # Catalogs the operator imports from the page sit beside that file, apart from the shipped ones. + app.state.imported_task_file_root = operator_settings_file.parent / "taskfiles" monitor = CaptureStatusMonitor(config, backend_instance_id=str(uuid.uuid4())) coordinator = RecordingSessionCoordinator( _recording_config_factory(monitor, config.station_label, recording_output_root), @@ -342,9 +351,26 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app.state.recording_context = RecordingContext( coordinator=coordinator, control_token=control_token, auto_fill_operator_name=auto_fill_operator_name ) + # The 3D view settings are read once, for the view itself and for the settings report. + visualization_config: VisualizationConfig | None = None + visualization_error: str | None = None + try: + visualization_config = visualization_from_env() + except Exception as exc: + visualization_error = str(exc) + app.state.settings = EffectiveSettings( + capture=config, + station_path=station_config_path_from_env(), + station=resolved_station, + station_error=station_config_error, + recording_output_root=recording_output_root, + visualization=visualization_config, + visualization_error=visualization_error, + task_file_root=task_file_root, + ) visualization: VisualizationScene | None = None try: - visualization = _optional_visualization(monitor) + visualization = _optional_visualization(monitor, visualization_config, visualization_error) app.state.visualization = visualization await visualization.start() # never raises: a failed start only reports the view unavailable yield @@ -363,20 +389,23 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: await monitor.stop() -def _optional_visualization(monitor: CaptureStatusMonitor) -> VisualizationScene: - """The optional 3D view; settings that cannot be read leave it unavailable instead of stopping the app.""" - try: - return VisualizationScene(monitor, visualization_from_env()) - except Exception as exc: - logger.error("3D view settings are invalid: %s", exc) - scene = VisualizationScene(monitor, VisualizationConfig(enabled=True)) - scene.mark_unavailable(f"3D view settings are invalid: {exc}") - return scene +def _optional_visualization( + monitor: CaptureStatusMonitor, config: VisualizationConfig | None, error: str | None +) -> VisualizationScene: + """The optional 3D view; settings that could not be read leave it unavailable instead of stopping the app.""" + if config is not None: + return VisualizationScene(monitor, config) + logger.error("3D view settings are invalid: %s", error) + scene = VisualizationScene(monitor, VisualizationConfig(enabled=True)) + scene.mark_unavailable(f"3D view settings are invalid: {error}") + return scene app = FastAPI(lifespan=lifespan) app.include_router(preview_router) app.include_router(task_file_router) +app.include_router(settings_router) +app.include_router(operator_settings_router) # PoC demo integration: browse/download recorded files, off unless # EGOCAP_RECORDINGS_ENABLED=1. Not part of the final system design -- see # recordings.py's own module docstring. diff --git a/examples/egocap/backend/egocap_web/operator_settings.py b/examples/egocap/backend/egocap_web/operator_settings.py new file mode 100644 index 0000000000..22e148363e --- /dev/null +++ b/examples/egocap/backend/egocap_web/operator_settings.py @@ -0,0 +1,326 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Operator-adjustable station setup kept by the backend, and the operator tools it knows about. + +Unlike the deployment settings (settings_routes.py), these are the operator's to change from the page: +camera type and roles, task catalog, and preselected task. They live in one JSON file under a hidden +``.egocap/`` directory in the recording output +(``EGOCAP_OPERATOR_SETTINGS_FILE`` overrides the location) so every browser sees the same value and it survives a +restart. A change needs the run's control token, the same credential that admits recording commands, so a stray +page cannot relabel a station. + +The tools list is a registry only. Each entry says whether this build can launch it; none can yet, and the page +shows them disabled with that reason rather than hiding them. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import tempfile +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +from .recording_command import control_token_valid +from .task_files import is_catalog_name + +logger = logging.getLogger("isaaccapture.egocap.operator_settings") + +router = APIRouter() + +_NO_STORE = {"Cache-Control": "no-store"} + +OPERATOR_SETTINGS_FILE_ENV = "EGOCAP_OPERATOR_SETTINGS_FILE" + +#: The camera types a station can be built with: GMSL cameras, or USB +#: OAK-D cameras read by CaptureServer. The value is a label for the operator and for tools; it does not itself +#: open any camera (CaptureServer and the station file own that). +CAMERA_TYPES = ("sensing_gmsl", "oak_d") + +#: The camera roles a station fills, whatever the camera type: the operator's view and each wrist. +CAMERA_ROLES = ("ego", "wrist_left", "wrist_right") + +_NOT_WIRED = "not wired in this build" + +#: Operator tools the page offers. ``applies_to`` names the camera type a tool is for, or ``None`` for all. +TOOLS: tuple[dict[str, Any], ...] = ( + { + "id": "headset_calibration", + "name": "Headset calibration", + "description": "Intrinsics and extrinsics of the headset cameras.", + "applies_to": None, + "available": False, + "unavailable_reason": _NOT_WIRED, + }, + { + "id": "glove_calibration", + "name": "Glove calibration", + "description": "Hand-tracking glove calibration against the headset controllers.", + "applies_to": None, + "available": False, + "unavailable_reason": _NOT_WIRED, + }, + { + "id": "fisheye_rectification", + "name": "Fisheye rectification", + "description": ( + "Rectification maps for the three SENSING GMSL cameras. Only fisheye lenses need this; " + "rectilinear cameras do not." + ), + "applies_to": "sensing_gmsl", + "available": False, + "unavailable_reason": _NOT_WIRED, + }, +) + + +class OperatorSettingsError(ValueError): + """The stored settings file exists but cannot be used (not JSON, not an object, or an unknown value).""" + + +@dataclass(frozen=True) +class OperatorSettings: + """The operator's station setup; every field is optional so a fresh station starts with nothing asserted.""" + + camera_type: str | None = None + #: Which offered camera (a CaptureServer camera name) fills each role; unassigned is + #: ``None``. Every role is always present so the page can show the gaps. + camera_roles: dict[str, str | None] = field(default_factory=lambda: {role: None for role in CAMERA_ROLES}) + #: The catalog the recording panel's task picker offers, by file name; ``None`` means free text only. + task_file: str | None = None + #: The task (by its catalog name) the picker preselects when the current task is still unset. + preselected_task: str | None = None + + def to_json(self) -> dict[str, Any]: + """The wire form, with the accepted camera types and roles so the page needs no second copy of either.""" + return { + "camera_type": self.camera_type, + "camera_types": list(CAMERA_TYPES), + "camera_roles": dict(self.camera_roles), + "roles": list(CAMERA_ROLES), + "task_file": self.task_file, + "preselected_task": self.preselected_task, + } + + def stored(self) -> dict[str, Any]: + """The fields a PUT may change, as written to the file.""" + return { + "camera_type": self.camera_type, + "camera_roles": dict(self.camera_roles), + "task_file": self.task_file, + "preselected_task": self.preselected_task, + } + + def updated(self, changes: dict[str, Any]) -> OperatorSettings: + """These settings with the fields named in ``changes`` replaced; a PUT names only what it changes. A + ``camera_roles`` change may itself name only some roles; the rest keep their assignment.""" + merged = self.stored() + merged.update({key: changes[key] for key in merged if key in changes and key != "camera_roles"}) + # A catalog change must not preserve a task from its predecessor. + if merged["task_file"] != self.task_file and "preselected_task" not in changes: + merged["preselected_task"] = None + if "camera_roles" in changes: + roles = changes["camera_roles"] + if not isinstance(roles, dict): + raise OperatorSettingsError("camera_roles must be an object of role to camera name or null") + merged["camera_roles"] = {**self.camera_roles, **roles} + return _parse(merged) + + +def _parse_roles(raw: Any) -> dict[str, str | None]: + """Role assignments: known roles only, each a non-empty camera name or null, no camera in two roles.""" + if raw is None: + return {role: None for role in CAMERA_ROLES} + if not isinstance(raw, dict): + raise OperatorSettingsError("camera_roles must be an object of role to camera name or null") + unknown = sorted(set(raw) - set(CAMERA_ROLES)) + if unknown: + raise OperatorSettingsError( + f"unknown camera role(s): {', '.join(unknown)}; roles are {', '.join(CAMERA_ROLES)}" + ) + roles: dict[str, str | None] = {} + for role in CAMERA_ROLES: + value = raw.get(role) + if value is not None and (not isinstance(value, str) or value.strip() == ""): + raise OperatorSettingsError(f"camera_roles.{role} must be a camera name or null") + roles[role] = value + assigned = [value for value in roles.values() if value is not None] + if len(assigned) != len(set(assigned)): + raise OperatorSettingsError("a camera cannot fill two roles") + return roles + + +def _optional_text(raw: dict[str, Any], key: str, *, check=None, what: str = "") -> str | None: + """Validate a nullable text field using its optional domain predicate.""" + value = raw.get(key) + if value is None: + return None + if not isinstance(value, str) or (check is not None and not check(value)): + raise OperatorSettingsError(f"{key} must be {what or 'a string'} or null") + return value + + +def _parse(raw: Any) -> OperatorSettings: + """A stored or submitted object as settings; an unknown value is an error, an unknown key is ignored.""" + if not isinstance(raw, dict): + raise OperatorSettingsError("operator settings must be an object") + camera_type = raw.get("camera_type") + if camera_type is not None and camera_type not in CAMERA_TYPES: + raise OperatorSettingsError(f"camera_type must be one of {', '.join(CAMERA_TYPES)} or null") + return OperatorSettings( + camera_type=camera_type, + camera_roles=_parse_roles(raw.get("camera_roles")), + task_file=_optional_text(raw, "task_file", check=is_catalog_name, what="a plain *.csv file name"), + preselected_task=_optional_text( + raw, "preselected_task", check=lambda v: v.strip() != "", what="a task name" + ), + ) + + +class OperatorSettingsStore: + """One JSON file holding the operator settings; absent means defaults, and writes replace it atomically.""" + + def __init__(self, path: Path) -> None: + """Bind storage and its mutation lock for this backend process.""" + self.path = path + self.lock = threading.RLock() + + def load(self) -> OperatorSettings: + """The stored settings, or the defaults when nothing has been saved yet.""" + try: + text = self.path.read_text(encoding="utf-8") + except FileNotFoundError: + return OperatorSettings() + except UnicodeError as exc: + raise OperatorSettingsError("operator settings are not valid UTF-8") from exc + try: + raw = json.loads(text) + except json.JSONDecodeError as exc: + raise OperatorSettingsError(f"{self.path} is not valid JSON: {exc}") from exc + return _parse(raw) + + def save(self, settings: OperatorSettings) -> None: + """Write the settings, through a temporary file so a crash never leaves a half-written one.""" + self.path.parent.mkdir(parents=True, exist_ok=True) + # Unique files prevent overlapping writers from sharing or replacing each other's staging file. + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=self.path.parent, prefix=f".{self.path.name}.", delete=False + ) as handle: + tmp = Path(handle.name) + try: + handle.write(json.dumps(settings.stored(), indent=2) + "\n") + handle.flush() + os.fsync(handle.fileno()) + handle.close() + os.replace(tmp, self.path) + finally: + tmp.unlink(missing_ok=True) + + def update(self, changes: dict[str, Any], root: Path | None, imported: Path | None) -> OperatorSettings: + """Validate and persist a partial change under the same lock as catalog removal.""" + from .task_files import TaskFileError, read_available_task_file + + # The whole transaction stays in one thread so cancellation cannot release its lock early. + with self.lock: + try: + current = self.load() + except (OperatorSettingsError, UnicodeError): + logger.warning("Replacing invalid operator settings at %s", self.path) + current = OperatorSettings() + settings = current.updated(changes) + # Validate task choices against the same imported-first catalog that the picker reads. + if settings.task_file is not None and root is not None: + try: + catalog = read_available_task_file(root, imported, settings.task_file) + except TaskFileError as exc: + raise OperatorSettingsError(str(exc)) from exc + if settings.preselected_task is not None and not any( + task.name == settings.preselected_task for task in catalog.tasks + ): + raise OperatorSettingsError("preselected_task must name a task in the selected catalog") + elif settings.task_file is None and settings.preselected_task is not None: + raise OperatorSettingsError("preselected_task needs a selected catalog") + self.save(settings) + return settings + + +def operator_settings_path(recording_output_root: Path) -> Path: + """Resolve the override or a hidden config directory excluded from the recordings browser.""" + override = os.environ.get(OPERATOR_SETTINGS_FILE_ENV) + return ( + Path(override).expanduser() if override else recording_output_root / ".egocap" / "operator_settings.json" + ) + + +def _store(request: Request) -> OperatorSettingsStore | None: + return getattr(request.app.state, "operator_settings_store", None) + + +@router.get("/api/operator-settings") +async def read_operator_settings(request: Request) -> JSONResponse: + """Return saved settings, or defaults with a repair warning if the saved data is invalid.""" + store = _store(request) + if store is None: + return JSONResponse( + {"detail": "operator settings are not available yet"}, status_code=503, headers=_NO_STORE + ) + try: + settings = await asyncio.to_thread(store.load) + except OperatorSettingsError: + # Defaults keep the form usable so an authenticated save can repair invalid storage. + logger.warning("Invalid operator settings at %s; showing defaults for repair", store.path) + return JSONResponse( + {**OperatorSettings().to_json(), "warning": "Saved settings are invalid. Save to replace them."}, + headers=_NO_STORE, + ) + return JSONResponse(settings.to_json(), headers=_NO_STORE) + + +@router.put("/api/operator-settings") +async def write_operator_settings(request: Request) -> JSONResponse: + """Change the operator settings. The body carries ``control_token`` and the fields to change (the rest keep + their stored value); a wrong token is 403 before the values are even looked at, and an invalid value is 400 + with the reason.""" + store = _store(request) + recording_context = getattr(request.app.state, "recording_context", None) + if store is None or recording_context is None: + return JSONResponse( + {"detail": "operator settings are not available yet"}, status_code=503, headers=_NO_STORE + ) + try: + body = await request.json() + except ValueError: + return JSONResponse({"detail": "body must be JSON"}, status_code=400, headers=_NO_STORE) + if not isinstance(body, dict): + return JSONResponse({"detail": "body must be an object"}, status_code=400, headers=_NO_STORE) + if not control_token_valid(body.get("control_token"), recording_context.control_token): + return JSONResponse({"detail": "control token rejected"}, status_code=403, headers=_NO_STORE) + try: + settings = await asyncio.to_thread( + store.update, + body, + getattr(request.app.state, "task_file_root", None), + getattr(request.app.state, "imported_task_file_root", None), + ) + except OperatorSettingsError as exc: + return JSONResponse({"detail": str(exc)}, status_code=400, headers=_NO_STORE) + except OSError as exc: + return JSONResponse( + {"detail": f"could not save operator settings: {exc}"}, status_code=503, headers=_NO_STORE + ) + return JSONResponse(settings.to_json(), headers=_NO_STORE) + + +@router.get("/api/tools") +async def list_tools() -> JSONResponse: + """The operator tools this build knows about, each with whether it can be launched.""" + return JSONResponse({"tools": list(TOOLS)}, headers=_NO_STORE) diff --git a/examples/egocap/backend/egocap_web/preview_routes.py b/examples/egocap/backend/egocap_web/preview_routes.py index 79c11378e0..fa2d247306 100644 --- a/examples/egocap/backend/egocap_web/preview_routes.py +++ b/examples/egocap/backend/egocap_web/preview_routes.py @@ -208,7 +208,7 @@ def _expected_origin(websocket: WebSocket) -> str: return f"{scheme}://{host}:{port}" -def _configured_allowed_origins() -> frozenset[str]: +def configured_allowed_origins() -> frozenset[str]: """Explicitly configured additional origins (comma-separated), for a deployment where the frontend is hosted separately from this backend or reached through a reverse proxy that changes the browser-visible @@ -217,7 +217,7 @@ def _configured_allowed_origins() -> frozenset[str]: return frozenset(entry.strip() for entry in raw.split(",") if entry.strip()) -def _trusted_hostnames() -> frozenset[str] | None: +def trusted_hostnames() -> frozenset[str] | None: """Pins the same-origin fallback below to a configured set of hostnames/addresses, or ``None`` if ``EGOCAP_TRUSTED_HOSTNAMES`` is unset. Unpinned, ``_expected_origin`` derives the "backend's own" @@ -246,14 +246,14 @@ def _origin_allowed(websocket: WebSocket) -> bool: block anything by itself). No substring matching, no wildcard, no trusting a forwarded header -- an exact match against an explicitly configured origin, or against the backend's own derived origin (only - when the request's Host is itself trusted, see ``_trusted_hostnames``).""" + when the request's Host is itself trusted, see ``trusted_hostnames``).""" origin = websocket.headers.get("origin") if not origin or origin == "null": return False - if origin in _configured_allowed_origins(): + if origin in configured_allowed_origins(): return True - trusted_hostnames = _trusted_hostnames() - if trusted_hostnames is not None and (websocket.url.hostname or "") not in trusted_hostnames: + trusted = trusted_hostnames() + if trusted is not None and (websocket.url.hostname or "") not in trusted: return False return origin == _expected_origin(websocket) diff --git a/examples/egocap/backend/egocap_web/recording_command.py b/examples/egocap/backend/egocap_web/recording_command.py index ad6d5b6063..1d042ffef3 100644 --- a/examples/egocap/backend/egocap_web/recording_command.py +++ b/examples/egocap/backend/egocap_web/recording_command.py @@ -186,7 +186,7 @@ def _new_batch_id() -> str: return f"egocap-batch-{uuid.uuid4()}" -def _control_token_valid(candidate: object, expected: str) -> bool: +def control_token_valid(candidate: object, expected: str) -> bool: """Timing-safe comparison against the server's one generated secret. Rejects a non-``str``, wrong-length, or non-ASCII candidate before ``compare_digest`` itself, which raises ``TypeError`` on non-ASCII @@ -430,7 +430,7 @@ async def apply(self, command: RecordingCommand) -> tuple[str, str | None]: place that actually needs current accepted sources and a committed instruction. See ``RecordingSessionCoordinator``'s own ``start_episode``/``set_task``/``new_batch`` docstrings.""" - if not _control_token_valid(command.control_token, self.control_token): + if not control_token_valid(command.control_token, self.control_token): return "rejected", "Enter the current recording control token" if command.payload.get("batch_id") != self.batch_id: return "rejected", "no matching batch" diff --git a/examples/egocap/backend/egocap_web/recordings.py b/examples/egocap/backend/egocap_web/recordings.py index 936d2d8e20..14e3f4f452 100644 --- a/examples/egocap/backend/egocap_web/recordings.py +++ b/examples/egocap/backend/egocap_web/recordings.py @@ -46,7 +46,7 @@ def _hidden(name: str) -> bool: return name.startswith(".") or name.endswith(".pending") -def _enabled() -> bool: +def enabled() -> bool: """Whether the whole feature is turned on -- see the module docstring's off-by-default rationale.""" return os.environ.get("EGOCAP_RECORDINGS_ENABLED", "").lower() in ("1", "true", "yes") @@ -98,7 +98,7 @@ def _hidden_path(base: Path, target: Path) -> bool: _NO_STORE_HEADERS = {"Cache-Control": "no-store"} -if _enabled(): +if enabled(): @router.get("/recordings/api/list", response_model=None) @router.get("/recordings/api/list/{path:path}", response_model=None) diff --git a/examples/egocap/backend/egocap_web/settings_routes.py b/examples/egocap/backend/egocap_web/settings_routes.py new file mode 100644 index 0000000000..b9d113fce2 --- /dev/null +++ b/examples/egocap/backend/egocap_web/settings_routes.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""GET /api/settings: the backend's effective configuration, read-only. + +Everything reported here was set before the backend started, through the environment or the station YAML +(see backend/README.md). The page can inspect it so a misconfigured station is visible from the browser instead +of only from the launch shell, but it cannot change it: a restart is the only way to apply a new value, and the +browser is never a configuration source (config.py states the same rule for capture). Secrets are reported as +present or absent, never by value. The application lifespan resolves the snapshot once into +``app.state.settings``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +from . import recordings +from .config import CaptureStatusConfig, VisualizationConfig +from .preview_protocol import CAMERA_SAMPLE_HZ, TRACKING_SAMPLE_HZ +from .preview_routes import configured_allowed_origins, trusted_hostnames +from .recording_command import control_token_valid +from .station_config import ResolvedStation + +router = APIRouter() + +_NO_STORE = {"Cache-Control": "no-store"} + + +def _enum_name(value: Any) -> Any: + """An enum member as its lower-case name; anything else as it is (the wire form avoids bare numbers).""" + name = getattr(value, "name", None) + return name.lower() if isinstance(name, str) else value + + +@dataclass(frozen=True) +class EffectiveSettings: + """What the backend is running with, resolved once at startup by the application lifespan.""" + + capture: CaptureStatusConfig + #: ``EGOCAP_STATION_CONFIG`` as given, or ``None`` when unset; ``station`` is ``None`` when unset or invalid. + station_path: Path | None + station: ResolvedStation | None + #: The public (path-free) reason the station file was rejected, or ``None``. + station_error: str | None + recording_output_root: Path + #: ``None`` when the 3D view settings could not be read; ``visualization_error`` then says why. + visualization: VisualizationConfig | None + visualization_error: str | None + task_file_root: Path + + def to_json(self) -> dict[str, Any]: + """The wire form: plain JSON types, enum names instead of numbers, secrets as booleans.""" + capture, station, viz = self.capture, self.station, self.visualization + # Origin policy and the recordings flag are read from the environment on every call, + # the same way their own modules read them, so this view never disagrees with the behaviour it describes. + trusted = trusted_hostnames() + return { + "capture": { + "server_url": capture.capture_server_url, + "station_label": capture.station_label, + "profile_label": capture.profile_label, + "sources": [_enum_name(source) for source in capture.sources], + "plugins": [_enum_name(plugin) for plugin in capture.plugins], + "cameras": [{"name": camera.name, "type": _enum_name(camera.type)} for camera in capture.cameras], + }, + "preview": {"camera_sample_hz": CAMERA_SAMPLE_HZ, "tracking_sample_hz": TRACKING_SAMPLE_HZ}, + "station": { + "path": str(self.station_path) if self.station_path is not None else None, + "error": self.station_error, + "id": station.station_id if station is not None else None, + "operator_name": station.operator_name if station is not None else None, + "auto_fill_operator_name": station.auto_fill_operator_name if station is not None else False, + "task_description": station.task_description if station is not None else None, + "data_dir": str(station.data_dir) + if station is not None and station.data_dir is not None + else None, + }, + "recording": {"output_root": str(self.recording_output_root)}, + "visualization": { + "error": self.visualization_error, + "enabled": viz.enabled if viz is not None else False, + "host": viz.host if viz is not None else None, + "port": viz.port if viz is not None else None, + "public_url": viz.public_url if viz is not None else None, + "device": viz.device if viz is not None else None, + "head_model": str(viz.head_model) if viz is not None and viz.head_model is not None else None, + "head_model_scale": viz.head_model_scale if viz is not None else None, + }, + "recordings_browser": {"enabled": recordings.enabled()}, + "task_files": {"dir": str(self.task_file_root), "available": self.task_file_root.is_dir()}, + "origins": { + "allowed": sorted(configured_allowed_origins()), + "trusted_hostnames": sorted(trusted) if trusted is not None else None, + }, + } + + +@router.get("/api/settings") +async def read_settings(request: Request) -> JSONResponse: + """The effective configuration as JSON; 503 until the lifespan has resolved it.""" + # Keep paths, operator identity and security policy behind the operator credential. + context = getattr(request.app.state, "recording_context", None) + if context is None: + return JSONResponse({"detail": "settings are not resolved yet"}, status_code=503, headers=_NO_STORE) + if not control_token_valid(request.headers.get("x-control-token"), context.control_token): + return JSONResponse({"detail": "control token rejected"}, status_code=403, headers=_NO_STORE) + settings = getattr(request.app.state, "settings", None) + if settings is None: + return JSONResponse({"detail": "settings are not resolved yet"}, status_code=503, headers=_NO_STORE) + return JSONResponse(settings.to_json(), headers=_NO_STORE) diff --git a/examples/egocap/backend/egocap_web/task_file_routes.py b/examples/egocap/backend/egocap_web/task_file_routes.py index 165d4827f8..6320948d1f 100644 --- a/examples/egocap/backend/egocap_web/task_file_routes.py +++ b/examples/egocap/backend/egocap_web/task_file_routes.py @@ -2,20 +2,28 @@ # # SPDX-License-Identifier: Apache-2.0 -"""GET /api/task-files and GET /api/task-files/{name}: the read-only catalog endpoints. +"""The task-catalog endpoints: the shipped catalogs plus the operator's imported ones. -A thin HTTP adapter over :mod:`task_files`; the filesystem work runs in a worker thread so it never blocks the -event loop. The catalog root is ``app.state.task_file_root``, set once by the application lifespan. +Two roots feed one listing. ``app.state.task_file_root`` holds the catalogs shipped with the application +(read-only, see data/taskfiles/README.md); ``app.state.imported_task_file_root`` holds the CSV files an operator +imported from the Settings page, which can also be removed again. An imported catalog with a shipped catalog's +name shadows it, so an operator can replace a shipped list without touching the installation. Imports and +removals carry the run's control token, like recording commands. The filesystem work runs in a worker thread so +it never blocks the event loop. """ from __future__ import annotations import asyncio +from pathlib import Path +from typing import Any from fastapi import APIRouter, Request from fastapi.responses import JSONResponse from . import task_files +from .operator_settings import OperatorSettingsError +from .recording_command import control_token_valid router = APIRouter() @@ -29,28 +37,141 @@ task_files.MalformedTaskFile: 500, task_files.TaskFilesUnavailable: 503, } +# The same file arriving from an operator is their mistake, not the server's: too large or malformed is a 400. +_IMPORT_STATUS = {**_STATUS, task_files.TaskFileTooLarge: 400, task_files.MalformedTaskFile: 400} -def _failure(exc: task_files.TaskFileError) -> JSONResponse: +def _failure(exc: task_files.TaskFileError, status: dict[type, int] = _STATUS) -> JSONResponse: """The error response for a task-file error: its status, a concise reason, never a path.""" - return JSONResponse({"detail": str(exc)}, status_code=_STATUS[type(exc)], headers=_NO_STORE) + return JSONResponse({"detail": str(exc)}, status_code=status[type(exc)], headers=_NO_STORE) + + +def _detail(message: str, status: int) -> JSONResponse: + return JSONResponse({"detail": message}, status_code=status, headers=_NO_STORE) + + +def _imported_root(request: Request) -> Path | None: + return getattr(request.app.state, "imported_task_file_root", None) + + +def _listing(shipped_root: Path, imported_root: Path | None) -> dict[str, list[str]]: + """Every catalog name once, plus which of them are imports; a not-yet-created imported root is empty.""" + shipped = task_files.list_task_files(shipped_root) + imported = ( + task_files.list_task_files(imported_root) if imported_root is not None and imported_root.is_dir() else [] + ) + return {"files": sorted(set(shipped) | set(imported)), "imported": imported} @router.get("/api/task-files") async def list_files(request: Request) -> JSONResponse: - """The sorted catalog file names.""" + """The sorted catalog file names, and which of them the operator imported.""" try: - names = await asyncio.to_thread(task_files.list_task_files, request.app.state.task_file_root) + listing = await asyncio.to_thread(_listing, request.app.state.task_file_root, _imported_root(request)) except task_files.TaskFileError as exc: return _failure(exc) - return JSONResponse({"files": names}, headers=_NO_STORE) + return JSONResponse(listing, headers=_NO_STORE) @router.get("/api/task-files/{name}") async def read_file(name: str, request: Request) -> JSONResponse: - """One catalog's task definitions in file order.""" + """One catalog's task definitions in file order, from the imported root first.""" try: - contents = await asyncio.to_thread(task_files.read_task_file, request.app.state.task_file_root, name) + contents = await asyncio.to_thread( + task_files.read_available_task_file, request.app.state.task_file_root, _imported_root(request), name + ) except task_files.TaskFileError as exc: return _failure(exc) return JSONResponse(contents.to_json(), headers=_NO_STORE) + + +async def _authorised_body(request: Request) -> tuple[dict[str, Any] | None, JSONResponse | None]: + """The JSON body once its control token is accepted, or the response that refuses it.""" + recording_context = getattr(request.app.state, "recording_context", None) + if recording_context is None: + return None, _detail("task file imports are not available yet", 503) + try: + body = await request.json() + except ValueError: + return None, _detail("body must be JSON", 400) + if not isinstance(body, dict): + return None, _detail("body must be an object", 400) + if not control_token_valid(body.get("control_token"), recording_context.control_token): + return None, _detail("control token rejected", 403) + return body, None + + +@router.post("/api/task-files/import") +async def import_file(request: Request) -> JSONResponse: + """Import browser-uploaded ``name`` and ``content`` after validating the catalog.""" + body, refused = await _authorised_body(request) + if body is None: + return refused # type: ignore[return-value] + imported_root = _imported_root(request) + if imported_root is None: + return _detail("task file imports are not available yet", 503) + name, content = body.get("name"), body.get("content") + if "path" in body: + return _detail("server path imports are disabled; choose a CSV file to upload", 400) + if not isinstance(name, str) or not isinstance(content, str): + return _detail("give name and content", 400) + raw = content.encode("utf-8") + store = getattr(request.app.state, "operator_settings_store", None) + if store is None: + return _detail("operator settings are not available yet", 503) + + def save_import() -> dict[str, Any] | None: + """Prevent an import from changing a selected catalog underneath its task selection.""" + with store.lock: + if store.load().task_file == name: + return None + contents = task_files.import_task_file(imported_root, name, raw) + return {"file": contents.to_json(), **_listing(request.app.state.task_file_root, imported_root)} + + try: + result = await asyncio.to_thread(save_import) + except OperatorSettingsError: + return _detail("repair operator settings before importing catalogs", 409) + except OSError: + return _detail("operator settings storage is unavailable", 503) + except task_files.TaskFileError as exc: + return _failure(exc, _IMPORT_STATUS) + if result is None: + return _detail("choose another catalog or free text before replacing the selected catalog", 409) + return JSONResponse(result, headers=_NO_STORE) + + +@router.delete("/api/task-files/{name}") +async def remove_file(name: str, request: Request) -> JSONResponse: + """Remove an unselected import, authenticating with the X-Control-Token header.""" + context = getattr(request.app.state, "recording_context", None) + if context is None: + return _detail("task file imports are not available yet", 503) + if not control_token_valid(request.headers.get("x-control-token"), context.control_token): + return _detail("control token rejected", 403) + imported_root = _imported_root(request) + if imported_root is None or not imported_root.is_dir(): + return _detail(f"no imported task file named {name}", 404) + store = getattr(request.app.state, "operator_settings_store", None) + if store is None: + return _detail("operator settings are not available yet", 503) + + def remove() -> dict[str, list[str]] | None: + """Hold the settings lock so selection cannot race a catalog removal.""" + with store.lock: + if store.load().task_file == name: + return None + task_files.remove_task_file(imported_root, name) + return _listing(request.app.state.task_file_root, imported_root) + + try: + listing = await asyncio.to_thread(remove) + except OperatorSettingsError: + return _detail("repair operator settings before removing catalogs", 409) + except OSError: + return _detail("operator settings storage is unavailable", 503) + except task_files.TaskFileError as exc: + return _failure(exc) + if listing is None: + return _detail("choose another catalog or free text before removing the selected catalog", 409) + return JSONResponse(listing, headers=_NO_STORE) diff --git a/examples/egocap/backend/egocap_web/task_files.py b/examples/egocap/backend/egocap_web/task_files.py index ca323b48e7..73ed24288f 100644 --- a/examples/egocap/backend/egocap_web/task_files.py +++ b/examples/egocap/backend/egocap_web/task_files.py @@ -2,10 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Read-only access to the task CSV catalogs in ``data/taskfiles/``. +"""Read shipped task CSV catalogs and manage operator-imported catalogs. -Two small functions, ``list_task_files`` and ``read_task_file``, take the catalog root explicitly and touch the -filesystem only when called. Two existing CSV layouts are supported, each by an explicit branch: +Catalog listing, reading, import and removal take their root explicitly and touch the filesystem only when +called. Two existing CSV layouts are supported, each by an explicit branch: * studio catalogs: a title record, a blank record, then ``Studio, Scenario, Task, Description`` headers; * Mid Train: headers on the first record (``Task Name``, ``Task Description`` and optional ``Primitive(s)``, @@ -19,11 +19,13 @@ import csv import io import logging +import os import re +import tempfile from dataclasses import dataclass from pathlib import Path -logger = logging.getLogger(__name__) +logger = logging.getLogger("isaaccapture.egocap.task_files") MAX_FILE_BYTES = 2 * 1024 * 1024 @@ -152,6 +154,19 @@ def read_task_file(root: Path, name: str) -> TaskFileContents: raise TaskFilesUnavailable("task file storage is unavailable") from exc if not inside_root: raise TaskFileNotFound(f"no task file named {name}") + return parse_task_file(name, raw) + + +def read_available_task_file(root: Path, imported: Path | None, name: str) -> TaskFileContents: + """Resolve a catalog with operator imports taking precedence over shipped files.""" + # One resolution rule keeps the picker and settings validation in agreement. + if imported is not None and imported.is_dir() and name in list_task_files(imported): + return read_task_file(imported, name) + return read_task_file(root, name) + + +def parse_task_file(name: str, raw: bytes) -> TaskFileContents: + """Parse catalog bytes (a file on disk or an upload) into task definitions, with the size bound applied.""" if len(raw) > MAX_FILE_BYTES: raise TaskFileTooLarge(f"{name} is larger than {MAX_FILE_BYTES} bytes") # Decode (a leading BOM is allowed), then parse with strict CSV: quoted newlines work, bad quoting is refused. @@ -166,6 +181,53 @@ def read_task_file(root: Path, name: str) -> TaskFileContents: return _parse(name, records) +def is_catalog_name(name: str) -> bool: + """Whether ``name`` could name a catalog: a plain ``*.csv`` name, not hidden, not an editor leftover.""" + plain = _is_plain_csv_name(name) and not name.startswith((".", "~")) + return plain and name not in _NOT_CATALOGS + + +def import_task_file(root: Path, name: str, raw: bytes) -> TaskFileContents: + """Store an operator-supplied catalog under ``root`` as ``name`` once it parses; a bad file is never written. + + The write goes through a temporary file so a crash leaves either the old catalog or the new one, never half. + """ + if not is_catalog_name(name): + raise InvalidTaskFileName("a task file name is a plain filename ending in .csv") + contents = parse_task_file(name, raw) + try: + root.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=root, prefix=f".{name}.", delete=False) as handle: + tmp = Path(handle.name) + try: + handle.write(raw) + handle.flush() + os.fsync(handle.fileno()) + handle.close() + tmp.replace(root / name) + finally: + tmp.unlink(missing_ok=True) + except OSError as exc: + logger.error("task file %s could not be stored under %s: %s", name, root, exc) + raise TaskFilesUnavailable("task file storage is unavailable") from exc + return contents + + +def remove_task_file(root: Path, name: str) -> None: + """Delete an imported catalog; a name that is not an imported catalog is absent, never a path to resolve.""" + if not _is_plain_csv_name(name): + raise InvalidTaskFileName("a task file name is a plain filename ending in .csv") + if name not in list_task_files(root): + raise TaskFileNotFound(f"no imported task file named {name}") + try: + (root / name).unlink() + except FileNotFoundError as exc: + raise TaskFileNotFound(f"no imported task file named {name}") from exc + except OSError as exc: + logger.error("task file %s could not be removed: %s", root / name, exc) + raise TaskFilesUnavailable("task file storage is unavailable") from exc + + def _headers(record: list[str]) -> list[str]: """Header cells trimmed and lowercased for case-insensitive matching.""" return [cell.strip().lower() for cell in record] diff --git a/examples/egocap/data/taskfiles/README.md b/examples/egocap/data/taskfiles/README.md index 76867056c6..5663c40527 100644 --- a/examples/egocap/data/taskfiles/README.md +++ b/examples/egocap/data/taskfiles/README.md @@ -7,8 +7,8 @@ SPDX-License-Identifier: Apache-2.0 # Task catalogs CSV task catalogs served read-only by the web backend at `GET /api/task-files` (names) and -`GET /api/task-files/{name}` (parsed task definitions, in file order). Only the files in this directory are -served; the backend never reads another location. Deployments must ship this directory with the application. +`GET /api/task-files/{name}` (parsed task definitions, in file order). Only files in the configured shipped and imported catalog directories are +served. Deployments must ship this directory with the application. Two layouts are supported: @@ -21,6 +21,10 @@ Two layouts are supported: Other columns (operators, sessions, completion tracking) are ignored. A task file lists available tasks; it does not select one. +Operators can add catalogs of their own from the page's Settings → Tasks (a CSV uploaded from their browser); those are stored in a `taskfiles/` directory beside the operator settings file (under the recording output +inside the hidden `.egocap/` configuration directory by default), listed together with the shipped ones, and can be removed again. An imported catalog with a +shipped catalog's name takes its place in the listing without modifying the shipped file. + Set `EGOCAP_TASK_FILES_DIR` to serve a different directory (for example the one the recorder reads its task library from); unset, the API reads this one. The summary file `Summary.csv`, hidden files and editor leftovers are never listed. Errors: 400 for a malformed name, 404 for a catalog that is not listed, 500 (with the reason) for a catalog on diff --git a/examples/egocap/frontend/src/App.test.tsx b/examples/egocap/frontend/src/App.test.tsx index a7c683e898..8ae3e89c73 100644 --- a/examples/egocap/frontend/src/App.test.tsx +++ b/examples/egocap/frontend/src/App.test.tsx @@ -14,6 +14,7 @@ import type { ReactElement } from "react"; import { afterEach, describe, expect, it, vi, type Mock } from "vitest"; import { act, cleanup, fireEvent, render, screen, within } from "@testing-library/react"; import App from "./App"; +import { OPERATOR_SETTINGS_FIXTURE, SETTINGS_FIXTURE } from "./settingsTestDoubles"; import { useCaptureSession, type AcceptedTracking, type CaptureSessionState } from "./useCaptureSession"; import { Axes, HandJointOrder, PositionUnit, QuaternionOrder, type PreviewSnapshot } from "./captureProtocol"; import type { ControllerDisplay, DecodedTracking, HandDisplay } from "./trackingDecoder"; @@ -462,3 +463,35 @@ describe("CaptureTree diagnostic vs never-had-data", () => { expect(screen.queryByText("No tracking sample yet.")).toBeNull(); }); }); + + +describe("Settings modal recording isolation", () => { + it("blocks shortcuts from focused modal buttons and restores controls after close", async () => { + const send = vi.fn().mockResolvedValue({ outcome: "ok", reason: null }); + vi.stubGlobal("fetch", vi.fn(async (url: string) => ({ + ok: true, json: async () => url === "/api/operator-settings" ? OPERATOR_SETTINGS_FIXTURE + : url === "/api/settings" ? SETTINGS_FIXTURE : url === "/api/tools" ? { tools: [] } : { files: [], imported: [] }, + }))); + mockState(makeState({ + sendRecordingCommand: send, + recordingSnapshot: { + batchId: "b1", episodePhase: "RECORDING", currentEpisodeId: "e1", currentEpisodeElapsedMs: 0, + taskText: "Fold towel", taskDetails: "", defaultOperator: null, autoFillOperatorName: false, + savedCount: 0, lastResult: null, readyToRecord: true, revision: 0, + }, + })); + const { container } = render(); + fireEvent.click(screen.getByRole("button", { name: "Settings" })); + const dialog = screen.getByRole("dialog", { name: "Settings" }); + const station = within(dialog).getByRole("button", { name: "Station" }); + fireEvent.click(station); + station.focus(); + for (const key of ["a", "w", "s", "r", "t", " "]) fireEvent.keyDown(window, { key }); + expect(send).not.toHaveBeenCalled(); + expect(container.querySelector(".op-body")?.hasAttribute("inert")).toBe(true); + fireEvent.click(within(dialog).getByRole("button", { name: "Close" })); + expect(container.querySelector(".op-body")?.hasAttribute("inert")).toBe(false); + fireEvent.keyDown(window, { key: "w" }); + expect(send).toHaveBeenCalledTimes(1); + }); +}); diff --git a/examples/egocap/frontend/src/App.tsx b/examples/egocap/frontend/src/App.tsx index e3e2428381..c16d4c220e 100644 --- a/examples/egocap/frontend/src/App.tsx +++ b/examples/egocap/frontend/src/App.tsx @@ -34,6 +34,8 @@ import { import { deviceTone, retainedTone, type Tone } from "./deviceHealth"; import { PoseViserPanel } from "./PoseViserPanel"; import { RecordingPanel } from "./RecordingPanel"; +import { SettingsPanel } from "./SettingsPanel"; +import { useOperatorSettings } from "./useOperatorSettings"; import { buildRecordingActions } from "./recordingActions"; import type { RecordingPhase } from "./recordingUiReducer"; @@ -204,10 +206,19 @@ function providerTone(status: ProviderStatus["status"]): Tone { return PROVIDER_TONE[status] ?? "neutral"; } -/** The page's top bar: app title, capture mode and feed-health status. +/** The page's top bar: app title, capture mode, feed-health status and the Settings button. * There is no page/tab navigation -- the generic capture display and the - * Ego Capture UI recording panel are always-present siblings on one page. */ -function AppHeader({ response, feedHealthLabel }: { response: CaptureStatusResponse | null; feedHealthLabel: string }) { + * Ego Capture UI recording panel are always-present siblings on one page, and + * Settings opens as a drawer over them rather than replacing them. */ +function AppHeader({ + response, + feedHealthLabel, + onOpenSettings, +}: { + response: CaptureStatusResponse | null; + feedHealthLabel: string; + onOpenSettings: () => void; +}) { return (
@@ -221,6 +232,9 @@ function AppHeader({ response, feedHealthLabel }: { response: CaptureStatusRespo Feed {feedHealthLabel} +
); @@ -468,11 +482,32 @@ export default function App() { return () => window.removeEventListener("resize", refit); }, []); const unavailable = response === null && feedHealth === "unavailable"; + const [settingsOpen, setSettingsOpen] = useState(false); + // The operator settings the page itself uses (camera roles). Re-read each time Settings closes, so a change + // saved there reaches the tiles without a reload. + const [settingsGeneration, setSettingsGeneration] = useState(0); + const operatorSettings = useOperatorSettings(settingsGeneration); + const closeSettings = () => { + setSettingsOpen(false); + setSettingsGeneration((n) => n + 1); + }; + // Camera roles use the capture streams currently available on main. + const cameraSources = (snapshot?.started?.streams ?? []).filter((s) => s.kind === "camera").map((s) => s.name); return (
- + setSettingsOpen(true)} /> + applyWidth(SIDEBAR_DEFAULT_PX, true)} + controlToken={controlToken} + cameraSources={cameraSources} + />
@@ -499,7 +534,7 @@ export default function App() { {/* The cameras are the page: they take every pixel the status bar above and the controls below leave. */}
- +
diff --git a/examples/egocap/frontend/src/CameraGrid.test.tsx b/examples/egocap/frontend/src/CameraGrid.test.tsx index 81b1993995..7f7d4617c0 100644 --- a/examples/egocap/frontend/src/CameraGrid.test.tsx +++ b/examples/egocap/frontend/src/CameraGrid.test.tsx @@ -96,3 +96,34 @@ describe("CameraGrid primary camera", () => { expect(container.querySelector(".camera-grid-items")?.className).toContain("camera-grid-items--n4"); }); }); + +describe("Camera role assignments", () => { + function twoCameras(): PreviewSnapshot { + const snapshot = makeSnapshot("1", 0); + return { ...snapshot, cameras: [...snapshot.cameras, { ...snapshot.cameras[0], stream_id: 2, name: "wrist_camera", url: "/stream2" }] }; + } + + it("gives the assigned ego camera the primary tile and shows role badges", () => { + const { container } = render(); + const primary = container.querySelectorAll(".camera-tile--primary"); + expect(primary).toHaveLength(1); + expect(primary[0].textContent).toContain("wrist_camera"); + expect(primary[0].textContent).toContain("Ego (head)"); + expect(screen.getByText("Left wrist")).toBeTruthy(); + }); + + it("restores the head-camera fallback after clearing the ego role", () => { + const snapshot = twoCameras(); + const { container, rerender } = render(); + expect(container.querySelector(".camera-tile--primary")?.textContent).toContain("wrist_camera"); + rerender(); + expect(container.querySelector(".camera-tile--primary")?.textContent).toContain("head_camera"); + expect(screen.queryByText("Ego (head)")).toBeNull(); + }); + + it("keeps an absent ego assignment from hiding the primary camera", () => { + const { container } = render(); + expect(container.querySelectorAll(".camera-tile--primary")).toHaveLength(1); + expect(container.querySelector(".camera-tile--primary")?.textContent).toContain("head_camera"); + }); +}); diff --git a/examples/egocap/frontend/src/CameraGrid.tsx b/examples/egocap/frontend/src/CameraGrid.tsx index 829e355a3b..416a453fd3 100644 --- a/examples/egocap/frontend/src/CameraGrid.tsx +++ b/examples/egocap/frontend/src/CameraGrid.tsx @@ -12,6 +12,8 @@ import { useEffect, useRef, useState } from "react"; import type { PreviewSnapshot } from "./captureProtocol"; import { TRACKING_STALE_AFTER_MS } from "./CaptureTree"; import { CameraRateTracker, formatCameraStats } from "./cameraRates"; +import { ROLE_LABELS } from "./settingsApi"; +import { roleOfCamera } from "./useOperatorSettings"; const CAMERA_ERROR_RETRY_LIMIT = 3; @@ -24,8 +26,10 @@ function CameraTile({ stale, stats, primary, + role, }: { camera: PreviewSnapshot["cameras"][number]; + role: string | null; /** The one camera the layout gives the most room (the operator's main view). */ primary: boolean; live: boolean; @@ -131,6 +135,7 @@ function CameraTile({ )}
{camera.name} + {role !== null && {ROLE_LABELS[role] ?? role}} {stats !== null && ( {stats} @@ -147,10 +152,13 @@ export function CameraGrid({ snapshot, live, receivedAtMonotonic, + cameraRoles = null, }: { snapshot: PreviewSnapshot | null; live: boolean; receivedAtMonotonic: number | null; + /** The assigned ego camera owns the primary tile when it is available. */ + cameraRoles?: Record | null; }) { const [, forceTick] = useState(0); useEffect(() => { @@ -186,9 +194,10 @@ export function CameraGrid({ ); } const elapsedSinceReceipt = receivedAtMonotonic === null ? 0 : performance.now() - receivedAtMonotonic; - // The head camera is the operator's main view when there is one by that name; otherwise the first camera is. + // A present ego assignment wins; an offline assignment retains the usual head-camera fallback. + const egoIndex = cameras.findIndex((cam) => cam.name === cameraRoles?.ego); const headIndex = cameras.findIndex((cam) => /head/i.test(cam.name)); - const primaryIndex = headIndex === -1 ? 0 : headIndex; + const primaryIndex = egoIndex !== -1 ? egoIndex : headIndex === -1 ? 0 : headIndex; return (

Cameras

@@ -223,6 +232,7 @@ export function CameraGrid({ live={live} stale={stale} stats={stats} + role={roleOfCamera(cameraRoles, cam.name)} /> ); })} diff --git a/examples/egocap/frontend/src/RecordingPanel.taskCatalog.test.tsx b/examples/egocap/frontend/src/RecordingPanel.taskCatalog.test.tsx new file mode 100644 index 0000000000..5e6063a162 --- /dev/null +++ b/examples/egocap/frontend/src/RecordingPanel.taskCatalog.test.tsx @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +/** The task editor's catalog picker: offered when the operator chose a catalog, fills the text, preselects a + * standing task into an empty draft only, and stays free text when no catalog is configured or loading fails. */ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { RecordingPanel } from "./RecordingPanel"; +import type { RecordingSnapshot } from "./recordingTypes"; +import type { TaskCatalog } from "./taskCatalog"; + +afterEach(cleanup); + +const operatorProps = { + // Edit task is offered only when a set-task action is wired; the tests below override it when they assert on it. + actions: { onSetTask: vi.fn().mockResolvedValue({ outcome: "ok", reason: null }) }, + operatorToken: "", + setOperatorToken: vi.fn(), + clearOperatorToken: vi.fn(), + controlToken: "", + setControlToken: vi.fn(), + clearControlToken: vi.fn(), +}; + +function snapshot(overrides: Partial = {}): RecordingSnapshot { + return { + batchId: "b1", + episodePhase: "IDLE", + currentEpisodeId: null, + currentEpisodeElapsedMs: null, + taskText: "", + taskDetails: "", + defaultOperator: null, + autoFillOperatorName: false, + savedCount: 0, + lastResult: null, + readyToRecord: true, + revision: 0, + ...overrides, + }; +} + +const CATALOG: TaskCatalog = { + fileName: "Kitchen.csv", + tasks: [ + { name: "wash_cup", instruction: "Wash the cup in the sink." }, + { name: "pour_water", instruction: "Pour water into the cup." }, + ], + preselected: { name: "pour_water", instruction: "Pour water into the cup." }, +}; + +// The operator-name field is also a textbox, so the task editor's input is found by its own class. +const taskInput = () => document.querySelector(".recording-task-editor-input") as HTMLInputElement; + +describe("RecordingPanel task catalog picker", () => { + it("offers the catalog's tasks and fills the text with the picked instruction", async () => { + render( CATALOG} />); + fireEvent.click(screen.getByRole("button", { name: /edit task \(t\)/i })); + const picker = (await screen.findByLabelText("Task from Kitchen.csv")) as HTMLSelectElement; + expect(picker.value).toBe("-1"); // the existing text matches no catalog task + expect(taskInput().value).toBe("Old task"); // an existing task is never overwritten by the preselection + fireEvent.change(picker, { target: { value: "0" } }); + expect(taskInput().value).toBe("Wash the cup in the sink."); + expect(picker.value).toBe("0"); + // Editing the wording keeps the text and drops the picker back to "custom". + fireEvent.change(taskInput(), { target: { value: "Wash the cup in the sink, then dry it." } }); + expect((screen.getByLabelText("Task from Kitchen.csv") as HTMLSelectElement).value).toBe("-1"); + }); + + it("preselects the standing task into an empty draft, ready to save with Enter", async () => { + const onSetTask = vi.fn().mockResolvedValue({ outcome: "ok", reason: null }); + render( CATALOG} />); + fireEvent.click(screen.getByRole("button", { name: /edit task \(t\)/i })); + await screen.findByLabelText("Task from Kitchen.csv"); + await waitFor(() => expect(taskInput().value).toBe("Pour water into the cup.")); + fireEvent.keyDown(taskInput(), { key: "Enter" }); + expect(onSetTask).toHaveBeenCalledWith({ batchId: "b1" }, "Pour water into the cup.", ""); + }); + + it("stays a plain text editor when no catalog is configured or the catalog cannot be loaded", async () => { + const { unmount } = render( null} />); + fireEvent.click(screen.getByRole("button", { name: /edit task \(t\)/i })); + expect(taskInput()).toBeTruthy(); + await Promise.resolve(); + expect(screen.queryByRole("combobox")).toBeNull(); + unmount(); + render( + { + throw new Error("backend down"); + }} + />, + ); + fireEvent.click(screen.getByRole("button", { name: /edit task \(t\)/i })); + await Promise.resolve(); + expect(screen.queryByRole("combobox")).toBeNull(); + expect(taskInput()).toBeTruthy(); + }); + + it("fetches the catalog each time the editor opens, so a catalog imported meanwhile is seen", async () => { + const load = vi.fn(async () => CATALOG); + render(); + fireEvent.click(screen.getByRole("button", { name: /edit task \(t\)/i })); + await screen.findByLabelText("Task from Kitchen.csv"); + fireEvent.click(screen.getByRole("button", { name: "Cancel" })); + fireEvent.click(screen.getByRole("button", { name: /edit task \(t\)/i })); + await screen.findByLabelText("Task from Kitchen.csv"); + expect(load).toHaveBeenCalledTimes(2); + }); +}); diff --git a/examples/egocap/frontend/src/RecordingPanel.test.tsx b/examples/egocap/frontend/src/RecordingPanel.test.tsx index 42a6437a17..bc5a2eb903 100644 --- a/examples/egocap/frontend/src/RecordingPanel.test.tsx +++ b/examples/egocap/frontend/src/RecordingPanel.test.tsx @@ -800,3 +800,21 @@ describe("RecordingPanel", () => { expect((screen.getByLabelText("Operator") as HTMLInputElement).value).toBe(""); }); }); + +describe("Settings owns recording shortcuts", () => { + it.each(["IDLE", "RECORDING"] as const)("blocks commands with %s behind the modal, then restores shortcuts", (episodePhase) => { + const command = vi.fn().mockResolvedValue({ outcome: "ok", reason: null }); + const actions = { onSetTask: command, onStartEpisode: command, onStopEpisode: command, onDiscardEpisode: command, onNewBatch: command }; + const recordingSnapshot = activeSnapshot({ episodePhase, currentEpisodeId: episodePhase === "RECORDING" ? "e1" : null }); + const { rerender } = render(<>); + screen.getByRole("button", { name: "Settings tab" }).focus(); + for (const key of ["s", "w", "a", "r", "t"]) fireEvent.keyDown(window, { key }); + (document.activeElement as HTMLElement).blur(); + for (const key of ["s", "w", "a", "r", "t"]) fireEvent.keyDown(window, { key }); + expect(command).not.toHaveBeenCalled(); + expect(document.querySelector(".recording-task-editor-input")).toBeNull(); + rerender(); + fireEvent.keyDown(window, { key: episodePhase === "RECORDING" ? "w" : "s" }); + expect(command).toHaveBeenCalledTimes(1); + }); +}); diff --git a/examples/egocap/frontend/src/RecordingPanel.tsx b/examples/egocap/frontend/src/RecordingPanel.tsx index 546e165bf8..92aec1031e 100644 --- a/examples/egocap/frontend/src/RecordingPanel.tsx +++ b/examples/egocap/frontend/src/RecordingPanel.tsx @@ -28,6 +28,7 @@ import { INITIAL_RECORDING_UI_STATE, recordingUiReducer, type CommandPhase, type import { RecordingCommandUncertainError } from "./useCaptureSession"; import { useVoiceRecognition, type VoiceRecognitionStatus } from "./useVoiceRecognition"; import { parseVoicePhrase } from "./voiceCommands"; +import { loadTaskCatalog as defaultLoadTaskCatalog, useTaskCatalog, type LoadTaskCatalog, type TaskCatalog } from "./taskCatalog"; import type { RecordingActions, RecordingBatchIdentity, @@ -68,6 +69,10 @@ export interface RecordingPanelProps { /** Told whenever this panel's own interaction phase changes (including the in-flight "stopping"/"starting" ones the * backend's snapshot does not show until the command finishes), so the page can mirror it in its status bar. */ onPhaseChange?: (phase: RecordingPhase) => void; + /** How the task editor's catalog is fetched; tests inject a fake, the page uses the backend. */ + loadTaskCatalog?: LoadTaskCatalog; + /** Settings owns keyboard and voice commands while its modal is open. */ + interactionBlocked?: boolean; } const NO_CONTROL_TOKEN_REASON = "Enter the recording control token before using recording controls"; @@ -131,6 +136,8 @@ export function RecordingPanel({ setControlToken, clearControlToken, onPhaseChange, + loadTaskCatalog = defaultLoadTaskCatalog, + interactionBlocked = false, }: RecordingPanelProps) { const [ui, dispatch] = useReducer(recordingUiReducer, INITIAL_RECORDING_UI_STATE); useEffect(() => { @@ -355,11 +362,16 @@ export function RecordingPanel({ // unreachable in practice; kept as a cheap defensive backstop. const utteranceTargetRef = useRef<{ batchId: string | null; episodeId: string | null } | "invalid" | null>(null); const handleUtteranceStart = useCallback(() => { + if (interactionBlocked) return; utteranceTargetRef.current = utteranceTargetRef.current !== null ? "invalid" : { batchId: recordingSnapshotRef.current?.batchId ?? null, episodeId: recordingSnapshotRef.current?.currentEpisodeId ?? null }; - }, []); + }, [interactionBlocked]); + // Opening Settings retires an utterance that might otherwise finish after the dialog closes. + useEffect(() => { + if (interactionBlocked) utteranceTargetRef.current = null; + }, [interactionBlocked]); // A recognizer SESSION boundary (initial enable, or any restart-after-end/disable/error) always // clears a leftover unfinished binding -- without this, an utterance that began but never // reached a final result before the session ended (e.g. Space disabled voice mid-utterance) @@ -389,6 +401,10 @@ export function RecordingPanel({ const handleVoiceResult = useCallback( (rawText: string) => { + if (interactionBlocked) { + utteranceTargetRef.current = null; + return; + } const intent = parseVoicePhrase(rawText); if (intent?.kind === "stopListening") { @@ -527,7 +543,7 @@ export function RecordingPanel({ // (useVoiceRecognition needs this callback to construct it), but its identity is stable // across renders (see useVoiceRecognition's own useCallback), so omitting it from these deps // never leaves a stale closure. - [eligibility, batchId, handleStart, handleStopSave, handleDiscard, submitTaskEdit, cancelTaskEdit], + [interactionBlocked, eligibility, batchId, handleStart, handleStopSave, handleDiscard, submitTaskEdit, cancelTaskEdit], ); const { status: voiceStatus, enabled: voiceEnabled, toggle: voiceToggle, errorMessage: voiceErrorMessage } = useVoiceRecognition(handleVoiceResult, { @@ -550,6 +566,8 @@ export function RecordingPanel({ // browser -- there is no toggle to flip and no reason shown, so // swallowing Space here would be pure interference. useEffect(() => { + // Modal ownership does not depend on where the browser left focus. + if (interactionBlocked) return; function onKeyDown(event: KeyboardEvent) { if (event.key !== " " || event.repeat || event.altKey || event.ctrlKey || event.metaKey) return; if (voiceStatus === "unsupported") return; @@ -559,7 +577,7 @@ export function RecordingPanel({ } window.addEventListener("keydown", onKeyDown); return () => window.removeEventListener("keydown", onKeyDown); - }, [voiceToggle, voiceStatus]); + }, [interactionBlocked, voiceToggle, voiceStatus]); // Buttons and shortcuts call the same guarded handlers above; this @@ -569,6 +587,8 @@ export function RecordingPanel({ // editor owns Enter/Escape itself, so this listener is skipped entirely // while it is open. useEffect(() => { + // Modal ownership does not depend on where the browser left focus. + if (interactionBlocked) return; function onKeyDown(event: KeyboardEvent) { if (event.repeat || event.altKey || event.ctrlKey || event.metaKey) return; if (ui.phase === "editingTask" || ui.phase === "savingTask" || focusOwnedByInput()) return; @@ -595,7 +615,7 @@ export function RecordingPanel({ } window.addEventListener("keydown", onKeyDown); return () => window.removeEventListener("keydown", onKeyDown); - }, [ui.phase, handleStart, handleStopSave, handleDiscard, beginTaskEdit, handleNewBatch]); + }, [interactionBlocked, ui.phase, handleStart, handleStopSave, handleDiscard, beginTaskEdit, handleNewBatch]); // Live elapsed = the backend's own monotonic figure as of its snapshot, // plus local time elapsed since THIS browser received that snapshot -- @@ -608,6 +628,15 @@ export function RecordingPanel({ : (recordingSnapshot?.lastResult?.durationMs ?? null); const taskEditBatchChanged = ui.taskEditBatchId !== null && batchId !== null && ui.taskEditBatchId !== batchId; const editingTask = ui.phase === "editingTask" || ui.phase === "savingTask"; + // The operator's catalog, fetched when the editor opens. A preselected task fills an empty draft once, so a + // station with a standing task needs one keypress (Enter) instead of a retype; an existing task is left alone. + const taskCatalog = useTaskCatalog(editingTask, loadTaskCatalog); + useEffect(() => { + if (ui.phase !== "editingTask" || taskCatalog?.preselected == null || ui.taskDraft !== "") return; + dispatch({ type: "UPDATE_TASK_DRAFT", value: taskCatalog.preselected.instruction }); + // Only the arrival of the catalog or the opening of the editor should prefill; typing must not re-trigger it. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [taskCatalog, ui.phase]); // `ui.phase` is this component's own interaction state machine, which // has no RECOVERY_REQUIRED value of its own -- with no command in // flight it reads "idle" even while the backend has a genuinely @@ -806,6 +835,7 @@ export function RecordingPanel({ : null } batchChanged={taskEditBatchChanged} + catalog={taskCatalog} onChange={(value) => dispatch({ type: "UPDATE_TASK_DRAFT", value })} onDetailsChange={(value) => dispatch({ type: "UPDATE_DETAILS_DRAFT", value })} onSubmit={submitTaskEdit} @@ -859,6 +889,7 @@ function InlineTaskEditor({ pending, saveBlockedReason, batchChanged, + catalog, onChange, onDetailsChange, onSubmit, @@ -872,6 +903,8 @@ function InlineTaskEditor({ * different operator actions to fix. */ saveBlockedReason: string | null; batchChanged: boolean; + /** The operator's chosen catalog, offered as a picker above the text; null keeps the editor free text. */ + catalog: TaskCatalog | null; onChange: (value: string) => void; onDetailsChange: (value: string) => void; onSubmit: () => void; @@ -889,8 +922,29 @@ function InlineTaskEditor({ onCancel(); } }; + // The picker fills the text field rather than replacing it: an operator can still adjust the wording. + const pickedIndex = catalog === null ? -1 : catalog.tasks.findIndex((task) => task.instruction === draft); return ( + {catalog !== null && catalog.tasks.length > 0 && ( + + )} { expect(voiceFeedbackText()).toMatch(/voice command target is unknown or changed/i); }); }); + +describe("Settings owns voice commands", () => { + it("blocks Space under the modal and restores it after close", () => { + const props = { ...defaultOperatorProps, recordingSnapshot: activeSnapshot() }; + const { rerender } = render(); + fireEvent.keyDown(window, { key: " " }); + expect(FakeRecognition.instances).toHaveLength(0); + rerender(); + fireEvent.keyDown(window, { key: " " }); + expect(FakeRecognition.instances).toHaveLength(1); + }); + + it("ignores voice results behind Settings and retires utterances crossing its open boundary", () => { + const onStartEpisode = vi.fn().mockResolvedValue({ outcome: "ok", reason: null }); + const props = { ...defaultOperatorProps, recordingSnapshot: activeSnapshot(), actions: { onStartEpisode } }; + const { rerender } = render(); + const recognizer = enableVoice(); + act(() => recognizer.emitStart()); + act(() => recognizer.emitInterim("record")); + rerender(); + act(() => recognizer.finalizeLast("record")); + act(() => recognizer.speak("record")); + act(() => recognizer.emitInterim("record")); + expect(onStartEpisode).not.toHaveBeenCalled(); + rerender(); + act(() => recognizer.finalizeLast("record")); + expect(onStartEpisode).not.toHaveBeenCalled(); + act(() => recognizer.speak("record")); + expect(onStartEpisode).toHaveBeenCalledTimes(1); + }); +}); diff --git a/examples/egocap/frontend/src/SettingsPanel.test.tsx b/examples/egocap/frontend/src/SettingsPanel.test.tsx new file mode 100644 index 0000000000..63ed35870e --- /dev/null +++ b/examples/egocap/frontend/src/SettingsPanel.test.tsx @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +/** The Settings dialog: tabs, the token-gated station/task changes, catalog import and removal, tools, display + * and the read-only deployment tab. */ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen, within, waitFor } from "@testing-library/react"; +import { SettingsPanel, type SettingsBackend, type StationSetup } from "./SettingsPanel"; +import { SettingsHttpError, type OperatorSettings, type TaskFileListing } from "./settingsApi"; +import { KITCHEN_FIXTURE, OPERATOR_SETTINGS_FIXTURE, SETTINGS_FIXTURE, TASK_FILES_FIXTURE, TOOLS_FIXTURE } from "./settingsTestDoubles"; + +afterEach(cleanup); + +const MID_TRAIN = { name: "Mid.csv", format: "mid_train_csv", tasks: [{ name: "fold_towel", instruction: "Fold a towel" }] }; + +/** A fake backend over in-memory state, so saves show up on the next load like the real one. */ +function fakeBackend(initial: Partial = {}): SettingsBackend & { operator: OperatorSettings; listing: TaskFileListing } { + const state = { + operator: initial.operator ?? OPERATOR_SETTINGS_FIXTURE, + listing: initial.taskFiles ?? TASK_FILES_FIXTURE, + tools: initial.tools ?? TOOLS_FIXTURE, + }; + return { + get operator() { + return state.operator; + }, + get listing() { + return state.listing; + }, + loadDeployment: vi.fn(async () => SETTINGS_FIXTURE), + loadStationSetup: vi.fn(async () => ({ operator: state.operator, tools: state.tools, taskFiles: state.listing })), + loadTaskFile: vi.fn(async (name: string) => (name === "Mid.csv" ? MID_TRAIN : KITCHEN_FIXTURE)), + saveOperator: vi.fn(async (_token: string, changes) => { + state.operator = { ...state.operator, ...changes }; + return state.operator; + }), + importTaskFile: vi.fn(async (_token: string, source) => { + const name = source.name; + state.listing = { files: [...new Set([...state.listing.files, name])].sort(), imported: [...state.listing.imported, name] }; + return { ...state.listing, file: { ...MID_TRAIN, name } }; + }), + removeTaskFile: vi.fn(async (_token: string, name: string) => { + state.listing = { files: state.listing.files.filter((f) => f !== name), imported: state.listing.imported.filter((f) => f !== name) }; + return state.listing; + }), + }; +} + +function renderPanel(overrides: Partial[0]> = {}, backend = fakeBackend()) { + const props = { + open: true, + onClose: vi.fn(), + sidebarWidth: 460, + sidebarDefault: 460, + onResetSidebar: vi.fn(), + controlToken: "tok" as string | null, + backend, + ...overrides, + }; + const utils = render(); + return { ...utils, props, backend }; +} + +const tab = (name: string) => screen.getByRole("button", { name }); + +describe("SettingsPanel shell", () => { + it("renders nothing while closed and does not fetch", () => { + const backend = fakeBackend(); + renderPanel({ open: false }, backend); + expect(screen.queryByRole("dialog")).toBeNull(); + expect(backend.loadDeployment).not.toHaveBeenCalled(); + expect(backend.loadStationSetup).not.toHaveBeenCalled(); + }); + + it("opens on the Station tab, switches tabs, and closes on Escape or Close", async () => { + const { props } = renderPanel(); + expect(await screen.findByText("Camera type")).toBeTruthy(); + fireEvent.click(tab("Display")); + expect(screen.getByText("Keyboard shortcuts")).toBeTruthy(); + expect(tab("Display").getAttribute("aria-current")).toBe("page"); + fireEvent.keyDown(window, { key: "Escape" }); + fireEvent.click(screen.getByRole("button", { name: "Close" })); + expect(props.onClose).toHaveBeenCalledTimes(2); + }); +}); + +describe("Station tab", () => { + it("saves the camera type with the control token, then has nothing left to save", async () => { + const { backend } = renderPanel(); + const station = await screen.findByRole("region", { name: "Camera type" }); + const save = within(station).getByRole("button", { name: "Save camera type" }) as HTMLButtonElement; + expect(save.disabled).toBe(true); + fireEvent.click(within(station).getByLabelText(/OAK-D cameras/)); + expect(save.disabled).toBe(false); + fireEvent.click(save); + expect(backend.saveOperator).toHaveBeenCalledWith("tok", { camera_type: "oak_d" }); + expect(((await within(station).findByRole("button", { name: "Save camera type" })) as HTMLButtonElement).disabled).toBe(true); + expect((within(station).getByLabelText(/OAK-D cameras/) as HTMLInputElement).checked).toBe(true); + }); + + it("cannot save without the control token and says where to enter it", async () => { + const { backend } = renderPanel({ controlToken: null }); + const station = await screen.findByRole("region", { name: "Camera type" }); + fireEvent.click(within(station).getByLabelText(/SENSING GMSL cameras/)); + const save = within(station).getByRole("button", { name: "Save camera type" }) as HTMLButtonElement; + expect(save.disabled).toBe(true); + expect(within(station).getByText(/Enter the control token in the recording panel/)).toBeTruthy(); + fireEvent.click(save); + expect(backend.saveOperator).not.toHaveBeenCalled(); + }); + + it("shows a rejected save in place and keeps the unsaved choice", async () => { + const backend = fakeBackend(); + backend.saveOperator = vi.fn(async () => { + throw new SettingsHttpError("control token rejected"); + }); + renderPanel({}, backend); + const station = await screen.findByRole("region", { name: "Camera type" }); + fireEvent.click(within(station).getByLabelText(/OAK-D cameras/)); + fireEvent.click(within(station).getByRole("button", { name: "Save camera type" })); + expect(await within(station).findByText(/Not saved: control token rejected/)).toBeTruthy(); + expect((within(station).getByLabelText(/OAK-D cameras/) as HTMLInputElement).checked).toBe(true); + }); + + it("assigns the session's cameras to roles, refuses one camera in two roles, swaps wrists, and saves only the roles", async () => { + const { backend } = renderPanel({ cameraSources: ["Ego", "Wrist A", "Wrist B"] }); + const roles = await screen.findByRole("region", { name: "Camera roles" }); + const ego = within(roles).getByLabelText("Ego (head)") as HTMLSelectElement; + const left = within(roles).getByLabelText("Left wrist") as HTMLSelectElement; + const right = within(roles).getByLabelText("Right wrist") as HTMLSelectElement; + const save = within(roles).getByRole("button", { name: "Save camera roles" }) as HTMLButtonElement; + expect(save.disabled).toBe(true); + fireEvent.change(ego, { target: { value: "Ego" } }); + fireEvent.change(left, { target: { value: "Wrist A" } }); + // A camera already used by another role cannot be picked again. + expect((within(right).getByRole("option", { name: "Wrist A" }) as HTMLOptionElement).disabled).toBe(true); + fireEvent.change(right, { target: { value: "Wrist B" } }); + fireEvent.click(within(roles).getByRole("button", { name: "Swap wrists" })); + expect(left.value).toBe("Wrist B"); + expect(right.value).toBe("Wrist A"); + fireEvent.click(save); + expect(backend.saveOperator).toHaveBeenCalledWith("tok", { camera_roles: { ego: "Ego", wrist_left: "Wrist B", wrist_right: "Wrist A" } }); + expect(((await within(roles).findByRole("button", { name: "Save camera roles" })) as HTMLButtonElement).disabled).toBe(true); + expect(screen.getByLabelText("Station readiness").textContent).toContain("3/3 roles assigned"); + }); + + it("keeps a saved camera the session no longer offers selectable, marked, so it can be cleared", async () => { + renderPanel( + { cameraSources: ["Ego"] }, + fakeBackend({ operator: { ...OPERATOR_SETTINGS_FIXTURE, camera_roles: { ego: "Ego", wrist_left: "Old wrist", wrist_right: null } } }), + ); + const roles = await screen.findByRole("region", { name: "Camera roles" }); + const left = within(roles).getByLabelText("Left wrist") as HTMLSelectElement; + expect(left.value).toBe("Old wrist"); + expect(within(left).getByRole("option", { name: "Old wrist (not in this session)" })).toBeTruthy(); + expect(screen.getByLabelText("Station readiness").textContent).toContain("1/2 assigned cameras in this session"); + }); +}); + +describe("Tasks tab", () => { + it("lists the catalogs, lets the operator choose the one the task editor offers, and shows its preselect card", async () => { + const { backend } = renderPanel(); + await screen.findByText("Camera type"); + fireEvent.click(tab("Tasks")); + const catalogs = await screen.findByRole("list", { name: "Task catalogs" }); + expect(within(catalogs).getByText("Kitchen.csv")).toBeTruthy(); + expect(screen.queryByRole("region", { name: "Preselected task" })).toBeNull(); + fireEvent.click(within(catalogs).getByLabelText("Use Kitchen.csv in the task editor")); + expect(backend.saveOperator).toHaveBeenCalledWith("tok", { task_file: "Kitchen.csv", preselected_task: null }); + const preselect = await screen.findByRole("region", { name: "Preselected task" }); + expect(await within(preselect).findByRole("option", { name: /wash_cup/ })).toBeTruthy(); + fireEvent.change(within(preselect).getByLabelText("Preselected task"), { target: { value: "pour_water" } }); + fireEvent.click(within(preselect).getByRole("button", { name: "Save preselected task" })); + expect(backend.saveOperator).toHaveBeenLastCalledWith("tok", { preselected_task: "pour_water" }); + expect(await within(catalogs).findByText("in task editor")).toBeTruthy(); + }); + + it("imports catalogs by file, then lists it as imported and reports its task count", async () => { + const { backend } = renderPanel(); + await screen.findByText("Camera type"); + fireEvent.click(tab("Tasks")); + const add = await screen.findByRole("region", { name: "Add a catalog" }); + const mid = new File(["Task Name,Task Description\nfold_towel,Fold a towel\n"], "Mid.csv", { type: "text/csv" }); + fireEvent.change(within(add).getByLabelText("Choose a CSV file"), { target: { files: [mid] } }); + expect(await within(add).findByText("Imported Mid.csv: 1 task.")).toBeTruthy(); + const catalogs = screen.getByRole("list", { name: "Task catalogs" }); + expect(within(catalogs).getByText("Mid.csv")).toBeTruthy(); + expect(within(catalogs).getByText("imported")).toBeTruthy(); + // The file picker uploads the chosen file's text. + const file = new File(["Task Name,Task Description\nfold_towel,Fold a towel\n"], "Garage.csv", { type: "text/csv" }); + fireEvent.change(within(add).getByLabelText("Choose a CSV file"), { target: { files: [file] } }); + expect(await within(add).findByText("Imported Garage.csv: 1 task.")).toBeTruthy(); + expect(backend.importTaskFile).toHaveBeenLastCalledWith("tok", { name: "Garage.csv", content: "Task Name,Task Description\nfold_towel,Fold a towel\n" }); + }); + + it("shows import errors and requires deselection before removing an imported catalog", async () => { + const backend = fakeBackend({ + operator: { ...OPERATOR_SETTINGS_FIXTURE, task_file: "Mid.csv" }, + taskFiles: { files: ["Kitchen.csv", "Mid.csv"], imported: ["Mid.csv"] }, + }); + backend.importTaskFile = vi.fn(async () => { + throw new SettingsHttpError("Bad.csv is neither a studio catalog nor a Mid Train file"); + }); + renderPanel({}, backend); + await screen.findByText("Camera type"); + fireEvent.click(tab("Tasks")); + const add = await screen.findByRole("region", { name: "Add a catalog" }); + const bad = new File(["bad"], "Bad.csv", { type: "text/csv" }); + fireEvent.change(within(add).getByLabelText("Choose a CSV file"), { target: { files: [bad] } }); + expect(await within(add).findByText(/Not imported: Bad.csv is neither/)).toBeTruthy(); + const catalogs = screen.getByRole("list", { name: "Task catalogs" }); + expect(within(catalogs).getAllByRole("button", { name: "Remove" })).toHaveLength(1); // only the import + expect(await screen.findByRole("region", { name: "Preselected task" })).toBeTruthy(); + expect((within(catalogs).getByRole("button", { name: "Remove" }) as HTMLButtonElement).disabled).toBe(true); + fireEvent.click(screen.getByRole("button", { name: "Free text only" })); + await waitFor(() => expect((within(catalogs).getByRole("button", { name: "Remove" }) as HTMLButtonElement).disabled).toBe(false)); + fireEvent.click(within(catalogs).getByRole("button", { name: "Remove" })); + expect(backend.removeTaskFile).toHaveBeenCalledWith("tok", "Mid.csv"); + await within(catalogs).findByText("Kitchen.csv"); + expect(within(catalogs).queryByText("Mid.csv")).toBeNull(); + expect(screen.queryByRole("region", { name: "Preselected task" })).toBeNull(); + }); + + it("is read-only without the control token", async () => { + renderPanel({ controlToken: null }); + await screen.findByText("Camera type"); + fireEvent.click(tab("Tasks")); + const catalogs = await screen.findByRole("list", { name: "Task catalogs" }); + expect((within(catalogs).getByLabelText("Use Kitchen.csv in the task editor") as HTMLInputElement).disabled).toBe(true); + expect((screen.getByLabelText("Choose a CSV file") as HTMLInputElement).disabled).toBe(true); + expect(screen.getAllByText(/Enter the control token in the recording panel/).length).toBeGreaterThan(0); + }); +}); + +describe("Tools tab", () => { + it("lists every tool disabled with the backend's reason, and scopes fisheye rectification to SENSING stations", async () => { + renderPanel({}, fakeBackend({ operator: { ...OPERATOR_SETTINGS_FIXTURE, camera_type: "oak_d" } })); + await screen.findByText("Camera type"); + fireEvent.click(tab("Tools")); + const tools = await screen.findByRole("region", { name: "Operator tools" }); + const buttons = within(tools).getAllByRole("button", { name: "Open" }) as HTMLButtonElement[]; + expect(buttons).toHaveLength(3); + expect(buttons.every((button) => button.disabled)).toBe(true); + expect(within(tools).getAllByText(/not wired in this build/)).toHaveLength(3); + expect(within(tools).getByText("Fisheye rectification")).toBeTruthy(); + expect(within(tools).getByText(/Only fisheye lenses need this/)).toBeTruthy(); + }); +}); + +describe("Display tab", () => { + it("offers a side-panel reset only when the width differs from the default, and lists the shortcuts", async () => { + const { props, unmount } = renderPanel({ sidebarWidth: 600 }); + await screen.findByText("Camera type"); + fireEvent.click(tab("Display")); + expect(screen.getByText("600 px")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "Reset side panel width" })); + expect(props.onResetSidebar).toHaveBeenCalledTimes(1); + expect(screen.getByText("Stop and save")).toBeTruthy(); + unmount(); + renderPanel({ sidebarWidth: 460, initialTab: "display" }); + expect(screen.getByText("460 px (default)")).toBeTruthy(); + expect((screen.getByRole("button", { name: "Reset side panel width" }) as HTMLButtonElement).disabled).toBe(true); + }); +}); + +describe("Advanced tab", () => { + it("shows the deployment settings grouped, with the variables they come from", async () => { + renderPanel({ initialTab: "advanced" }); + expect(await screen.findByText("ws://127.0.0.1:8082")).toBeTruthy(); + expect(screen.getByText("head, controllers, hands")).toBeTruthy(); + expect(screen.getByText("head_human_camera (human)")).toBeTruthy(); + expect(screen.getByText("not configured")).toBeTruthy(); + expect(screen.queryByText("LiveKit credentials")).toBeNull(); + expect(screen.getByText("127.0.0.1:8085")).toBeTruthy(); + expect(screen.getByText(/EGOCAP_CAPTURE_SERVER_URL/)).toBeTruthy(); + }); + + it("reports a station file error and a failed load in place", async () => { + const backend = fakeBackend(); + backend.loadDeployment = vi.fn(async () => ({ + ...SETTINGS_FIXTURE, + station: { ...SETTINGS_FIXTURE.station, path: "/etc/egocap/stations.yaml", error: "station configuration is invalid; see backend logs" }, + })); + const { unmount } = renderPanel({ initialTab: "advanced" }, backend); + expect(await screen.findByText("/etc/egocap/stations.yaml")).toBeTruthy(); + expect(screen.getByText(/station configuration is invalid/)).toBeTruthy(); + unmount(); + const failing = fakeBackend(); + failing.loadDeployment = vi.fn(async () => { + throw new SettingsHttpError("settings are not resolved yet"); + }); + renderPanel({ initialTab: "advanced" }, failing); + expect(await screen.findByText(/Deployment settings unavailable: settings are not resolved yet/)).toBeTruthy(); + }); +}); + +describe("Deployment credential", () => { + it("waits for a token and reloads deployment settings when one is entered", async () => { + const { backend, props, rerender } = renderPanel({ controlToken: null, initialTab: "advanced" }); + expect(screen.getByText(/to view deployment settings/)).toBeTruthy(); + expect(backend.loadDeployment).not.toHaveBeenCalled(); + rerender(); + await waitFor(() => expect(backend.loadDeployment).toHaveBeenCalledWith(expect.any(AbortSignal), "new-token")); + expect(await screen.findByRole("region", { name: "Capture" })).toBeTruthy(); + }); +}); diff --git a/examples/egocap/frontend/src/SettingsPanel.tsx b/examples/egocap/frontend/src/SettingsPanel.tsx new file mode 100644 index 0000000000..5c39bfb5ab --- /dev/null +++ b/examples/egocap/frontend/src/SettingsPanel.tsx @@ -0,0 +1,754 @@ +// SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * The Settings dialog: a tabbed modal over the page. + * + * Tabs, in the order an operator sets up a station: Station (camera type and roles), Tasks (the catalogs the task picker + * offers, imports, and the preselected task), Tools (calibration and rectification), Display (this browser's + * choices and the keyboard shortcuts), Advanced (what the deployment is running with, read-only). Station and Tasks + * are saved by the backend for every browser and need the run's control token to change, as recording + * commands do; Tools lists unavailable stubs; Display is this browser's only; Advanced values are set where the backend is launched. + */ +import { useCallback, useEffect, useRef, useState, type ReactNode } from "react"; +import { + CAMERA_TYPE_LABELS, + ROLE_LABELS, + fetchOperatorSettings, + fetchSettings, + fetchTaskFile, + fetchTaskFiles, + fetchTools, + importTaskFile, + removeTaskFile, + saveOperatorSettings, + type EffectiveSettings, + type OperatorSettings, + type OperatorSettingsChange, + type TaskFileContents, + type TaskFileListing, + type TaskFileSource, + type ToolInfo, +} from "./settingsApi"; + +export interface StationSetup { + operator: OperatorSettings; + tools: ToolInfo[]; + taskFiles: TaskFileListing; +} + +/** The backend calls the dialog makes; tests inject fakes, the page uses the real ones. */ +export interface SettingsBackend { + loadDeployment: (signal: AbortSignal, controlToken?: string | null) => Promise; + loadStationSetup: (signal: AbortSignal) => Promise; + loadTaskFile: (name: string, signal: AbortSignal) => Promise; + saveOperator: (controlToken: string, changes: OperatorSettingsChange) => Promise; + importTaskFile: (controlToken: string, source: TaskFileSource) => Promise; + removeTaskFile: (controlToken: string, name: string) => Promise; +} + +export const REAL_SETTINGS_BACKEND: SettingsBackend = { + loadDeployment: fetchSettings, + loadStationSetup: async (signal) => { + const [operator, tools, taskFiles] = await Promise.all([fetchOperatorSettings(signal), fetchTools(signal), fetchTaskFiles(signal)]); + return { operator, tools, taskFiles }; + }, + loadTaskFile: fetchTaskFile, + saveOperator: saveOperatorSettings, + importTaskFile, + removeTaskFile, +}; + +export type SettingsTab = "station" | "tasks" | "tools" | "display" | "advanced"; + +const TABS: ReadonlyArray<{ id: SettingsTab; label: string }> = [ + { id: "station", label: "Station" }, + { id: "tasks", label: "Tasks" }, + { id: "tools", label: "Tools" }, + { id: "display", label: "Display" }, + { id: "advanced", label: "Advanced" }, +]; + +export interface SettingsPanelProps { + open: boolean; + onClose: () => void; + /** The current side-panel width and its default, for the Display tab. */ + sidebarWidth: number; + sidebarDefault: number; + onResetSidebar: () => void; + /** The run's control token once the operator has entered it in the recording panel; null until then. */ + controlToken: string | null; + /** The cameras the current session offers, by name. */ + cameraSources?: string[]; + initialTab?: SettingsTab; + backend?: SettingsBackend; +} + +type Loaded = { kind: "loading" } | { kind: "error"; message: string } | { kind: "loaded"; value: T }; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** Loads `load` each time the dialog opens, so a backend restart shows its new values; abandons a load the + * dialog closed on. */ +function useLoaded(open: boolean, load: (signal: AbortSignal) => Promise): [Loaded, (update: (value: T) => T) => void] { + const [state, setState] = useState>({ kind: "loading" }); + useEffect(() => { + if (!open) return; + const controller = new AbortController(); + setState({ kind: "loading" }); + load(controller.signal).then( + (value) => setState({ kind: "loaded", value }), + (error: unknown) => { + if (!controller.signal.aborted) setState({ kind: "error", message: errorMessage(error) }); + }, + ); + return () => controller.abort(); + }, [open, load]); + const update = (change: (value: T) => T) => setState((current) => (current.kind === "loaded" ? { kind: "loaded", value: change(current.value) } : current)); + return [state, update]; +} + +/** A dash for an absent value, so the row still shows the setting exists. */ +function orDash(value: string | number | null | undefined): string { + return value === null || value === undefined || value === "" ? "—" : String(value); +} + +function onOff(value: boolean): string { + return value ? "On" : "Off"; +} + +function list(values: readonly string[]): string { + return values.length === 0 ? "none" : values.join(", "); +} + +function Row({ label, value, mono = false }: { label: string; value: string; mono?: boolean }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +/** One titled block of a tab. */ +function Card({ title, lead, children }: { title: string; lead?: string; children: ReactNode }) { + return ( +
+

{title}

+ {lead !== undefined &&

{lead}

} + {children} +
+ ); +} + +function TokenHint({ controlToken }: { controlToken: string | null }) { + if (controlToken !== null) return null; + return

Enter the control token in the recording panel to change these.

; +} + +/** The pending/failed state of one save-type action, shown next to its button. */ +function useAction(): [{ busy: boolean; failure: string | null }, (work: () => Promise) => void] { + const [busy, setBusy] = useState(false); + const [failure, setFailure] = useState(null); + const run = (work: () => Promise) => { + setBusy(true); + setFailure(null); + work().then( + () => setBusy(false), + (error: unknown) => { + setBusy(false); + setFailure(errorMessage(error)); + }, + ); + }; + return [{ busy, failure }, run]; +} + +// --- Station --------------------------------------------------------------------------------------------------- + +function StationTab({ + setup, + controlToken, + backend, + cameraSources, + onOperator, +}: { + setup: Loaded; + controlToken: string | null; + backend: SettingsBackend; + cameraSources: string[]; + onOperator: (operator: OperatorSettings) => void; +}) { + if (setup.kind !== "loaded") return ; + const { operator } = setup.value; + return ( + <> + + + + + + + + + + ); +} + +/** The station's readiness at a glance: what is set, what is still to do. */ +function ReadyLine({ operator, cameraSources }: { operator: OperatorSettings; cameraSources: string[] }) { + const assigned = operator.roles.filter((role) => operator.camera_roles[role] !== null); + const offered = assigned.filter((role) => cameraSources.includes(operator.camera_roles[role] ?? "")); + const items: Array<[boolean, string]> = [ + [operator.camera_type !== null, operator.camera_type === null ? "Camera type not set" : `Camera type: ${CAMERA_TYPE_LABELS[operator.camera_type]?.split(" — ")[0] ?? operator.camera_type}`], + [assigned.length === operator.roles.length, `${assigned.length}/${operator.roles.length} roles assigned`], + [assigned.length > 0 && offered.length === assigned.length, cameraSources.length === 0 ? "No cameras offered by the session yet" : `${offered.length}/${assigned.length} assigned cameras in this session`], + [operator.task_file !== null, operator.task_file === null ? "No task catalog chosen" : `Task catalog: ${operator.task_file}`], + ]; + return ( +

+ {items.map(([ok, text]) => ( + + {ok ? "✓" : "○"} {text} + + ))} +

+ ); +} + +/** One select per role over the cameras the session offers; a camera already saved for a role that the session + * no longer offers stays selectable, so an operator can see (and clear) it. */ +function CameraRoles({ + operator, + cameraSources, + controlToken, + save, + onSaved, +}: { + operator: OperatorSettings; + cameraSources: string[]; + controlToken: string | null; + save: SettingsBackend["saveOperator"]; + onSaved: (operator: OperatorSettings) => void; +}) { + const [roles, setRoles] = useState>({ ...operator.camera_roles }); + const [{ busy, failure }, run] = useAction(); + const dirty = operator.roles.some((role) => roles[role] !== operator.camera_roles[role]); + const options = [...new Set([...cameraSources, ...Object.values(roles).filter((name): name is string => name !== null)])]; + const taken = (role: string) => new Set(operator.roles.filter((other) => other !== role).map((other) => roles[other])); + const swapWrists = () => setRoles((current) => ({ ...current, wrist_left: current.wrist_right ?? null, wrist_right: current.wrist_left ?? null })); + return ( + <> + {options.length === 0 &&

The session offers no cameras yet; roles can be assigned once CaptureServer lists them.

} +
+ {operator.roles.map((role) => ( + + + + ))} +
+
+ + + {failure !== null && Not saved: {failure}} +
+ + + ); +} + +function FragmentRow({ role, children }: { role: string; children: ReactNode }) { + return ( + <> + + {children} + + ); +} + +function CameraTypeChoice({ + operator, + controlToken, + save, + onSaved, +}: { + operator: OperatorSettings; + controlToken: string | null; + save: SettingsBackend["saveOperator"]; + onSaved: (operator: OperatorSettings) => void; +}) { + const [choice, setChoice] = useState(operator.camera_type); + const [{ busy, failure }, run] = useAction(); + const dirty = choice !== operator.camera_type; + return ( +
+ Choose the camera type + {operator.camera_types.map((type) => ( + + ))} + +
+ + {failure !== null && Not saved: {failure}} +
+
+ ); +} + +// --- Tasks ----------------------------------------------------------------------------------------------------- + +function TasksTab({ + setup, + controlToken, + backend, + onSetup, +}: { + setup: Loaded; + controlToken: string | null; + backend: SettingsBackend; + onSetup: (update: (value: StationSetup) => StationSetup) => void; +}) { + if (setup.kind !== "loaded") return ; + const { operator, taskFiles } = setup.value; + const canChange = controlToken !== null; + return ( + <> + + + controlToken !== null && + backend.saveOperator(controlToken, { task_file: name, preselected_task: null }).then((saved) => onSetup((value) => ({ ...value, operator: saved }))) + } + onRemove={(name) => + controlToken !== null && + backend.removeTaskFile(controlToken, name).then((listing) => + onSetup((value) => ({ + ...value, + taskFiles: listing, + // A removed catalog cannot stay the picker's source. + operator: value.operator.task_file === name ? { ...value.operator, task_file: null, preselected_task: null } : value.operator, + })), + ) + } + /> + + + + (controlToken === null ? Promise.reject(new Error("no control token")) : backend.importTaskFile(controlToken, source))} + onImported={(result) => onSetup((value) => ({ ...value, taskFiles: { files: result.files, imported: result.imported } }))} + /> + + {operator.task_file !== null && ( + + (controlToken === null ? Promise.resolve() : backend.saveOperator(controlToken, { preselected_task: task }).then((saved) => onSetup((value) => ({ ...value, operator: saved }))))} + /> + + )} + + ); +} + +function CatalogList({ + listing, + chosen, + canChange, + onChoose, + onRemove, +}: { + listing: TaskFileListing; + chosen: string | null; + canChange: boolean; + onChoose: (name: string | null) => false | Promise; + onRemove: (name: string) => false | Promise; +}) { + const [{ busy, failure }, run] = useAction(); + if (listing.files.length === 0) return

No catalogs yet. Add one below.

; + const imported = new Set(listing.imported); + return ( + <> +
    + {listing.files.map((name) => ( +
  • + + {imported.has(name) && ( + + )} +
    • + ))} +
+
+ + {failure !== null && {failure}} +
+ + ); +} + +function ImportCatalog({ + canChange, + doImport, + onImported, +}: { + canChange: boolean; + doImport: (source: TaskFileSource) => Promise; + onImported: (result: TaskFileListing & { file: TaskFileContents }) => void; +}) { + const [done, setDone] = useState(null); + const [{ busy, failure }, run] = useAction(); + const fileInput = useRef(null); + const finish = (result: TaskFileListing & { file: TaskFileContents }) => { + onImported(result); + setDone(`Imported ${result.file.name}: ${result.file.tasks.length} task${result.file.tasks.length === 1 ? "" : "s"}.`); + }; + return ( + <> +
+ { + const file = e.target.files?.[0]; + if (file === undefined) return; + run(() => file.text().then((content) => doImport({ name: file.name, content })).then(finish)); + e.target.value = ""; + }} + /> + +
+ {done !== null && failure === null &&

{done}

} + {failure !== null &&

Not imported: {failure}

} + {!canChange &&

Enter the control token in the recording panel to import.

} + + ); +} + +function PreselectedTask({ + fileName, + current, + canChange, + loadTaskFile, + save, +}: { + fileName: string; + current: string | null; + canChange: boolean; + loadTaskFile: SettingsBackend["loadTaskFile"]; + save: (task: string | null) => Promise; +}) { + const [file, setFile] = useState>({ kind: "loading" }); + const [choice, setChoice] = useState(current); + const [{ busy, failure }, run] = useAction(); + useEffect(() => { + const controller = new AbortController(); + loadTaskFile(fileName, controller.signal).then( + (value) => setFile({ kind: "loaded", value }), + (error: unknown) => { + if (!controller.signal.aborted) setFile({ kind: "error", message: errorMessage(error) }); + }, + ); + return () => controller.abort(); + }, [fileName, loadTaskFile]); + if (file.kind !== "loaded") return ; + const known = file.value.tasks.some((task) => task.name === current); + return ( + <> + + {current !== null && !known &&

The saved task "{current}" is not in {fileName} any more.

} +
+ + {failure !== null && Not saved: {failure}} +
+ + ); +} + +// --- Tools ----------------------------------------------------------------------------------------------------- + +function ToolsTab({ setup }: { setup: Loaded }) { + if (setup.kind !== "loaded") return ; + const { tools, operator } = setup.value; + return ( + +
    + {tools.map((tool) => { + const forOtherCameras = tool.applies_to !== null && operator.camera_type !== null && tool.applies_to !== operator.camera_type; + const reason = !tool.available ? tool.unavailable_reason : forOtherCameras ? `for ${CAMERA_TYPE_LABELS[tool.applies_to ?? ""]?.split(" — ")[0] ?? tool.applies_to} stations` : null; + return ( +
  • +
    + {tool.name} + + {tool.description} + {reason !== null && ` — ${reason}.`} + +
    + +
    • + ); + })} +
+ + ); +} + +// --- Display --------------------------------------------------------------------------------------------------- + +const SHORTCUTS: ReadonlyArray<[string, string]> = [ + ["s", "Record"], + ["w", "Stop and save"], + ["a", "Discard"], + ["t", "Edit task"], + ["r", "New batch"], + ["Space", "Voice commands on/off"], + ["Enter / Esc", "Save / cancel the task editor"], +]; + +function DisplayTab({ sidebarWidth, sidebarDefault, onResetSidebar }: { sidebarWidth: number; sidebarDefault: number; onResetSidebar: () => void }) { + return ( + <> + +
+ +
+
+ + Drag the divider between the cameras and the side panel to change it. +
+ + +
+ {SHORTCUTS.map(([key, action]) => ( +
+
{action}
+
+ {key} +
+
+ ))} +
+ + + ); +} + +// --- Advanced -------------------------------------------------------------------------------------------------- + +function AdvancedTab({ deployment }: { deployment: Loaded }) { + if (deployment.kind !== "loaded") return ; + const { capture, preview, station, recording, visualization, recordings_browser, task_files, origins } = deployment.value; + const viewAddress = visualization.host === null ? null : `${visualization.host}:${orDash(visualization.port)}`; + return ( + <> +

Set where the backend is launched (environment or station file); a change applies on restart.

+ +
+ + + + + + `${camera.name} (${camera.type})`))} /> + + +
+ + +
+ + {station.error !== null && } + + + + + +
+ + +
+ + + + +
+ + +
+ + {visualization.error !== null && } + + + + + +
+ + +
+ + +
+ + + ); +} + +// --- Shell ----------------------------------------------------------------------------------------------------- + +function LoadState({ state, what }: { state: Loaded; what: string }) { + if (state.kind === "loading") return

Loading…

; + if (state.kind === "error") return

{what} unavailable: {state.message}

; + return null; +} + +export function SettingsPanel({ + open, + onClose, + sidebarWidth, + sidebarDefault, + onResetSidebar, + controlToken, + cameraSources = [], + initialTab = "station", + backend = REAL_SETTINGS_BACKEND, +}: SettingsPanelProps) { + const [tab, setTab] = useState(initialTab); + const loadDeployment = useCallback((signal: AbortSignal) => backend.loadDeployment(signal, controlToken), [backend, controlToken]); + const [deployment] = useLoaded(open && controlToken !== null, loadDeployment); + const [setup, updateSetup] = useLoaded(open, backend.loadStationSetup); + + // Escape closes, as for any dialog. + useEffect(() => { + if (!open) return; + const onKey = (event: KeyboardEvent) => { + if (event.key === "Escape") onClose(); + }; + window.addEventListener("keydown", onKey); + return () => window.removeEventListener("keydown", onKey); + }, [open, onClose]); + + if (!open) return null; + + const setOperator = (operator: OperatorSettings) => updateSetup((value) => ({ ...value, operator })); + + return ( + <> +