diff --git a/examples/egocap/AGENTS.md b/examples/egocap/AGENTS.md index 9555ed4913..12bd65d9ae 100644 --- a/examples/egocap/AGENTS.md +++ b/examples/egocap/AGENTS.md @@ -72,3 +72,8 @@ Run all three test suites per "Run the tests yourself" above — check each pack - Bound mutation request streams before JSON parsing, even when Content-Length is absent or false; return only deliberate public error messages. - Dialogs must move, contain and restore focus as well as suppress recording shortcuts. - Concurrent coverage runs must use separate `COVERAGE_FILE` paths so one suite cannot overwrite another suite’s measurements. +## Derived output never touches the raw recording + +An aligned or otherwise derived copy of a recording must live in its own tree and read the raw session +read-only. Check that the output path is not the raw session, inside it or containing it (resolving links) +before any file is written. An existing derived episode is refused, never replaced. diff --git a/examples/egocap/LIVEKIT_PREVIEW.md b/examples/egocap/LIVEKIT_PREVIEW.md index d4f829df6a..26d2a795da 100644 --- a/examples/egocap/LIVEKIT_PREVIEW.md +++ b/examples/egocap/LIVEKIT_PREVIEW.md @@ -37,6 +37,9 @@ livekit: [streams file reference](capture_server/README.md#the---livekit-streams-file). Preview works with any of them. Recording needs a monotonic basis, `monotonic_us` or `capture_monotonic_us`, and `width`, `height` and `fps`; recording is refused while any offered stream lacks them. +- Automatic time alignment of a kept episode needs the stream's time to be the sensor's capture event, so use + `capture_monotonic_us`. With `monotonic_us`, which is when the frame was ready, the take is recorded but not + aligned. Start CaptureServer with the file (add `--mock` for synthetic tracking): diff --git a/examples/egocap/README.md b/examples/egocap/README.md index 37d07401f0..c89d3ffde5 100644 --- a/examples/egocap/README.md +++ b/examples/egocap/README.md @@ -56,7 +56,11 @@ pytest fixture; the rest of `tests/backend` needs no Node/npm. Three processes, each in its own terminal, from `egocap/`: ```bash -# 1. A standalone mock CaptureServer -- synthetic tracking and camera frames, no hardware +# 1. A standalone mock CaptureServer -- synthetic tracking and camera frames, no hardware. +# This starts with no synthetic delay (--controller-delay-ms/--glove-delay-ms/--delay-jitter-ms add +# one), and its cameras are plain CaptureServer JPEG streams, which carry no camera clock evidence: +# a recorded take is kept but automatic time alignment refuses it. Aligned output needs the cameras +# published through LiveKit (see LIVEKIT_PREVIEW.md). uv run capture-server --mock # 2. The backend, connecting to CaptureServer and serving both frontends -- note the diff --git a/examples/egocap/backend/egocap_web/app.py b/examples/egocap/backend/egocap_web/app.py index e8072fb891..66bf679dbb 100644 --- a/examples/egocap/backend/egocap_web/app.py +++ b/examples/egocap/backend/egocap_web/app.py @@ -8,12 +8,14 @@ from __future__ import annotations +import asyncio import logging import os import secrets import sys import time import uuid +from collections.abc import Callable from contextlib import asynccontextmanager from dataclasses import replace from pathlib import Path @@ -32,13 +34,19 @@ VideoSourceSpec, ) +from .auto_align import AutoAligner from .capture_preview import MAX_CAMERA_JPEG_BYTES from .capture_status import CaptureStatusMonitor from .config import VisualizationConfig, config_from_env, visualization_from_env from .livekit_preview import API_KEY_ENV, API_SECRET_ENV, subscriber_token from .livekit_preview import router as livekit_preview_router from .livekit_receiver import RECEIVER_FRAMES_PER_STREAM -from .livekit_recording import LiveKitRecordingSource, recordable_livekit_streams, unrecordable_reason +from .livekit_recording import ( + LiveKitRecordingSource, + recordable_livekit_streams, + sensor_capture_names, + unrecordable_reason, +) from .operator_settings import OperatorSettingsStore, operator_settings_path from .operator_settings import router as operator_settings_router from .preview_routes import router as preview_router @@ -91,6 +99,8 @@ # LiveKit stream's decoded frame when that is larger. _INGRESS_QUEUE_BYTES = 64 * 1024 * 1024 _INGRESS_QUEUE_CAPACITY = 4096 +#: How long shutdown waits for queued time alignments before leaving them for the next start. +_ALIGNER_SHUTDOWN_SECONDS = 30.0 # LiveKit frames are decoded RGB (megabytes each, ~15 MB at 2560x1984), so # the JPEG allowance above holds only a few and a short consumer stall would # fail the take. Each recordable stream therefore adds this many of its own @@ -159,7 +169,12 @@ def _recorder_livekit_token(room: str) -> str: return subscriber_token(api_key=api_key, signing_key=signing_key, room=room, now=int(time.time())) -def _recording_config_factory(monitor: CaptureStatusMonitor, station_label: str | None, output_root: Path): +def _recording_config_factory( + monitor: CaptureStatusMonitor, + station_label: str | None, + output_root: Path, + on_video_sources: Callable[[frozenset[str], frozenset[str]], None] | None = None, +): """Builds the real ``ServiceConfig`` lazily, read fresh each time the coordinator actually opens/reopens a session (not once at import time) -- so it reflects whichever cameras CaptureServer has ACTUALLY accepted @@ -181,6 +196,13 @@ def factory() -> ServiceConfig: VideoSourceSpec(s.name, s.width, s.height, s.fps, intrinsics=False) for s in recordable_livekit_streams(monitor.native.started) ) + if on_video_sources is not None: + # Tells the aligner, once per session open, which cameras this session records and which of them + # have a sensor capture time (only LiveKit streams stamped that way do). + on_video_sources( + frozenset(v.name for v in video_sources), + sensor_capture_names(recordable_livekit_streams(monitor.native.started)), + ) # A supported profile must record actual tracking AND the # selected cameras -- pico_controller and every configured camera # are required for a KEPT result. manus_glove is NOT required: @@ -393,15 +415,28 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app.state.imported_task_file_root = operator_settings_file.parent / "taskfiles" monitor = CaptureStatusMonitor(config, backend_instance_id=str(uuid.uuid4())) livekit_source = LiveKitRecordingSource( - lambda: recordable_livekit_streams(monitor.native.started), _recorder_livekit_token + lambda: recordable_livekit_streams(monitor.native.started), + _recorder_livekit_token, + ) + # Kept episodes are time-aligned in the background when the station configures an index camera. + aligner = ( + AutoAligner(recording_output_root, resolved_station.alignment_index) + if resolved_station is not None and resolved_station.alignment_index is not None + else None ) coordinator = RecordingSessionCoordinator( - _recording_config_factory(monitor, config.station_label, recording_output_root), + _recording_config_factory( + monitor, + config.station_label, + recording_output_root, + aligner.note_video_sources if aligner is not None else None, + ), queue_capacity=_INGRESS_QUEUE_CAPACITY, queue_bytes=lambda: _ingress_queue_bytes(monitor.native.started), readiness_check=_recording_readiness_check(monitor, station_config_error, config.capture_server_url), initial_context=initial_context, livekit_source=livekit_source, + tracking_latency=resolved_station.latency if resolved_station is not None else None, ) monitor.set_recording(coordinator) await monitor.start() @@ -435,6 +470,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: visualization_error=visualization_error, task_file_root=task_file_root, ) + # Kept episodes are time-aligned in the background when the station configures an index camera. + aligner: AutoAligner | None = None + aligner_task: asyncio.Task[None] | None = None + if aligner is not None: + coordinator.on_episode_kept = aligner.submit + aligner_task = asyncio.create_task(aligner.run()) visualization: VisualizationScene | None = None try: visualization = _optional_visualization(monitor, visualization_config, visualization_error) @@ -451,6 +492,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # retire any in-flight take cleanly. An unreleased close is # surfaced, not silently discarded. close_result = await coordinator.close(final=True) + if aligner is not None and aligner_task is not None: + # After the recorder closed, so its last kept episode is queued; bounded so shutdown is too. + await aligner.close(aligner_task, _ALIGNER_SHUTDOWN_SECONDS) if not close_result.released: logger.error("recording coordinator did not fully release on shutdown: %s", close_result.issues) await monitor.stop() diff --git a/examples/egocap/backend/egocap_web/auto_align.py b/examples/egocap/backend/egocap_web/auto_align.py new file mode 100644 index 0000000000..28e967dc89 --- /dev/null +++ b/examples/egocap/backend/egocap_web/auto_align.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Time-aligns each kept episode in the background, right after its take is finalized. + +One worker handles episodes in the order they were kept. Each alignment runs as a low-priority child process (the +same ``recording/tools/align_episode.py`` an operator can run by hand), so its CPU and memory never compete with a +live take inside the recorder's own process, and a shutdown can stop it. An episode that cannot be aligned is +logged with the reason and left as recorded; the raw episode is never modified. Output is published by one rename, +so an alignment that is killed leaves no partial result. At startup the worker re-queues the kept episodes that +still have no aligned copy. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import shutil +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +from egocap.recording.aligned_writer import episode_qualification +from egocap.recording.episode_layout import ( + FINALIZATION_KEEP, + MANIFEST_DIR, + destination_path, + read_manifest, +) +from recording_service import Disposition, EpisodeResult + +logger = logging.getLogger("isaaccapture.egocap_web.auto_align") + +#: The command-line aligner, run once per episode. +_ALIGN_TOOL = Path(__file__).resolve().parents[2] / "recording" / "tools" / "align_episode.py" +#: A staging directory untouched this long belongs to a run that is no longer going. +_STALE_STAGING_SECONDS = 24 * 3600 + + +@dataclass(frozen=True) +class _Job: + """One kept episode to align: its recording session, its index there, and where it was kept.""" + + session_id: str + episode_index: int + final: Path + + @property + def aligned(self) -> Path: + """Where this episode's aligned copy goes, beside its group.""" + return self.final.parent.with_name(f"{self.final.parent.name}_aligned") / self.final.name + + @property + def refused_marker(self) -> Path: + """A hidden file in the aligned session recording that this episode was refused; deleting it retries.""" + return self.aligned.parent / f".{self.final.name}.refused" + + +class AutoAligner: + """Aligns kept episodes one at a time on its own worker task.""" + + def __init__(self, output_root: Path, index_source: str) -> None: + """``output_root`` holds the recording sessions; ``index_source`` is the camera that sets the timeline.""" + self._output_root = output_root + self._index_source = index_source + # Unbounded: a job is three small fields, and the number waiting is the number of kept episodes. + self._queue: asyncio.Queue[_Job] = asyncio.Queue() + self._process: asyncio.subprocess.Process | None = None + # False while the open session cannot be aligned; set once per session open. + self._session_alignable = True + + def note_video_sources(self, recorded: frozenset[str], qualified: frozenset[str]) -> None: + """Note whether the session being opened can be aligned; if not, its episodes are not queued. + + It can when the index camera is among the cameras recorded with a sensor capture time and every recorded + camera has one, since an episode with any other camera is refused whole. + """ + alignable = self._index_source in qualified and recorded <= qualified + if not alignable and self._session_alignable: + logger.warning( + "automatic time alignment is off: index source %r and every recorded camera need a sensor " + "capture time (recorded: %s; with one: %s)", + self._index_source, + sorted(recorded), + sorted(qualified), + ) + self._session_alignable = alignable + + def submit(self, result: EpisodeResult) -> None: + """Queue a finished take for alignment; anything other than a kept episode is ignored.""" + if result.disposition is Disposition.KEPT and result.final_path is not None and self._session_alignable: + self._queue.put_nowait(_Job(result.session_id, result.episode_index, result.final_path)) + + def scan(self) -> list[_Job]: + """Every kept episode, in any session, that has no aligned copy and whose evidence allows one. + + Skips episodes already refused (see ``_Job.refused_marker``), and an episode recorded without qualified + timing evidence (an older recording, or a station with no tracking delay set), so history is not retried at + every start. Also removes staging directories left by a run that was killed long ago. Blocking file work. + """ + jobs: list[_Job] = [] + groups: set[Path] = set() + for path in sorted(self._output_root.glob(f"*/{MANIFEST_DIR}/episode_*.json")): + try: + manifest = read_manifest(path) + session = path.parent.parent + final = ( + self._output_root / destination_path(manifest.assignment) + if manifest.assignment + else session / manifest.final_path + ) + job = _Job(manifest.session_id, manifest.episode_index, final) + groups.add(final.parent) + eligible = ( + manifest.storage_phase == "FINALIZED" + and manifest.finalization == FINALIZATION_KEEP + and not job.aligned.exists() + and not job.refused_marker.exists() + and all( + episode_qualification( + final.parent, int(final.name.removeprefix("episode_")), self._index_source + ).values() + ) + ) + except Exception: # noqa: BLE001, S112 - an unreadable record is simply not eligible + continue + if eligible: + jobs.append(job) + for group in groups: + for stale in group.parent.glob(f".{group.name}_aligned.staging-*"): + with contextlib.suppress(OSError): + if time.time() - stale.stat().st_mtime > _STALE_STAGING_SECONDS: + shutil.rmtree(stale, ignore_errors=True) + return jobs + + async def run(self) -> None: + """Re-queue unfinished work, then align queued episodes until cancelled; failures never stop the worker.""" + try: + for job in await asyncio.to_thread(self.scan): + self._queue.put_nowait(job) + except Exception as exc: # noqa: BLE001 - recovery is best effort; live episodes are still aligned + logger.warning("time-alignment recovery failed: %s", exc) + while True: + job = await self._queue.get() + try: + await self._align(job) + except Exception as exc: # noqa: BLE001 - one episode's failure must not end the worker + logger.warning("%s episode %d was not time-aligned: %s", job.session_id, job.episode_index, exc) + finally: + self._queue.task_done() + + async def close(self, task: asyncio.Task[None], timeout: float) -> None: + """Let the queued episodes finish for up to ``timeout`` seconds, then stop the worker ``task``. + + Call after the recorder has closed, so the last kept episode is already queued. An alignment still running + at the deadline is killed; it publishes only by one rename, so nothing partial appears, and the episode is + picked up again at the next start. + """ + try: + await asyncio.wait_for(self._queue.join(), timeout) + except asyncio.TimeoutError: + logger.warning("time alignment still has queued episodes at shutdown; they are retried at next start") + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + async def _align(self, job: _Job) -> None: + """Align one kept episode in a child process; raises with the reason when it is refused or fails.""" + if job.aligned.exists(): + return + # The manifest stays in the recording session; a routed episode sits elsewhere, under its slot. + command = [ + "nice", + "-n", + "10", + sys.executable, + str(_ALIGN_TOOL), + str(job.final.parent), + "--episode", + str(int(job.final.name.removeprefix("episode_"))), + "--index-source", + self._index_source, + "--manifest-session", + str(self._output_root / job.session_id), + "--manifest-episode", + str(job.episode_index), + ] + process = await asyncio.create_subprocess_exec( + *command, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE + ) + self._process = process + try: + _, stderr = await process.communicate() + except asyncio.CancelledError: + # Cancelled at shutdown: stop the child rather than leave it running past the recorder. + process.kill() + await process.wait() + raise + finally: + self._process = None + reason = stderr.decode(errors="replace").strip() + if process.returncode == 1: + # Exit 1 is a refusal, which is the same every time, so remember it instead of retrying at every start; + # any other failure (an unreadable manifest, a killed child) stays retryable. + job.refused_marker.parent.mkdir(parents=True, exist_ok=True) + job.refused_marker.write_text(reason + "\n", encoding="utf-8") + if process.returncode != 0: + raise RuntimeError(reason or f"aligner exited with status {process.returncode}") + logger.info("%s episode %d was time-aligned", job.session_id, job.episode_index) diff --git a/examples/egocap/backend/egocap_web/livekit_recording.py b/examples/egocap/backend/egocap_web/livekit_recording.py index f44b43d8de..6638ddfb1c 100644 --- a/examples/egocap/backend/egocap_web/livekit_recording.py +++ b/examples/egocap/backend/egocap_web/livekit_recording.py @@ -33,7 +33,7 @@ from recording_service import PixelFormat, VideoSample from .livekit_receiver import LIVEKIT_MISSING, LiveKitFrameReceiver, livekit_available -from .recording_adapter import RecordingIngress +from .recording_adapter import MANUS_GLOVE_SOURCE, PICO_CONTROLLER_SOURCE, RecordingIngress logger = logging.getLogger("isaaccapture.egocap.livekit_recording") @@ -64,13 +64,38 @@ # What each tracking source's saved time means today; none is a physical acquisition time. _TRACKING_SEMANTICS: dict[str, str] = { - "recorded_time": "the host's poll time for every tracking source", + "recorded_time": "the host's poll time for every tracking source, less a configured latency where one is set", "controllers": "device value is the pose-evaluation time (a predicted XrTime), not an acquisition time", "manus_gloves": "device value is host monotonic time at push; the SDK publishTime is not used", "physical_alignment": "unavailable: no tracking source carries a mapped device measurement time", } +def sensor_capture_names(streams: Sequence[p.LiveKitCameraStream]) -> frozenset[str]: + """The names of the recordable streams whose time is the sensor's capture event, which alignment requires.""" + return frozenset(s.name for s in streams if _RECORDABLE_BASES.get(s.timestamp_basis) == "sensor_capture") + + +def tracking_event_time(latency: Mapping[str, float]) -> dict[str, dict[str, Any]]: + """What each tracking source's saved time is, and whether it is a qualified event time. + + A source with a configured latency is saved as its host poll time minus that latency: an estimate, marked as + one with the latency used, never a measurement of the device. One without a latency is only the host poll + time and stays unqualified. + """ + return { + name: { + "meaning": "host_poll_minus_latency", + "estimated": True, + "latency_s": latency[name], + "qualified": True, + } + if name in latency + else {"meaning": "host_poll_time", "qualified": False} + for name in (PICO_CONTROLLER_SOURCE, MANUS_GLOVE_SOURCE) + } + + def uncertainty_budget_ns() -> int: """The largest clock-mapping uncertainty a take may be recorded with, from the environment or the default.""" raw = os.environ.get(BUDGET_ENV) @@ -283,6 +308,8 @@ async def _run(self, stream_id: int) -> None: pixel_format=PixelFormat.RGB8, capture_time=capture_time, receive_time=receive_time, + # The mapping that placed this frame, read once here: a later refresh must not relabel it. + continuity=("clock_mapping", mapping.epoch), ) if not self._ingress.offer(sample): # The ingress refuses once it has failed or stopped; it reports its own reason. @@ -507,8 +534,18 @@ def timing_evidence(self) -> dict[str, Any] | None: "exchanges_rejected": clock_sync.rejected, "mapping_current_at_stop": clock_sync.mapping() is not None, }, - # Tracking samples carry the host's poll time for every source; device measurement times are unmapped. - "tracking_time": "host_poll", + # Per recorded source: what its saved time means and whether it is a qualified event time. A camera is + # qualified only when its time is the sensor's capture event; a frame-ready time is not. + "event_time": { + # A stream is recorded under its own name (see start()), which is the key readers look it up by. + **{ + s.name: { + "meaning": _RECORDABLE_BASES[s.timestamp_basis], + "qualified": _RECORDABLE_BASES[s.timestamp_basis] == "sensor_capture", + } + for s in self._streams + }, + }, "tracking_semantics": _TRACKING_SEMANTICS, } diff --git a/examples/egocap/backend/egocap_web/recording_adapter.py b/examples/egocap/backend/egocap_web/recording_adapter.py index 8e19c9b769..f06e20dca4 100644 --- a/examples/egocap/backend/egocap_web/recording_adapter.py +++ b/examples/egocap/backend/egocap_web/recording_adapter.py @@ -32,7 +32,7 @@ import asyncio import contextlib -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Literal import numpy as np @@ -71,6 +71,7 @@ FieldSpec("x_button", "bool"), FieldSpec("y_button", "bool"), ), + "PicoTrackingFrame", ) MANUS_GLOVE_SPEC = NumericSourceSpec( MANUS_GLOVE_SOURCE, @@ -78,6 +79,7 @@ FieldSpec("left_hand_pose", "float64", (25, 4, 4)), FieldSpec("right_hand_pose", "float64", (25, 4, 4)), ), + "ManusFrame", ) @@ -323,10 +325,12 @@ def convert_camera_image( capture_time=image.capture_time_ns / 1e9, receive_time=image.receive_time_ns / 1e9, intrinsics=intrinsics, + # The transport session says the connection was unbroken; it says nothing about the camera behind it. + continuity=("capture_session", image.session_id), ) -def convert_capture_frame(frame: CaptureFrame) -> list[NumericSample]: +def convert_capture_frame(frame: CaptureFrame, latency: Mapping[str, float] | None = None) -> list[NumericSample]: """Decodes one raw tracking payload and converts it into zero, one or two ``NumericSample``s (``pico_controller``/``manus_glove``), each omitted entirely (not fabricated) when its own whole-row validity @@ -334,15 +338,21 @@ def convert_capture_frame(frame: CaptureFrame) -> list[NumericSample]: the descriptor once per attempt (``validate_tracking_descriptor``) -- this does not re-validate it per frame.""" tracking = capture_protocol.decode_tracking_frame(frame.payload) - capture_time = frame.capture_time_ns / 1e9 + poll_time = frame.capture_time_ns / 1e9 receive_time = frame.receive_time_ns / 1e9 + latency = latency or {} + # Host poll and connection continuity only: a device restart behind a live connection is not visible here. + continuity = ("host_clock", frame.session_id) samples: list[NumericSample] = [] pico_fields = build_pico_controller_fields(tracking) if pico_fields is not None: - samples.append(NumericSample(PICO_CONTROLLER_SOURCE, capture_time, receive_time, pico_fields)) + # A configured latency is taken off the poll time once, here: the saved time is the estimated event time. + capture_time = poll_time - latency.get(PICO_CONTROLLER_SOURCE, 0.0) + samples.append(NumericSample(PICO_CONTROLLER_SOURCE, capture_time, receive_time, pico_fields, continuity)) manus_fields = build_manus_glove_fields(tracking) if manus_fields is not None: - samples.append(NumericSample(MANUS_GLOVE_SOURCE, capture_time, receive_time, manus_fields)) + capture_time = poll_time - latency.get(MANUS_GLOVE_SOURCE, 0.0) + samples.append(NumericSample(MANUS_GLOVE_SOURCE, capture_time, receive_time, manus_fields, continuity)) return samples @@ -360,13 +370,15 @@ def _raw_item_bytes(raw_item: RawItem) -> int: return len(raw_item.jpeg) if isinstance(raw_item, CameraImage) else len(raw_item.payload) -def _convert_raw_item(raw_item: RawItem) -> list[NumericSample | VideoSample]: +def convert_raw_item( + raw_item: RawItem, latency: Mapping[str, float] | None = None +) -> list[NumericSample | VideoSample]: """The default, real conversion dispatch -- CPU-bound (JPEG decode, pose math), so ``RecordingIngress`` always runs this inside its bounded consumer (off the synchronous capture-callback path that calls ``offer()``), via a worker executor, never before the queue.""" if isinstance(raw_item, CaptureFrame): - return list(convert_capture_frame(raw_item)) + return list(convert_capture_frame(raw_item, latency)) if isinstance(raw_item, CameraImage): return [convert_camera_image(raw_item, raw_item.name)] if isinstance(raw_item, VideoSample): @@ -403,7 +415,7 @@ def __init__( submit: Callable[[NumericSample | VideoSample], object], queue_capacity: int, queue_bytes: int | None = None, - convert: Callable[[RawItem], list[NumericSample | VideoSample]] = _convert_raw_item, + convert: Callable[[RawItem], list[NumericSample | VideoSample]] = convert_raw_item, full_retry_deadline_seconds: float = 2.0, full_retry_backoff_seconds: float = 0.01, in_flight_timeout_seconds: float = 30.0, diff --git a/examples/egocap/backend/egocap_web/recording_session.py b/examples/egocap/backend/egocap_web/recording_session.py index 2b331e75da..2e051c9959 100644 --- a/examples/egocap/backend/egocap_web/recording_session.py +++ b/examples/egocap/backend/egocap_web/recording_session.py @@ -59,9 +59,11 @@ from typing import TYPE_CHECKING, Any, TypeVar from capture_server.client import CameraImage, CaptureFrame -from egocap_web.recording_adapter import RecordingIngress, validate_tracking_descriptor +from egocap_web.livekit_recording import tracking_event_time +from egocap_web.recording_adapter import RecordingIngress, convert_raw_item, validate_tracking_descriptor from recording_service import ( CloseResult, + Disposition, EpisodeInfo, EpisodeResult, PendingEpisode, @@ -187,11 +189,14 @@ def __init__( readiness_check: ReadinessCheck, initial_context: TaskContext | None = None, livekit_source: LiveKitRecordingSource | None = None, + tracking_latency: dict[str, float] | None = None, ) -> None: self._config_factory = config_factory # Optional LiveKit video for each take: joined before storage opens, fed after the ingress starts, left # before the ingress is retired. self._livekit_source = livekit_source + # The station's estimated delay per tracking source, applied to every take's tracking times. + self._tracking_latency = dict(tracking_latency or {}) #: The clock-mapping evidence of the take in progress, gathered when its video source stops. self._timing_evidence: dict[str, Any] | None = None self._queue_capacity = queue_capacity @@ -240,6 +245,8 @@ def __init__( # methods. None (the default) means no one is listening yet. self.on_episode_started: Callable[[], None] | None = None self.on_episode_result: Callable[[EpisodeResult], None] | None = None + #: Called with each episode finalized as kept, including one recovered after an interruption. + self.on_episode_kept: Callable[[EpisodeResult], None] | None = None self.on_batch_rotated: Callable[[], None] | None = None @property @@ -472,6 +479,7 @@ async def _start_episode_with_storage(self, frozen_context: TaskContext) -> Epis queue_bytes=self._queue_bytes() if callable(self._queue_bytes) else self._queue_bytes, submit=partial(service.submit_sample, episode_id), on_failure=lambda message: self._schedule_failure_settlement(service, episode_id, message), + convert=partial(convert_raw_item, latency=self._tracking_latency), ) await ingress.start() self._current_ingress = ingress @@ -555,6 +563,12 @@ def _emit_result(self, result: EpisodeResult) -> None: coordinator's own retained operation task.""" if self.on_episode_result is not None: self.on_episode_result(result) + self._emit_kept(result) + + def _emit_kept(self, result: EpisodeResult) -> None: + """Tell ``on_episode_kept`` about an episode that was kept.""" + if result.disposition is Disposition.KEPT and self.on_episode_kept is not None: + self.on_episode_kept(result) async def stop_episode(self, episode_id: str) -> EpisodeResult: service = self._require_open() @@ -646,11 +660,15 @@ async def _retire_current_ingress(self) -> _IngressRetirement: ingress = self._current_ingress if ingress is None: return _IngressRetirement(settled=True, failure=None) + # What the tracking times are, whatever media the take has: the same latency its conversion applied. + evidence: dict[str, Any] = {"event_time": tracking_event_time(self._tracking_latency)} + if self._livekit_source is not None: + # Read before the room is left: the clock mappings the take was recorded under, merged with the above. + media = self._livekit_source.timing_evidence() + if media is not None: + evidence = {**media, "event_time": {**media.get("event_time", {}), **evidence["event_time"]}} + self._timing_evidence = evidence if self._livekit_source is not None: - # Read before the room is left: the clock mappings the take was recorded under. - evidence = self._livekit_source.timing_evidence() - if evidence is not None: - self._timing_evidence = evidence # The producer stops first, so nothing is offered to an ingress that is draining. await self._livekit_source.stop() settled = await ingress.stop() @@ -674,7 +692,9 @@ def _require_matching_active_episode(self, service: _RecordingService, episode_i async def recover_episode(self, episode_id: str) -> EpisodeResult: service = self._require_open() - return await self._run_operation(OperationKind.RECOVERING, partial(service.recover_episode, episode_id)) + result = await self._run_operation(OperationKind.RECOVERING, partial(service.recover_episode, episode_id)) + self._emit_kept(result) + return result async def discard_pending(self, episode_id: str) -> EpisodeResult: service = self._require_open() diff --git a/examples/egocap/backend/egocap_web/station_config.py b/examples/egocap/backend/egocap_web/station_config.py index 53f873185b..71daeab774 100644 --- a/examples/egocap/backend/egocap_web/station_config.py +++ b/examples/egocap/backend/egocap_web/station_config.py @@ -17,8 +17,8 @@ Only the fields in :class:`ResolvedStation` are consumed; everything else in the YAML (hardware, visualization, or any other section) is read by nobody here and never passed downstream. The ``cameras`` field is itself a small ``name``/``type`` projection of ``video_sources`` -- the -fuller per-source declaration (``frame_type``, ``latency``, ``params``) is reference data for -other tooling, not consumed here. +fuller per-source declaration (``frame_type``, ``params``) is reference data for other tooling, not consumed +here; the one exception is the ``alignment`` section. """ from __future__ import annotations @@ -37,6 +37,8 @@ #: Matches capture_server.protocol's own BeginRequest.cameras limit -- checked here too so an #: over-limit YAML is a configuration error, not a later capture-startup failure. _MAX_CAMERAS = 3 +#: The tracking sources the recorder takes a latency for. +_TRACKING_SOURCES = ("pico_controller", "manus_glove") class StationConfigError(ValueError): @@ -57,6 +59,11 @@ class ResolvedStation: task_description: str | None data_dir: Path | None cameras: tuple[p.CameraSelection, ...] + #: Estimated seconds of device-to-host delay per tracking source, from ``alignment.tracking_latency_s``. Only + #: a source named there is recorded as a qualified estimate; the others cannot be time-aligned. + latency: dict[str, float] + #: The camera that sets the timeline for automatic time alignment of kept episodes; None turns it off. + alignment_index: str | None class _DuplicateKeyLoader(yaml.SafeLoader): @@ -130,6 +137,40 @@ def _resolve_cameras(station_id: str, entry: dict) -> tuple[p.CameraSelection, . return tuple(cameras) +def _resolve_latency(station_id: str, entry: dict) -> dict[str, float]: + """The finite, non-negative seconds in ``alignment.tracking_latency_s``, keyed by tracking source.""" + section = entry.get("alignment") or {} + if not isinstance(section, dict): + raise StationConfigError(f"station {station_id!r}: 'alignment' must be a mapping") + configured = section.get("tracking_latency_s") or {} + if not isinstance(configured, dict): + raise StationConfigError(f"station {station_id!r}: alignment.tracking_latency_s must be a mapping") + latency: dict[str, float] = {} + for name, seconds in configured.items(): + if name not in _TRACKING_SOURCES: + raise StationConfigError(f"station {station_id!r}: tracking_latency_s names unknown source {name!r}") + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or not 0 <= seconds < float("inf"): + raise StationConfigError( + f"station {station_id!r}: tracking_latency_s.{name} must be a finite number of seconds >= 0" + ) + latency[name] = float(seconds) + return latency + + +def _resolve_alignment_index(station_id: str, entry: dict) -> str | None: + """The index camera for automatic alignment: ``alignment.index_source``, which has no default. + + None (alignment off) when it is not set or ``alignment.enabled`` is false. It must name a camera the + station records with a sensor capture time, which is checked each time a session opens. + """ + section = entry.get("alignment") or {} + if not isinstance(section, dict): + raise StationConfigError(f"station {station_id!r}: 'alignment' must be a mapping") + if not _bool_field(section, "enabled", default=True): + return None + return _str_field(section, "index_source") or None + + def _resolve_one_station(station_id: str, entry: Any, config_dir: Path) -> ResolvedStation: """Build a :class:`ResolvedStation` from one station's parsed mapping.""" if not isinstance(entry, dict): @@ -162,6 +203,8 @@ def _resolve_one_station(station_id: str, entry: Any, config_dir: Path) -> Resol task_description=_str_field(meta, "task_description"), data_dir=data_dir, cameras=_resolve_cameras(station_id, entry), + latency=_resolve_latency(station_id, entry), + alignment_index=_resolve_alignment_index(station_id, entry), ) diff --git a/examples/egocap/capture_server/capture_server/launch.py b/examples/egocap/capture_server/capture_server/launch.py index 234ba35311..0a6a403685 100644 --- a/examples/egocap/capture_server/capture_server/launch.py +++ b/examples/egocap/capture_server/capture_server/launch.py @@ -125,6 +125,7 @@ def build_mock_source_factory( seed: int, ic_faults: set[str] | frozenset[str] = frozenset(), camera_faults: set[str] | frozenset[str] = frozenset(), + delays: dict[str, float] | None = None, ) -> SourceFactory: """The default ``CaptureServer`` source factory, wired to the two mock SDK files. ``ic_faults``/``camera_faults`` are captured here, once, @@ -141,7 +142,11 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . begin, sdk=mock_isaac_capture, session_factory=partial( - mock_isaac_capture.TeleopSession, scenario=scenario, seed=seed, faults=set(ic_faults) + mock_isaac_capture.TeleopSession, + scenario=scenario, + seed=seed, + faults=set(ic_faults), + **(delays or {}), ), ) @@ -409,6 +414,7 @@ def build_source_factory( scenario: str = "demo", seed: int = 0, faults: frozenset[str] = frozenset(), + delays: dict[str, float] | None = None, ) -> SourceFactory: """Select the acquisition implementation once, from ``config.mock``. @@ -422,5 +428,5 @@ def build_source_factory( if scenario not in SCENARIOS: raise ValueError(f"unknown mock scenario {scenario!r}; choose one of {', '.join(SCENARIOS)}") return build_mock_source_factory( - scenario=scenario, seed=seed, ic_faults=set(faults), camera_faults=set(faults) + scenario=scenario, seed=seed, ic_faults=set(faults), camera_faults=set(faults), delays=delays ) diff --git a/examples/egocap/capture_server/capture_server/mock/mock_isaac_capture.py b/examples/egocap/capture_server/capture_server/mock/mock_isaac_capture.py index 4eb74dfdbf..8029b29dd4 100644 --- a/examples/egocap/capture_server/capture_server/mock/mock_isaac_capture.py +++ b/examples/egocap/capture_server/capture_server/mock/mock_isaac_capture.py @@ -60,6 +60,7 @@ from __future__ import annotations import math +import random from dataclasses import dataclass, field from enum import Enum, IntEnum from functools import lru_cache @@ -682,9 +683,17 @@ def __init__( seed: int = 0, faults: set[str] | frozenset[str] | None = None, clock: Clock | None = None, + controller_delay_s: float = 0.0, + glove_delay_s: float = 0.0, + delay_jitter_s: float = 0.0, ) -> None: self.config = config self.frame_count = 0 + # Synthetic device-to-host delay: a source's content is what the operator was doing this long before the + # poll, plus seeded bounded jitter, so a configured latency can be checked against a known truth. + self._delays = {"controller": controller_delay_s, "glove": glove_delay_s} + self._jitter_s = delay_jitter_s + self._delay_rng = random.Random(seed) self._operator = _Operator(scenario=scenario, seed=seed) self._faults: set[str] = set(faults) if faults else set() self._clock = clock or MonotonicClock() @@ -749,6 +758,13 @@ def _elapsed(self) -> float: self._frozen_t = None return now - self._enter_t0 + def _delayed(self, t: float, source: str) -> float: + """The instant ``source``'s content is from: ``t`` less its synthetic delay and jitter, never before 0.""" + delay = self._delays[source] + # A source with no delay is read at the poll instant; jitter on it would only skew it late. + jitter = self._delay_rng.uniform(-self._jitter_s, self._jitter_s) if self._jitter_s and delay else 0.0 + return max(0.0, t - max(0.0, delay + jitter)) + def step( self, *, @@ -771,8 +787,12 @@ def step( raise RuntimeError("mock fatal_acquisition_error fault: acquisition worker exited") t = self._elapsed() - head_pose, left_hand, right_hand = self._operator.core(t) - inputs = self._operator.inputs(t) + head_pose, _, _ = self._operator.core(t) + # Each source reads the operator at its own delayed instant (the head is not delayed). + t_controller, t_glove = self._delayed(t, "controller"), self._delayed(t, "glove") + _, left_ctrl, right_ctrl = self._operator.core(t_controller) + _, left_hand, right_hand = self._operator.core(t_glove) + inputs = self._operator.inputs(t_controller) result: dict[str, OptionalTensorGroup] = {} for leaf_key, (source_node, selector) in self.config.pipeline.outputs.items(): @@ -786,7 +806,7 @@ def step( # all (which would simply not appear in this dict). result[leaf_key] = OptionalTensorGroup(len(ControllerInputIndex)) continue - hand = left_hand if side == "left" else right_hand + hand = left_ctrl if side == "left" else right_ctrl valid = "controller_pose_invalid" not in self._faults result[leaf_key] = self._controller_group(hand, inputs, side, valid=valid) elif isinstance(source_node, HandsSource): diff --git a/examples/egocap/capture_server/capture_server/server.py b/examples/egocap/capture_server/capture_server/server.py index 2c1ede6191..b710767a9e 100644 --- a/examples/egocap/capture_server/capture_server/server.py +++ b/examples/egocap/capture_server/capture_server/server.py @@ -2036,6 +2036,12 @@ def _build_arg_parser() -> argparse.ArgumentParser: parser.add_argument("--seed", type=int, default=_MOCK_DEFAULTS["seed"], help="mock only") parser.add_argument("--scenario", default=_MOCK_DEFAULTS["scenario"], help="mock only") parser.add_argument("--connect-delay", type=float, default=_MOCK_DEFAULTS["connect_delay"], help="mock only") + for flag, help_text in ( + ("controller-delay-ms", "controller content is this old at the poll"), + ("glove-delay-ms", "glove content is this old at the poll"), + ("delay-jitter-ms", "seeded +/- jitter on both delays"), + ): + parser.add_argument(f"--{flag}", type=float, default=0.0, help=f"mock only: synthetic delay, {help_text}") parser.add_argument( "--fault", action="append", @@ -2124,7 +2130,14 @@ def _station_cameras(args: argparse.Namespace) -> dict[str, StationCamera] | Non # Defaults of the mock-only CLI options; any other value with real acquisition is an error. -_MOCK_DEFAULTS = {"seed": 0, "scenario": "demo", "connect_delay": 0.0} +_MOCK_DEFAULTS = { + "seed": 0, + "scenario": "demo", + "connect_delay": 0.0, + "controller_delay_ms": 0.0, + "glove_delay_ms": 0.0, + "delay_jitter_ms": 0.0, +} def _mock_only_options_used(args: argparse.Namespace) -> list[str]: @@ -2245,7 +2258,15 @@ async def _run_cli(args: argparse.Namespace) -> int: ) # The one place the implementation is chosen, before any session is accepted. source_factory = build_source_factory( - config, scenario=args.scenario, seed=args.seed, faults=frozenset(args.fault) + config, + scenario=args.scenario, + seed=args.seed, + faults=frozenset(args.fault), + delays={ + "controller_delay_s": args.controller_delay_ms / 1000, + "glove_delay_s": args.glove_delay_ms / 1000, + "delay_jitter_s": args.delay_jitter_ms / 1000, + }, ) if args.livekit_streams is not None: from .livekit_streams import load_livekit_streams diff --git a/examples/egocap/recording/README.md b/examples/egocap/recording/README.md index 780e7707ff..29f4e9f26e 100644 --- a/examples/egocap/recording/README.md +++ b/examples/egocap/recording/README.md @@ -127,9 +127,9 @@ What the recording service writes into each routed group: ├── / │ └── / │ ├── zarr.json # dataset-level attrs -│ ├── config.yaml # {"meta": {"egocap_station_name": ...}} -- station identity -│ │ # only, NOT the fuller video_sources/numeric_sources/recording -│ │ # schema `core/config_io.py`'s default config illustrates +│ ├── config.yaml # meta.egocap_station_name plus the source declarations time +│ │ # alignment reads (video_sources/numeric_sources names and frame +│ │ # types, recording defaults) │ ├── head_camera_intrinsics.json, head_camera_extrinsics.json │ ├── left_wrist_camera_intrinsics.json, right_wrist_camera_intrinsics.json │ ├── wrist_calibration_offsets.json @@ -143,10 +143,10 @@ Calibration files appear only when present at `EGOCAP_CALIBRATION_DIR` (which al resolved value, see above -- absence there is a skip, not a configuration requirement); the service does not write a separate `data_check_config.yaml`. -A data-loading pipeline's source-config reader requires `video_sources`/`numeric_sources`, -source names, frame types/defaults, and latency to run its alignment step; the per-group -`config.yaml` above does not carry those fields, so this reader needs that configuration -supplied separately, not read from the recorded output. A Zarr-to-LeRobot converter reads fixed, +The per-group `config.yaml` carries the source declarations time alignment reads (the +`video_sources`/`numeric_sources` names, frame types and recording defaults); a data-loading +pipeline's source-config reader that also needs per-source latency or other settings must have +them supplied separately, not read from the recorded output. A Zarr-to-LeRobot converter reads fixed, hardcoded group paths instead: `pico_controller/{left,right,head}_pose_interp` and `manus_glove/{left,right}_hand_pose_interp`. Renaming a wire stream is harmless as long as the backend still maps it to the established recorded source name; renaming the recorded group diff --git a/examples/egocap/recording/egocap/analytics/count_daily_trajectories.py b/examples/egocap/recording/egocap/analytics/count_daily_trajectories.py index 3ee6432a8e..298708d679 100644 --- a/examples/egocap/recording/egocap/analytics/count_daily_trajectories.py +++ b/examples/egocap/recording/egocap/analytics/count_daily_trajectories.py @@ -161,10 +161,10 @@ def compute_station_data_breakdown( for session_dir in sorted(studio_dir.iterdir()): if not session_dir.is_dir(): continue - # Skip sibling ``_aligned`` folders the background aligner + # Skip sibling ``_aligned`` folders the background aligner (and its hidden staging folders) # writes — they are processed copies of the raw session, not # separately collected data (else each aligned episode double-counts). - if session_dir.name.endswith("_aligned"): + if session_dir.name.endswith("_aligned") or session_dir.name.startswith("."): continue if _read_session_station_name(session_dir) != station: continue @@ -223,10 +223,10 @@ def count_trajectories( for session_dir in sorted(studio_dir.iterdir()): if not session_dir.is_dir(): continue - # Skip sibling ``_aligned`` folders the background aligner + # Skip sibling ``_aligned`` folders the background aligner (and its hidden staging folders) # writes — they are processed copies of the raw session, not new # trajectories (else each aligned episode double-counts the raw one). - if session_dir.name.endswith("_aligned"): + if session_dir.name.endswith("_aligned") or session_dir.name.startswith("."): continue session_attrs = _read_zarr_json(session_dir / "zarr.json") diff --git a/examples/egocap/recording/egocap/data_processing/update_episode_task_description.py b/examples/egocap/recording/egocap/data_processing/update_episode_task_description.py index cc12591410..a7db6dfa12 100644 --- a/examples/egocap/recording/egocap/data_processing/update_episode_task_description.py +++ b/examples/egocap/recording/egocap/data_processing/update_episode_task_description.py @@ -108,6 +108,25 @@ class EpisodeTaskUpdateResult: config_yaml_updated: bool = False +def _update_aligned_copy( + aligned_root: Path, indices: list[int], task_description: str, task_name: str, studio: str +) -> None: + """Apply the same task, name and studio metadata to an aligned copy's root, episodes and ``config.yaml``.""" + fields = {"task_description": task_description} + if task_name: + fields["task_name"] = task_name + if studio: + fields["studio"] = studio + store = zarr.open(str(aligned_root), mode="a") + store.attrs.update(fields) + for n in indices: + if f"episode_{n}" in store: + store[f"episode_{n}"].attrs.update(fields) + _update_session_config_yaml( + aligned_root, task_description=task_description, task_name=task_name, studio=studio + ) + + def update_episode_task_description( zarr_root: Path, episodes: str, @@ -169,6 +188,10 @@ def update_episode_task_description( task_name=task_name, studio=studio_stored, ) + # The time-aligned copy beside the session carries the same metadata, so it is updated with it. + aligned_root = zarr_root.with_name(f"{zarr_root.name}_aligned") + if aligned_root.is_dir(): + _update_aligned_copy(aligned_root, indices, task_description, task_name, studio_stored) try: refresh_session_egocap_zarr_aggregates(zarr_root) diff --git a/examples/egocap/recording/egocap/eod/eod_studio_session_move.py b/examples/egocap/recording/egocap/eod/eod_studio_session_move.py index cb11c80b46..721ddb430d 100644 --- a/examples/egocap/recording/egocap/eod/eod_studio_session_move.py +++ b/examples/egocap/recording/egocap/eod/eod_studio_session_move.py @@ -82,7 +82,13 @@ def relocate_session_to_canonical_studio( if dest.exists(): raise OSError(f"EOD session relocate: destination already exists (refuse overwrite): {dest}") dest.parent.mkdir(parents=True, exist_ok=True) + aligned, aligned_dest = session.with_name(f"{session.name}_aligned"), dest.with_name(f"{dest.name}_aligned") + # The aligned copy follows its session; refuse before moving anything if it would collide. + if aligned.is_dir() and aligned_dest.exists(): + raise OSError(f"EOD session relocate: destination already exists (refuse overwrite): {aligned_dest}") shutil.move(str(session), str(dest)) + if aligned.is_dir(): + shutil.move(str(aligned), str(aligned_dest)) return dest diff --git a/examples/egocap/recording/egocap/recording/aligned_writer.py b/examples/egocap/recording/egocap/recording/aligned_writer.py new file mode 100644 index 0000000000..d3184821af --- /dev/null +++ b/examples/egocap/recording/egocap/recording/aligned_writer.py @@ -0,0 +1,385 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Writes a time-aligned copy of a recorded episode, in the aligned dataset layout downstream readers expect. + +The aligned session sits beside the raw one and mirrors it: the same ``episode_`` groups, the same per-source +groups, the same videos. What differs is the time alignment (see :mod:`alignment`): each episode gains +``valid_indices`` and ``valid_timestamps``; each camera gains ``aligned_frame_indices``; each resampled field is +written as ``_interp`` and its raw array is left out (readers use the ``_interp`` name); every other raw +array, the attributes, the session's ``config.yaml`` and sidecars, and the videos are carried over unchanged. The +aligned root's ``zarr.json`` records which source was the index. + +This module writes the files; the caller chooses the times, qualification and the staging that makes publication +all-or-nothing. +""" + +from __future__ import annotations + +import json +import os +import shutil +import tempfile +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import numpy as np +import yaml +import zarr + +from .alignment import AlignedEpisode, align_episode, check_continuity, qualified_sources, source_rules +from .episode_layout import ( + FINALIZATION_KEEP, + ROUTE_ATTR, + EpisodeManifest, + read_identity, + route_id, +) + +__all__ = [ + "ALIGNED_SESSION_SIDECARS", + "AlignedWriteError", + "align_kept_episode", + "check_kept_episode", + "publish_aligned_episode", + "validate_aligned_episode", + "write_aligned_episode", +] + +#: Session-level files carried into the aligned session when the raw session has them. +ALIGNED_SESSION_SIDECARS = ( + "config.yaml", + "data_check_config.yaml", + "head_camera_intrinsics.json", + "head_camera_extrinsics.json", + "wrist_calibration_offsets.json", + "left_wrist_camera_intrinsics.json", + "right_wrist_camera_intrinsics.json", + "rectification.json", +) +#: Video container suffixes that are carried into the aligned episode. +_VIDEO_SUFFIXES = (".mp4", ".mkv", ".avi", ".mov") +#: Arrays at most this large are read back by validation to catch stale or empty chunks. +_CHECK_MAX_BYTES = 10 * 1024 * 1024 + + +class AlignedWriteError(RuntimeError): + """The aligned episode could not be written or did not validate.""" + + +def _copy_attrs(source: Any, target: Any) -> None: + """Carry every attribute of a zarr group over to the aligned group.""" + for key, value in source.attrs.items(): + target.attrs[key] = value + + +def _link_or_copy(source: Path, target: Path) -> None: + """Put a video in the aligned episode: a hard link when the filesystem allows it, else a copy.""" + # A video from an earlier alignment is replaced, not merged with. + if os.path.lexists(target): + target.unlink() + try: + os.link(source, target) + except OSError: + shutil.copy2(source, target) + + +def _check_separate(raw_session: Path, aligned_session: Path) -> None: + """Refuse an output that is the raw session or shares a tree with it, before any file is touched. + + Re-aligning an episode deletes the aligned copy first; if that copy were the raw episode, the recording + would be destroyed. Paths are resolved, so a symlink to the raw session is refused too. + """ + raw, aligned = raw_session.resolve(), aligned_session.resolve() + if aligned == raw or raw in aligned.parents or aligned in raw.parents: + raise AlignedWriteError("the aligned session must be separate from the raw session") + + +def _write_root(raw_session: Path, aligned_session: Path, index_source: str) -> None: + """Create the aligned root: its attributes, the session sidecars, and the index source in ``zarr.json``.""" + aligned_session.mkdir(parents=True, exist_ok=True) + aligned_root = zarr.open_group(str(aligned_session), mode="a") + # The raw session is only read: a raw root with no metadata of its own has no attributes to carry over. + try: + raw_attrs = dict(zarr.open_group(str(raw_session), mode="r").attrs) + except (FileNotFoundError, zarr.errors.GroupNotFoundError): + raw_attrs = {} + for key, value in raw_attrs.items(): + aligned_root.attrs[key] = value + # The index source is a root attribute so a reader knows which camera's frames define the timeline. One root + # has one, so an episode on a different index is refused rather than relabelling the ones already there. + existing = aligned_root.attrs.get("index_source") + if existing not in (None, index_source): + raise AlignedWriteError(f"the aligned session uses index source {existing!r}, not {index_source!r}") + aligned_root.attrs["index_source"] = index_source + for name in ALIGNED_SESSION_SIDECARS: + source, target = raw_session / name, aligned_session / name + # The session config is always refreshed from the raw session, as the aligned dataset format does; any + # other existing sidecar is left alone, so re-aligning never overwrites what is already there. + if source.is_file() and (name == "config.yaml" or not target.exists()): + shutil.copy2(source, target) + + +def _write_source(raw_group: Any, target_group: Any, aligned_source: Any) -> None: + """Write one source: its attributes, the raw arrays that are not resampled, and what the alignment adds.""" + _copy_attrs(raw_group, target_group) + for name in raw_group.array_keys(): + # A resampled field is read as ``_interp``; its raw array is not kept. + if name not in aligned_source.interpolated: + target_group[name] = raw_group[name][:] + for name, values in aligned_source.interpolated.items(): + target_group[f"{name}_interp"] = values + if aligned_source.aligned_frame_indices is not None: + target_group["aligned_frame_indices"] = aligned_source.aligned_frame_indices + + +def validate_aligned_episode(aligned_session: Path, episode_index: int, index_source: str) -> list[str]: + """Problems with a written aligned episode; empty means it is as expected. + + Checks the arrays every aligned episode has, that the index source has ``aligned_frame_indices``, and that the + small arrays can be read back and are not empty (a stale chunk from an earlier, differently shaped alignment + only shows when the data is read). + """ + problems: list[str] = [] + key = f"episode_{episode_index}" + root = zarr.open_group(str(aligned_session), mode="r") + if key not in root: + return [f"{key} group missing from the aligned session"] + episode = root[key] + for required in ("valid_indices", "valid_timestamps"): + if required not in episode: + problems.append(f"{key}/{required} missing") + if index_source not in episode: + problems.append(f"{key}/{index_source} group missing") + elif "aligned_frame_indices" not in episode[index_source]: + problems.append(f"{key}/{index_source}/aligned_frame_indices missing") + + def check(group: Any, prefix: str) -> None: + """Read each small array under ``group`` and report one that fails or is empty.""" + for name in group.array_keys(): + array = group[name] + if array.nbytes > _CHECK_MAX_BYTES: + continue + try: + data = np.asarray(array[:]) + except Exception as exc: # a stale chunk raises something zarr-specific + problems.append(f"{prefix}/{name} unreadable: {exc}") + continue + if data.size == 0: + problems.append(f"{prefix}/{name} is empty") + for name in group.group_keys(): + check(group[name], f"{prefix}/{name}") + + check(episode, key) + return problems + + +def write_aligned_episode( + raw_session: Path, + aligned_session: Path, + episode_index: int, + index_source: str, + *, + downsample: int = 1, + qualified: Mapping[str, bool], + continuity: Mapping[str, Mapping[str, object]] | None, +) -> AlignedEpisode: + """Align one recorded episode and write it into ``aligned_session``; returns the alignment. + + ``qualified`` says, per source, whether its recorded capture time is a qualified event time (a measured one, + or a configured estimate the recorder already applied). A source that is not, or is not listed, refuses the + alignment, so nothing is written for it. The times aligned on are the recorded capture times, used unchanged. + ``continuity`` is the episode's stored per-source continuity evidence (None if it recorded none): a source + whose evidence is missing, unknown or records a break refuses it too. + Raises ``AlignedWriteError`` if the written episode does not validate. + """ + _check_separate(raw_session, aligned_session) + config = yaml.safe_load((raw_session / "config.yaml").read_text(encoding="utf-8")) or {} + rules = source_rules(config) + cameras = [e["name"] for e in config.get("video_sources") or []] + key = f"episode_{episode_index}" + # The episode group is opened on its own: the raw session's root need not have metadata of its own. + raw_episode = zarr.open_group(str(raw_session / key), mode="r") + + # Only declared sources the episode actually recorded take part, as the valid range spans what is present. + present = _participating(raw_episode, rules) + unqualified = [name for name in [index_source, *present] if not qualified.get(name, False)] + if unqualified: + raise AlignedWriteError(f"sources without a qualified event time: {sorted(set(unqualified))}") + times = {name: raw_episode[name]["capture_time"][:] for name in present} + # The evidence is checked against the rows actually written: a break at a row the source lacks is refused. + broken = check_continuity(continuity, [index_source, *present], {name: len(t) for name, t in times.items()}) + if broken: + raise AlignedWriteError("; ".join(broken)) + fields = {name: {a: raw_episode[name][a][:] for a in raw_episode[name].array_keys()} for name in present} + aligned = align_episode( + index_source, + times, + [c for c in cameras if c in present], + fields, + {name: rules[name] for name in present}, + downsample=downsample, + ) + + _write_root(raw_session, aligned_session, index_source) + root = zarr.open_group(str(aligned_session), mode="a") + # An existing aligned episode is never replaced. + if key in root: + raise AlignedWriteError(f"{key} already exists in the aligned session") + episode = root.require_group(key) + _copy_attrs(raw_episode, episode) + episode["valid_indices"] = aligned.valid_indices + episode["valid_timestamps"] = aligned.valid_timestamps + for name in present: + _write_source(raw_episode[name], episode.require_group(name), aligned.sources[name]) + + # The videos are the same files: linked, so the aligned episode costs no extra disk. + raw_dir, aligned_dir = raw_session / key, aligned_session / key + for video in sorted(raw_dir.iterdir()) if raw_dir.is_dir() else []: + if video.suffix in _VIDEO_SUFFIXES: + _link_or_copy(video, aligned_dir / video.name) + + problems = validate_aligned_episode(aligned_session, episode_index, index_source) + if problems: + raise AlignedWriteError("; ".join(problems)) + return aligned + + +def publish_aligned_episode( + raw_session: Path, + aligned_session: Path, + episode_index: int, + index_source: str, + *, + downsample: int = 1, + qualified: Mapping[str, bool], + continuity: Mapping[str, Mapping[str, object]] | None, +) -> AlignedEpisode: + """Write the aligned episode privately, then move it into ``aligned_session`` with one rename. + + The episode is built and validated in a private staging directory beside the aligned session (same + filesystem, so the videos stay linked), so a reader sees the finished episode or none. Each attempt owns its + staging directory, so concurrent runs never touch each other's work. An existing aligned episode is never + replaced: it is refused. A crashed run leaves its staging directory behind. Arguments and refusals are those + of :func:`write_aligned_episode`. + """ + _check_separate(raw_session, aligned_session) + key = f"episode_{episode_index}" + final = aligned_session / key + # Refuse before doing any work; the rename below fails too if one appears meanwhile. + if final.exists(): + raise AlignedWriteError(f"{final} already exists; it is not replaced") + aligned_session.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=f".{aligned_session.name}.staging-", dir=aligned_session.parent)) + try: + _check_separate(raw_session, staging) + aligned = write_aligned_episode( + raw_session, + staging, + episode_index, + index_source, + downsample=downsample, + qualified=qualified, + continuity=continuity, + ) + # The aligned root and sidecars are idempotent; the episode itself moves in by one rename. + _write_root(raw_session, aligned_session, index_source) + os.rename(staging / key, final) + finally: + # Nothing of a failed or refused write reaches the aligned session. + shutil.rmtree(staging, ignore_errors=True) + return aligned + + +def check_kept_episode(raw_session: Path, episode_index: int, manifest: EpisodeManifest) -> list[str]: + """Why ``manifest`` is not the record of the finalized, kept episode at ``raw_session/episode_``. + + The stored evidence is trusted only for the exact episode it describes, so this binds them: the episode's + own stamp must name the manifest's episode, the take must be finalized and kept, the written row counts must + match the manifest's, and the location must be the manifest's: its routed slot and group when it was routed, + its own index otherwise. A matching integer in a file name is never taken as identity. Empty means it is. + """ + problems: list[str] = [] + episode_dir = raw_session / f"episode_{episode_index}" + # The episode directory carries the id of the episode that owns it. + if read_identity(episode_dir) != manifest.episode_id: + problems.append("the episode directory is not the one this manifest describes (episode id differs)") + # Only a finalized take whose disposition was to keep its data is a kept episode. + if manifest.storage_phase != "FINALIZED" or manifest.finalization != FINALIZATION_KEEP: + problems.append( + f"the manifest is not a finalized kept episode ({manifest.storage_phase}, {manifest.finalization})" + ) + if manifest.assignment is not None: + # A routed episode sits in its group under its slot, which need not equal its recording index. + group = manifest.assignment["group"] + stamp = _attribute(raw_session / "zarr.json", ROUTE_ATTR) + if episode_index != manifest.assignment["slot"] or stamp != route_id(manifest.session_id, group): + problems.append("the episode is not at the routed location the manifest records") + elif episode_index != manifest.episode_index: + problems.append("the episode index is not the manifest's") + # The written rows must be exactly the rows the manifest sealed. + try: + episode = zarr.open_group(str(episode_dir), mode="r") + for name, count in manifest.counts.items(): + # A source with sealed rows must still be there; one that wrote none may be absent. + if name not in episode: + if count.get("written"): + problems.append(f"{name}: recorded rows are missing from the episode") + elif int(episode[name]["capture_time"].shape[0]) != count.get("written"): + problems.append(f"{name}: the rows on disk differ from the sealed count") + except (OSError, zarr.errors.GroupNotFoundError) as exc: + problems.append(f"the episode cannot be read: {exc}") + return problems + + +def _attribute(metadata_file: Path, name: str) -> Any: + """One attribute of a zarr group's metadata file, or None when the file or the attribute is absent.""" + try: + return json.loads(metadata_file.read_text(encoding="utf-8")).get("attributes", {}).get(name) + except (OSError, json.JSONDecodeError): + return None + + +def _participating(episode: Any, rules: Mapping[str, Any]) -> list[str]: + """The declared sources an episode recorded with at least one row; an absent or empty one takes no part.""" + return [name for name in rules if name in episode and episode[name]["capture_time"].shape[0] > 0] + + +def episode_qualification(raw_session: Path, episode_index: int, index_source: str) -> dict[str, bool]: + """Whether the episode's own timing evidence qualifies the index source and each source that took part.""" + config = yaml.safe_load((raw_session / "config.yaml").read_text(encoding="utf-8")) or {} + episode = zarr.open_group(str(raw_session / f"episode_{episode_index}"), mode="r") + names = sorted({index_source, *_participating(episode, source_rules(config))}) + return qualified_sources(dict(episode.attrs).get("timing"), names) + + +def align_kept_episode( + raw_session: Path, + episode_index: int, + index_source: str, + manifest: EpisodeManifest, + *, + aligned_session: Path | None = None, + downsample: int = 1, +) -> AlignedEpisode: + """Align one kept episode from what the recorder stored about it, and publish the result. + + ``manifest`` must be the record of exactly this finalized, kept episode (see :func:`check_kept_episode`); + its continuity evidence is used only then. Which sources have a qualified event time is read from the + episode's own ``timing`` attribute, so an episode recorded without that evidence is refused, never assumed + qualified. The aligned session defaults to ``_aligned`` beside the raw one. + """ + problems = check_kept_episode(raw_session, episode_index, manifest) + if problems: + raise AlignedWriteError("; ".join(problems)) + return publish_aligned_episode( + raw_session, + aligned_session or raw_session.with_name(f"{raw_session.name}_aligned"), + episode_index, + index_source, + downsample=downsample, + qualified=episode_qualification(raw_session, episode_index, index_source), + continuity=manifest.continuity, + ) diff --git a/examples/egocap/recording/egocap/recording/alignment.py b/examples/egocap/recording/egocap/recording/alignment.py new file mode 100644 index 0000000000..0c8569ece2 --- /dev/null +++ b/examples/egocap/recording/egocap/recording/alignment.py @@ -0,0 +1,345 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Time alignment of a recorded episode, producing the aligned-dataset arrays downstream consumers read. + +The recorded sources are resampled onto one camera's frame times (the index source). The rules are fixed by the +aligned dataset format and are implemented exactly as stated, because consumers read the result by array name and +expect these values: + +* the valid range runs from the latest first time to the earliest last time over all sources; an index frame is + kept when it is in that range and on the downsampling stride (``valid_indices``, ``valid_timestamps``); +* a camera other than the index source contributes the position of its frame nearest each kept time + (``aligned_frame_indices``), found by one forward walk, so the earlier frame wins a tie and a frame may serve + two kept times; +* the index source's own fields are only filtered by ``valid_indices``; every other source's fields are + interpolated at the kept times by their declared rule, written as ``_interp``: ``pose`` (4x4 + transforms: linear position, slerp rotation), ``linear`` (element-wise, also used for a 4x4 field with extra + leading dimensions) or ``nearest`` (the value at the nearest sample); ``raw`` fields are not resampled. + +There is no file access here. The times passed in are the ones to align on: each source's recorded capture time, +used exactly as given and never adjusted here. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Literal + +import numpy as np +from scipy import interpolate +from scipy.spatial import transform as st + +__all__ = [ + "AlignedEpisode", + "AlignedSource", + "AlignmentError", + "FRAME_FIELD_RULES", + "FieldRule", + "InsufficientSamplesError", + "align_episode", + "find_nearest_timestamp_indices", + "interpolate_field", + "check_continuity", + "qualified_sources", + "source_rules", +] + +#: How one field is resampled onto the kept times. +FieldRule = Literal["pose", "linear", "nearest", "raw"] + + +class AlignmentError(ValueError): + """The episode cannot be aligned: bad input, or no frame of the index source in the common range.""" + + +class InsufficientSamplesError(AlignmentError): + """Fewer than two samples with strictly increasing times remain, so nothing can be interpolated.""" + + +def qualified_sources(timing: Mapping[str, object] | None, sources: Sequence[str]) -> dict[str, bool]: + """Which sources' recorded times are qualified event times, from an episode's ``timing`` attribute. + + ``timing["event_time"][source]["qualified"]`` is the recorded statement, per source, of whether its saved time + is a qualified event time (a sensor's capture event, or a device measurement time). Only a literal True counts; + a source the attribute does not list, an entry without it, or no attribute at all is False. A clock mapping + says which clock a time is on, not what it measures, so it is never taken as qualification. A configured + latency estimate is applied when the sample is recorded and marked in this attribute; it is not applied here. + """ + event_time = (timing or {}).get("event_time") + entries = event_time if isinstance(event_time, Mapping) else {} + # Fail closed: anything but an explicit True is not qualified. + return { + name: isinstance(entries.get(name), Mapping) and entries[name].get("qualified") is True for name in sources + } + + +def _complete(entry: Mapping[str, object], rows: int | None) -> bool: + """Whether a source's evidence has its initial identity and a well-formed list of break rows.""" + identity, breaks = entry.get("identity"), entry.get("breaks") + if not isinstance(identity, (str, int)) or isinstance(identity, bool) or not isinstance(breaks, list): + return False + # A break is a row index: a non-negative integer, and a row the source actually has. + return all( + isinstance(b, int) and not isinstance(b, bool) and b >= 0 and (rows is None or b < rows) for b in breaks + ) + + +#: Continuity scopes that vouch for a source being one unbroken stretch. A clock mapping's epoch covers the +#: publisher's times; ``host_clock`` is a source stamped on the recording host's own clock, covering its host +#: poll and connection but not the device behind it. A transport session alone is not accepted. +ACCEPTED_CONTINUITY_SCOPES = ("clock_mapping", "host_clock") + + +def check_continuity( + continuity: Mapping[str, Mapping[str, object]] | None, + sources: Sequence[str], + rows: Mapping[str, int] | None = None, +) -> list[str]: + """Why the recorded continuity evidence does not allow aligning ``sources``; empty means it does. + + ``continuity`` is the per-source evidence stored with the episode (None when the episode recorded none). A + source is refused when there is no evidence for it, the evidence is incomplete, its scope is not one that + vouches for the source, or it records a break: interpolating across a restart would produce values the + sensor never reported. Nothing here defaults to clean: the initial identity and an explicit list of break rows + must both be present and well formed, and with ``rows`` (the written row count per source) every break must be + a row of the source. + """ + if continuity is None: + return ["no continuity evidence was recorded for this episode"] + problems: list[str] = [] + for name in sources: + entry = continuity.get(name) + if not isinstance(entry, Mapping): + problems.append(f"{name}: no continuity evidence") + elif entry.get("known") is not True: + problems.append(f"{name}: continuity is unknown") + elif entry.get("scope") not in ACCEPTED_CONTINUITY_SCOPES: + problems.append( + f"{name}: its continuity evidence ({entry.get('scope')!r}) does not cover the source itself" + ) + elif not _complete(entry, None if rows is None else rows.get(name)): + problems.append(f"{name}: continuity evidence is incomplete or malformed") + elif entry["breaks"]: + problems.append(f"{name}: has a restart inside the episode (rows {list(entry['breaks'])})") + return problems + + +def find_nearest_timestamp_indices(source_times: np.ndarray, target_times: np.ndarray) -> np.ndarray: + """Position in ``source_times`` of the sample nearest each target, by one forward walk. + + Both arrays are sorted. The walk only moves forward, and advances only when the next sample is strictly closer, + so a tie keeps the earlier sample and one sample can be nearest to several targets. + """ + m, n = len(source_times), len(target_times) + indices = np.empty(n, dtype=int) + j = 0 + for i in range(n): + # Move on only for a strictly closer sample; equal distance stays on the earlier one. + while j < m - 1 and abs(source_times[j + 1] - target_times[i]) < abs(source_times[j] - target_times[i]): + j += 1 + indices[i] = j + return indices + + +def _clean(times: np.ndarray, values: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Drop each sample whose time is not later than the sample before it, as recorded.""" + drop = [i for i in range(1, len(times)) if times[i] <= times[i - 1]] + if drop: + times, values = np.delete(times, drop), np.delete(values, drop, axis=0) + # An interpolator needs two distinct times to span anything. + if len(times) < 2: + raise InsufficientSamplesError( + f"interpolation needs at least 2 samples with strictly increasing times; {len(times)} remain" + ) + return times, values + + +def _linear(times: np.ndarray, values: np.ndarray, targets: np.ndarray) -> np.ndarray: + """Element-wise linear interpolation along the first axis; targets outside the sample range are an error.""" + times, values = _clean(times, values) + return interpolate.interp1d(times, values, kind="linear", axis=0)(targets) + + +def _pose(times: np.ndarray, values: np.ndarray, targets: np.ndarray) -> np.ndarray: + """Rigid transforms ``(N, 4, 4)``: linear position and slerp rotation, with a bottom row of [0, 0, 0, 1].""" + times, values = _clean(times, values) + positions = interpolate.interp1d(times, values[:, :3, 3], kind="linear", axis=0)(targets) + rotations = st.Slerp(times, st.Rotation.from_matrix(values[:, :3, :3]))(targets).as_matrix() + out = np.zeros((len(targets), 4, 4)) + out[:, :3, :3], out[:, :3, 3], out[:, 3, 3] = rotations, positions, 1.0 + return out + + +def interpolate_field( + rule: FieldRule, times: np.ndarray, values: np.ndarray, targets: np.ndarray +) -> np.ndarray | None: + """One field resampled at ``targets``; None for a ``raw`` field, which is not resampled. + + A ``pose`` field that is not a plain ``(N, 4, 4)`` array is interpolated linearly instead, element by element. + Targets outside the range of ``times`` are an error (they cannot occur for the kept times of an alignment). + """ + times, values = np.asarray(times, dtype=np.float64), np.asarray(values) + if rule == "raw": + return None + if rule == "nearest": + return values[find_nearest_timestamp_indices(times, targets)] + # A pose rule only applies to a plain stack of transforms; any other shape falls back to element-wise. + if rule == "pose" and values.ndim == 3 and values.shape[1:] == (4, 4): + return _pose(times, values, targets) + if rule in ("pose", "linear"): + return _linear(times, values, targets) + raise AlignmentError(f"unknown field rule {rule!r}") + + +@dataclass(frozen=True) +class AlignedSource: + """What the alignment adds for one source.""" + + #: For a camera: positions of the frames matched to the kept times (the index source's own kept positions). + aligned_frame_indices: np.ndarray | None = None + #: ```` -> values at the kept times; written as ``_interp``. + interpolated: dict[str, np.ndarray] = field(default_factory=dict) + + +@dataclass(frozen=True) +class AlignedEpisode: + """The result of aligning one episode.""" + + index_source: str + #: Boolean mask over the index source's frames: in the valid range and on the downsampling stride. + valid_indices: np.ndarray + #: The index source's times at the kept frames. + valid_timestamps: np.ndarray + #: The valid range that was used, as (first, last). + valid_range: tuple[float, float] + sources: dict[str, AlignedSource] + + +#: How each field of a frame type is resampled when an episode is time aligned. A session's config.yaml names the +#: frame type of each source. Head and controller poses are rigid transforms (slerp); the glove's joint transforms +#: are interpolated element-wise, so their shape never selects slerp; triggers and sensor values are continuous; +#: a count is the nearest sample's; buttons and flags are not resampled, so they are absent here (raw). +FRAME_FIELD_RULES: dict[str, dict[str, FieldRule]] = { + "RGBCameraFrame": {"pose": "pose"}, + "NumericFrame": {}, + "PicoTrackingFrame": { + "head_pose": "pose", + "left_pose": "pose", + "right_pose": "pose", + "left_trigger": "linear", + "right_trigger": "linear", + }, + "ManusFrame": { + **{f"{side}_hand_pose": "linear" for side in ("left", "right")}, + **{ + f"{side}_sensor_{kind}": "linear" + for side in ("left", "right") + for kind in ("position", "orientation", "rotation") + }, + **{f"{side}_sensor_count": "nearest" for side in ("left", "right")}, + }, +} + + +def source_rules(config: Mapping[str, object]) -> dict[str, dict[str, FieldRule]]: + """Each declared source's field rules, from the source declarations in a session's ``config.yaml``. + + A source declares its ``frame_type``; one that declares none takes the session's default for its kind. A frame + type with no known rules is an error, since silently resampling nothing would write an aligned episode that + looks complete and is not. + """ + recording = config.get("recording") + defaults = { + "video_sources": (recording or {}).get("video_frame_type", "RGBCameraFrame") + if isinstance(recording, Mapping) + else "RGBCameraFrame", + "numeric_sources": (recording or {}).get("numeric_frame_type", "NumericFrame") + if isinstance(recording, Mapping) + else "NumericFrame", + } + rules: dict[str, dict[str, FieldRule]] = {} + for section, default in defaults.items(): + entries = config.get(section) + # An absent or malformed section declares no sources. + for entry in entries if isinstance(entries, (list, tuple)) else []: + if not isinstance(entry, Mapping) or not isinstance(entry.get("name"), str): + raise AlignmentError(f"{section}: every source needs a name") + frame_type = entry.get("frame_type", default) + if frame_type not in FRAME_FIELD_RULES: + raise AlignmentError(f"{entry['name']}: no alignment rules for frame type {frame_type!r}") + rules[entry["name"]] = dict(FRAME_FIELD_RULES[frame_type]) + return rules + + +def _check_inputs(index_source: str, times: Mapping[str, np.ndarray], downsample: int) -> None: + """Refuse input that cannot be aligned: a missing index source, a bad stride, or unusable times.""" + if index_source not in times: + raise AlignmentError(f"the index source {index_source!r} has no times") + if downsample < 1: + raise AlignmentError("downsample must be at least 1") + for name, source_times in times.items(): + arr = np.asarray(source_times) + # The forward walk and the range logic assume sorted, finite times. + if arr.ndim != 1 or arr.size == 0 or not np.all(np.isfinite(arr)): + raise AlignmentError(f"{name}: needs a non-empty list of finite times") + if np.any(np.diff(arr) < 0): + raise AlignmentError(f"{name}: times go backwards") + + +def align_episode( + index_source: str, + times: Mapping[str, np.ndarray], + cameras: Sequence[str], + fields: Mapping[str, Mapping[str, np.ndarray]], + rules: Mapping[str, Mapping[str, FieldRule]], + *, + downsample: int = 1, +) -> AlignedEpisode: + """Align every source in ``times`` onto ``index_source``. + + ``times`` maps each source to the times to align on (sorted, finite); ``cameras`` names the sources that are + cameras; ``fields[source][name]`` are a source's arrays and ``rules[source][name]`` how each is resampled (a + field with no rule is not resampled). Raises ``AlignmentError`` if the index source has no frame in the valid + range, so an alignment is never an empty success. + """ + _check_inputs(index_source, times, downsample) + index_times = np.asarray(times[index_source], dtype=np.float64) + + # The valid range: every source has data on both sides of a kept time. + lo = max(float(np.min(t)) for t in times.values()) + hi = min(float(np.max(t)) for t in times.values()) + in_range = (index_times >= lo) & (index_times <= hi) + # Downsampling keeps every n-th index frame, counted from the first frame (not from the first kept one). + stride = np.zeros_like(in_range, dtype=bool) + stride[::downsample] = True + valid = in_range & stride + kept = index_times[valid] + if kept.size == 0: + raise AlignmentError(f"no frame of {index_source!r} is in the common range [{lo}, {hi}]") + + sources: dict[str, AlignedSource] = {} + for name, source_times in times.items(): + source_times = np.asarray(source_times, dtype=np.float64) + interpolated: dict[str, np.ndarray] = {} + for field_name, rule in rules.get(name, {}).items(): + # A rule for a field that was not recorded, or a raw field, adds nothing. + if field_name not in fields.get(name, {}) or rule == "raw": + continue + values = np.asarray(fields[name][field_name]) + if name == index_source: + # The index source already sits on the kept times: only filter it. + interpolated[field_name] = values[valid] + else: + interpolated[field_name] = interpolate_field(rule, source_times, values, kept) + frames = None + if name in cameras: + frames = ( + np.flatnonzero(valid) + if name == index_source + else find_nearest_timestamp_indices(source_times, kept) + ) + sources[name] = AlignedSource(frames, interpolated) + return AlignedEpisode(index_source, valid, kept, (lo, hi), sources) diff --git a/examples/egocap/recording/egocap/recording/episode_layout.py b/examples/egocap/recording/egocap/recording/episode_layout.py index 5128cfb473..693f4a53e4 100644 --- a/examples/egocap/recording/egocap/recording/episode_layout.py +++ b/examples/egocap/recording/egocap/recording/episode_layout.py @@ -36,10 +36,10 @@ from .constants import PENDING_SUFFIX MANIFEST_DIR = ".recording" -MANIFEST_SCHEMA_VERSION = 3 +MANIFEST_SCHEMA_VERSION = 4 #: Older records are decoded onto the current fields when read and rewritten at the current version only when #: next saved; nothing is migrated at startup. -READABLE_SCHEMA_VERSIONS = (1, 2, 3) +READABLE_SCHEMA_VERSIONS = (1, 2, 3, 4) _EPISODE_NAME = re.compile(r"^episode_(\d+)(\.pending)?$") #: What finalization does with a checked take, recorded durably so recovery never re-derives it from policy: @@ -80,6 +80,10 @@ class EpisodeManifest: # Digest of the configuration without the capture-QC policy: what a CHECKED take (its decision is already # stored with its own policy) must still match to be finished. None on manifests written before it existed. layout_digest: str | None = None + # Per source, whether its samples were one unbroken stretch: {"scope", "identity", "breaks", "known"} (see + # recording_service.models.NumericSample.continuity). None on manifests written before it existed, which says + # nothing about continuity; it never means "no breaks". + continuity: dict[str, dict[str, Any]] | None = None issues: list[str] = field(default_factory=list) duration_seconds: float = 0.0 # Optional free-text context the operator attached to the task; "" on manifests written before it existed. @@ -178,7 +182,8 @@ def write_manifest(session_path: Path, manifest: EpisodeManifest) -> None: #: Sidecar record of the calibration assets preserved in a session: name -> sha256 (null if absent at creation). CALIBRATION_RECORD = "calibration_assets.json" -#: Session-root (and, once routed, group-root) metadata snapshot: {"meta": {"egocap_station_name": ...}}. +#: Session-root (and, once routed, group-root) config snapshot: the station identity under ``meta`` plus the +#: source declarations (names, frame types, recording defaults) that time alignment reads. #: Written once at session creation and never rewritten on reopen/recovery (see #: RecordingService._prepare_session_assets); downstream tooling such as #: analytics/count_daily_trajectories.py reads it to map a session back to its station. diff --git a/examples/egocap/recording/egocap/recording/episode_store.py b/examples/egocap/recording/egocap/recording/episode_store.py index c768ab7285..4ce27c8a81 100644 --- a/examples/egocap/recording/egocap/recording/episode_store.py +++ b/examples/egocap/recording/egocap/recording/episode_store.py @@ -47,10 +47,15 @@ class FieldSpec: @dataclass(frozen=True) class NumericSourceSpec: - """A numeric source (pose tracker, glove, ...) and the fields recorded per row.""" + """A numeric source (pose tracker, glove, ...) and the fields recorded per row. + + ``frame_type`` names the frame definition that says how each field is resampled when the episode is time + aligned (see ``alignment.FRAME_FIELD_RULES``); None leaves the session's default numeric frame type. + """ name: str fields: tuple[FieldSpec, ...] + frame_type: str | None = None @dataclass(frozen=True) diff --git a/examples/egocap/recording/egocap/stations/ego_stations.example.yaml b/examples/egocap/recording/egocap/stations/ego_stations.example.yaml index 8cb4017bf0..8195eff79b 100644 --- a/examples/egocap/recording/egocap/stations/ego_stations.example.yaml +++ b/examples/egocap/recording/egocap/stations/ego_stations.example.yaml @@ -6,8 +6,8 @@ # backend/egocap_web/station_config.py) reads the FIRST station entry under "stations" and # consumes meta.egocap_station_name, meta.operator_name, meta.auto_fill_operator_name, # meta.task_description, recording.data_dir, and each video_sources entry's name/type (as the -# cameras requested from CaptureServer). numeric_sources and the rest of each video_sources entry -# (frame_type, latency, params) are reference data for other tooling, not consumed here -- +# cameras requested from CaptureServer) and the alignment section. numeric_sources and the rest of each +# video_sources entry (frame_type, latency, params) are reference data for other tooling, not consumed here -- # CaptureServer owns actually opening the selected cameras/devices. Each video_sources name MUST # match one of the names the running CaptureServer instance actually knows about (its own # --camera list, mock or real) -- an unknown name rejects the whole capture session, not just @@ -38,6 +38,17 @@ stations: # then this only prefills the field's draft text; the operator still has to press "Set # operator" before any recording command is accepted. auto_fill_operator_name: false + # Each kept episode is time-aligned in the background on index_source's frames; with no index_source nothing + # is aligned, and enabled: false also turns it off. Alignment needs every recorded camera, the index + # included, to have a sensor capture time (a LiveKit stream with timestamp_basis capture_monotonic_us); the + # plain CaptureServer cameras of the mock do not, so episodes recorded with them are not aligned. + alignment: + enabled: true + index_source: head_human_camera + # Estimated device-to-host delay in seconds for each tracking source; it is taken off that source's host + # poll time when recorded, and a source not listed is not time-aligned. Measure it on the station and + # uncomment; no value is assumed. + # tracking_latency_s: {pico_controller: , manus_glove: } video_sources: - type: oak_camera name: head_human_camera diff --git a/examples/egocap/recording/recording_service/models.py b/examples/egocap/recording/recording_service/models.py index 2fd4aa17dd..6f21d12829 100644 --- a/examples/egocap/recording/recording_service/models.py +++ b/examples/egocap/recording/recording_service/models.py @@ -166,6 +166,12 @@ class NumericSample: capture_time: float receive_time: float fields: Mapping[str, Any] + #: What stretch of the source this sample belongs to, as (scope, identity): the scope says what the identity + #: covers. ``"clock_mapping"``: the mapping epoch that placed the time. ``"host_clock"``: the capture + #: connection of a source polled on this host's clock, not the device behind it. ``"capture_session"``: the + #: transport session, which says nothing about the device. A change of identity within a take is a break. + #: None means the producer gave no evidence, which is not a claim of continuity. + continuity: tuple[str, str | int] | None = None @dataclass(frozen=True) @@ -178,6 +184,8 @@ class VideoSample: capture_time: float receive_time: float intrinsics: np.ndarray | None = None + #: As on ``NumericSample``: (scope, identity) of the stretch this frame belongs to, or None for no evidence. + continuity: tuple[str, str | int] | None = None @dataclass(frozen=True) diff --git a/examples/egocap/recording/recording_service/recording_service.py b/examples/egocap/recording/recording_service/recording_service.py index bf02de37d6..123bc71220 100644 --- a/examples/egocap/recording/recording_service/recording_service.py +++ b/examples/egocap/recording/recording_service/recording_service.py @@ -142,7 +142,9 @@ def _digest(config: ServiceConfig) -> str: qc["capture_algorithm_version"] = ALGORITHM_VERSION # a changed algorithm is a changed configuration canonical = { **({"routing": True} if config.routing is not None else {}), # routed or not; each take freezes its studio - "numeric": [asdict(s) for s in config.numeric_sources], + # A frame type says how a source's fields are resampled, not how its data is laid out, so it is left out: + # setting or changing one never makes a take recorded earlier look like a different configuration. + "numeric": [{k: v for k, v in asdict(s).items() if k != "frame_type"} for s in config.numeric_sources], "video": [asdict(s) for s in config.video_sources], "qc": qc, "latency": config.latency, @@ -164,6 +166,8 @@ def __init__(self, manifest: EpisodeManifest, sources: list[str], store: Episode self.admission_closed = False self.accepted = dict.fromkeys(sources, 0) self.written = dict.fromkeys(sources, 0) + # Per source: whether its accepted rows were one unbroken stretch (see note_continuity). + self.continuity: dict[str, dict[str, Any]] = {} self.span: dict[str, list[float]] = {} self.worker: asyncio.Task[Any] | None = None self.operation_task: asyncio.Task[Any] | None = None @@ -185,6 +189,38 @@ def episode_id(self) -> str: """The stable episode identity.""" return self.manifest.episode_id + def note_continuity(self, source: str, row: int, continuity: tuple[str, str | int] | None) -> None: + """Record the continuity evidence of one accepted row of ``source``. + + The first row fixes the scope and identity; a later row with a different identity is a break at that row + (rows are positions among the source's accepted rows, which equal its written rows once sealed). A row + with no evidence, or one whose scope changes, makes the source's continuity unknown: a take cannot vouch + for stretches it was never told about. + """ + entry = self.continuity.setdefault(source, {"scope": None, "identity": None, "breaks": [], "known": True}) + if continuity is None: + entry["known"] = False + return + scope, identity = continuity + if entry["scope"] is None: + entry["scope"], entry["identity"], entry["last"] = scope, identity, identity + return + if scope != entry["scope"]: + entry["known"] = False + elif identity != entry["last"]: + entry["breaks"].append(row) + entry["last"] = identity + + def continuity_record(self) -> dict[str, dict[str, Any]]: + """The evidence for every source of the take, as stored in the manifest. + + A source that accepted no rows has no evidence and is recorded as unknown, never as an empty clean list. + """ + unknown = {"scope": None, "identity": None, "breaks": [], "known": False} + return { + s: {k: v for k, v in self.continuity.get(s, unknown).items() if k != "last"} for s in self.accepted + } + def duration(self) -> float: """Maximum per-source capture-time span; never subtracts timestamps from different sources.""" return max((last - first for first, last in self.span.values()), default=0.0) @@ -251,6 +287,8 @@ def submit_sample(self, episode_id: str, sample: NumericSample | VideoSample) -> copied = self._copy(sample) # the only fallible step: finish it before any bookkeeping changes index = take.accepted[sample.source] take.accepted[sample.source] = index + 1 + # Evidence is noted for the accepted row only, never for an offer that came back FULL and will be retried. + take.note_continuity(sample.source, index, sample.continuity) take.queue.append(copied) take.wake.set() return SubmissionResult(SubmissionStatus.ACCEPTED, sample.source, index) @@ -358,6 +396,21 @@ def _validate(self, sample: NumericSample | VideoSample) -> None: for stamp in (sample.capture_time, sample.receive_time): if not isinstance(stamp, int | float) or not np.isfinite(stamp): raise ValidationError(f"{sample.source}: timestamps must be finite numbers") + # Continuity evidence is stored in the manifest as JSON, so its shape is checked before any bookkeeping + # changes: a malformed value must leave the counters, the queue and the evidence untouched. + evidence = sample.continuity + if evidence is not None: + valid = ( + isinstance(evidence, tuple) + and len(evidence) == 2 + and isinstance(evidence[0], str) + and isinstance(evidence[1], str | int) + and not isinstance(evidence[1], bool) + ) + if not valid: + raise ValidationError( + f"{sample.source}: continuity must be a (scope, identity) pair of str and str|int" + ) if isinstance(sample, NumericSample): if kind != "numeric": raise ValidationError(f"{sample.source} is a video source") @@ -403,6 +456,7 @@ def _copy(sample: NumericSample | VideoSample) -> NumericSample | VideoSample: float(sample.capture_time), float(sample.receive_time), {k: np.array(v, copy=True) for k, v in sample.fields.items()}, + sample.continuity, ) intrinsics = ( None if sample.intrinsics is None else np.array(sample.intrinsics, dtype=np.float64, copy=True) @@ -414,6 +468,7 @@ def _copy(sample: NumericSample | VideoSample) -> NumericSample | VideoSample: float(sample.capture_time), float(sample.receive_time), intrinsics, + sample.continuity, ) # -- task and call ownership -------------------------------------------------- @@ -651,11 +706,7 @@ def _prepare_session_assets(self, session_path: Path, is_new: bool) -> None: record[asset.name] = None self._issues.append(f"optional calibration asset {asset.name!r} not found at {source}") write_json_atomic(record_path, record) - if self._config.station_label: - write_yaml_atomic( - session_path / SESSION_CONFIG_FILE, - {"meta": {"egocap_station_name": self._config.station_label}}, - ) + write_yaml_atomic(session_path / SESSION_CONFIG_FILE, self._session_config()) return record = json.loads(record_path.read_text()) if record_path.is_file() else {} for asset in assets: @@ -667,6 +718,35 @@ def _prepare_session_assets(self, session_path: Path, is_new: bool) -> None: raise ConfigurationError(problem) self._issues.append("optional " + problem) + def _session_config(self) -> dict[str, Any]: + """The session's ``config.yaml``: the station identity and the source declarations time alignment reads. + + Each source is declared by name with the frame type that says how its fields are resampled. A latency is + written only for a source the configuration gives one, since a latency that was never measured must not + be recorded as zero. + """ + config = self._config + data: dict[str, Any] = {} + # The station identity analytics tools map a recorded session back to its station with. + if config.station_label: + data["meta"] = {"egocap_station_name": config.station_label} + # The default frame types for sources that declare none. + data["recording"] = {"video_frame_type": "RGBCameraFrame", "numeric_frame_type": "NumericFrame"} + latency = config.latency or {} + + def declare(name: str, frame_type: str | None = None) -> dict[str, Any]: + """One source's declaration; a frame type or latency appears only when it is known.""" + entry: dict[str, Any] = {"name": name} + if frame_type is not None: + entry["frame_type"] = frame_type + if name in latency: + entry["latency"] = latency[name] + return entry + + data["video_sources"] = [declare(s.name) for s in config.video_sources] + data["numeric_sources"] = [declare(s.name, s.frame_type) for s in config.numeric_sources] + return data + def _seed_group_assets(self, session_path: Path, group: str) -> None: """Copy the session's frozen config.yaml and preserved calibration assets into a routed group. @@ -893,6 +973,7 @@ async def _finish_impl(self, take: _Take) -> EpisodeResult: counts={ s: {"accepted": take.accepted[s], "written": take.written[s]} for s in take.accepted }, + continuity=take.continuity_record(), duration_seconds=take.duration(), storage_phase="SEALED", ) diff --git a/examples/egocap/recording/tools/align_episode.py b/examples/egocap/recording/tools/align_episode.py new file mode 100755 index 0000000000..a657869704 --- /dev/null +++ b/examples/egocap/recording/tools/align_episode.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Time-align one kept episode and publish the aligned copy beside its session. + +The raw session is only read. The episode's stored continuity evidence comes from its manifest, and which +sources have a qualified event time from the episode's own ``timing`` attribute; an episode that lacks either is +refused with the reason, never assumed good. + +Usage (from any directory outside ``egocap`` itself):: + + /.venv/bin/python recording/tools/align_episode.py SESSION_DIR --episode N --index-source NAME \ + [--manifest-session DIR] [--manifest-episode N] [--aligned-session DIR] [--downsample K] + +``--manifest-session`` is where the episode's manifest lives when that is not ``SESSION_DIR`` (a routed +episode's manifest stays in the session it was recorded in), and ``--manifest-episode`` its index there when that +differs from ``--episode`` (a routed episode sits under its slot). The manifest is used only if it describes +exactly this finalized, kept episode. Exit status 0 means the aligned episode was +published; 1 means it was refused (the same input is refused every time); 2 means the manifest could not be read +(retry later). The reason is on stderr. +""" + +import argparse +import sys +from pathlib import Path + +from egocap.recording.aligned_writer import AlignedWriteError, align_kept_episode +from egocap.recording.alignment import AlignmentError +from egocap.recording.episode_layout import manifest_path, read_manifest + + +def main(argv: list[str] | None = None) -> int: + """Align the named episode and report the result; returns the process exit status.""" + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("session", type=Path, help="the raw session directory holding episode_N") + parser.add_argument("--episode", type=int, required=True, help="the episode's index in that directory") + parser.add_argument("--index-source", required=True, help="the camera whose frames define the timeline") + parser.add_argument("--manifest-session", type=Path, help="the session whose manifest describes the episode") + parser.add_argument( + "--manifest-episode", + type=int, + help="the episode's index in that session's manifests, when it differs from --episode (a routed episode)", + ) + parser.add_argument("--aligned-session", type=Path, help="where to publish (default: _aligned)") + parser.add_argument("--downsample", type=int, default=1, help="keep every K-th index frame (default 1)") + args = parser.parse_args(argv) + + # The manifest carries the continuity evidence; without a readable one the episode cannot be vouched for. + try: + manifest_episode = args.episode if args.manifest_episode is None else args.manifest_episode + manifest = read_manifest(manifest_path(args.manifest_session or args.session, manifest_episode)) + except (OSError, ValueError) as exc: + print(f"cannot read the episode's manifest: {exc}", file=sys.stderr) + return 2 + try: + aligned = align_kept_episode( + args.session, + args.episode, + args.index_source, + manifest, + aligned_session=args.aligned_session, + downsample=args.downsample, + ) + except (AlignedWriteError, AlignmentError) as exc: + print(f"not aligned: {exc}", file=sys.stderr) + return 1 + kept, total = int(aligned.valid_indices.sum()), len(aligned.valid_indices) + print(f"aligned episode_{args.episode}: {kept} of {total} frames") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/egocap/tests/backend/test_app_livekit_recording.py b/examples/egocap/tests/backend/test_app_livekit_recording.py index 1f9f9a9646..6096cd2e90 100644 --- a/examples/egocap/tests/backend/test_app_livekit_recording.py +++ b/examples/egocap/tests/backend/test_app_livekit_recording.py @@ -54,11 +54,34 @@ def test_config_factory_declares_and_requires_each_recordable_livekit_stream(tmp assert "cam16" in config.qc.required_sources +def test_config_factory_reports_recorded_and_sensor_captured_names_each_time_it_builds(tmp_path): + capture = p.LiveKitTimestampBasis.SENSOR_CAPTURE_MONOTONIC_US + ready = p.LiveKitTimestampBasis.PUBLISHER_MONOTONIC_US + monitor = _monitor([_stream(16, capture), _stream(17, ready)]) + reported = [] + factory = app_module._recording_config_factory(monitor, None, tmp_path, lambda *names: reported.append(names)) + factory() + factory() + assert reported == [(frozenset({"cam16", "cam17"}), frozenset({"cam16"}))] * 2 + + +def test_config_factory_reports_no_names_when_nothing_is_recordable(tmp_path): + monitor = SimpleNamespace(camera_descriptors=(), native=SimpleNamespace(started=None)) + reported = [] + app_module._recording_config_factory(monitor, None, tmp_path, lambda *names: reported.append(names))() + assert reported == [(frozenset(), frozenset())] + + def test_config_factory_without_a_started_session_has_no_livekit_sources(tmp_path): monitor = SimpleNamespace(camera_descriptors=(), native=SimpleNamespace(started=None)) assert app_module._recording_config_factory(monitor, None, tmp_path)().video_sources == () +def test_config_factory_does_not_carry_a_latency(tmp_path): + monitor = SimpleNamespace(camera_descriptors=(), native=SimpleNamespace(started=None)) + assert app_module._recording_config_factory(monitor, None, tmp_path)().latency is None + + def test_recorder_token_is_signed_when_the_key_pair_is_set(monkeypatch): monkeypatch.setenv("LIVEKIT_API_KEY", "key") monkeypatch.setenv("LIVEKIT_API_SECRET", "secret") @@ -237,3 +260,12 @@ def test_readiness_accepts_livekit_recording_with_a_loopback_capture_server(acce _ready_monitor([_stream(16, capture)]), None, "ws://127.0.0.1:8082" ) assert check() == (True, None) + + +def test_config_factory_numeric_sources_carry_their_frame_types(tmp_path): + monitor = SimpleNamespace(camera_descriptors=(), native=SimpleNamespace(started=None)) + config = app_module._recording_config_factory(monitor, None, tmp_path)() + assert {s.name: s.frame_type for s in config.numeric_sources} == { + "pico_controller": "PicoTrackingFrame", + "manus_glove": "ManusFrame", + } diff --git a/examples/egocap/tests/backend/test_auto_align.py b/examples/egocap/tests/backend/test_auto_align.py new file mode 100644 index 0000000000..113ca0d7d1 --- /dev/null +++ b/examples/egocap/tests/backend/test_auto_align.py @@ -0,0 +1,470 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Background alignment of kept episodes: what is queued, scanned, aligned in a child process, and refused.""" + +import asyncio +import json +import logging +import os +import time +from pathlib import Path + +import pytest +import yaml +import zarr + +from egocap.recording.episode_layout import ( + FINALIZATION_KEEP, + FINALIZATION_RETAIN, + EpisodeManifest, + episode_dir_names, + write_manifest, +) +from egocap_web import auto_align +from egocap_web.auto_align import AutoAligner +from recording_service import Disposition, EpisodeResult +from tests.recording.unit.test_align_kept_episode import INDEX, build_raw, make_manifest + +STUB = """ +import json, os, sys, time +from pathlib import Path + +argv = sys.argv[1:] +with open(os.environ["STUB_LOG"], "a") as log: + log.write(json.dumps(argv) + "\\n") +group = Path(argv[0]) +slot = argv[argv.index("--episode") + 1] +if "hang" in group.name: + Path(os.environ["STUB_LOG"] + ".pid").write_text(str(os.getpid())) + time.sleep(60) +if "bad" in group.name: + print("no qualified latency", file=sys.stderr) + sys.exit(1) +if "crash" in group.name: + sys.exit(3) +(group.with_name(group.name + "_aligned") / f"episode_{slot}").mkdir(parents=True) +""" + + +def _result(index=3, disposition=Disposition.KEPT, final_path="default", session_id="session_a"): + if final_path == "default": + final_path = Path("/routed/group") / "episode_000007" + return EpisodeResult( + session_id=session_id, + episode_id=f"ep{index}", + episode_index=index, + disposition=disposition, + final_path=final_path, + pending_path=None, + counts=(), + duration_seconds=1.0, + quality=None, + ) + + +@pytest.fixture +def stub_tool(tmp_path, monkeypatch): + tool = tmp_path / "stub_tool" / "align_stub.py" + tool.parent.mkdir() + tool.write_text(STUB, encoding="utf-8") + log = tmp_path / "stub_tool" / "calls.log" + monkeypatch.setattr(auto_align, "_ALIGN_TOOL", tool) + monkeypatch.setenv("STUB_LOG", str(log)) + return log + + +def _stub_calls(log): + return [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] + + +@pytest.mark.parametrize( + "result", + [ + _result(disposition=Disposition.DISCARDED), + _result(final_path=None), + ], +) +def test_results_that_are_not_a_kept_episode_with_a_path_are_ignored(result): + aligner = AutoAligner(Path("/out"), "head") + aligner.submit(result) + assert aligner._queue.empty() + + +def test_kept_results_are_queued_before_any_session_has_reported_its_sources(): + aligner = AutoAligner(Path("/out"), "head") + aligner.submit(_result()) + assert aligner._queue.qsize() == 1 + + +def test_kept_results_are_queued_when_the_index_source_and_every_recorded_camera_are_qualified(): + aligner = AutoAligner(Path("/out"), "head") + aligner.note_video_sources(frozenset({"head", "wrist"}), frozenset({"head", "wrist", "other"})) + aligner.submit(_result()) + assert aligner._queue.qsize() == 1 + + +def test_kept_results_are_not_queued_when_the_index_source_is_recorded_without_a_sensor_capture_time(): + aligner = AutoAligner(Path("/out"), "head") + aligner.note_video_sources(frozenset({"head"}), frozenset()) + aligner.submit(_result()) + assert aligner._queue.empty() + + +def test_kept_results_are_not_queued_when_the_index_source_is_not_recorded(): + aligner = AutoAligner(Path("/out"), "head") + aligner.note_video_sources(frozenset({"wrist"}), frozenset({"wrist"})) + aligner.submit(_result()) + assert aligner._queue.empty() + + +def test_kept_results_are_not_queued_when_another_recorded_camera_is_not_qualified(): + aligner = AutoAligner(Path("/out"), "head") + aligner.note_video_sources(frozenset({"head", "wrist"}), frozenset({"head"})) + aligner.submit(_result()) + assert aligner._queue.empty() + + +def test_the_not_alignable_warning_is_logged_once_per_transition(caplog): + aligner = AutoAligner(Path("/out"), "head") + with caplog.at_level(logging.WARNING, logger="isaaccapture"): + aligner.note_video_sources(frozenset({"wrist"}), frozenset({"wrist"})) + aligner.note_video_sources(frozenset(), frozenset()) + assert len(caplog.records) == 1 + assert "'head'" in caplog.text and "sensor capture time" in caplog.text + aligner.note_video_sources(frozenset({"head"}), frozenset({"head"})) + aligner.note_video_sources(frozenset({"wrist"}), frozenset()) + assert len(caplog.records) == 2 + + +def test_queueing_resumes_when_a_later_session_is_alignable(): + aligner = AutoAligner(Path("/out"), "head") + aligner.note_video_sources(frozenset({"head"}), frozenset()) + aligner.note_video_sources(frozenset({"head"}), frozenset({"head"})) + aligner.submit(_result()) + assert aligner._queue.qsize() == 1 + + +async def test_cancelling_an_idle_worker_ends_it_cleanly(tmp_path): + task = asyncio.create_task(AutoAligner(tmp_path / "out", "head").run()) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task.cancelled() + + +def _write_evidence(group_dir, slot, *, qualified): + group_dir.mkdir(parents=True, exist_ok=True) + (group_dir / "config.yaml").write_text(yaml.safe_dump({"video_sources": [{"name": "head"}]}), encoding="utf-8") + episode = zarr.open_group(str(group_dir / f"episode_{slot}"), mode="w") + episode.attrs["timing"] = {"event_time": {"head": {"qualified": qualified}}} + + +def _write_session_manifest(output_root, session, index, *, aligned=False, qualified=True, **overrides): + pending, final = episode_dir_names(index) + fields = { + "session_id": session, + "episode_id": f"e{index}", + "episode_index": index, + "config_digest": "d", + "task": "t", + "operator": "o", + "sources": [], + "pending_path": pending, + "final_path": final, + "storage_phase": "FINALIZED", + "finalization": FINALIZATION_KEEP, + } + fields.update(overrides) + (output_root / session).mkdir(parents=True, exist_ok=True) + write_manifest(output_root / session, EpisodeManifest(**fields)) + assignment = fields.get("assignment") + group = output_root / assignment["group"] if assignment else output_root / session + slot = assignment["slot"] if assignment else index + if qualified is not None: + _write_evidence(group, slot, qualified=qualified) + if aligned: + (group.with_name(f"{group.name}_aligned") / f"episode_{slot}").mkdir(parents=True) + return final + + +def test_scan_finds_only_kept_episodes_without_an_aligned_copy(tmp_path): + _write_session_manifest(tmp_path, "s1", 0) + _write_session_manifest(tmp_path, "s1", 1, aligned=True) + _write_session_manifest(tmp_path, "s1", 2, finalization=FINALIZATION_RETAIN) + _write_session_manifest(tmp_path, "s1", 3, storage_phase="WRITING") + aligner = AutoAligner(tmp_path, "head") + jobs = aligner.scan() + assert jobs == [auto_align._Job("s1", 0, tmp_path / "s1" / episode_dir_names(0)[1])] + + +def test_scan_finds_older_sessions_work_even_when_the_newest_session_is_empty(tmp_path): + _write_session_manifest(tmp_path, "s1", 0) + _write_session_manifest(tmp_path, "s2", 0) + (tmp_path / "s3" / auto_align.MANIFEST_DIR).mkdir(parents=True) + aligner = AutoAligner(tmp_path, "head") + jobs = aligner.scan() + assert [job.session_id for job in jobs] == ["s1", "s2"] + + +def test_scan_skips_an_episode_whose_timing_is_not_qualified(tmp_path): + _write_session_manifest(tmp_path, "s1", 0, qualified=False) + _write_session_manifest(tmp_path, "s1", 1) + aligner = AutoAligner(tmp_path, "head") + jobs = aligner.scan() + assert [job.episode_index for job in jobs] == [1] + + +def test_scan_skips_an_episode_with_no_readable_evidence(tmp_path): + _write_session_manifest(tmp_path, "s1", 0, qualified=None) + aligner = AutoAligner(tmp_path, "head") + assert aligner.scan() == [] + + +def test_scan_skips_an_unreadable_manifest_and_keeps_the_rest(tmp_path): + _write_session_manifest(tmp_path, "s1", 1) + (tmp_path / "s1" / auto_align.MANIFEST_DIR / "episode_0.json").write_text("{not json") + aligner = AutoAligner(tmp_path, "head") + jobs = aligner.scan() + assert [job.episode_index for job in jobs] == [1] + + +def test_scan_uses_the_routed_destination_for_a_routed_episode(tmp_path): + _write_session_manifest(tmp_path, "s1", 0, studio="Lab", assignment={"group": "Lab/20260101", "slot": 4}) + aligner = AutoAligner(tmp_path, "head") + jobs = aligner.scan() + assert jobs == [auto_align._Job("s1", 0, tmp_path / "Lab/20260101/episode_4")] + + +def test_scan_skips_a_routed_episode_that_is_already_aligned(tmp_path): + _write_session_manifest( + tmp_path, "s1", 0, aligned=True, studio="Lab", assignment={"group": "Lab/20260101", "slot": 4} + ) + aligner = AutoAligner(tmp_path, "head") + jobs = aligner.scan() + assert jobs == [] + + +@pytest.mark.parametrize("create_root", [False, True]) +def test_scan_with_no_sessions_finds_nothing(tmp_path, create_root): + root = tmp_path / "out" + if create_root: + root.mkdir() + aligner = AutoAligner(root, "head") + assert aligner.scan() == [] + + +def test_scan_is_not_vetoed_by_a_declared_source_the_episode_did_not_record(tmp_path): + _write_session_manifest(tmp_path, "s1", 0) + (tmp_path / "s1" / "config.yaml").write_text( + yaml.safe_dump({"video_sources": [{"name": "head"}, {"name": "optional_camera"}]}), encoding="utf-8" + ) + aligner = AutoAligner(tmp_path, "head") + jobs = aligner.scan() + assert [job.episode_index for job in jobs] == [0] + + +def test_the_logger_is_in_the_isaaccapture_tree(): + assert auto_align.logger.name == "isaaccapture.egocap_web.auto_align" + + +async def test_a_kept_episode_is_aligned_by_a_child_process_with_its_slot_and_manifest(tmp_path, stub_tool): + aligner = AutoAligner(tmp_path / "out", "head") + aligner.submit(_result(index=3, final_path=tmp_path / "out" / "Lab" / "g" / "episode_000007")) + task = asyncio.create_task(aligner.run()) + await aligner.close(task, 30) + (argv,) = _stub_calls(stub_tool) + assert argv == [ + str(tmp_path / "out" / "Lab" / "g"), + "--episode", + "7", + "--index-source", + "head", + "--manifest-session", + str(tmp_path / "out" / "session_a"), + "--manifest-episode", + "3", + ] + assert (tmp_path / "out" / "Lab" / "g_aligned" / "episode_7").is_dir() + + +async def test_an_episode_that_already_has_an_aligned_copy_is_not_aligned_again(tmp_path, stub_tool): + (tmp_path / "group_aligned" / "episode_7").mkdir(parents=True) + aligner = AutoAligner(tmp_path, "head") + aligner.submit(_result(final_path=tmp_path / "group" / "episode_7")) + task = asyncio.create_task(aligner.run()) + await aligner.close(task, 30) + assert _stub_calls(stub_tool) == [] + + +async def test_many_submits_are_all_aligned_in_order_and_close_drains_them(tmp_path, stub_tool): + aligner = AutoAligner(tmp_path, "head") + for index in range(6): + aligner.submit(_result(index=index, final_path=tmp_path / f"g{index}" / f"episode_{index}")) + task = asyncio.create_task(aligner.run()) + await aligner.close(task, 60) + assert [call[0] for call in _stub_calls(stub_tool)] == [str(tmp_path / f"g{i}") for i in range(6)] + assert aligner._queue.empty() and task.cancelled() + + +async def test_a_refusal_writes_a_marker_and_the_next_episode_is_still_aligned(tmp_path, stub_tool, caplog): + aligner = AutoAligner(tmp_path, "head") + aligner.submit(_result(index=1, final_path=tmp_path / "bad" / "episode_1")) + aligner.submit(_result(index=2, final_path=tmp_path / "good" / "episode_2")) + task = asyncio.create_task(aligner.run()) + with caplog.at_level(logging.INFO, logger=auto_align.logger.name): + await aligner.close(task, 30) + assert (tmp_path / "bad_aligned" / ".episode_1.refused").read_text() == "no qualified latency\n" + assert (tmp_path / "good_aligned" / "episode_2").is_dir() + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings) == 1 and "session_a episode 1" in warnings[0] and "no qualified latency" in warnings[0] + assert any("session_a episode 2 was time-aligned" in r.getMessage() for r in caplog.records) + + +async def test_a_crash_is_logged_without_a_marker_and_the_next_episode_is_still_aligned( + tmp_path, stub_tool, caplog +): + aligner = AutoAligner(tmp_path, "head") + aligner.submit(_result(index=1, final_path=tmp_path / "crash" / "episode_1")) + aligner.submit(_result(index=2, final_path=tmp_path / "good" / "episode_2")) + task = asyncio.create_task(aligner.run()) + with caplog.at_level(logging.WARNING, logger=auto_align.logger.name): + await aligner.close(task, 30) + assert not (tmp_path / "crash_aligned").exists() + assert "exited with status 3" in caplog.text + assert (tmp_path / "good_aligned" / "episode_2").is_dir() + + +async def test_a_refused_episode_is_not_queued_again_until_its_marker_is_deleted(tmp_path, stub_tool): + _write_session_manifest(tmp_path, "bad", 0) + aligner = AutoAligner(tmp_path, "head") + (job,) = aligner.scan() + aligner.submit(_result(index=0, session_id="bad", final_path=job.final)) + task = asyncio.create_task(aligner.run()) + await aligner.close(task, 30) + assert job.refused_marker.is_file() and aligner.scan() == [] + job.refused_marker.unlink() + assert aligner.scan() == [job] + + +async def test_the_worker_aligns_what_the_startup_scan_finds_without_a_submit(tmp_path, stub_tool): + _write_session_manifest(tmp_path, "s1", 0) + _write_session_manifest(tmp_path, "s1", 1, aligned=True) + aligner = AutoAligner(tmp_path, "head") + task = asyncio.create_task(aligner.run()) + for _ in range(300): + if (tmp_path / "s1_aligned" / "episode_0").is_dir(): + break + await asyncio.sleep(0.05) + await aligner.close(task, 30) + assert len(_stub_calls(stub_tool)) == 1 + + +async def test_a_failing_scan_is_logged_and_the_worker_still_aligns_submitted_episodes( + tmp_path, stub_tool, monkeypatch, caplog +): + def broken_scan(): + raise OSError("disk unavailable") + + aligner = AutoAligner(tmp_path, "head") + monkeypatch.setattr(aligner, "scan", broken_scan) + aligner.submit(_result(index=2, final_path=tmp_path / "good" / "episode_2")) + task = asyncio.create_task(aligner.run()) + with caplog.at_level(logging.WARNING, logger=auto_align.logger.name): + await aligner.close(task, 30) + assert "recovery failed: disk unavailable" in caplog.text + assert (tmp_path / "good_aligned" / "episode_2").is_dir() + + +async def test_close_kills_an_alignment_still_running_at_the_deadline_and_returns(tmp_path, stub_tool, caplog): + aligner = AutoAligner(tmp_path, "head") + aligner.submit(_result(index=1, final_path=tmp_path / "hang" / "episode_1")) + task = asyncio.create_task(aligner.run()) + pid_file = Path(str(stub_tool) + ".pid") + for _ in range(300): + if pid_file.exists() and pid_file.read_text(): + break + await asyncio.sleep(0.05) + pid = int(pid_file.read_text()) + with caplog.at_level(logging.WARNING, logger=auto_align.logger.name): + await asyncio.wait_for(aligner.close(task, 0.2), 10) + assert "still has queued episodes" in caplog.text + assert task.cancelled() and aligner._process is None + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + assert not (tmp_path / "hang_aligned").exists() + + +async def test_a_kept_episode_is_aligned_for_real_through_the_command_line_tool(tmp_path): + session = tmp_path / "out" / "s1" + build_raw(session) + write_manifest(session, make_manifest(session, session_id="s1")) + aligner = AutoAligner(tmp_path / "out", INDEX) + assert [job.episode_index for job in aligner.scan()] == [0] + task = asyncio.create_task(aligner.run()) + for _ in range(1200): + if (tmp_path / "out" / "s1_aligned" / "episode_0").is_dir(): + break + await asyncio.sleep(0.1) + await aligner.close(task, 120) + assert (tmp_path / "out" / "s1_aligned" / "episode_0").is_dir() + assert aligner.scan() == [] + + +async def test_an_episode_the_tool_refuses_is_marked_with_its_reason_and_left_unaligned(tmp_path): + session = tmp_path / "out" / "s1" + build_raw(session) + write_manifest(session, make_manifest(session, session_id="s1")) + aligner = AutoAligner(tmp_path / "out", "not_a_camera") + aligner.submit(_result(index=0, session_id="s1", final_path=session / "episode_0")) + task = asyncio.create_task(aligner.run()) + await aligner.close(task, 120) + marker = tmp_path / "out" / "s1_aligned" / ".episode_0.refused" + assert marker.read_text().strip() and not (tmp_path / "out" / "s1_aligned" / "episode_0").exists() + + +async def test_an_unreadable_manifest_leaves_no_marker_and_a_later_scan_queues_the_episode_again(tmp_path): + session = tmp_path / "out" / "s1" + build_raw(session) + write_manifest(session, make_manifest(session, session_id="s1")) + aligner = AutoAligner(tmp_path / "out", INDEX) + (job,) = aligner.scan() + manifest = session / auto_align.MANIFEST_DIR / "episode_0.json" + original = manifest.read_text() + aligner.submit(_result(index=0, session_id="s1", final_path=job.final)) + manifest.write_text("{not json") + task = asyncio.create_task(aligner.run()) + await aligner.close(task, 120) + assert not job.refused_marker.exists() and not job.aligned.exists() + manifest.write_text(original) + assert aligner.scan() == [job] + + +async def test_a_missing_manifest_exits_2_without_a_marker(tmp_path): + session = tmp_path / "out" / "s1" + build_raw(session) + aligner = AutoAligner(tmp_path / "out", INDEX) + aligner.submit(_result(index=0, session_id="s1", final_path=session / "episode_0")) + task = asyncio.create_task(aligner.run()) + await aligner.close(task, 120) + assert not (tmp_path / "out" / "s1_aligned").exists() + + +def _old(path, days=2): + stamp = time.time() - days * 86400 + os.utime(path, (stamp, stamp)) + + +def test_scan_removes_old_staging_directories_beside_a_group_and_keeps_fresh_ones(tmp_path): + _write_session_manifest(tmp_path, "s1", 0) + stale = tmp_path / ".s1_aligned.staging-abc" + fresh = tmp_path / ".s1_aligned.staging-def" + unrelated = tmp_path / ".other_aligned.staging-xyz" + for directory in (stale, fresh, unrelated): + (directory / "episode_0").mkdir(parents=True) + _old(stale) + _old(unrelated) + AutoAligner(tmp_path, "head").scan() + assert not stale.exists() and fresh.is_dir() and unrelated.is_dir() diff --git a/examples/egocap/tests/backend/test_livekit_recording.py b/examples/egocap/tests/backend/test_livekit_recording.py index 31ce77d2f6..8e8421b696 100644 --- a/examples/egocap/tests/backend/test_livekit_recording.py +++ b/examples/egocap/tests/backend/test_livekit_recording.py @@ -19,6 +19,7 @@ from egocap_web.livekit_recording import ( BUDGET_ENV, LiveKitRecordingFeed, + sensor_capture_names, uncertainty_budget_ns, unrecordable_reason, ) @@ -112,12 +113,17 @@ def __init__(self, offset_ns: int = -10_000_000_000, uncertainty_ns: int = 1_000 self.uncertainty_ns = uncertainty_ns self.discontinuity = False self.available = True + self.epoch = 1 def mapping(self) -> ClockMapping | None: if not self.available: return None return ClockMapping( - epoch=1, offset_ns=self.offset_ns, rtt_ns=2_000, uncertainty_ns=self.uncertainty_ns, accepted_at_ns=0 + epoch=self.epoch, + offset_ns=self.offset_ns, + rtt_ns=2_000, + uncertainty_ns=self.uncertainty_ns, + accepted_at_ns=0, ) @@ -195,6 +201,27 @@ async def test_mapping_offset_places_capture_time_on_the_local_clock(): assert [m.capture_time for m in ingress.offered] == [11.0] +async def test_each_frame_carries_the_epoch_of_the_mapping_that_placed_it(): + clock = FakeClockSync() + frames = [_received(16, 1_000_000), _received(16, 2_000_000), _received(16, 3_000_000)] + ingress = FakeIngress() + + def new_epoch(c): + c.epoch = 5 + + # The mapping is replaced (new epoch, same offset) while the second frame is being served. + receiver = _FlipAfterFirst({16: frames}, clock, new_epoch) + feed = _feed(receiver, [_stream(16, "Head left")], ingress, {16: "head_left"}) + feed.start() + await _settle() + await feed.stop() + assert [m.continuity for m in ingress.offered] == [ + ("clock_mapping", 1), + ("clock_mapping", 5), + ("clock_mapping", 5), + ] + + @pytest.mark.parametrize( ("value", "expected_ns"), [ @@ -860,3 +887,16 @@ def test_unrecordable_reason_refuses_a_capture_server_on_another_host(url): def test_a_remote_capture_server_does_not_affect_a_session_without_livekit_streams(): assert unrecordable_reason(_started(object()), "ws://192.168.1.20:8082") is None + + +def test_sensor_capture_names_keeps_only_streams_stamped_at_sensor_capture(): + streams = [ + _stream(1, "sensor", CAPTURE), + _stream(2, "frame_ready", p.LiveKitTimestampBasis.PUBLISHER_MONOTONIC_US), + _stream(3, "unix", p.LiveKitTimestampBasis.PUBLISHER_PROCESSING_UNIX_US), + ] + assert sensor_capture_names(streams) == frozenset({"sensor"}) + + +def test_sensor_capture_names_is_empty_without_streams(): + assert sensor_capture_names([]) == frozenset() diff --git a/examples/egocap/tests/backend/test_livekit_recording_source.py b/examples/egocap/tests/backend/test_livekit_recording_source.py index 5234d1e988..35989cc8f4 100644 --- a/examples/egocap/tests/backend/test_livekit_recording_source.py +++ b/examples/egocap/tests/backend/test_livekit_recording_source.py @@ -424,8 +424,8 @@ def clock_sync_with(history, *, discontinuity=False, rejected=0, current=True): ) -async def prepared_source(streams): - source = make_source(streams) +async def prepared_source(streams, **kwargs): + source = make_source(streams, **kwargs) await prepare_with_frames(source, streams) return source, FakeReceiver.instances[0] @@ -479,12 +479,51 @@ async def test_timing_evidence_describes_the_mappings_the_take_was_placed_under( "exchanges_rejected": 4, "mapping_current_at_stop": False, }, - "tracking_time": "host_poll", + "event_time": { + "cam16": {"meaning": "sensor_capture", "qualified": True}, + "cam17": {"meaning": "frame_ready", "qualified": False}, + }, "tracking_semantics": lrec._TRACKING_SEMANTICS, } assert json.loads(json.dumps(evidence)) == evidence +async def test_event_time_is_keyed_by_recorded_source_and_qualifies_only_sensor_capture(monkeypatch): + streams = [livekit_stream(16), livekit_stream(17, basis=MONO)] + source, receiver = await prepared_source(streams) + receiver.clock_sync = clock_sync_with(history_of(1)) + event_time = source.timing_evidence()["event_time"] + # Recorded sources are named after their streams; the key is that source name, never a stream id. + assert {k: v["qualified"] for k, v in event_time.items()} == { + "cam16": True, + "cam17": False, + } + assert event_time["cam16"]["meaning"] == "sensor_capture" and event_time["cam17"]["meaning"] == "frame_ready" + assert 16 not in event_time and 17 not in event_time + + +def test_a_tracking_latency_makes_that_source_a_marked_estimate_and_others_stay_unqualified(): + event_time = lrec.tracking_event_time({"pico_controller": 0.02}) + assert event_time["pico_controller"] == { + "meaning": "host_poll_minus_latency", + "estimated": True, + "latency_s": 0.02, + "qualified": True, + } + assert event_time["manus_glove"] == {"meaning": "host_poll_time", "qualified": False} + + +def test_a_zero_latency_is_still_a_qualified_estimate(): + assert lrec.tracking_event_time({"manus_glove": 0.0})["manus_glove"]["latency_s"] == 0.0 + assert lrec.tracking_event_time({"manus_glove": 0.0})["manus_glove"]["qualified"] is True + + +async def test_the_sources_timing_evidence_lists_only_its_cameras_not_tracking(): + source, receiver = await prepared_source([livekit_stream(16)]) + receiver.clock_sync = clock_sync_with(history_of(1)) + assert set(source.timing_evidence()["event_time"]) == {"cam16"} + + async def test_timing_evidence_records_the_configured_uncertainty_budget(monkeypatch): monkeypatch.setenv(lrec.BUDGET_ENV, "2.5") source, receiver = await prepared_source([livekit_stream(16)]) diff --git a/examples/egocap/tests/backend/test_recording_adapter.py b/examples/egocap/tests/backend/test_recording_adapter.py index 9c18cf11e6..9d5c385fea 100644 --- a/examples/egocap/tests/backend/test_recording_adapter.py +++ b/examples/egocap/tests/backend/test_recording_adapter.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import dataclasses import threading import numpy as np @@ -25,17 +26,20 @@ from capture_server import protocol as capture_protocol from capture_server.client import CameraImage, CaptureFrame from egocap_web.recording_adapter import ( + MANUS_GLOVE_SPEC, + PICO_CONTROLLER_SPEC, RecordingAdapterError, RecordingIngress, - _convert_raw_item, _raw_item_bytes, build_manus_glove_fields, build_pico_controller_fields, button_pressed, convert_camera_image, + convert_capture_frame, convert_controller_aim_pose, convert_hand_pose, convert_head_pose, + convert_raw_item, oxr_fingertips_to_manus_skeleton, pose7d_to_matrix, validate_tracking_descriptor, @@ -319,7 +323,10 @@ def test_validate_tracking_descriptor_rejects_mismatched_fields(): class _FakeCameraImage: - def __init__(self, rgb: np.ndarray, capture_time_ns: int, receive_time_ns: int, name: str = "cam"): + def __init__( + self, rgb: np.ndarray, capture_time_ns: int, receive_time_ns: int, name: str = "cam", session_id: str = "s" + ): + self.session_id = session_id self._rgb = rgb self.capture_time_ns = capture_time_ns self.receive_time_ns = receive_time_ns @@ -340,6 +347,12 @@ def test_convert_camera_image_is_rgb8_with_seconds_timestamps(): assert sample.image is rgb +def test_convert_camera_image_records_the_transport_session_as_its_continuity(): + image = _FakeCameraImage(np.zeros((4, 4, 3), dtype=np.uint8), 1, 2, session_id="sess-a") + sample = convert_camera_image(image, "head_human_camera") # type: ignore[arg-type] + assert sample.continuity == ("capture_session", "sess-a") + + # --- bounded ingress ----------------------------------------------------------- @@ -663,9 +676,9 @@ def _video_sample(shape=(10, 10, 3), source="head_left") -> VideoSample: ) -def test_convert_raw_item_passes_a_video_sample_through_unchanged(): +def testconvert_raw_item_passes_a_video_sample_through_unchanged(): sample = _video_sample() - converted = _convert_raw_item(sample) + converted = convert_raw_item(sample) assert len(converted) == 1 assert converted[0] is sample @@ -730,7 +743,7 @@ def test_pico_and_manus_samples_are_both_stamped_with_the_one_host_poll_frame_ti # The recorded tracking time is when the host polled the devices (HOST_POLL), not a device measurement time: # every sample from one frame carries that frame's single timestamp, whichever source it is. frame = _tracking_frame(capture_time_ns=5_000_123_456_789, receive_time_ns=5_000_130_000_000) - samples = _convert_raw_item(frame) + samples = convert_raw_item(frame) assert sorted(s.source for s in samples) == ["manus_glove", "pico_controller"] for sample in samples: @@ -816,7 +829,7 @@ def offer(self, item): (video,) = ingress.offered assert isinstance(video, VideoSample) - tracking = _convert_raw_item(_tracking_frame(capture_time_ns=event_ns + 1_500_000, receive_time_ns=now_ns)) + tracking = convert_raw_item(_tracking_frame(capture_time_ns=event_ns + 1_500_000, receive_time_ns=now_ns)) assert tracking # Both are seconds on the PC's monotonic clock; their gap is the synthetic 1.5 ms (the host poll lags the @@ -826,3 +839,63 @@ def offer(self, item): assert abs(video.capture_time - event_ns / S) <= tolerance_s for sample in tracking: assert abs((sample.capture_time - video.capture_time) - truth_gap_s) <= tolerance_s + + +def test_tracking_specs_name_the_frame_types_that_select_their_alignment_rules(): + from egocap.recording.alignment import FRAME_FIELD_RULES + + assert PICO_CONTROLLER_SPEC.frame_type == "PicoTrackingFrame" + assert MANUS_GLOVE_SPEC.frame_type == "ManusFrame" + assert {PICO_CONTROLLER_SPEC.frame_type, MANUS_GLOVE_SPEC.frame_type} <= set(FRAME_FIELD_RULES) + + +def test_every_tracking_sample_of_a_frame_carries_the_host_clock_connection_as_its_continuity(): + frame = _tracking_frame(capture_time_ns=5_000_000_000, receive_time_ns=5_000_100_000) + samples = convert_raw_item(frame) + assert sorted(s.source for s in samples) == ["manus_glove", "pico_controller"] + assert [s.continuity for s in samples] == [("host_clock", "s")] * 2 + + +def _capture_times(samples): + return {s.source: s.capture_time for s in samples} + + +def test_no_latency_leaves_the_poll_time_as_the_capture_time(): + frame = _tracking_frame(capture_time_ns=5_000_000_000, receive_time_ns=5_040_000_000) + for samples in (convert_raw_item(frame), convert_raw_item(frame, {}), convert_capture_frame(frame, None)): + assert set(_capture_times(samples).values()) == {5.0} + + +def test_a_latency_is_taken_off_the_poll_time_per_source_and_receive_time_is_unchanged(): + frame = _tracking_frame(capture_time_ns=5_000_000_000, receive_time_ns=5_040_000_000) + samples = convert_raw_item(frame, {"pico_controller": 0.02, "manus_glove": 0.0}) + assert _capture_times(samples) == pytest.approx({"pico_controller": 4.98, "manus_glove": 5.0}) + assert {s.receive_time for s in samples} == {5.04} + + +def test_a_later_receive_time_does_not_move_the_adjusted_capture_times(): + latency = {"pico_controller": 0.02, "manus_glove": 0.03} + prompt = _tracking_frame(capture_time_ns=5_000_000_000, receive_time_ns=5_001_000_000) + delayed = _tracking_frame(capture_time_ns=5_000_000_000, receive_time_ns=5_250_000_000) + assert _capture_times(convert_raw_item(prompt, latency)) == _capture_times(convert_raw_item(delayed, latency)) + + +def test_the_latency_is_applied_once_to_each_frame(): + latency = {"pico_controller": 0.02, "manus_glove": 0.02} + times = [ + _capture_times(convert_raw_item(_tracking_frame(t, t + 1_000_000), latency))["pico_controller"] + for t in (1_000_000_000, 2_000_000_000) + ] + assert times == pytest.approx([0.98, 1.98]) + + +def test_a_latency_does_not_change_the_tracking_continuity(): + samples = convert_raw_item(_tracking_frame(5_000_000_000, 5_000_100_000), {"pico_controller": 0.02}) + assert {s.continuity for s in samples} == {("host_clock", "s")} + + +def test_a_new_capture_connection_changes_the_tracking_continuity_identity(): + first = convert_raw_item(_tracking_frame(1_000_000_000, 1_000_100_000)) + other = dataclasses.replace(_tracking_frame(2_000_000_000, 2_000_100_000), session_id="t") + assert {s.continuity for s in convert_raw_item(other)} == {("host_clock", "t")} + assert {s.continuity for s in first} == {("host_clock", "s")} diff --git a/examples/egocap/tests/backend/test_recording_session.py b/examples/egocap/tests/backend/test_recording_session.py index e31174e73c..c64c62cfbc 100644 --- a/examples/egocap/tests/backend/test_recording_session.py +++ b/examples/egocap/tests/backend/test_recording_session.py @@ -32,6 +32,7 @@ from egocap_web.recording_session import CoordinatorBusyError, RecordingSessionCoordinator, SessionStatus from recording_service import ( Disposition, + EpisodeResult, FieldSpec, NumericSourceSpec, RecordingPhase, @@ -57,9 +58,14 @@ def factory() -> ServiceConfig: return factory -def make_coordinator(tmp_path: Path, *, readiness_check=ALWAYS_READY, **overrides) -> RecordingSessionCoordinator: +def make_coordinator( + tmp_path: Path, *, readiness_check=ALWAYS_READY, tracking_latency=None, **overrides +) -> RecordingSessionCoordinator: return RecordingSessionCoordinator( - make_config_factory(tmp_path, **overrides), queue_capacity=8, readiness_check=readiness_check + make_config_factory(tmp_path, **overrides), + queue_capacity=8, + readiness_check=readiness_check, + tracking_latency=tracking_latency, ) @@ -737,3 +743,211 @@ async def test_a_final_close_keeps_admission_blocked_but_an_ordinary_close_reope coordinator.set_task("too late") with pytest.raises(CoordinatorBusyError): await coordinator.start_episode() + + +async def test_the_configured_latency_reaches_the_converted_tracking_sample_times(tmp_path: Path, monkeypatch): + from tests.backend.test_recording_adapter import default_tracking_values + + converters = [] + real = coordinator_module.RecordingIngress + + def capture_convert(*args, **kwargs): + converters.append(kwargs["convert"]) + return real(*args, **kwargs) + + monkeypatch.setattr(coordinator_module, "RecordingIngress", capture_convert) + coordinator = make_coordinator( + tmp_path, + numeric_sources=(PICO_CONTROLLER_SPEC, MANUS_GLOVE_SPEC), + tracking_latency={"pico_controller": 0.02}, + ) + await coordinator.open() + coordinator.set_task("t") + await coordinator.start_episode() + frame = CaptureFrame( + session_id="s", + description=None, # type: ignore[arg-type] + stream_id=1, + sequence=0, + capture_time_ns=5_000_000_000, + receive_time_ns=5_040_000_000, + payload=capture_protocol.encode_tracking_frame(default_tracking_values()), + ) + times = {s.source: s.capture_time for s in converters[-1](frame)} + assert times["pico_controller"] == pytest.approx(4.98) and times["manus_glove"] == 5.0 + await coordinator.close() + + +class _MediaSource: + def __init__(self, evidence): + self.evidence = evidence + + async def prepare(self): + pass + + def start(self, ingress, on_failure): + pass + + async def stop(self): + pass + + def timing_evidence(self): + return self.evidence + + +async def _kept_timing(tmp_path: Path, *, latency=None, livekit_source=None): + coordinator = RecordingSessionCoordinator( + make_config_factory(tmp_path, numeric_sources=(PICO_CONTROLLER_SPEC, MANUS_GLOVE_SPEC)), + queue_capacity=8, + readiness_check=ALWAYS_READY, + livekit_source=livekit_source, + tracking_latency=latency, + ) + await coordinator.open() + coordinator.set_task("t") + episode = await coordinator.start_episode() + coordinator.on_capture_frame(_fake_tracking_frame(capture_time_ns=5_000_000_000)) + result = await coordinator.stop_episode(episode.episode_id) + await coordinator.close() + group = zarr.open_group(str(result.final_path), mode="r") + return dict(group.attrs).get("timing"), group + + +async def test_a_configured_latency_is_recorded_as_a_qualified_estimate_matching_the_stored_times(tmp_path: Path): + timing, group = await _kept_timing(tmp_path, latency={"pico_controller": 0.02, "manus_glove": 0.0}) + assert timing["event_time"] == { + "pico_controller": { + "meaning": "host_poll_minus_latency", + "estimated": True, + "latency_s": 0.02, + "qualified": True, + }, + "manus_glove": { + "meaning": "host_poll_minus_latency", + "estimated": True, + "latency_s": 0.0, + "qualified": True, + }, + } + assert group["pico_controller"]["capture_time"][0] == pytest.approx(4.98) + assert group["manus_glove"]["capture_time"][0] == pytest.approx(5.0) + + +async def test_the_coordinator_keeps_its_own_copy_of_the_tracking_latency(tmp_path: Path): + latency = {"pico_controller": 0.02} + coordinator = make_coordinator(tmp_path, tracking_latency=latency) + latency["pico_controller"] = 9.0 + assert coordinator._tracking_latency == {"pico_controller": 0.02} + assert make_coordinator(tmp_path)._tracking_latency == {} + + +async def test_a_missing_latency_is_recorded_as_an_unqualified_host_poll_time(tmp_path: Path): + timing, _ = await _kept_timing(tmp_path) + assert timing["event_time"] == { + "pico_controller": {"meaning": "host_poll_time", "qualified": False}, + "manus_glove": {"meaning": "host_poll_time", "qualified": False}, + } + + +async def test_tracking_timing_is_recorded_when_the_media_source_has_no_evidence(tmp_path: Path): + timing, _ = await _kept_timing(tmp_path, latency={"pico_controller": 0.02}, livekit_source=_MediaSource(None)) + assert timing["event_time"]["pico_controller"]["qualified"] is True + assert timing["event_time"]["manus_glove"]["qualified"] is False + + +async def test_tracking_timing_is_merged_with_the_media_evidence_and_keeps_its_cameras(tmp_path: Path): + media = { + "clock_mapping": {"mappings": []}, + "event_time": { + "cam16": {"meaning": "sensor_capture", "qualified": True}, + "pico_controller": {"meaning": "stale", "qualified": False}, + }, + } + timing, _ = await _kept_timing(tmp_path, latency={"pico_controller": 0.02}, livekit_source=_MediaSource(media)) + assert timing["clock_mapping"] == {"mappings": []} + assert timing["event_time"]["cam16"] == {"meaning": "sensor_capture", "qualified": True} + assert timing["event_time"]["pico_controller"]["meaning"] == "host_poll_minus_latency" + assert timing["event_time"]["manus_glove"] == {"meaning": "host_poll_time", "qualified": False} + + +async def test_a_kept_stop_calls_the_kept_hook_once_and_leaves_the_result_hook_alone(tmp_path: Path): + coordinator = make_coordinator(tmp_path) + kept, results = [], [] + coordinator.on_episode_kept = kept.append + coordinator.on_episode_result = results.append + await coordinator.open() + coordinator.set_task("t") + episode = await coordinator.start_episode() + result = await coordinator.stop_episode(episode.episode_id) + assert kept == [result] and results == [result] + await coordinator.close() + + +async def test_a_discarded_episode_does_not_call_the_kept_hook_but_still_reports_its_result(tmp_path: Path): + coordinator = make_coordinator(tmp_path) + kept, results = [], [] + coordinator.on_episode_kept = kept.append + coordinator.on_episode_result = results.append + await coordinator.open() + coordinator.set_task("t") + episode = await coordinator.start_episode() + result = await coordinator.discard_episode(episode.episode_id) + assert kept == [] and results == [result] + await coordinator.close() + + +async def test_a_kept_stop_without_hooks_set_succeeds(tmp_path: Path): + coordinator = make_coordinator(tmp_path) + await coordinator.open() + coordinator.set_task("t") + episode = await coordinator.start_episode() + assert (await coordinator.stop_episode(episode.episode_id)).disposition is Disposition.KEPT + await coordinator.close() + + +def _recovered_result(disposition): + return EpisodeResult( + session_id="s", + episode_id="s/episode_0", + episode_index=0, + disposition=disposition, + final_path=Path("/out/s/episode_0"), + pending_path=None, + counts=(), + duration_seconds=1.0, + quality=None, + ) + + +@pytest.mark.parametrize( + ("disposition", "expected_calls"), + [(Disposition.KEPT, 1), (Disposition.RECOVERY_REQUIRED, 0), (Disposition.DISCARDED, 0)], +) +async def test_a_recovered_episode_calls_the_kept_hook_only_when_it_was_kept( + tmp_path: Path, monkeypatch, disposition, expected_calls +): + coordinator = make_coordinator(tmp_path) + result = _recovered_result(disposition) + + class RecoveringService: + async def recover_episode(self, episode_id): + return result + + kept, results = [], [] + coordinator.on_episode_kept = kept.append + coordinator.on_episode_result = results.append + monkeypatch.setattr(coordinator, "_require_open", lambda: RecoveringService()) + assert await coordinator.recover_episode("s/episode_0") is result + assert kept == [result] * expected_calls and results == [] + + +async def test_a_recovered_episode_without_a_kept_hook_succeeds(tmp_path: Path, monkeypatch): + coordinator = make_coordinator(tmp_path) + result = _recovered_result(Disposition.KEPT) + + class RecoveringService: + async def recover_episode(self, episode_id): + return result + + monkeypatch.setattr(coordinator, "_require_open", lambda: RecoveringService()) + assert await coordinator.recover_episode("s/episode_0") is result diff --git a/examples/egocap/tests/backend/test_recording_session_livekit.py b/examples/egocap/tests/backend/test_recording_session_livekit.py index 314a4581e6..8ea2d5ce32 100644 --- a/examples/egocap/tests/backend/test_recording_session_livekit.py +++ b/examples/egocap/tests/backend/test_recording_session_livekit.py @@ -186,7 +186,11 @@ async def test_the_sources_timing_evidence_is_given_to_the_service_once_before_s tmp_path, monkeypatch ): events: list[str] = [] - first = {"clock_mapping": {"mappings": [1]}} + first = {"clock_mapping": {"mappings": [1]}, "event_time": {"cam16": {"qualified": True}}} + tracking = { + "pico_controller": {"meaning": "host_poll_time", "qualified": False}, + "manus_glove": {"meaning": "host_poll_time", "qualified": False}, + } calls = spy_on_service(monkeypatch, events) coordinator = make_coordinator(tmp_path, EvidenceSource(events, [first, None])) await coordinator.open() @@ -194,18 +198,24 @@ async def test_the_sources_timing_evidence_is_given_to_the_service_once_before_s one = await coordinator.start_episode() await coordinator.stop_episode(one.episode_id) - assert calls == [(one.episode_id, first)] + merged = {**first, "event_time": {**first["event_time"], **tracking}} + assert calls == [(one.episode_id, merged)] assert events.index("timing_evidence") < events.index("set_timing") < events.index("stop_episode") two = await coordinator.start_episode() await coordinator.stop_episode(two.episode_id) - assert calls == [(one.episode_id, first)] # the second take's source had none; nothing carried over + # the second take's source had none: only its tracking times are recorded, nothing carried over + assert calls[1:] == [(two.episode_id, {"event_time": tracking})] await coordinator.close() async def test_a_discarded_takes_evidence_does_not_describe_the_next_take(tmp_path, monkeypatch): events: list[str] = [] calls = spy_on_service(monkeypatch, events) + tracking = { + "pico_controller": {"meaning": "host_poll_time", "qualified": False}, + "manus_glove": {"meaning": "host_poll_time", "qualified": False}, + } discarded = {"clock_mapping": {"mappings": ["discarded take"]}} coordinator = make_coordinator(tmp_path, EvidenceSource(events, [discarded, None])) await coordinator.open() @@ -216,13 +226,18 @@ async def test_a_discarded_takes_evidence_does_not_describe_the_next_take(tmp_pa two = await coordinator.start_episode() await coordinator.stop_episode(two.episode_id) - assert calls == [] # the second take's source had none, and nothing was inherited from the discarded one + # the second take's source had none: only its tracking times are recorded, nothing inherited + assert calls == [(two.episode_id, {"event_time": tracking})] await coordinator.close() async def test_an_interrupted_takes_evidence_does_not_describe_the_next_take(tmp_path, monkeypatch): events: list[str] = [] calls = spy_on_service(monkeypatch, events) + tracking = { + "pico_controller": {"meaning": "host_poll_time", "qualified": False}, + "manus_glove": {"meaning": "host_poll_time", "qualified": False}, + } interrupted = {"clock_mapping": {"mappings": ["interrupted take"]}} coordinator = make_coordinator(tmp_path, EvidenceSource(events, [interrupted, None])) await coordinator.open() @@ -237,5 +252,5 @@ async def test_an_interrupted_takes_evidence_does_not_describe_the_next_take(tmp two = await coordinator.start_episode() await coordinator.stop_episode(two.episode_id) - assert calls == [] + assert calls == [(two.episode_id, {"event_time": tracking})] await coordinator.close() diff --git a/examples/egocap/tests/backend/test_station_config.py b/examples/egocap/tests/backend/test_station_config.py index d5df52a5d0..494987edca 100644 --- a/examples/egocap/tests/backend/test_station_config.py +++ b/examples/egocap/tests/backend/test_station_config.py @@ -48,6 +48,8 @@ def test_flat_file_with_station_name_uses_it_as_id(tmp_path): task_description=None, data_dir=tmp_path / "recordings", cameras=(), + latency={}, + alignment_index=None, ) @@ -256,3 +258,78 @@ def test_the_shipped_template_resolves_a_real_station_label(): assert resolved.station_id == "station_01" assert resolved.station_label == "Station 01" assert resolved.auto_fill_operator_name is False + + +def _latency_yaml(**seconds): + return "alignment:\n tracking_latency_s:\n" + "".join( + f" {name}: {value}\n" for name, value in seconds.items() + ) + + +def test_tracking_source_latencies_are_resolved_in_seconds(tmp_path): + path = _write(tmp_path, _latency_yaml(pico_controller=0.05, manus_glove=1)) + assert load_station_config(path).latency == {"pico_controller": 0.05, "manus_glove": 1.0} + + +def test_numeric_sources_latency_is_ignored(tmp_path): + path = _write( + tmp_path, + "numeric_sources:\n - name: pico_controller\n latency: 0.1\n - name: manus_glove\n - just_a_string\n", # noqa: E501 + ) + assert load_station_config(path).latency == {} + + +def test_absent_or_empty_tracking_latency_resolves_to_no_latency(tmp_path): + assert load_station_config(_write(tmp_path, "alignment:\n tracking_latency_s:\n")).latency == {} + assert load_station_config(_write(tmp_path, "alignment: {}\n")).latency == {} + + +@pytest.mark.parametrize("value", ["true", "'0.1'", "-0.1", ".inf", ".nan", "[1]"]) +def test_an_invalid_tracking_latency_is_a_station_config_error(tmp_path, value): + path = _write(tmp_path, _latency_yaml(pico_controller=value)) + with pytest.raises(StationConfigError, match="tracking_latency_s.pico_controller must be"): + load_station_config(path) + + +def test_a_tracking_latency_for_an_unknown_source_is_a_station_config_error(tmp_path): + with pytest.raises(StationConfigError, match="unknown source 'head'"): + load_station_config(_write(tmp_path, _latency_yaml(head=0.1))) + + +@pytest.mark.parametrize( + ("text", "match"), + [ + ("alignment: [1]\n", "'alignment' must be a mapping"), + ("alignment:\n tracking_latency_s: [0.1]\n", "tracking_latency_s must be a mapping"), + ], +) +def test_a_non_mapping_alignment_or_tracking_latency_is_a_station_config_error(tmp_path, text, match): + with pytest.raises(StationConfigError, match=match): + load_station_config(_write(tmp_path, text)) + + +_CAMERAS = "video_sources:\n - type: oak_camera\n name: head\n - type: oak_camera\n name: wrist\n" + + +def test_alignment_is_off_without_an_index_source_even_with_cameras(tmp_path): + assert load_station_config(_write(tmp_path, _CAMERAS)).alignment_index is None + + +def test_alignment_index_source_can_be_named(tmp_path): + path = _write(tmp_path, _CAMERAS + "alignment:\n index_source: wrist\n") + assert load_station_config(path).alignment_index == "wrist" + + +def test_alignment_can_be_disabled(tmp_path): + path = _write(tmp_path, _CAMERAS + "alignment:\n enabled: false\n index_source: head\n") + assert load_station_config(path).alignment_index is None + + +def test_alignment_is_off_without_an_alignment_section(tmp_path): + assert load_station_config(_write(tmp_path, "meta: {}\n")).alignment_index is None + + +@pytest.mark.parametrize("section", ["[a, b]", "yes", "3"]) +def test_a_non_mapping_alignment_section_is_a_station_config_error(tmp_path, section): + with pytest.raises(StationConfigError, match="alignment"): + load_station_config(_write(tmp_path, _CAMERAS + f"alignment: {section}\n")) diff --git a/examples/egocap/tests/backend/test_viser_scene.py b/examples/egocap/tests/backend/test_viser_scene.py index ae30fd255d..60b5e46ea1 100644 --- a/examples/egocap/tests/backend/test_viser_scene.py +++ b/examples/egocap/tests/backend/test_viser_scene.py @@ -738,6 +738,185 @@ async def stop(self): assert events == ["scene stop"] and created[0].stopped # capture is still stopped +_HEAD_VIDEO = "video_sources:\n - type: oak_camera\n name: head\n" +_HEAD_ALIGNED = _HEAD_VIDEO + "alignment:\n index_source: head\n" + + +class _FakeAligner: + instances = [] + + def __init__(self, output_root, index_source): + self.output_root, self.index_source = output_root, index_source + self.submitted = [] + self.cancelled = False + self.closed_with = None + _FakeAligner.instances.append(self) + + def submit(self, result): + self.submitted.append(result) + + def note_video_sources(self, recorded, qualified): + self.noted = (recorded, qualified) + + async def run(self): + import asyncio + + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + self.cancelled = True + raise + + async def close(self, task, timeout): + import asyncio + + self.closed_with = timeout + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +async def _lifespan_with_station(monkeypatch, tmp_path, station_yaml): + import egocap_web.app as app_module + + config = tmp_path / "station.yaml" + config.write_text(station_yaml, encoding="utf-8") + monkeypatch.setenv("EGOCAP_STATION_CONFIG", str(config)) + monkeypatch.setattr(app_module, "CaptureStatusMonitor", _FakeMonitor) + monkeypatch.setattr(app_module, "AutoAligner", _FakeAligner) + _FakeAligner.instances.clear() + return app_module + + +async def test_the_lifespan_runs_the_aligner_for_the_index_camera_and_cancels_it_at_shutdown( + monkeypatch, tmp_path +): + import asyncio + + app_module = await _lifespan_with_station( + monkeypatch, + tmp_path, + f"recording:\n data_dir: {tmp_path}/rec\n" + _HEAD_ALIGNED, + ) + app = app_module.FastAPI() + async with app_module.lifespan(app): + (aligner,) = _FakeAligner.instances + await asyncio.sleep(0) # let the worker start so its cancellation is observable + assert aligner.index_source == "head" and aligner.output_root == tmp_path / "rec" + assert app.state.recording_context.coordinator.on_episode_kept == aligner.submit + assert aligner.cancelled and aligner.closed_with == app_module._ALIGNER_SHUTDOWN_SECONDS + + +async def test_the_aligner_is_created_once_and_is_the_factorys_video_source_callback(monkeypatch, tmp_path): + app_module = await _lifespan_with_station( + monkeypatch, + tmp_path, + _HEAD_ALIGNED, + ) + factory_args = [] + real_factory = app_module._recording_config_factory + + def spy(*args, **kwargs): + factory_args.append(args) + return real_factory(*args, **kwargs) + + monkeypatch.setattr(app_module, "_recording_config_factory", spy) + async with app_module.lifespan(app_module.FastAPI()): + (aligner,) = _FakeAligner.instances + (args,) = factory_args + assert args[3] == aligner.note_video_sources + + +async def test_no_video_source_callback_is_passed_without_an_index_camera(monkeypatch, tmp_path): + app_module = await _lifespan_with_station(monkeypatch, tmp_path, "meta: {}\n") + factory_args = [] + real_factory = app_module._recording_config_factory + + def spy(*args, **kwargs): + factory_args.append(args) + return real_factory(*args, **kwargs) + + monkeypatch.setattr(app_module, "_recording_config_factory", spy) + async with app_module.lifespan(app_module.FastAPI()): + assert _FakeAligner.instances == [] and factory_args[0][3] is None + + +async def test_the_kept_hook_is_the_aligner_and_the_result_hook_is_left_alone(monkeypatch, tmp_path): + app_module = await _lifespan_with_station( + monkeypatch, + tmp_path, + _HEAD_ALIGNED, + ) + previous_calls = [] + real_context = app_module.RecordingContext + + def context_with_previous(**kwargs): + context = real_context(**kwargs) + context.coordinator.on_episode_result = previous_calls.append + return context + + monkeypatch.setattr(app_module, "RecordingContext", context_with_previous) + app = app_module.FastAPI() + async with app_module.lifespan(app): + (aligner,) = _FakeAligner.instances + coordinator = app.state.recording_context.coordinator + assert coordinator.on_episode_kept == aligner.submit + coordinator.on_episode_result("result") + assert previous_calls == ["result"] and aligner.submitted == [] + + +async def test_a_kept_result_emitted_while_the_recorder_closes_is_aligned_before_the_aligner_closes( + monkeypatch, tmp_path +): + app_module = await _lifespan_with_station( + monkeypatch, + tmp_path, + _HEAD_ALIGNED, + ) + from recording_service import Disposition, EpisodeResult + + last = EpisodeResult( + session_id="s", + episode_id="e", + episode_index=0, + disposition=Disposition.KEPT, + final_path=tmp_path / "episode_0", + pending_path=None, + counts=(), + duration_seconds=1.0, + quality=None, + ) + events = [] + real_close = app_module.RecordingSessionCoordinator.close + + async def close_after_a_last_result(self, *args, **kwargs): + self.on_episode_kept(last) + events.append("coordinator closed") + return await real_close(self, *args, **kwargs) + + monkeypatch.setattr(app_module.RecordingSessionCoordinator, "close", close_after_a_last_result) + async with app_module.lifespan(app_module.FastAPI()): + (aligner,) = _FakeAligner.instances + original_close = aligner.close + + async def recording_close(task, timeout): + events.append(f"aligner closed with {len(aligner.submitted)} result") + await original_close(task, timeout) + + aligner.close = recording_close + assert events == ["coordinator closed", "aligner closed with 1 result"] + + +async def test_the_lifespan_starts_no_aligner_when_alignment_is_off(monkeypatch, tmp_path): + app_module = await _lifespan_with_station( + monkeypatch, + tmp_path, + _HEAD_VIDEO + "alignment:\n index_source: head\n enabled: false\n", + ) + async with app_module.lifespan(app_module.FastAPI()): + assert _FakeAligner.instances == [] + + async def test_invalid_3d_view_settings_leave_it_unavailable_and_capture_running(monkeypatch): monkeypatch.setenv("EGOCAP_VISER_ENABLED", "1") monkeypatch.setenv("EGOCAP_VISER_PORT", "not-a-number") diff --git a/examples/egocap/tests/capture_server/test_cli.py b/examples/egocap/tests/capture_server/test_cli.py index 3ff524c6f3..2dfec3cd9c 100644 --- a/examples/egocap/tests/capture_server/test_cli.py +++ b/examples/egocap/tests/capture_server/test_cli.py @@ -28,6 +28,23 @@ def test_mock_only_options_are_rejected_with_real_acquisition(capsys): assert "only apply with --mock" in capsys.readouterr().err +def test_delay_options_are_rejected_without_mock(capsys): + for flags in (["--controller-delay-ms", "5"], ["--glove-delay-ms", "5"], ["--delay-jitter-ms", "5"]): + with pytest.raises(SystemExit) as excinfo: + main(flags) + assert excinfo.value.code == 2 + assert "only apply with --mock" in capsys.readouterr().err + + +def test_delay_options_are_parsed_in_milliseconds_and_default_to_zero(): + defaults = _build_arg_parser().parse_args(["--mock"]) + assert (defaults.controller_delay_ms, defaults.glove_delay_ms, defaults.delay_jitter_ms) == (0.0, 0.0, 0.0) + args = _build_arg_parser().parse_args( + ["--mock", "--controller-delay-ms", "120", "--glove-delay-ms", "45.5", "--delay-jitter-ms", "8"] + ) + assert (args.controller_delay_ms, args.glove_delay_ms, args.delay_jitter_ms) == (120.0, 45.5, 8.0) + + def test_arg_parser_defaults(): args = _build_arg_parser().parse_args(["--mock"]) assert args.host == "127.0.0.1" diff --git a/examples/egocap/tests/capture_server/test_mock_sdks.py b/examples/egocap/tests/capture_server/test_mock_sdks.py index f2b8c76e26..5a5745adca 100644 --- a/examples/egocap/tests/capture_server/test_mock_sdks.py +++ b/examples/egocap/tests/capture_server/test_mock_sdks.py @@ -216,6 +216,92 @@ def monotonic(self) -> float: np.testing.assert_array_equal(a["head"][HeadInputIndex.POSITION], b["head"][HeadInputIndex.POSITION]) +class _SettableClock: + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + +def _step_at(elapsed: float, **delays): + clock = _SettableClock() + with mic.TeleopSession(_full_config(), scenario="demo", seed=2, clock=clock, **delays) as session: + clock.now = elapsed + return session.step() + + +def _controller_position(result): + return result["controller_left"][ControllerInputIndex.GRIP_POSITION] + + +def _hand_positions(result): + return result["hand_left"][HandInputIndex.JOINT_POSITIONS] + + +def _head_position(result): + return result["head"][HeadInputIndex.POSITION] + + +def _equal(a, b): + np.testing.assert_allclose(a, b, atol=1e-12) + + +def test_zero_delay_output_matches_the_session_built_without_delay_options(): + plain, zero = _step_at(5.0), _step_at(5.0, controller_delay_s=0.0, glove_delay_s=0.0, delay_jitter_s=0.0) + for read in (_head_position, _controller_position, _hand_positions): + _equal(read(plain), read(zero)) + for index in ControllerInputIndex: + _equal(plain["controller_right"][index], zero["controller_right"][index]) + + +def test_a_delayed_controller_shows_the_pose_from_the_delay_earlier_and_the_head_is_unchanged(): + delayed, earlier, now = _step_at(5.0, controller_delay_s=0.3), _step_at(4.7), _step_at(5.0) + _equal(_controller_position(delayed), _controller_position(earlier)) + assert not np.allclose(_controller_position(delayed), _controller_position(now)) + _equal(_head_position(delayed), _head_position(now)) + + +def test_the_glove_and_controller_delays_are_independent(): + glove_only = _step_at(5.0, glove_delay_s=0.3) + controller_only = _step_at(5.0, controller_delay_s=0.3) + now, earlier = _step_at(5.0), _step_at(4.7) + _equal(_hand_positions(glove_only), _hand_positions(earlier)) + _equal(_controller_position(glove_only), _controller_position(now)) + _equal(_controller_position(controller_only), _controller_position(earlier)) + _equal(_hand_positions(controller_only), _hand_positions(now)) + + +def _entered_session(**delays): + session = mic.TeleopSession(_full_config(), scenario="demo", seed=7, clock=_SettableClock(), **delays) + return session.__enter__() + + +def test_jitter_is_deterministic_for_a_seed_and_bounded_by_the_jitter(): + kwargs = {"controller_delay_s": 0.2, "delay_jitter_s": 0.05} + session_a, session_b = _entered_session(**kwargs), _entered_session(**kwargs) + a = [session_a._delayed(10.0, "controller") for _ in range(50)] + b = [session_b._delayed(10.0, "controller") for _ in range(50)] + assert a == b + assert len(set(a)) > 1 + assert all(10.0 - 0.25 - 1e-9 <= value <= 10.0 - 0.15 + 1e-9 for value in a) + + +def test_a_delay_never_reaches_before_the_start_of_the_session(): + session = _entered_session(controller_delay_s=0.01, glove_delay_s=5.0, delay_jitter_s=0.5) + assert all(session._delayed(0.1, "controller") >= 0.0 for _ in range(50)) + assert session._delayed(1.0, "glove") == 0.0 + + +def test_a_source_without_a_delay_reads_the_current_instant_exactly(): + assert _entered_session(controller_delay_s=0.2)._delayed(3.0, "glove") == 3.0 + + +def test_jitter_never_moves_a_source_that_has_no_delay(): + session = _entered_session(controller_delay_s=0.2, delay_jitter_s=0.5) + assert all(session._delayed(3.0, "glove") == 3.0 for _ in range(50)) + + def test_startup_error_fault_raises_on_enter(): session = mic.TeleopSession(_full_config(), scenario="demo", seed=0, faults={"startup_error"}) with pytest.raises(RuntimeError, match="startup_error"): diff --git a/examples/egocap/tests/capture_server/test_setup_selection.py b/examples/egocap/tests/capture_server/test_setup_selection.py index 87c4e6f590..71ce640513 100644 --- a/examples/egocap/tests/capture_server/test_setup_selection.py +++ b/examples/egocap/tests/capture_server/test_setup_selection.py @@ -126,3 +126,24 @@ def test_real_setup_never_loads_the_mock_sdk_files(): "print('OK')\n" ) assert result.returncode == 0 and "OK" in result.stdout, result.stderr + + +async def test_configured_delays_reach_the_mock_session(monkeypatch): + from capture_server.mock import mock_isaac_capture + + seen = {} + + class RecordingSession(mock_isaac_capture.TeleopSession): + def __init__(self, *args, **kwargs): + seen.update(kwargs) + super().__init__(*args, **kwargs) + + monkeypatch.setattr(mock_isaac_capture, "TeleopSession", RecordingSession) + delays = {"controller_delay_s": 0.12, "glove_delay_s": 0.05, "delay_jitter_s": 0.01} + begin = p.BeginRequest(tracking=p.TrackingConfig(sources=(p.SourceType.HEAD,))) + acq = build_source_factory(ServerConfig(mock=True), delays=delays)(begin, ()) + try: + await acq.enter() + finally: + await acq.close() + assert {k: seen[k] for k in delays} == delays diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/config.yaml b/examples/egocap/tests/recording/fixtures/aligned_writer/config.yaml new file mode 100644 index 0000000000..7d5cc67634 --- /dev/null +++ b/examples/egocap/tests/recording/fixtures/aligned_writer/config.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +meta: + egocap_station_name: station_a +numeric_sources: +- frame_type: PicoTrackingFrame + name: pico_controller +- frame_type: ManusFrame + name: glove +recording: + numeric_frame_type: NumericFrame + video_frame_type: RGBCameraFrame +video_sources: +- name: head_camera +- name: wrist_camera diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/golden_attrs_sidecars.json b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_attrs_sidecars.json new file mode 100644 index 0000000000..65113a9075 --- /dev/null +++ b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_attrs_sidecars.json @@ -0,0 +1,713 @@ +{ + "entries": [ + { + "path": "config.yaml", + "kind": "file", + "size": 288, + "sha256": "e4142ea4adc900fb4870b4c31a465f0bf885ea925ee762f8aa688db5cfa82fbb" + }, + { + "path": "episode_0/head_camera.mp4", + "kind": "file", + "size": 544, + "sha256": "9f2ff969f02f9dd114ead58b074424d237b6a2109790434998549c09f6eaae36" + }, + { + "path": "episode_0/wrist_camera.mp4", + "kind": "file", + "size": 560, + "sha256": "140f5fa6e646b0bc7ac525867ac191a2b8ea76ccc17b46c702daa36e619be6d6" + }, + { + "path": "head_camera_intrinsics.json", + "kind": "file", + "size": 62, + "sha256": "3b2689f670b93236bafc6d6571e4c38d2c478f6190ba95a4e0d0a60e05b3001a" + }, + { + "path": "zarr.json", + "kind": "root_json", + "zarr_format": 3, + "node_type": "group", + "attributes": { + "task_description": "pick and place", + "operator": { + "id": 7, + "tags": [ + "a", + "b" + ] + }, + "index_source": "head_camera" + } + }, + { + "path": "episode_0", + "kind": "group", + "attributes": { + "task_description": "episode task", + "timing": { + "clock": "monotonic", + "offset_s": 0.25 + } + } + }, + { + "path": "episode_0/glove", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/glove/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a0", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a1", + "info_chunks": [ + 35, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_sensor_count_interp", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a2", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_sensor_rotation_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4 + ], + "attributes": {}, + "npz_key": "a3", + "info_chunks": [ + 35, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a4", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/right_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a5", + "info_chunks": [ + 35, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/head_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a6", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a7", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a8", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a9", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a10", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller", + "kind": "group", + "attributes": { + "handedness": "both" + } + }, + { + "path": "episode_0/pico_controller/a_button", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a11", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/body_joints_valid", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a12", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a13", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/head_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a14", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/left_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a15", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/left_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a16", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a17", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/right_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a18", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/right_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a19", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/wrist_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a20", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a21", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a22", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a23", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a24", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/valid_indices", + "kind": "array", + "dtype": "bool", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a25", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/valid_timestamps", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a26", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + } + ], + "expected_tree_listing": [ + "config.yaml", + "episode_0", + "episode_0/glove", + "episode_0/glove/capture_time", + "episode_0/glove/capture_time/c", + "episode_0/glove/capture_time/c/0", + "episode_0/glove/capture_time/zarr.json", + "episode_0/glove/left_hand_pose_interp", + "episode_0/glove/left_hand_pose_interp/c", + "episode_0/glove/left_hand_pose_interp/c/0", + "episode_0/glove/left_hand_pose_interp/c/0/0", + "episode_0/glove/left_hand_pose_interp/c/0/0/0", + "episode_0/glove/left_hand_pose_interp/c/0/0/0/0", + "episode_0/glove/left_hand_pose_interp/zarr.json", + "episode_0/glove/left_sensor_count_interp", + "episode_0/glove/left_sensor_count_interp/c", + "episode_0/glove/left_sensor_count_interp/c/0", + "episode_0/glove/left_sensor_count_interp/zarr.json", + "episode_0/glove/left_sensor_rotation_interp", + "episode_0/glove/left_sensor_rotation_interp/c", + "episode_0/glove/left_sensor_rotation_interp/c/0", + "episode_0/glove/left_sensor_rotation_interp/c/0/0", + "episode_0/glove/left_sensor_rotation_interp/zarr.json", + "episode_0/glove/receive_time", + "episode_0/glove/receive_time/c", + "episode_0/glove/receive_time/c/0", + "episode_0/glove/receive_time/zarr.json", + "episode_0/glove/right_hand_pose_interp", + "episode_0/glove/right_hand_pose_interp/c", + "episode_0/glove/right_hand_pose_interp/c/0", + "episode_0/glove/right_hand_pose_interp/c/0/0", + "episode_0/glove/right_hand_pose_interp/c/0/0/0", + "episode_0/glove/right_hand_pose_interp/c/0/0/0/0", + "episode_0/glove/right_hand_pose_interp/zarr.json", + "episode_0/glove/zarr.json", + "episode_0/head_camera", + "episode_0/head_camera.mp4", + "episode_0/head_camera/aligned_frame_indices", + "episode_0/head_camera/aligned_frame_indices/c", + "episode_0/head_camera/aligned_frame_indices/c/0", + "episode_0/head_camera/aligned_frame_indices/zarr.json", + "episode_0/head_camera/capture_time", + "episode_0/head_camera/capture_time/c", + "episode_0/head_camera/capture_time/c/0", + "episode_0/head_camera/capture_time/zarr.json", + "episode_0/head_camera/frame_id", + "episode_0/head_camera/frame_id/c", + "episode_0/head_camera/frame_id/c/0", + "episode_0/head_camera/frame_id/zarr.json", + "episode_0/head_camera/pose_interp", + "episode_0/head_camera/pose_interp/c", + "episode_0/head_camera/pose_interp/c/0", + "episode_0/head_camera/pose_interp/c/0/0", + "episode_0/head_camera/pose_interp/c/0/0/0", + "episode_0/head_camera/pose_interp/zarr.json", + "episode_0/head_camera/receive_time", + "episode_0/head_camera/receive_time/c", + "episode_0/head_camera/receive_time/c/0", + "episode_0/head_camera/receive_time/zarr.json", + "episode_0/head_camera/zarr.json", + "episode_0/pico_controller", + "episode_0/pico_controller/a_button", + "episode_0/pico_controller/a_button/c", + "episode_0/pico_controller/a_button/c/0", + "episode_0/pico_controller/a_button/zarr.json", + "episode_0/pico_controller/body_joints_valid", + "episode_0/pico_controller/body_joints_valid/c", + "episode_0/pico_controller/body_joints_valid/c/0", + "episode_0/pico_controller/body_joints_valid/zarr.json", + "episode_0/pico_controller/capture_time", + "episode_0/pico_controller/capture_time/c", + "episode_0/pico_controller/capture_time/c/0", + "episode_0/pico_controller/capture_time/zarr.json", + "episode_0/pico_controller/head_pose_interp", + "episode_0/pico_controller/head_pose_interp/c", + "episode_0/pico_controller/head_pose_interp/c/0", + "episode_0/pico_controller/head_pose_interp/c/0/0", + "episode_0/pico_controller/head_pose_interp/c/0/0/0", + "episode_0/pico_controller/head_pose_interp/zarr.json", + "episode_0/pico_controller/left_pose_interp", + "episode_0/pico_controller/left_pose_interp/c", + "episode_0/pico_controller/left_pose_interp/c/0", + "episode_0/pico_controller/left_pose_interp/c/0/0", + "episode_0/pico_controller/left_pose_interp/c/0/0/0", + "episode_0/pico_controller/left_pose_interp/zarr.json", + "episode_0/pico_controller/left_trigger_interp", + "episode_0/pico_controller/left_trigger_interp/c", + "episode_0/pico_controller/left_trigger_interp/c/0", + "episode_0/pico_controller/left_trigger_interp/zarr.json", + "episode_0/pico_controller/receive_time", + "episode_0/pico_controller/receive_time/c", + "episode_0/pico_controller/receive_time/c/0", + "episode_0/pico_controller/receive_time/zarr.json", + "episode_0/pico_controller/right_pose_interp", + "episode_0/pico_controller/right_pose_interp/c", + "episode_0/pico_controller/right_pose_interp/c/0", + "episode_0/pico_controller/right_pose_interp/c/0/0", + "episode_0/pico_controller/right_pose_interp/c/0/0/0", + "episode_0/pico_controller/right_pose_interp/zarr.json", + "episode_0/pico_controller/right_trigger_interp", + "episode_0/pico_controller/right_trigger_interp/c", + "episode_0/pico_controller/right_trigger_interp/c/0", + "episode_0/pico_controller/right_trigger_interp/zarr.json", + "episode_0/pico_controller/zarr.json", + "episode_0/valid_indices", + "episode_0/valid_indices/c", + "episode_0/valid_indices/c/0", + "episode_0/valid_indices/zarr.json", + "episode_0/valid_timestamps", + "episode_0/valid_timestamps/c", + "episode_0/valid_timestamps/c/0", + "episode_0/valid_timestamps/zarr.json", + "episode_0/wrist_camera", + "episode_0/wrist_camera.mp4", + "episode_0/wrist_camera/aligned_frame_indices", + "episode_0/wrist_camera/aligned_frame_indices/c", + "episode_0/wrist_camera/aligned_frame_indices/c/0", + "episode_0/wrist_camera/aligned_frame_indices/zarr.json", + "episode_0/wrist_camera/capture_time", + "episode_0/wrist_camera/capture_time/c", + "episode_0/wrist_camera/capture_time/c/0", + "episode_0/wrist_camera/capture_time/zarr.json", + "episode_0/wrist_camera/frame_id", + "episode_0/wrist_camera/frame_id/c", + "episode_0/wrist_camera/frame_id/c/0", + "episode_0/wrist_camera/frame_id/zarr.json", + "episode_0/wrist_camera/pose_interp", + "episode_0/wrist_camera/pose_interp/c", + "episode_0/wrist_camera/pose_interp/c/0", + "episode_0/wrist_camera/pose_interp/c/0/0", + "episode_0/wrist_camera/pose_interp/c/0/0/0", + "episode_0/wrist_camera/pose_interp/zarr.json", + "episode_0/wrist_camera/receive_time", + "episode_0/wrist_camera/receive_time/c", + "episode_0/wrist_camera/receive_time/c/0", + "episode_0/wrist_camera/receive_time/zarr.json", + "episode_0/wrist_camera/zarr.json", + "episode_0/zarr.json", + "head_camera_intrinsics.json", + "zarr.json" + ] +} diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/golden_attrs_sidecars.npz b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_attrs_sidecars.npz new file mode 100644 index 0000000000..21cb1e7d60 Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_attrs_sidecars.npz differ diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds1.json b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds1.json new file mode 100644 index 0000000000..bf6cd1c4bd --- /dev/null +++ b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds1.json @@ -0,0 +1,1373 @@ +{ + "entries": [ + { + "path": "config.yaml", + "kind": "file", + "size": 288, + "sha256": "e4142ea4adc900fb4870b4c31a465f0bf885ea925ee762f8aa688db5cfa82fbb" + }, + { + "path": "episode_0/head_camera.mp4", + "kind": "file", + "size": 544, + "sha256": "9f2ff969f02f9dd114ead58b074424d237b6a2109790434998549c09f6eaae36" + }, + { + "path": "episode_0/wrist_camera.mp4", + "kind": "file", + "size": 560, + "sha256": "140f5fa6e646b0bc7ac525867ac191a2b8ea76ccc17b46c702daa36e619be6d6" + }, + { + "path": "episode_1/head_camera.mp4", + "kind": "file", + "size": 544, + "sha256": "520464427520ebe79e6c809f6632ca024236bb44f540a604f1068d5c63ea1fd4" + }, + { + "path": "episode_1/wrist_camera.mp4", + "kind": "file", + "size": 560, + "sha256": "fc228cf71fa912a8e41b9840c11fad07f032a8d671a008b42b6d90b38020745a" + }, + { + "path": "zarr.json", + "kind": "root_json", + "zarr_format": 3, + "node_type": "group", + "attributes": { + "index_source": "head_camera" + } + }, + { + "path": "episode_0", + "kind": "group", + "attributes": { + "task_description": "episode task", + "timing": { + "clock": "monotonic", + "offset_s": 0.25 + } + } + }, + { + "path": "episode_0/glove", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/glove/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a0", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a1", + "info_chunks": [ + 35, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_sensor_count_interp", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a2", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_sensor_rotation_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4 + ], + "attributes": {}, + "npz_key": "a3", + "info_chunks": [ + 35, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a4", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/right_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a5", + "info_chunks": [ + 35, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/head_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a6", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a7", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a8", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a9", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a10", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller", + "kind": "group", + "attributes": { + "handedness": "both" + } + }, + { + "path": "episode_0/pico_controller/a_button", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a11", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/body_joints_valid", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a12", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a13", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/head_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a14", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/left_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a15", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/left_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a16", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a17", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/right_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a18", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/right_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a19", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/wrist_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a20", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a21", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a22", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a23", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a24", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/valid_indices", + "kind": "array", + "dtype": "bool", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a25", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/valid_timestamps", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a26", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1", + "kind": "group", + "attributes": { + "task_description": "episode task", + "timing": { + "clock": "monotonic", + "offset_s": 0.25 + } + } + }, + { + "path": "episode_1/glove", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_1/glove/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a27", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/left_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a28", + "info_chunks": [ + 35, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/left_sensor_count_interp", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a29", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/left_sensor_rotation_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4 + ], + "attributes": {}, + "npz_key": "a30", + "info_chunks": [ + 35, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a31", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/right_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a32", + "info_chunks": [ + 35, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_1/head_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a33", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a34", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a35", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a36", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a37", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller", + "kind": "group", + "attributes": { + "handedness": "both" + } + }, + { + "path": "episode_1/pico_controller/a_button", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a38", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/body_joints_valid", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a39", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a40", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/head_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a41", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/left_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a42", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/left_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a43", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a44", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/right_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a45", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/right_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a46", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_1/wrist_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a47", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a48", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a49", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 35, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a50", + "info_chunks": [ + 35, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a51", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/valid_indices", + "kind": "array", + "dtype": "bool", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a52", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/valid_timestamps", + "kind": "array", + "dtype": "float64", + "shape": [ + 35 + ], + "attributes": {}, + "npz_key": "a53", + "info_chunks": [ + 35 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + } + ], + "expected_tree_listing": [ + "config.yaml", + "episode_0", + "episode_0/glove", + "episode_0/glove/capture_time", + "episode_0/glove/capture_time/c", + "episode_0/glove/capture_time/c/0", + "episode_0/glove/capture_time/zarr.json", + "episode_0/glove/left_hand_pose_interp", + "episode_0/glove/left_hand_pose_interp/c", + "episode_0/glove/left_hand_pose_interp/c/0", + "episode_0/glove/left_hand_pose_interp/c/0/0", + "episode_0/glove/left_hand_pose_interp/c/0/0/0", + "episode_0/glove/left_hand_pose_interp/c/0/0/0/0", + "episode_0/glove/left_hand_pose_interp/zarr.json", + "episode_0/glove/left_sensor_count_interp", + "episode_0/glove/left_sensor_count_interp/c", + "episode_0/glove/left_sensor_count_interp/c/0", + "episode_0/glove/left_sensor_count_interp/zarr.json", + "episode_0/glove/left_sensor_rotation_interp", + "episode_0/glove/left_sensor_rotation_interp/c", + "episode_0/glove/left_sensor_rotation_interp/c/0", + "episode_0/glove/left_sensor_rotation_interp/c/0/0", + "episode_0/glove/left_sensor_rotation_interp/zarr.json", + "episode_0/glove/receive_time", + "episode_0/glove/receive_time/c", + "episode_0/glove/receive_time/c/0", + "episode_0/glove/receive_time/zarr.json", + "episode_0/glove/right_hand_pose_interp", + "episode_0/glove/right_hand_pose_interp/c", + "episode_0/glove/right_hand_pose_interp/c/0", + "episode_0/glove/right_hand_pose_interp/c/0/0", + "episode_0/glove/right_hand_pose_interp/c/0/0/0", + "episode_0/glove/right_hand_pose_interp/c/0/0/0/0", + "episode_0/glove/right_hand_pose_interp/zarr.json", + "episode_0/glove/zarr.json", + "episode_0/head_camera", + "episode_0/head_camera.mp4", + "episode_0/head_camera/aligned_frame_indices", + "episode_0/head_camera/aligned_frame_indices/c", + "episode_0/head_camera/aligned_frame_indices/c/0", + "episode_0/head_camera/aligned_frame_indices/zarr.json", + "episode_0/head_camera/capture_time", + "episode_0/head_camera/capture_time/c", + "episode_0/head_camera/capture_time/c/0", + "episode_0/head_camera/capture_time/zarr.json", + "episode_0/head_camera/frame_id", + "episode_0/head_camera/frame_id/c", + "episode_0/head_camera/frame_id/c/0", + "episode_0/head_camera/frame_id/zarr.json", + "episode_0/head_camera/pose_interp", + "episode_0/head_camera/pose_interp/c", + "episode_0/head_camera/pose_interp/c/0", + "episode_0/head_camera/pose_interp/c/0/0", + "episode_0/head_camera/pose_interp/c/0/0/0", + "episode_0/head_camera/pose_interp/zarr.json", + "episode_0/head_camera/receive_time", + "episode_0/head_camera/receive_time/c", + "episode_0/head_camera/receive_time/c/0", + "episode_0/head_camera/receive_time/zarr.json", + "episode_0/head_camera/zarr.json", + "episode_0/pico_controller", + "episode_0/pico_controller/a_button", + "episode_0/pico_controller/a_button/c", + "episode_0/pico_controller/a_button/c/0", + "episode_0/pico_controller/a_button/zarr.json", + "episode_0/pico_controller/body_joints_valid", + "episode_0/pico_controller/body_joints_valid/c", + "episode_0/pico_controller/body_joints_valid/c/0", + "episode_0/pico_controller/body_joints_valid/zarr.json", + "episode_0/pico_controller/capture_time", + "episode_0/pico_controller/capture_time/c", + "episode_0/pico_controller/capture_time/c/0", + "episode_0/pico_controller/capture_time/zarr.json", + "episode_0/pico_controller/head_pose_interp", + "episode_0/pico_controller/head_pose_interp/c", + "episode_0/pico_controller/head_pose_interp/c/0", + "episode_0/pico_controller/head_pose_interp/c/0/0", + "episode_0/pico_controller/head_pose_interp/c/0/0/0", + "episode_0/pico_controller/head_pose_interp/zarr.json", + "episode_0/pico_controller/left_pose_interp", + "episode_0/pico_controller/left_pose_interp/c", + "episode_0/pico_controller/left_pose_interp/c/0", + "episode_0/pico_controller/left_pose_interp/c/0/0", + "episode_0/pico_controller/left_pose_interp/c/0/0/0", + "episode_0/pico_controller/left_pose_interp/zarr.json", + "episode_0/pico_controller/left_trigger_interp", + "episode_0/pico_controller/left_trigger_interp/c", + "episode_0/pico_controller/left_trigger_interp/c/0", + "episode_0/pico_controller/left_trigger_interp/zarr.json", + "episode_0/pico_controller/receive_time", + "episode_0/pico_controller/receive_time/c", + "episode_0/pico_controller/receive_time/c/0", + "episode_0/pico_controller/receive_time/zarr.json", + "episode_0/pico_controller/right_pose_interp", + "episode_0/pico_controller/right_pose_interp/c", + "episode_0/pico_controller/right_pose_interp/c/0", + "episode_0/pico_controller/right_pose_interp/c/0/0", + "episode_0/pico_controller/right_pose_interp/c/0/0/0", + "episode_0/pico_controller/right_pose_interp/zarr.json", + "episode_0/pico_controller/right_trigger_interp", + "episode_0/pico_controller/right_trigger_interp/c", + "episode_0/pico_controller/right_trigger_interp/c/0", + "episode_0/pico_controller/right_trigger_interp/zarr.json", + "episode_0/pico_controller/zarr.json", + "episode_0/valid_indices", + "episode_0/valid_indices/c", + "episode_0/valid_indices/c/0", + "episode_0/valid_indices/zarr.json", + "episode_0/valid_timestamps", + "episode_0/valid_timestamps/c", + "episode_0/valid_timestamps/c/0", + "episode_0/valid_timestamps/zarr.json", + "episode_0/wrist_camera", + "episode_0/wrist_camera.mp4", + "episode_0/wrist_camera/aligned_frame_indices", + "episode_0/wrist_camera/aligned_frame_indices/c", + "episode_0/wrist_camera/aligned_frame_indices/c/0", + "episode_0/wrist_camera/aligned_frame_indices/zarr.json", + "episode_0/wrist_camera/capture_time", + "episode_0/wrist_camera/capture_time/c", + "episode_0/wrist_camera/capture_time/c/0", + "episode_0/wrist_camera/capture_time/zarr.json", + "episode_0/wrist_camera/frame_id", + "episode_0/wrist_camera/frame_id/c", + "episode_0/wrist_camera/frame_id/c/0", + "episode_0/wrist_camera/frame_id/zarr.json", + "episode_0/wrist_camera/pose_interp", + "episode_0/wrist_camera/pose_interp/c", + "episode_0/wrist_camera/pose_interp/c/0", + "episode_0/wrist_camera/pose_interp/c/0/0", + "episode_0/wrist_camera/pose_interp/c/0/0/0", + "episode_0/wrist_camera/pose_interp/zarr.json", + "episode_0/wrist_camera/receive_time", + "episode_0/wrist_camera/receive_time/c", + "episode_0/wrist_camera/receive_time/c/0", + "episode_0/wrist_camera/receive_time/zarr.json", + "episode_0/wrist_camera/zarr.json", + "episode_0/zarr.json", + "episode_1", + "episode_1/glove", + "episode_1/glove/capture_time", + "episode_1/glove/capture_time/c", + "episode_1/glove/capture_time/c/0", + "episode_1/glove/capture_time/zarr.json", + "episode_1/glove/left_hand_pose_interp", + "episode_1/glove/left_hand_pose_interp/c", + "episode_1/glove/left_hand_pose_interp/c/0", + "episode_1/glove/left_hand_pose_interp/c/0/0", + "episode_1/glove/left_hand_pose_interp/c/0/0/0", + "episode_1/glove/left_hand_pose_interp/c/0/0/0/0", + "episode_1/glove/left_hand_pose_interp/zarr.json", + "episode_1/glove/left_sensor_count_interp", + "episode_1/glove/left_sensor_count_interp/c", + "episode_1/glove/left_sensor_count_interp/c/0", + "episode_1/glove/left_sensor_count_interp/zarr.json", + "episode_1/glove/left_sensor_rotation_interp", + "episode_1/glove/left_sensor_rotation_interp/c", + "episode_1/glove/left_sensor_rotation_interp/c/0", + "episode_1/glove/left_sensor_rotation_interp/c/0/0", + "episode_1/glove/left_sensor_rotation_interp/zarr.json", + "episode_1/glove/receive_time", + "episode_1/glove/receive_time/c", + "episode_1/glove/receive_time/c/0", + "episode_1/glove/receive_time/zarr.json", + "episode_1/glove/right_hand_pose_interp", + "episode_1/glove/right_hand_pose_interp/c", + "episode_1/glove/right_hand_pose_interp/c/0", + "episode_1/glove/right_hand_pose_interp/c/0/0", + "episode_1/glove/right_hand_pose_interp/c/0/0/0", + "episode_1/glove/right_hand_pose_interp/c/0/0/0/0", + "episode_1/glove/right_hand_pose_interp/zarr.json", + "episode_1/glove/zarr.json", + "episode_1/head_camera", + "episode_1/head_camera.mp4", + "episode_1/head_camera/aligned_frame_indices", + "episode_1/head_camera/aligned_frame_indices/c", + "episode_1/head_camera/aligned_frame_indices/c/0", + "episode_1/head_camera/aligned_frame_indices/zarr.json", + "episode_1/head_camera/capture_time", + "episode_1/head_camera/capture_time/c", + "episode_1/head_camera/capture_time/c/0", + "episode_1/head_camera/capture_time/zarr.json", + "episode_1/head_camera/frame_id", + "episode_1/head_camera/frame_id/c", + "episode_1/head_camera/frame_id/c/0", + "episode_1/head_camera/frame_id/zarr.json", + "episode_1/head_camera/pose_interp", + "episode_1/head_camera/pose_interp/c", + "episode_1/head_camera/pose_interp/c/0", + "episode_1/head_camera/pose_interp/c/0/0", + "episode_1/head_camera/pose_interp/c/0/0/0", + "episode_1/head_camera/pose_interp/zarr.json", + "episode_1/head_camera/receive_time", + "episode_1/head_camera/receive_time/c", + "episode_1/head_camera/receive_time/c/0", + "episode_1/head_camera/receive_time/zarr.json", + "episode_1/head_camera/zarr.json", + "episode_1/pico_controller", + "episode_1/pico_controller/a_button", + "episode_1/pico_controller/a_button/c", + "episode_1/pico_controller/a_button/c/0", + "episode_1/pico_controller/a_button/zarr.json", + "episode_1/pico_controller/body_joints_valid", + "episode_1/pico_controller/body_joints_valid/c", + "episode_1/pico_controller/body_joints_valid/c/0", + "episode_1/pico_controller/body_joints_valid/zarr.json", + "episode_1/pico_controller/capture_time", + "episode_1/pico_controller/capture_time/c", + "episode_1/pico_controller/capture_time/c/0", + "episode_1/pico_controller/capture_time/zarr.json", + "episode_1/pico_controller/head_pose_interp", + "episode_1/pico_controller/head_pose_interp/c", + "episode_1/pico_controller/head_pose_interp/c/0", + "episode_1/pico_controller/head_pose_interp/c/0/0", + "episode_1/pico_controller/head_pose_interp/c/0/0/0", + "episode_1/pico_controller/head_pose_interp/zarr.json", + "episode_1/pico_controller/left_pose_interp", + "episode_1/pico_controller/left_pose_interp/c", + "episode_1/pico_controller/left_pose_interp/c/0", + "episode_1/pico_controller/left_pose_interp/c/0/0", + "episode_1/pico_controller/left_pose_interp/c/0/0/0", + "episode_1/pico_controller/left_pose_interp/zarr.json", + "episode_1/pico_controller/left_trigger_interp", + "episode_1/pico_controller/left_trigger_interp/c", + "episode_1/pico_controller/left_trigger_interp/c/0", + "episode_1/pico_controller/left_trigger_interp/zarr.json", + "episode_1/pico_controller/receive_time", + "episode_1/pico_controller/receive_time/c", + "episode_1/pico_controller/receive_time/c/0", + "episode_1/pico_controller/receive_time/zarr.json", + "episode_1/pico_controller/right_pose_interp", + "episode_1/pico_controller/right_pose_interp/c", + "episode_1/pico_controller/right_pose_interp/c/0", + "episode_1/pico_controller/right_pose_interp/c/0/0", + "episode_1/pico_controller/right_pose_interp/c/0/0/0", + "episode_1/pico_controller/right_pose_interp/zarr.json", + "episode_1/pico_controller/right_trigger_interp", + "episode_1/pico_controller/right_trigger_interp/c", + "episode_1/pico_controller/right_trigger_interp/c/0", + "episode_1/pico_controller/right_trigger_interp/zarr.json", + "episode_1/pico_controller/zarr.json", + "episode_1/valid_indices", + "episode_1/valid_indices/c", + "episode_1/valid_indices/c/0", + "episode_1/valid_indices/zarr.json", + "episode_1/valid_timestamps", + "episode_1/valid_timestamps/c", + "episode_1/valid_timestamps/c/0", + "episode_1/valid_timestamps/zarr.json", + "episode_1/wrist_camera", + "episode_1/wrist_camera.mp4", + "episode_1/wrist_camera/aligned_frame_indices", + "episode_1/wrist_camera/aligned_frame_indices/c", + "episode_1/wrist_camera/aligned_frame_indices/c/0", + "episode_1/wrist_camera/aligned_frame_indices/zarr.json", + "episode_1/wrist_camera/capture_time", + "episode_1/wrist_camera/capture_time/c", + "episode_1/wrist_camera/capture_time/c/0", + "episode_1/wrist_camera/capture_time/zarr.json", + "episode_1/wrist_camera/frame_id", + "episode_1/wrist_camera/frame_id/c", + "episode_1/wrist_camera/frame_id/c/0", + "episode_1/wrist_camera/frame_id/zarr.json", + "episode_1/wrist_camera/pose_interp", + "episode_1/wrist_camera/pose_interp/c", + "episode_1/wrist_camera/pose_interp/c/0", + "episode_1/wrist_camera/pose_interp/c/0/0", + "episode_1/wrist_camera/pose_interp/c/0/0/0", + "episode_1/wrist_camera/pose_interp/zarr.json", + "episode_1/wrist_camera/receive_time", + "episode_1/wrist_camera/receive_time/c", + "episode_1/wrist_camera/receive_time/c/0", + "episode_1/wrist_camera/receive_time/zarr.json", + "episode_1/wrist_camera/zarr.json", + "episode_1/zarr.json", + "zarr.json" + ] +} diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds1.npz b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds1.npz new file mode 100644 index 0000000000..4670316e16 Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds1.npz differ diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds2.json b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds2.json new file mode 100644 index 0000000000..c1d0b82031 --- /dev/null +++ b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds2.json @@ -0,0 +1,1373 @@ +{ + "entries": [ + { + "path": "config.yaml", + "kind": "file", + "size": 288, + "sha256": "e4142ea4adc900fb4870b4c31a465f0bf885ea925ee762f8aa688db5cfa82fbb" + }, + { + "path": "episode_0/head_camera.mp4", + "kind": "file", + "size": 544, + "sha256": "9f2ff969f02f9dd114ead58b074424d237b6a2109790434998549c09f6eaae36" + }, + { + "path": "episode_0/wrist_camera.mp4", + "kind": "file", + "size": 560, + "sha256": "140f5fa6e646b0bc7ac525867ac191a2b8ea76ccc17b46c702daa36e619be6d6" + }, + { + "path": "episode_1/head_camera.mp4", + "kind": "file", + "size": 544, + "sha256": "520464427520ebe79e6c809f6632ca024236bb44f540a604f1068d5c63ea1fd4" + }, + { + "path": "episode_1/wrist_camera.mp4", + "kind": "file", + "size": 560, + "sha256": "fc228cf71fa912a8e41b9840c11fad07f032a8d671a008b42b6d90b38020745a" + }, + { + "path": "zarr.json", + "kind": "root_json", + "zarr_format": 3, + "node_type": "group", + "attributes": { + "index_source": "head_camera" + } + }, + { + "path": "episode_0", + "kind": "group", + "attributes": { + "task_description": "episode task", + "timing": { + "clock": "monotonic", + "offset_s": 0.25 + } + } + }, + { + "path": "episode_0/glove", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/glove/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a0", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a1", + "info_chunks": [ + 18, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_sensor_count_interp", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a2", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/left_sensor_rotation_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4 + ], + "attributes": {}, + "npz_key": "a3", + "info_chunks": [ + 18, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a4", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/glove/right_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a5", + "info_chunks": [ + 18, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/head_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a6", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a7", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a8", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a9", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/head_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a10", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller", + "kind": "group", + "attributes": { + "handedness": "both" + } + }, + { + "path": "episode_0/pico_controller/a_button", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a11", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/body_joints_valid", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a12", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a13", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/head_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a14", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/left_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a15", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/left_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a16", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a17", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/right_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a18", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/pico_controller/right_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a19", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_0/wrist_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a20", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a21", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a22", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a23", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/wrist_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a24", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/valid_indices", + "kind": "array", + "dtype": "bool", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a25", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_0/valid_timestamps", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a26", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1", + "kind": "group", + "attributes": { + "task_description": "episode task", + "timing": { + "clock": "monotonic", + "offset_s": 0.25 + } + } + }, + { + "path": "episode_1/glove", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_1/glove/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a27", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/left_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a28", + "info_chunks": [ + 18, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/left_sensor_count_interp", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a29", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/left_sensor_rotation_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4 + ], + "attributes": {}, + "npz_key": "a30", + "info_chunks": [ + 18, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 54 + ], + "attributes": {}, + "npz_key": "a31", + "info_chunks": [ + 54 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/glove/right_hand_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 5, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a32", + "info_chunks": [ + 18, + 5, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_1/head_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a33", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a34", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a35", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a36", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/head_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a37", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller", + "kind": "group", + "attributes": { + "handedness": "both" + } + }, + { + "path": "episode_1/pico_controller/a_button", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a38", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/body_joints_valid", + "kind": "array", + "dtype": "bool", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a39", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a40", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/head_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a41", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/left_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a42", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/left_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a43", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 56 + ], + "attributes": {}, + "npz_key": "a44", + "info_chunks": [ + 56 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/right_pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a45", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/pico_controller/right_trigger_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a46", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera", + "kind": "group", + "attributes": {} + }, + { + "path": "episode_1/wrist_camera/aligned_frame_indices", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a47", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera/capture_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a48", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera/frame_id", + "kind": "array", + "dtype": "int64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a49", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera/pose_interp", + "kind": "array", + "dtype": "float64", + "shape": [ + 18, + 4, + 4 + ], + "attributes": {}, + "npz_key": "a50", + "info_chunks": [ + 18, + 4, + 4 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/wrist_camera/receive_time", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a51", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/valid_indices", + "kind": "array", + "dtype": "bool", + "shape": [ + 36 + ], + "attributes": {}, + "npz_key": "a52", + "info_chunks": [ + 36 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + }, + { + "path": "episode_1/valid_timestamps", + "kind": "array", + "dtype": "float64", + "shape": [ + 18 + ], + "attributes": {}, + "npz_key": "a53", + "info_chunks": [ + 18 + ], + "info_codecs": [ + "BytesCodec", + "ZstdCodec" + ] + } + ], + "expected_tree_listing": [ + "config.yaml", + "episode_0", + "episode_0/glove", + "episode_0/glove/capture_time", + "episode_0/glove/capture_time/c", + "episode_0/glove/capture_time/c/0", + "episode_0/glove/capture_time/zarr.json", + "episode_0/glove/left_hand_pose_interp", + "episode_0/glove/left_hand_pose_interp/c", + "episode_0/glove/left_hand_pose_interp/c/0", + "episode_0/glove/left_hand_pose_interp/c/0/0", + "episode_0/glove/left_hand_pose_interp/c/0/0/0", + "episode_0/glove/left_hand_pose_interp/c/0/0/0/0", + "episode_0/glove/left_hand_pose_interp/zarr.json", + "episode_0/glove/left_sensor_count_interp", + "episode_0/glove/left_sensor_count_interp/c", + "episode_0/glove/left_sensor_count_interp/c/0", + "episode_0/glove/left_sensor_count_interp/zarr.json", + "episode_0/glove/left_sensor_rotation_interp", + "episode_0/glove/left_sensor_rotation_interp/c", + "episode_0/glove/left_sensor_rotation_interp/c/0", + "episode_0/glove/left_sensor_rotation_interp/c/0/0", + "episode_0/glove/left_sensor_rotation_interp/zarr.json", + "episode_0/glove/receive_time", + "episode_0/glove/receive_time/c", + "episode_0/glove/receive_time/c/0", + "episode_0/glove/receive_time/zarr.json", + "episode_0/glove/right_hand_pose_interp", + "episode_0/glove/right_hand_pose_interp/c", + "episode_0/glove/right_hand_pose_interp/c/0", + "episode_0/glove/right_hand_pose_interp/c/0/0", + "episode_0/glove/right_hand_pose_interp/c/0/0/0", + "episode_0/glove/right_hand_pose_interp/c/0/0/0/0", + "episode_0/glove/right_hand_pose_interp/zarr.json", + "episode_0/glove/zarr.json", + "episode_0/head_camera", + "episode_0/head_camera.mp4", + "episode_0/head_camera/aligned_frame_indices", + "episode_0/head_camera/aligned_frame_indices/c", + "episode_0/head_camera/aligned_frame_indices/c/0", + "episode_0/head_camera/aligned_frame_indices/zarr.json", + "episode_0/head_camera/capture_time", + "episode_0/head_camera/capture_time/c", + "episode_0/head_camera/capture_time/c/0", + "episode_0/head_camera/capture_time/zarr.json", + "episode_0/head_camera/frame_id", + "episode_0/head_camera/frame_id/c", + "episode_0/head_camera/frame_id/c/0", + "episode_0/head_camera/frame_id/zarr.json", + "episode_0/head_camera/pose_interp", + "episode_0/head_camera/pose_interp/c", + "episode_0/head_camera/pose_interp/c/0", + "episode_0/head_camera/pose_interp/c/0/0", + "episode_0/head_camera/pose_interp/c/0/0/0", + "episode_0/head_camera/pose_interp/zarr.json", + "episode_0/head_camera/receive_time", + "episode_0/head_camera/receive_time/c", + "episode_0/head_camera/receive_time/c/0", + "episode_0/head_camera/receive_time/zarr.json", + "episode_0/head_camera/zarr.json", + "episode_0/pico_controller", + "episode_0/pico_controller/a_button", + "episode_0/pico_controller/a_button/c", + "episode_0/pico_controller/a_button/c/0", + "episode_0/pico_controller/a_button/zarr.json", + "episode_0/pico_controller/body_joints_valid", + "episode_0/pico_controller/body_joints_valid/c", + "episode_0/pico_controller/body_joints_valid/c/0", + "episode_0/pico_controller/body_joints_valid/zarr.json", + "episode_0/pico_controller/capture_time", + "episode_0/pico_controller/capture_time/c", + "episode_0/pico_controller/capture_time/c/0", + "episode_0/pico_controller/capture_time/zarr.json", + "episode_0/pico_controller/head_pose_interp", + "episode_0/pico_controller/head_pose_interp/c", + "episode_0/pico_controller/head_pose_interp/c/0", + "episode_0/pico_controller/head_pose_interp/c/0/0", + "episode_0/pico_controller/head_pose_interp/c/0/0/0", + "episode_0/pico_controller/head_pose_interp/zarr.json", + "episode_0/pico_controller/left_pose_interp", + "episode_0/pico_controller/left_pose_interp/c", + "episode_0/pico_controller/left_pose_interp/c/0", + "episode_0/pico_controller/left_pose_interp/c/0/0", + "episode_0/pico_controller/left_pose_interp/c/0/0/0", + "episode_0/pico_controller/left_pose_interp/zarr.json", + "episode_0/pico_controller/left_trigger_interp", + "episode_0/pico_controller/left_trigger_interp/c", + "episode_0/pico_controller/left_trigger_interp/c/0", + "episode_0/pico_controller/left_trigger_interp/zarr.json", + "episode_0/pico_controller/receive_time", + "episode_0/pico_controller/receive_time/c", + "episode_0/pico_controller/receive_time/c/0", + "episode_0/pico_controller/receive_time/zarr.json", + "episode_0/pico_controller/right_pose_interp", + "episode_0/pico_controller/right_pose_interp/c", + "episode_0/pico_controller/right_pose_interp/c/0", + "episode_0/pico_controller/right_pose_interp/c/0/0", + "episode_0/pico_controller/right_pose_interp/c/0/0/0", + "episode_0/pico_controller/right_pose_interp/zarr.json", + "episode_0/pico_controller/right_trigger_interp", + "episode_0/pico_controller/right_trigger_interp/c", + "episode_0/pico_controller/right_trigger_interp/c/0", + "episode_0/pico_controller/right_trigger_interp/zarr.json", + "episode_0/pico_controller/zarr.json", + "episode_0/valid_indices", + "episode_0/valid_indices/c", + "episode_0/valid_indices/c/0", + "episode_0/valid_indices/zarr.json", + "episode_0/valid_timestamps", + "episode_0/valid_timestamps/c", + "episode_0/valid_timestamps/c/0", + "episode_0/valid_timestamps/zarr.json", + "episode_0/wrist_camera", + "episode_0/wrist_camera.mp4", + "episode_0/wrist_camera/aligned_frame_indices", + "episode_0/wrist_camera/aligned_frame_indices/c", + "episode_0/wrist_camera/aligned_frame_indices/c/0", + "episode_0/wrist_camera/aligned_frame_indices/zarr.json", + "episode_0/wrist_camera/capture_time", + "episode_0/wrist_camera/capture_time/c", + "episode_0/wrist_camera/capture_time/c/0", + "episode_0/wrist_camera/capture_time/zarr.json", + "episode_0/wrist_camera/frame_id", + "episode_0/wrist_camera/frame_id/c", + "episode_0/wrist_camera/frame_id/c/0", + "episode_0/wrist_camera/frame_id/zarr.json", + "episode_0/wrist_camera/pose_interp", + "episode_0/wrist_camera/pose_interp/c", + "episode_0/wrist_camera/pose_interp/c/0", + "episode_0/wrist_camera/pose_interp/c/0/0", + "episode_0/wrist_camera/pose_interp/c/0/0/0", + "episode_0/wrist_camera/pose_interp/zarr.json", + "episode_0/wrist_camera/receive_time", + "episode_0/wrist_camera/receive_time/c", + "episode_0/wrist_camera/receive_time/c/0", + "episode_0/wrist_camera/receive_time/zarr.json", + "episode_0/wrist_camera/zarr.json", + "episode_0/zarr.json", + "episode_1", + "episode_1/glove", + "episode_1/glove/capture_time", + "episode_1/glove/capture_time/c", + "episode_1/glove/capture_time/c/0", + "episode_1/glove/capture_time/zarr.json", + "episode_1/glove/left_hand_pose_interp", + "episode_1/glove/left_hand_pose_interp/c", + "episode_1/glove/left_hand_pose_interp/c/0", + "episode_1/glove/left_hand_pose_interp/c/0/0", + "episode_1/glove/left_hand_pose_interp/c/0/0/0", + "episode_1/glove/left_hand_pose_interp/c/0/0/0/0", + "episode_1/glove/left_hand_pose_interp/zarr.json", + "episode_1/glove/left_sensor_count_interp", + "episode_1/glove/left_sensor_count_interp/c", + "episode_1/glove/left_sensor_count_interp/c/0", + "episode_1/glove/left_sensor_count_interp/zarr.json", + "episode_1/glove/left_sensor_rotation_interp", + "episode_1/glove/left_sensor_rotation_interp/c", + "episode_1/glove/left_sensor_rotation_interp/c/0", + "episode_1/glove/left_sensor_rotation_interp/c/0/0", + "episode_1/glove/left_sensor_rotation_interp/zarr.json", + "episode_1/glove/receive_time", + "episode_1/glove/receive_time/c", + "episode_1/glove/receive_time/c/0", + "episode_1/glove/receive_time/zarr.json", + "episode_1/glove/right_hand_pose_interp", + "episode_1/glove/right_hand_pose_interp/c", + "episode_1/glove/right_hand_pose_interp/c/0", + "episode_1/glove/right_hand_pose_interp/c/0/0", + "episode_1/glove/right_hand_pose_interp/c/0/0/0", + "episode_1/glove/right_hand_pose_interp/c/0/0/0/0", + "episode_1/glove/right_hand_pose_interp/zarr.json", + "episode_1/glove/zarr.json", + "episode_1/head_camera", + "episode_1/head_camera.mp4", + "episode_1/head_camera/aligned_frame_indices", + "episode_1/head_camera/aligned_frame_indices/c", + "episode_1/head_camera/aligned_frame_indices/c/0", + "episode_1/head_camera/aligned_frame_indices/zarr.json", + "episode_1/head_camera/capture_time", + "episode_1/head_camera/capture_time/c", + "episode_1/head_camera/capture_time/c/0", + "episode_1/head_camera/capture_time/zarr.json", + "episode_1/head_camera/frame_id", + "episode_1/head_camera/frame_id/c", + "episode_1/head_camera/frame_id/c/0", + "episode_1/head_camera/frame_id/zarr.json", + "episode_1/head_camera/pose_interp", + "episode_1/head_camera/pose_interp/c", + "episode_1/head_camera/pose_interp/c/0", + "episode_1/head_camera/pose_interp/c/0/0", + "episode_1/head_camera/pose_interp/c/0/0/0", + "episode_1/head_camera/pose_interp/zarr.json", + "episode_1/head_camera/receive_time", + "episode_1/head_camera/receive_time/c", + "episode_1/head_camera/receive_time/c/0", + "episode_1/head_camera/receive_time/zarr.json", + "episode_1/head_camera/zarr.json", + "episode_1/pico_controller", + "episode_1/pico_controller/a_button", + "episode_1/pico_controller/a_button/c", + "episode_1/pico_controller/a_button/c/0", + "episode_1/pico_controller/a_button/zarr.json", + "episode_1/pico_controller/body_joints_valid", + "episode_1/pico_controller/body_joints_valid/c", + "episode_1/pico_controller/body_joints_valid/c/0", + "episode_1/pico_controller/body_joints_valid/zarr.json", + "episode_1/pico_controller/capture_time", + "episode_1/pico_controller/capture_time/c", + "episode_1/pico_controller/capture_time/c/0", + "episode_1/pico_controller/capture_time/zarr.json", + "episode_1/pico_controller/head_pose_interp", + "episode_1/pico_controller/head_pose_interp/c", + "episode_1/pico_controller/head_pose_interp/c/0", + "episode_1/pico_controller/head_pose_interp/c/0/0", + "episode_1/pico_controller/head_pose_interp/c/0/0/0", + "episode_1/pico_controller/head_pose_interp/zarr.json", + "episode_1/pico_controller/left_pose_interp", + "episode_1/pico_controller/left_pose_interp/c", + "episode_1/pico_controller/left_pose_interp/c/0", + "episode_1/pico_controller/left_pose_interp/c/0/0", + "episode_1/pico_controller/left_pose_interp/c/0/0/0", + "episode_1/pico_controller/left_pose_interp/zarr.json", + "episode_1/pico_controller/left_trigger_interp", + "episode_1/pico_controller/left_trigger_interp/c", + "episode_1/pico_controller/left_trigger_interp/c/0", + "episode_1/pico_controller/left_trigger_interp/zarr.json", + "episode_1/pico_controller/receive_time", + "episode_1/pico_controller/receive_time/c", + "episode_1/pico_controller/receive_time/c/0", + "episode_1/pico_controller/receive_time/zarr.json", + "episode_1/pico_controller/right_pose_interp", + "episode_1/pico_controller/right_pose_interp/c", + "episode_1/pico_controller/right_pose_interp/c/0", + "episode_1/pico_controller/right_pose_interp/c/0/0", + "episode_1/pico_controller/right_pose_interp/c/0/0/0", + "episode_1/pico_controller/right_pose_interp/zarr.json", + "episode_1/pico_controller/right_trigger_interp", + "episode_1/pico_controller/right_trigger_interp/c", + "episode_1/pico_controller/right_trigger_interp/c/0", + "episode_1/pico_controller/right_trigger_interp/zarr.json", + "episode_1/pico_controller/zarr.json", + "episode_1/valid_indices", + "episode_1/valid_indices/c", + "episode_1/valid_indices/c/0", + "episode_1/valid_indices/zarr.json", + "episode_1/valid_timestamps", + "episode_1/valid_timestamps/c", + "episode_1/valid_timestamps/c/0", + "episode_1/valid_timestamps/zarr.json", + "episode_1/wrist_camera", + "episode_1/wrist_camera.mp4", + "episode_1/wrist_camera/aligned_frame_indices", + "episode_1/wrist_camera/aligned_frame_indices/c", + "episode_1/wrist_camera/aligned_frame_indices/c/0", + "episode_1/wrist_camera/aligned_frame_indices/zarr.json", + "episode_1/wrist_camera/capture_time", + "episode_1/wrist_camera/capture_time/c", + "episode_1/wrist_camera/capture_time/c/0", + "episode_1/wrist_camera/capture_time/zarr.json", + "episode_1/wrist_camera/frame_id", + "episode_1/wrist_camera/frame_id/c", + "episode_1/wrist_camera/frame_id/c/0", + "episode_1/wrist_camera/frame_id/zarr.json", + "episode_1/wrist_camera/pose_interp", + "episode_1/wrist_camera/pose_interp/c", + "episode_1/wrist_camera/pose_interp/c/0", + "episode_1/wrist_camera/pose_interp/c/0/0", + "episode_1/wrist_camera/pose_interp/c/0/0/0", + "episode_1/wrist_camera/pose_interp/zarr.json", + "episode_1/wrist_camera/receive_time", + "episode_1/wrist_camera/receive_time/c", + "episode_1/wrist_camera/receive_time/c/0", + "episode_1/wrist_camera/receive_time/zarr.json", + "episode_1/wrist_camera/zarr.json", + "episode_1/zarr.json", + "zarr.json" + ] +} diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds2.npz b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds2.npz new file mode 100644 index 0000000000..ee91d56130 Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/aligned_writer/golden_basic_ds2.npz differ diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/head_camera_intrinsics.json b/examples/egocap/tests/recording/fixtures/aligned_writer/head_camera_intrinsics.json new file mode 100644 index 0000000000..4e8f5563e1 --- /dev/null +++ b/examples/egocap/tests/recording/fixtures/aligned_writer/head_camera_intrinsics.json @@ -0,0 +1,6 @@ +{ + "fx": 600.0, + "fy": 601.0, + "cx": 320.0, + "cy": 240.0 +} diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/raw_inputs.npz b/examples/egocap/tests/recording/fixtures/aligned_writer/raw_inputs.npz new file mode 100644 index 0000000000..0fca63c22a Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/aligned_writer/raw_inputs.npz differ diff --git a/examples/egocap/tests/recording/fixtures/aligned_writer/scenarios.json b/examples/egocap/tests/recording/fixtures/aligned_writer/scenarios.json new file mode 100644 index 0000000000..970791901d --- /dev/null +++ b/examples/egocap/tests/recording/fixtures/aligned_writer/scenarios.json @@ -0,0 +1,84 @@ +{ + "episode_attrs": { + "task_description": "episode task", + "timing": { + "clock": "monotonic", + "offset_s": 0.25 + } + }, + "source_attrs": { + "pico_controller": { + "handedness": "both" + } + }, + "index_source": "head_camera", + "scenarios": { + "basic_ds1": { + "raw_attrs": {}, + "sidecars": [], + "steps": [ + [ + 0, + 1 + ], + [ + 1, + 1 + ] + ] + }, + "basic_ds2": { + "raw_attrs": {}, + "sidecars": [], + "steps": [ + [ + 0, + 2 + ], + [ + 1, + 2 + ] + ] + }, + "realign": { + "raw_attrs": {}, + "sidecars": [], + "steps": [ + [ + 0, + 1 + ], + [ + 1, + 1 + ], + [ + 0, + 2 + ] + ] + }, + "attrs_sidecars": { + "raw_attrs": { + "task_description": "pick and place", + "operator": { + "id": 7, + "tags": [ + "a", + "b" + ] + } + }, + "sidecars": [ + "head_camera_intrinsics.json" + ], + "steps": [ + [ + 0, + 1 + ] + ] + } + } +} diff --git a/examples/egocap/tests/recording/fixtures/alignment/01_jittered_index_offset_wrist.npz b/examples/egocap/tests/recording/fixtures/alignment/01_jittered_index_offset_wrist.npz new file mode 100644 index 0000000000..d0caafd3a4 Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/alignment/01_jittered_index_offset_wrist.npz differ diff --git a/examples/egocap/tests/recording/fixtures/alignment/03_tracking_late_start_early_end.npz b/examples/egocap/tests/recording/fixtures/alignment/03_tracking_late_start_early_end.npz new file mode 100644 index 0000000000..dd832e404a Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/alignment/03_tracking_late_start_early_end.npz differ diff --git a/examples/egocap/tests/recording/fixtures/alignment/04_duplicate_timestamps.npz b/examples/egocap/tests/recording/fixtures/alignment/04_duplicate_timestamps.npz new file mode 100644 index 0000000000..de599be331 Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/alignment/04_duplicate_timestamps.npz differ diff --git a/examples/egocap/tests/recording/fixtures/alignment/04b_backwards_timestamp.npz b/examples/egocap/tests/recording/fixtures/alignment/04b_backwards_timestamp.npz new file mode 100644 index 0000000000..d651c61fac Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/alignment/04b_backwards_timestamp.npz differ diff --git a/examples/egocap/tests/recording/fixtures/alignment/05_irregular_tracking_with_gap.npz b/examples/egocap/tests/recording/fixtures/alignment/05_irregular_tracking_with_gap.npz new file mode 100644 index 0000000000..2842c94a18 Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/alignment/05_irregular_tracking_with_gap.npz differ diff --git a/examples/egocap/tests/recording/fixtures/alignment/06_single_sample_source.npz b/examples/egocap/tests/recording/fixtures/alignment/06_single_sample_source.npz new file mode 100644 index 0000000000..26d5f5fab9 Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/alignment/06_single_sample_source.npz differ diff --git a/examples/egocap/tests/recording/fixtures/alignment/07_no_common_range.npz b/examples/egocap/tests/recording/fixtures/alignment/07_no_common_range.npz new file mode 100644 index 0000000000..8eb3d97870 Binary files /dev/null and b/examples/egocap/tests/recording/fixtures/alignment/07_no_common_range.npz differ diff --git a/examples/egocap/tests/recording/unit/test_align_kept_episode.py b/examples/egocap/tests/recording/unit/test_align_kept_episode.py new file mode 100644 index 0000000000..5836a561ae --- /dev/null +++ b/examples/egocap/tests/recording/unit/test_align_kept_episode.py @@ -0,0 +1,490 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +import json +import shutil +import sys +from pathlib import Path + +import numpy as np +import pytest +import zarr + +from egocap.recording.aligned_writer import ( + AlignedWriteError, + align_kept_episode, + check_kept_episode, + episode_qualification, + publish_aligned_episode, +) +from egocap.recording.episode_layout import ( + FINALIZATION_KEEP, + IDENTITY_ATTR, + ROUTE_ATTR, + EpisodeManifest, + episode_dir_names, + manifest_path, + route_id, + write_manifest, +) + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "aligned_writer" +META = json.loads((FIXTURES / "scenarios.json").read_text()) +INDEX = META["index_source"] +SOURCES = ("head_camera", "wrist_camera", "pico_controller", "glove") +CAMERAS = ("head_camera", "wrist_camera") +TOOLS = Path(__file__).resolve().parents[3] / "recording" / "tools" + + +def clean_continuity(): + return {name: {"scope": "clock_mapping", "identity": 0, "breaks": [], "known": True} for name in SOURCES} + + +def good_timing(): + return {"event_time": {name: {"meaning": "capture", "qualified": True} for name in SOURCES}} + + +def build_raw(root, timing="default"): + """A raw session in the recording service's layout, with the episode's ``timing`` attribute set.""" + scenario = META["scenarios"]["basic_ds1"] + root.mkdir(parents=True) + group = zarr.open_group(str(root), mode="w") + for key, value in scenario["raw_attrs"].items(): + group.attrs[key] = value + with np.load(FIXTURES / "raw_inputs.npz") as inputs: + arrays = {k: inputs[k] for k in inputs.files} + for episode in (0, 1): + episode_group = group.require_group(f"episode_{episode}") + episode_group.attrs[IDENTITY_ATTR] = f"e{episode}" + for key, value in META["episode_attrs"].items(): + episode_group.attrs[key] = value + if timing == "default": + episode_group.attrs["timing"] = good_timing() + elif timing is not None: + episode_group.attrs["timing"] = timing + for key, array in arrays.items(): + tag, source, name = key.split("__") + if tag != f"e{episode}": + continue + source_group = episode_group.require_group(source) + for attr, value in META["source_attrs"].get(source, {}).items(): + source_group.attrs[attr] = value + source_group[name] = array + for camera in CAMERAS: + (root / f"episode_{episode}" / f"{camera}.mp4").write_bytes(f"video {camera} {episode} ".encode() * 16) + shutil.copy2(FIXTURES / "config.yaml", root / "config.yaml") + for name in scenario["sidecars"]: + shutil.copy2(FIXTURES / name, root / name) + + +def written_counts(root, index=0): + """The row counts the recorder would have sealed: what is really on disk for the episode.""" + episode = zarr.open_group(str(root / f"episode_{index}"), mode="r") + rows = {name: int(episode[name]["capture_time"].shape[0]) for name in episode.group_keys()} + return {name: {"accepted": n, "written": n} for name, n in rows.items()} + + +def make_manifest(root, index=0, **overrides): + """A finalized, kept manifest of the unrouted episode ``index`` whose counts match the rows on disk.""" + pending, final = episode_dir_names(index) + fields = { + "session_id": "s", + "episode_id": f"e{index}", + "episode_index": index, + "config_digest": "d", + "task": "t", + "operator": "o", + "sources": [], + "pending_path": pending, + "final_path": final, + "continuity": clean_continuity(), + "storage_phase": "FINALIZED", + "finalization": FINALIZATION_KEEP, + "counts": written_counts(root, index), + } + fields.update(overrides) + return EpisodeManifest(**fields) + + +def store_manifest(session, index=0, root=None): + write_manifest(session, make_manifest(root or session, index)) + + +def snapshot(root): + """Files by hash plus every zarr array's contents.""" + entries = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + entries[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + group = zarr.open_group(str(root), mode="r") + for episode in sorted(group.group_keys()): + for source in sorted(group[episode].group_keys()): + for name in sorted(group[episode][source].array_keys()): + entries[f"{episode}/{source}/{name}"] = group[episode][source][name][:].tobytes() + return entries + + +def tree_files(root): + return sorted(p.relative_to(root).as_posix() for p in root.rglob("*")) + + +@pytest.fixture +def raw(tmp_path): + build_raw(tmp_path / "raw") + return tmp_path / "raw" + + +def test_success_publishes_beside_the_raw_session_and_equals_publish(tmp_path, raw): + got = align_kept_episode(raw, 0, INDEX, make_manifest(raw)) + publish_aligned_episode( + raw, + tmp_path / "ref", + 0, + INDEX, + qualified=dict.fromkeys(SOURCES, True), + continuity=clean_continuity(), + ) + aligned = tmp_path / "raw_aligned" + assert aligned.is_dir() + assert snapshot(aligned) == snapshot(tmp_path / "ref") + assert got.valid_indices.any() + + +def test_explicit_aligned_session_is_used(tmp_path, raw): + target = tmp_path / "elsewhere" + align_kept_episode(raw, 0, INDEX, make_manifest(raw), aligned_session=target) + assert (target / "episode_0").is_dir() + assert not (tmp_path / "raw_aligned").exists() + + +def test_downsample_is_passed_through(tmp_path, raw): + full = align_kept_episode(raw, 0, INDEX, make_manifest(raw), aligned_session=tmp_path / "a") + half = align_kept_episode(raw, 0, INDEX, make_manifest(raw), aligned_session=tmp_path / "b", downsample=2) + assert 0 < int(half.valid_indices.sum()) < int(full.valid_indices.sum()) + + +def bad_timings(): + event = good_timing()["event_time"] + return { + "missing": (None, SOURCES), + "no_event_time": ({}, SOURCES), + "source_absent": ({"event_time": {k: v for k, v in event.items() if k != "glove"}}, ("glove",)), + "false": ({"event_time": {**event, "glove": {"qualified": False}}}, ("glove",)), + "string_true": ({"event_time": {**event, "glove": {"qualified": "true"}}}, ("glove",)), + } + + +@pytest.mark.parametrize("case", sorted(bad_timings())) +def test_unqualified_timing_is_refused_and_nothing_is_written(tmp_path, case): + timing, unqualified = bad_timings()[case] + build_raw(tmp_path / "raw", timing=timing) + before = snapshot(tmp_path / "raw") + with pytest.raises(AlignedWriteError, match="qualified") as info: + align_kept_episode(tmp_path / "raw", 0, INDEX, make_manifest(tmp_path / "raw")) + for name in unqualified: + assert name in str(info.value) + assert not (tmp_path / "raw_aligned").exists() + assert snapshot(tmp_path / "raw") == before + + +def test_a_source_with_recorded_rows_missing_from_the_episode_is_refused_and_nothing_is_written(tmp_path, raw): + manifest = make_manifest(raw) + assert manifest.counts["glove"]["written"] > 0 + shutil.rmtree(raw / "episode_0" / "glove") + before = snapshot(raw) + files = tree_files(raw) + with pytest.raises(AlignedWriteError, match="recorded rows are missing"): + align_kept_episode(raw, 0, INDEX, manifest) + assert not (tmp_path / "raw_aligned").exists() + assert tree_files(raw) == files and snapshot(raw) == before + + +def test_missing_continuity_is_refused(tmp_path, raw): + with pytest.raises(AlignedWriteError): + align_kept_episode(raw, 0, INDEX, make_manifest(raw, continuity=None)) + assert not (tmp_path / "raw_aligned").exists() + + +def test_a_continuity_break_is_refused(tmp_path, raw): + continuity = clean_continuity() + continuity["glove"]["breaks"] = [1] + with pytest.raises(AlignedWriteError): + align_kept_episode(raw, 0, INDEX, make_manifest(raw, continuity=continuity)) + assert not (tmp_path / "raw_aligned").exists() + + +def refused(tmp_path, raw, manifest, index=0, match=None): + """Align is refused before any output and leaves the raw session untouched.""" + before = snapshot(raw) + with pytest.raises(AlignedWriteError, match=match): + align_kept_episode(raw, index, INDEX, manifest) + assert not (tmp_path / "raw_aligned").exists() + assert snapshot(raw) == before + + +def test_another_episodes_manifest_with_the_same_index_is_refused(tmp_path, raw): + other = make_manifest(raw, episode_id="someone-else") + refused(tmp_path, raw, other, match="episode id differs") + + +def test_an_episode_without_an_identity_stamp_is_refused(tmp_path, raw): + zarr.open_group(str(raw / "episode_0"), mode="r+").attrs.pop(IDENTITY_ATTR) + refused(tmp_path, raw, make_manifest(raw), match="episode id differs") + + +@pytest.mark.parametrize( + ("phase", "finalization"), + [ + ("WRITING", FINALIZATION_KEEP), + ("SEALED", FINALIZATION_KEEP), + ("CHECKED", FINALIZATION_KEEP), + ("FINALIZED", "REMOVE"), + ("FINALIZED", "RETAIN"), + ("FINALIZED", None), + ], +) +def test_a_manifest_that_is_not_a_finalized_kept_episode_is_refused(tmp_path, raw, phase, finalization): + manifest = make_manifest(raw, storage_phase=phase, finalization=finalization) + refused(tmp_path, raw, manifest, match="not a finalized kept episode") + + +@pytest.mark.parametrize("delta", [1, -1]) +def test_a_written_count_that_differs_from_the_disk_is_refused(tmp_path, raw, delta): + manifest = make_manifest(raw) + manifest.counts["glove"]["written"] += delta + refused(tmp_path, raw, manifest, match="glove: the rows on disk differ") + + +def test_a_source_that_wrote_no_rows_may_be_absent_from_the_episode(raw): + manifest = make_manifest(raw) + manifest.counts["not_recorded"] = {"accepted": 3, "written": 0} + assert check_kept_episode(raw, 0, manifest) == [] + + +def test_an_unrouted_index_mismatch_is_refused(tmp_path, raw): + manifest = make_manifest(raw, episode_index=1) + refused(tmp_path, raw, manifest, match="episode index is not the manifest's") + + +def test_an_unreadable_episode_is_a_problem_not_an_exception(raw): + problems = check_kept_episode(raw, 7, make_manifest(raw)) + assert any("cannot be read" in p for p in problems) + assert any("episode id differs" in p for p in problems) + + +def test_a_missing_manifest_attribute_file_reads_as_no_stamp(tmp_path): + # Malformed route metadata (not JSON) is the same as no stamp. + build_raw(tmp_path / "raw") + manifest = make_manifest(tmp_path / "raw", assignment={"group": "Studio/20260101", "slot": 0}, session_id="s") + (tmp_path / "raw" / "zarr.json").write_text("{not json") + problems = check_kept_episode(tmp_path / "raw", 0, manifest) + assert "the episode is not at the routed location the manifest records" in problems + + +GROUP = "Studio/20260101-000000" + + +def build_routed(tmp_path, slot=3, stamp="own"): + """A routed layout: the group directory is the raw session, its episode sits under ``slot``. + + The manifest is indexed by the recording index (5), which differs from the slot. + """ + build_raw(tmp_path / "raw") + group_dir = tmp_path / "raw" + if slot != 0: + (group_dir / "episode_0").rename(group_dir / f"episode_{slot}") + attrs = zarr.open_group(str(group_dir), mode="r+").attrs + if stamp == "own": + attrs[ROUTE_ATTR] = route_id("s", GROUP) + elif stamp == "other": + attrs[ROUTE_ATTR] = route_id("another-session", GROUP) + manifest = make_manifest( + group_dir, + slot, + episode_index=5, + pending_path="episode_5.pending", + final_path="episode_5", + assignment={"group": GROUP, "slot": slot}, + ) + manifest.episode_id = "e0" + return group_dir, manifest + + +def test_a_routed_episode_aligns_at_its_slot(tmp_path): + group_dir, manifest = build_routed(tmp_path) + got = align_kept_episode(group_dir, 3, INDEX, manifest) + assert (tmp_path / "raw_aligned" / "episode_3").is_dir() + assert got.valid_indices.any() + + +def test_a_routed_episode_at_another_slot_is_refused(tmp_path): + group_dir, manifest = build_routed(tmp_path) + manifest.assignment["slot"] = 4 + refused(tmp_path, group_dir, manifest, index=3, match="not at the routed location") + + +@pytest.mark.parametrize("stamp", ["missing", "other"]) +def test_a_routed_episode_without_this_sessions_route_stamp_is_refused(tmp_path, stamp): + group_dir, manifest = build_routed(tmp_path, stamp=stamp) + refused(tmp_path, group_dir, manifest, index=3, match="not at the routed location") + + +def test_a_routed_episode_named_by_its_recording_index_is_refused(tmp_path): + group_dir, manifest = build_routed(tmp_path) + problems = check_kept_episode(group_dir, 5, manifest) + assert "the episode is not at the routed location the manifest records" in problems + + +# The CLI lives in tools/, which is not a package. +sys.path.insert(0, str(TOOLS)) +import align_episode # noqa: E402 + + +def run_cli(*args): + return align_episode.main([str(a) for a in args]) + + +def test_cli_success_prints_the_kept_count(tmp_path, raw, capsys): + store_manifest(raw) + assert run_cli(raw, "--episode", 0, "--index-source", INDEX) == 0 + out = capsys.readouterr().out + assert out.startswith("aligned episode_0: ") and " of " in out + assert (tmp_path / "raw_aligned" / "episode_0").is_dir() + + +def test_cli_missing_manifest_exits_2(raw, capsys): + assert run_cli(raw, "--episode", 0, "--index-source", INDEX) == 2 + assert "cannot read the episode's manifest" in capsys.readouterr().err + + +def test_cli_garbage_manifest_exits_2(raw, capsys): + path = manifest_path(raw, 0) + path.parent.mkdir() + path.write_text("{not json") + assert run_cli(raw, "--episode", 0, "--index-source", INDEX) == 2 + assert "cannot read the episode's manifest" in capsys.readouterr().err + + +def test_cli_refused_episode_exits_1(tmp_path, capsys): + build_raw(tmp_path / "raw", timing=None) + store_manifest(tmp_path / "raw") + assert run_cli(tmp_path / "raw", "--episode", 0, "--index-source", INDEX) == 1 + assert "not aligned:" in capsys.readouterr().err + assert not (tmp_path / "raw_aligned").exists() + + +def test_cli_manifest_session_option(tmp_path, raw, capsys): + other = tmp_path / "other" + other.mkdir() + store_manifest(other, root=raw) + assert run_cli(raw, "--episode", 0, "--index-source", INDEX, "--manifest-session", other) == 0 + assert "aligned episode_0" in capsys.readouterr().out + + +def test_cli_aligned_session_and_downsample_are_honoured(tmp_path, raw, capsys): + store_manifest(raw) + run_cli(raw, "--episode", 0, "--index-source", INDEX, "--aligned-session", tmp_path / "x") + full = capsys.readouterr().out + assert ( + run_cli( + raw, "--episode", 0, "--index-source", INDEX, "--aligned-session", tmp_path / "y", "--downsample", 2 + ) + == 0 + ) + half = capsys.readouterr().out + + def kept(line): + return int(line.split(": ")[1].split(" of ")[0]) + + assert (tmp_path / "x" / "episode_0").is_dir() and (tmp_path / "y" / "episode_0").is_dir() + assert not (tmp_path / "raw_aligned").exists() + assert kept(half) < kept(full) + + +def test_cli_missing_episode_is_an_argument_error(raw): + with pytest.raises(SystemExit) as info: + run_cli(raw, "--index-source", INDEX) + assert info.value.code == 2 + + +def test_cli_routed_episode_with_a_separate_manifest_index(tmp_path, capsys): + group_dir, manifest = build_routed(tmp_path) + sessions = tmp_path / "recorded" + sessions.mkdir() + write_manifest(sessions, manifest) + args = (group_dir, "--episode", 3, "--index-source", INDEX, "--manifest-session", sessions) + assert run_cli(*args, "--manifest-episode", 5) == 0 + assert "aligned episode_3" in capsys.readouterr().out + assert (tmp_path / "raw_aligned" / "episode_3").is_dir() + + +def test_cli_routed_episode_without_the_manifest_episode_cannot_find_it(tmp_path, capsys): + group_dir, manifest = build_routed(tmp_path) + sessions = tmp_path / "recorded" + sessions.mkdir() + write_manifest(sessions, manifest) + args = (group_dir, "--episode", 3, "--index-source", INDEX, "--manifest-session", sessions) + assert run_cli(*args) == 2 + assert "cannot read the episode's manifest" in capsys.readouterr().err + + +def test_cli_refuses_a_manifest_that_does_not_bind(tmp_path, raw, capsys): + write_manifest(raw, make_manifest(raw, episode_id="someone-else")) + assert run_cli(raw, "--episode", 0, "--index-source", INDEX) == 1 + assert "episode id differs" in capsys.readouterr().err + assert not (tmp_path / "raw_aligned").exists() + + +def test_qualification_reports_each_declared_source_from_the_episodes_own_timing(tmp_path): + timing = {"event_time": {"head_camera": {"qualified": True}, "pico_controller": {"qualified": False}}} + build_raw(tmp_path / "raw", timing=timing) + assert episode_qualification(tmp_path / "raw", 1, INDEX) == { + "head_camera": True, + "wrist_camera": False, + "pico_controller": False, + "glove": False, + } + + +def test_qualification_is_false_for_every_source_when_the_episode_has_no_timing(tmp_path): + build_raw(tmp_path / "raw", timing=None) + assert set(episode_qualification(tmp_path / "raw", 0, INDEX).values()) == {False} + + +def test_qualification_covers_every_declared_source_when_all_are_qualified(raw): + assert episode_qualification(raw, 0, INDEX) == dict.fromkeys(SOURCES, True) + + +def drop_rows(raw, source): + """Leave the source's group in the episode with its arrays emptied.""" + group = zarr.open_group(str(raw / "episode_0"), mode="r+")[source] + for name in list(group.array_keys()): + array = group[name] + del group[name] + group[name] = array[:0] + + +def test_an_absent_optional_source_is_left_out_of_the_qualification_and_alignment_proceeds(tmp_path, raw): + shutil.rmtree(raw / "episode_0" / "glove") + assert "glove" not in episode_qualification(raw, 0, INDEX) + align_kept_episode(raw, 0, INDEX, make_manifest(raw)) + aligned = zarr.open_group(str(tmp_path / "raw_aligned" / "episode_0"), mode="r") + assert "glove" not in aligned and "pico_controller" in aligned + + +def test_a_source_with_no_rows_is_excluded_from_the_output_and_the_qualification_and_does_not_veto(tmp_path, raw): + drop_rows(raw, "glove") + timing = good_timing() + timing["event_time"]["glove"] = {"qualified": False} + zarr.open_group(str(raw / "episode_0"), mode="r+").attrs["timing"] = timing + assert "glove" not in episode_qualification(raw, 0, INDEX) + align_kept_episode(raw, 0, INDEX, make_manifest(raw)) + aligned = zarr.open_group(str(tmp_path / "raw_aligned" / "episode_0"), mode="r") + assert "glove" not in aligned and "pico_controller" in aligned + + +def test_the_index_source_is_always_in_the_qualification(raw): + shutil.rmtree(raw / "episode_0" / INDEX) + assert INDEX in episode_qualification(raw, 0, INDEX) diff --git a/examples/egocap/tests/recording/unit/test_aligned_reader_contract.py b/examples/egocap/tests/recording/unit/test_aligned_reader_contract.py new file mode 100644 index 0000000000..096698ae2c --- /dev/null +++ b/examples/egocap/tests/recording/unit/test_aligned_reader_contract.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +# The read pattern of a downstream consumer of an aligned session, re-implemented here from its description: +# open the aligned root; per episode read the boolean ``valid_indices`` and ``valid_timestamps``; per source +# read every ``_interp`` array (one row per aligned timestamp) and, for each camera, the integer +# ``aligned_frame_indices`` (also one per aligned timestamp) that select frames of the ``.mp4`` stored +# beside the arrays; the root attribute ``index_source`` names the camera whose frames define the timeline. + +import json +import shutil +from pathlib import Path + +import cv2 +import numpy as np +import pytest +import zarr + +from egocap.recording.aligned_writer import write_aligned_episode + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "aligned_writer" +META = json.loads((FIXTURES / "scenarios.json").read_text()) +INDEX = META["index_source"] +CAMERAS = ("head_camera", "wrist_camera") +QUALIFIED = dict.fromkeys((*CAMERAS, "pico_controller", "glove"), True) +CONTINUITY = {name: {"scope": "clock_mapping", "identity": 0, "breaks": [], "known": True} for name in QUALIFIED} +RAW_ATTRS = {"task_description": "pick and place", "operator": {"id": 7, "tags": ["a", "b"]}} + + +def build_raw(root): + """A raw session with non-empty root attributes and real videos, one frame per recorded camera sample.""" + group = zarr.open_group(str(root), mode="w") + for key, value in RAW_ATTRS.items(): + group.attrs[key] = value + with np.load(FIXTURES / "raw_inputs.npz") as inputs: + arrays = {k: inputs[k] for k in inputs.files} + for episode in (0, 1): + episode_group = group.require_group(f"episode_{episode}") + for key, value in META["episode_attrs"].items(): + episode_group.attrs[key] = value + for key, array in arrays.items(): + tag, source, name = key.split("__") + if tag == f"e{episode}": + episode_group.require_group(source)[name] = array + for camera in CAMERAS: + frames = len(episode_group[camera]["capture_time"][:]) + writer = cv2.VideoWriter( + str(root / f"episode_{episode}" / f"{camera}.mp4"), cv2.VideoWriter_fourcc(*"mp4v"), 30, (64, 48) + ) + for _ in range(frames): + writer.write(np.zeros((48, 64, 3), np.uint8)) + writer.release() + shutil.copy2(FIXTURES / "config.yaml", root / "config.yaml") + + +def video_frame_count(path): + capture = cv2.VideoCapture(str(path)) + try: + return int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) + finally: + capture.release() + + +@pytest.fixture(scope="module") +def aligned(tmp_path_factory): + base = tmp_path_factory.mktemp("session") + build_raw(base / "raw") + for episode in (0, 1): + write_aligned_episode( + base / "raw", base / "aligned", episode, INDEX, qualified=QUALIFIED, continuity=CONTINUITY + ) + return base / "aligned" + + +@pytest.mark.parametrize("episode", [0, 1]) +def test_a_consumer_reads_the_aligned_episode_by_array_name(aligned, episode): + root = zarr.open_group(str(aligned), mode="r") + assert root.attrs["index_source"] == INDEX + # The raw root's own attributes are carried over beside the index source. + assert {k: root.attrs[k] for k in RAW_ATTRS} == RAW_ATTRS + + group = root[f"episode_{episode}"] + valid_indices = np.asarray(group["valid_indices"]) + timestamps = np.asarray(group["valid_timestamps"]) + assert valid_indices.dtype == np.bool_ + assert timestamps.ndim == 1 and np.all(np.diff(timestamps) > 0) + rows = timestamps.shape[0] + + interp_seen = 0 + for source in group.group_keys(): + for name in group[source].array_keys(): + if name.endswith("_interp"): + interp_seen += 1 + assert np.asarray(group[source][name]).shape[0] == rows, (source, name) + # A resampled field is read under its ``_interp`` name only. + assert name[: -len("_interp")] not in group[source], (source, name) + assert interp_seen + + for camera in CAMERAS: + indices = np.asarray(group[camera]["aligned_frame_indices"]) + assert np.issubdtype(indices.dtype, np.integer) and indices.shape == (rows,) + video = aligned / f"episode_{episode}" / f"{camera}.mp4" + assert video.is_file() + assert indices.min() >= 0 and indices.max() < video_frame_count(video) + # The index camera plays every frame of the valid range once, in order. + index_frames = np.asarray(group[INDEX]["aligned_frame_indices"]) + assert np.array_equal(np.diff(index_frames), np.ones(rows - 1, dtype=index_frames.dtype)) diff --git a/examples/egocap/tests/recording/unit/test_aligned_writer.py b/examples/egocap/tests/recording/unit/test_aligned_writer.py new file mode 100644 index 0000000000..91f364bca3 --- /dev/null +++ b/examples/egocap/tests/recording/unit/test_aligned_writer.py @@ -0,0 +1,449 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import dataclasses +import hashlib +import json +import os +import shutil +from pathlib import Path + +import numpy as np +import pytest +import yaml +import zarr + +from egocap.recording import aligned_writer +from egocap.recording.aligned_writer import AlignedWriteError, validate_aligned_episode, write_aligned_episode + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "aligned_writer" +META = json.loads((FIXTURES / "scenarios.json").read_text()) +INDEX = META["index_source"] +QUALIFIED = dict.fromkeys(("head_camera", "wrist_camera", "pico_controller", "glove"), True) +CAMERAS = ("head_camera", "wrist_camera") + + +def clean_continuity(sources=tuple(QUALIFIED)): + """Continuity evidence that vouches for every source: the clock mapping, no restart, known.""" + return {name: {"scope": "clock_mapping", "identity": 0, "breaks": [], "known": True} for name in sources} + + +def video_bytes(source, episode): + return (f"dummy video {source} episode {episode} ".encode()) * 16 + + +def build_raw(root, scenario, drop_sources=()): + """The raw session a scenario starts from, in the layout the recording service writes.""" + root.mkdir(parents=True) + group = zarr.open_group(str(root), mode="w") + for key, value in scenario["raw_attrs"].items(): + group.attrs[key] = value + with np.load(FIXTURES / "raw_inputs.npz") as inputs: + arrays = {k: inputs[k] for k in inputs.files} + for episode in (0, 1): + episode_group = group.require_group(f"episode_{episode}") + for key, value in META["episode_attrs"].items(): + episode_group.attrs[key] = value + for key, array in arrays.items(): + tag, source, name = key.split("__") + if tag != f"e{episode}" or source in drop_sources: + continue + source_group = episode_group.require_group(source) + for attr, value in META["source_attrs"].get(source, {}).items(): + source_group.attrs[attr] = value + source_group[name] = array + for camera in CAMERAS: + (root / f"episode_{episode}" / f"{camera}.mp4").write_bytes(video_bytes(camera, episode)) + shutil.copy2(FIXTURES / "config.yaml", root / "config.yaml") + for name in scenario["sidecars"]: + shutil.copy2(FIXTURES / name, root / name) + + +def write_steps(raw, aligned, steps): + for episode, downsample in steps: + write_aligned_episode( + raw, aligned, episode, INDEX, downsample=downsample, qualified=QUALIFIED, continuity=clean_continuity() + ) + + +def snapshot(root): + """Everything a reader of an aligned session sees: files, zarr nodes and their contents.""" + entries = {} + for path in sorted(root.rglob("*")): + rel = path.relative_to(root) + # Chunk files and node metadata belong to the zarr nodes, which are compared through zarr below. + if path.is_file() and path.name != "zarr.json" and "c" not in rel.parts[:-1]: + entries[rel.as_posix()] = ("file", path.stat().st_size, hashlib.sha256(path.read_bytes()).hexdigest()) + root_json = json.loads((root / "zarr.json").read_text()) + entries["zarr.json"] = ("root_json", root_json["attributes"]) + + def walk(group, prefix): + for name in sorted(group.group_keys()): + entries[f"{prefix}{name}"] = ("group", dict(group[name].attrs)) + walk(group[name], f"{prefix}{name}/") + for name in sorted(group.array_keys()): + array = group[name] + entries[f"{prefix}{name}"] = ( + "array", + str(array.dtype), + tuple(array.shape), + dict(array.attrs), + array[:], + ) + + walk(zarr.open_group(str(root), mode="r"), "") + return entries + + +def golden(name): + manifest = json.loads((FIXTURES / f"golden_{name}.json").read_text())["entries"] + with np.load(FIXTURES / f"golden_{name}.npz") as store: + values = {k: store[k] for k in store.files} + entries = {} + for e in manifest: + if e["kind"] == "file": + entries[e["path"]] = ("file", e["size"], e["sha256"]) + elif e["kind"] == "root_json": + entries[e["path"]] = ("root_json", e["attributes"]) + elif e["kind"] == "group": + entries[e["path"]] = ("group", e["attributes"]) + else: + entries[e["path"]] = ( + "array", + e["dtype"], + tuple(e["shape"]), + e["attributes"], + values[e["npz_key"]], + ) + return entries + + +# "realign" writes episode 0 twice, which is refused (see the test below), so it has no written tree. +@pytest.mark.parametrize("name", sorted(set(META["scenarios"]) - {"realign"})) +def test_written_tree_equals_the_expected_tree(tmp_path, name): + scenario = META["scenarios"][name] + build_raw(tmp_path / "raw", scenario) + write_steps(tmp_path / "raw", tmp_path / "aligned", scenario["steps"]) + + ours, expected = snapshot(tmp_path / "aligned"), golden(name) + # Sidecars are carried over byte for byte, whatever header a fixture file has gained. + for sidecar in ("config.yaml", *scenario["sidecars"]): + raw_bytes = (tmp_path / "raw" / sidecar).read_bytes() + expected[sidecar] = ("file", len(raw_bytes), hashlib.sha256(raw_bytes).hexdigest()) + + assert set(ours) == set(expected) + for path, want in expected.items(): + got = ours[path] + assert got[0] == want[0], path + if want[0] != "array": + assert got[1:] == want[1:], path + continue + assert got[1:4] == want[1:4], path + if want[1] in ("bool", "int64"): + assert np.array_equal(got[4], want[4]), path + else: + assert np.allclose(got[4], want[4], rtol=0, atol=1e-12), path + + +def test_videos_are_hard_linked_to_the_raw_ones(tmp_path): + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"]) + write_steps(tmp_path / "raw", tmp_path / "aligned", [(0, 1)]) + for camera in CAMERAS: + raw = tmp_path / "raw" / "episode_0" / f"{camera}.mp4" + aligned = tmp_path / "aligned" / "episode_0" / f"{camera}.mp4" + assert os.stat(raw).st_ino == os.stat(aligned).st_ino + assert aligned.read_bytes() == video_bytes(camera, 0) + + +def test_videos_are_copied_when_linking_is_not_possible(tmp_path, monkeypatch): + real_link = os.link + + def no_link(source, target, *args, **kwargs): + if str(source).endswith(".mp4"): + raise OSError("cross-device link") + return real_link(source, target, *args, **kwargs) + + monkeypatch.setattr(aligned_writer.os, "link", no_link) + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"]) + write_steps(tmp_path / "raw", tmp_path / "aligned", [(0, 1)]) + for camera in CAMERAS: + raw = tmp_path / "raw" / "episode_0" / f"{camera}.mp4" + aligned = tmp_path / "aligned" / "episode_0" / f"{camera}.mp4" + assert os.stat(raw).st_ino != os.stat(aligned).st_ino + assert aligned.read_bytes() == video_bytes(camera, 0) + + +@pytest.mark.parametrize( + "qualified", + [ + {**QUALIFIED, "glove": False}, + {**QUALIFIED, INDEX: False}, + {name: ok for name, ok in QUALIFIED.items() if name != "glove"}, + ], + ids=["unqualified", "unqualified_index", "not_listed"], +) +def test_an_unqualified_or_unlisted_source_refuses_and_writes_nothing(tmp_path, qualified): + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"]) + with pytest.raises(AlignedWriteError, match="without a qualified event time"): + write_aligned_episode( + tmp_path / "raw", tmp_path / "aligned", 0, INDEX, qualified=qualified, continuity=clean_continuity() + ) + assert not (tmp_path / "aligned").exists() + + +def test_an_unqualified_source_is_refused_even_when_its_config_declares_a_latency(tmp_path): + raw = tmp_path / "raw" + build_raw(raw, META["scenarios"]["basic_ds1"]) + config = yaml.safe_load((raw / "config.yaml").read_text()) + for entry in config["numeric_sources"]: + entry["latency"] = 0.02 + (raw / "config.yaml").write_text(yaml.safe_dump(config)) + with pytest.raises(AlignedWriteError, match="without a qualified event time"): + write_aligned_episode( + raw, + tmp_path / "aligned", + 0, + INDEX, + qualified={**QUALIFIED, "glove": False}, + continuity=clean_continuity(), + ) + assert not (tmp_path / "aligned").exists() + + +def test_a_declared_source_absent_from_the_episode_is_skipped(tmp_path): + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"], drop_sources=("glove",)) + # Not recorded, so its qualification is not asked for either. + qualified = {name: ok for name, ok in QUALIFIED.items() if name != "glove"} + write_aligned_episode( + tmp_path / "raw", tmp_path / "aligned", 0, INDEX, qualified=qualified, continuity=clean_continuity() + ) + episode = zarr.open_group(str(tmp_path / "aligned"), mode="r")["episode_0"] + assert sorted(episode.group_keys()) == ["head_camera", "pico_controller", "wrist_camera"] + + +def test_an_episode_already_in_the_aligned_session_is_refused_and_left_unchanged(tmp_path): + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"]) + write_steps(tmp_path / "raw", tmp_path / "aligned", [(0, 1)]) + before = raw_listing(tmp_path / "aligned") + shape = zarr.open_group(str(tmp_path / "aligned"), mode="r")["episode_0"]["valid_timestamps"].shape + with pytest.raises(AlignedWriteError, match="episode_0 already exists in the aligned session"): + write_steps(tmp_path / "raw", tmp_path / "aligned", [(0, 2)]) + assert raw_listing(tmp_path / "aligned") == before + assert zarr.open_group(str(tmp_path / "aligned"), mode="r")["episode_0"]["valid_timestamps"].shape == shape + + +def test_an_existing_target_video_is_replaced_by_the_link(tmp_path): + source, target = tmp_path / "raw.mp4", tmp_path / "aligned.mp4" + source.write_bytes(b"new") + target.write_bytes(b"old") + aligned_writer._link_or_copy(source, target) + assert target.read_bytes() == b"new" + assert os.stat(source).st_ino == os.stat(target).st_ino + + +def test_an_existing_aligned_sidecar_is_not_overwritten(tmp_path): + build_raw(tmp_path / "raw", META["scenarios"]["attrs_sidecars"]) + (tmp_path / "aligned").mkdir() + (tmp_path / "aligned" / "head_camera_intrinsics.json").write_text("kept: true\n") + write_steps(tmp_path / "raw", tmp_path / "aligned", [(0, 1)]) + assert (tmp_path / "aligned" / "head_camera_intrinsics.json").read_text() == "kept: true\n" + assert any((tmp_path / "aligned" / name).is_file() for name in aligned_writer.ALIGNED_SESSION_SIDECARS) + + +def test_an_edited_aligned_config_is_replaced_by_the_raw_one(tmp_path): + build_raw(tmp_path / "raw", META["scenarios"]["realign"]) + write_steps(tmp_path / "raw", tmp_path / "aligned", [(0, 1)]) + (tmp_path / "aligned" / "config.yaml").write_text("edited: true\n") + write_steps(tmp_path / "raw", tmp_path / "aligned", [(1, 1)]) + assert (tmp_path / "aligned" / "config.yaml").read_bytes() == (tmp_path / "raw" / "config.yaml").read_bytes() + + +def raw_listing(root): + """Every path under the raw session with its size and content hash, to compare before and after.""" + return { + path.relative_to(root).as_posix(): (path.stat().st_size, hashlib.sha256(path.read_bytes()).hexdigest()) + if path.is_file() + else None + for path in sorted(root.rglob("*")) + } + + +def test_the_aligned_root_is_written_from_a_raw_root_without_zarr_json_and_the_raw_session_gains_no_files( + tmp_path, +): + raw = tmp_path / "raw" + build_raw(raw, META["scenarios"]["basic_ds1"]) + (raw / "zarr.json").unlink() + before = raw_listing(raw) + aligned_writer._write_root(raw, tmp_path / "aligned", INDEX) + assert raw_listing(raw) == before + assert not (raw / "zarr.json").exists() + assert dict(zarr.open_group(str(tmp_path / "aligned"), mode="r").attrs) == {"index_source": INDEX} + + +def test_a_raw_session_without_a_root_zarr_json_aligns_and_is_left_untouched(tmp_path): + raw = tmp_path / "raw" + build_raw(raw, META["scenarios"]["basic_ds1"]) + (raw / "zarr.json").unlink() + before = raw_listing(raw) + write_steps(raw, tmp_path / "aligned", [(0, 1)]) + assert raw_listing(raw) == before + assert not (raw / "zarr.json").exists() + assert zarr.open_group(str(tmp_path / "aligned"), mode="r").attrs["index_source"] == INDEX + + +@pytest.fixture +def raw_session(tmp_path): + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"]) + return tmp_path / "raw" + + +def test_the_aligned_session_must_be_separate_from_the_raw_session(tmp_path, raw_session): + (tmp_path / "alias").symlink_to(raw_session, target_is_directory=True) + (tmp_path / "outer").mkdir() + (tmp_path / "outer" / "inner_alias").symlink_to(raw_session, target_is_directory=True) + outside = tmp_path / "elsewhere" + outside.mkdir() + (outside / "up").symlink_to(tmp_path, target_is_directory=True) + targets = { + "same_path": raw_session, + "inside_raw": raw_session / "aligned", + "inside_episode": raw_session / "episode_0" / "aligned", + "containing_raw": tmp_path, + "symlink_alias": tmp_path / "alias", + "inside_through_symlink": tmp_path / "alias" / "aligned", + "containing_through_symlink": outside / "up", + } + before = raw_listing(raw_session) + for label, target in targets.items(): + with pytest.raises(AlignedWriteError, match="must be separate"): + write_aligned_episode( + raw_session, target, 0, INDEX, qualified=QUALIFIED, continuity=clean_continuity() + ) + # Nothing is created in, or removed from, the raw session, and its content is byte for byte the same. + assert raw_listing(raw_session) == before, label + assert not (raw_session / "aligned").exists() + assert (raw_session / "episode_0" / "head_camera.mp4").read_bytes() == video_bytes("head_camera", 0) + + +@pytest.fixture +def aligned_session(tmp_path): + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"]) + write_steps(tmp_path / "raw", tmp_path / "aligned", [(0, 1)]) + return tmp_path / "aligned" + + +def problems(session, episode=0): + return validate_aligned_episode(session, episode, INDEX) + + +def test_a_written_episode_validates(aligned_session): + assert problems(aligned_session) == [] + + +def test_a_missing_episode_group_is_reported(aligned_session): + assert problems(aligned_session, 7) == ["episode_7 group missing from the aligned session"] + + +@pytest.mark.parametrize("name", ["valid_indices", "valid_timestamps"]) +def test_a_missing_episode_array_is_reported(aligned_session, name): + del zarr.open_group(str(aligned_session), mode="a")["episode_0"][name] + assert problems(aligned_session) == [f"episode_0/{name} missing"] + + +def test_a_missing_index_source_group_is_reported(aligned_session): + del zarr.open_group(str(aligned_session), mode="a")["episode_0"][INDEX] + assert problems(aligned_session) == [f"episode_0/{INDEX} group missing"] + + +def test_a_missing_aligned_frame_indices_is_reported(aligned_session): + del zarr.open_group(str(aligned_session), mode="a")["episode_0"][INDEX]["aligned_frame_indices"] + assert problems(aligned_session) == [f"episode_0/{INDEX}/aligned_frame_indices missing"] + + +def test_an_empty_array_is_reported(aligned_session): + zarr.open_group(str(aligned_session), mode="a")["episode_0"]["glove"]["empty"] = np.zeros(0) + assert problems(aligned_session) == ["episode_0/glove/empty is empty"] + + +def corrupt_chunk(session): + chunk = session / "episode_0" / "valid_timestamps" / "c" / "0" + chunk.write_bytes(b"not a zstd frame") + + +def test_an_unreadable_array_is_reported(aligned_session): + corrupt_chunk(aligned_session) + (found,) = problems(aligned_session) + assert found.startswith("episode_0/valid_timestamps unreadable") + + +def test_a_large_array_is_not_read_back(aligned_session, monkeypatch): + corrupt_chunk(aligned_session) + monkeypatch.setattr(aligned_writer, "_CHECK_MAX_BYTES", 0) + assert problems(aligned_session) == [] + + +def test_a_write_that_does_not_validate_raises(tmp_path, monkeypatch): + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"]) + real = aligned_writer.align_episode + + def empty_timestamps(*args, **kwargs): + return dataclasses.replace(real(*args, **kwargs), valid_timestamps=np.zeros(0)) + + monkeypatch.setattr(aligned_writer, "align_episode", empty_timestamps) + with pytest.raises(AlignedWriteError, match="valid_timestamps is empty"): + write_aligned_episode( + tmp_path / "raw", tmp_path / "aligned", 0, INDEX, qualified=QUALIFIED, continuity=clean_continuity() + ) + + +def with_entry(name, **changes): + """Clean evidence with one source's entry changed.""" + evidence = clean_continuity() + evidence[name] = {**evidence[name], **changes} + return evidence + + +@pytest.mark.parametrize( + "continuity", + [ + None, + {k: v for k, v in clean_continuity().items() if k != "glove"}, + with_entry("glove", known=False), + with_entry("glove", scope="capture_session"), + with_entry("glove", breaks=[12]), + with_entry(INDEX, breaks=[3]), + ], + ids=["none", "source_missing", "unknown", "capture_session_scope", "breaks", "index_breaks"], +) +def test_missing_or_broken_continuity_refuses_and_writes_nothing(tmp_path, continuity): + raw = tmp_path / "raw" + build_raw(raw, META["scenarios"]["basic_ds1"]) + before = raw_listing(raw) + with pytest.raises(AlignedWriteError): + write_aligned_episode(raw, tmp_path / "aligned", 0, INDEX, qualified=QUALIFIED, continuity=continuity) + assert not (tmp_path / "aligned").exists() + assert raw_listing(raw) == before + + +@pytest.mark.parametrize("source", ["glove", INDEX]) +def test_a_break_beyond_the_episodes_rows_is_refused_and_writes_nothing(tmp_path, source): + raw = tmp_path / "raw" + build_raw(raw, META["scenarios"]["basic_ds1"]) + before = raw_listing(raw) + with pytest.raises(AlignedWriteError, match=rf"{source}: continuity evidence is incomplete or malformed"): + write_aligned_episode( + raw, tmp_path / "aligned", 0, INDEX, qualified=QUALIFIED, continuity=with_entry(source, breaks=[10**6]) + ) + assert not (tmp_path / "aligned").exists() + assert raw_listing(raw) == before + + +def test_partial_evidence_without_identity_or_breaks_is_refused(tmp_path): + raw = tmp_path / "raw" + build_raw(raw, META["scenarios"]["basic_ds1"]) + partial = {name: {"scope": "clock_mapping", "known": True} for name in QUALIFIED} + with pytest.raises(AlignedWriteError, match="incomplete or malformed"): + write_aligned_episode(raw, tmp_path / "aligned", 0, INDEX, qualified=QUALIFIED, continuity=partial) + assert not (tmp_path / "aligned").exists() diff --git a/examples/egocap/tests/recording/unit/test_alignment.py b/examples/egocap/tests/recording/unit/test_alignment.py new file mode 100644 index 0000000000..8cffbd8589 --- /dev/null +++ b/examples/egocap/tests/recording/unit/test_alignment.py @@ -0,0 +1,543 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import numpy as np +import pytest + +from egocap.recording import alignment +from egocap.recording.alignment import ( + FRAME_FIELD_RULES, + AlignmentError, + InsufficientSamplesError, + align_episode, + check_continuity, + find_nearest_timestamp_indices, + interpolate_field, + qualified_sources, + source_rules, +) + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "alignment" +CAMERAS = ["head_camera", "wrist_camera"] +# The declarations a session's config.yaml carries for the golden scenarios' sources. +CONFIG = { + "recording": {"video_frame_type": "RGBCameraFrame", "numeric_frame_type": "NumericFrame"}, + "video_sources": [{"name": name} for name in CAMERAS], + "numeric_sources": [ + {"name": "pico_controller", "frame_type": "PicoTrackingFrame"}, + {"name": "manus_glove", "frame_type": "ManusFrame"}, + ], +} +RULES = alignment.source_rules(CONFIG) +GLOVE_FIELDS = ("left_hand_pose", "right_hand_pose", "left_sensor_count") + + +def load(name): + with np.load(FIXTURES / f"{name}.npz") as z: + return {k: z[k] for k in z.files} + + +def inputs(data, rules=RULES): + times = {k.split("__")[1]: v for k, v in data.items() if k.startswith("times__")} + fields = {} + for k, v in data.items(): + if k.startswith("field__"): + _, source, field_name = k.split("__") + fields.setdefault(source, {})[field_name] = v + return times, fields + + +def align(data, ds=1, rules=RULES, **kwargs): + times, fields = inputs(data) + return align_episode("head_camera", times, CAMERAS, fields, rules, downsample=ds, **kwargs) + + +def by_output_key(result): + out = {"valid_indices": result.valid_indices, "valid_timestamps": result.valid_timestamps} + for name, source in result.sources.items(): + if source.aligned_frame_indices is not None: + out[f"aligned_frame_indices__{name}"] = source.aligned_frame_indices + for field_name, values in source.interpolated.items(): + out[f"interp__{name}__{field_name}"] = values + return out + + +def assert_matches_expected(data, ds, rules=RULES): + ours = by_output_key(align(data, ds, rules)) + ref = {k.split("__", 1)[1]: v for k, v in data.items() if k.startswith(f"out{ds}__")} + ref1 = {k.split("__", 1)[1]: v for k, v in data.items() if k.startswith("out1__")} + if ds > 1: + # The expected glove arrays for a stride are the matching rows of the full-rate run. + rows = np.searchsorted(np.flatnonzero(ref1["valid_indices"]), np.flatnonzero(ref["valid_indices"])) + for field_name in GLOVE_FIELDS: + ref[f"interp__manus_glove__{field_name}"] = ref1[f"interp__manus_glove__{field_name}"][rows] + assert set(ours) == set(ref) + for key, expected in ref.items(): + assert ours[key].shape == expected.shape, key + if expected.dtype.kind in "biu": + assert np.array_equal(ours[key], expected), key + else: + assert np.allclose(ours[key], expected, rtol=0, atol=1e-12), key + + +@pytest.mark.parametrize( + ("scenario", "ds"), + [ + ("01_jittered_index_offset_wrist", 1), + ("01_jittered_index_offset_wrist", 2), + ("01_jittered_index_offset_wrist", 3), + ("03_tracking_late_start_early_end", 1), + ("04_duplicate_timestamps", 1), + ("05_irregular_tracking_with_gap", 1), + ], +) +def test_matches_expected_alignment_outputs(scenario, ds): + assert_matches_expected(load(scenario), ds) + + +def test_late_tracking_sets_the_valid_range(): + data = load("03_tracking_late_start_early_end") + result = align(data) + assert result.valid_range[0] == data["times__pico_controller"][0] + assert result.valid_range[1] == data["times__pico_controller"][-1] + assert not result.valid_indices[0] + + +def test_single_sample_source_is_refused_for_its_linear_fields(): + # A single sample cannot be interpolated, so the glove's linear fields are refused rather than skipped. + data = load("06_single_sample_source") + assert "out1__interp__manus_glove__left_hand_pose" not in data + with pytest.raises(InsufficientSamplesError): + align(data) + without_hand_poses = {**RULES, "manus_glove": {"left_sensor_count": "nearest"}} + assert_matches_expected(data, 1, without_hand_poses) + + +def test_backwards_timestamp_is_refused(): + data = load("04b_backwards_timestamp") + assert "out1__valid_indices" in data + with pytest.raises(AlignmentError, match="backwards"): + align(data) + + +def test_no_common_range_is_refused(): + data = load("07_no_common_range") + assert not data["out1__valid_indices"].any() + times, fields = inputs(data) + with pytest.raises(AlignmentError, match="common range"): + align_episode("head_camera", times, ["head_camera"], fields, RULES) + + +def test_nearest_tie_goes_to_the_earlier_sample(): + assert find_nearest_timestamp_indices(np.array([0.0, 2.0]), np.array([1.0])).tolist() == [0] + + +def test_a_sample_can_serve_two_targets(): + out = find_nearest_timestamp_indices(np.array([0.0, 10.0]), np.array([0.1, 0.2, 9.9])) + assert out.tolist() == [0, 0, 1] + + +def test_single_sample_source_is_nearest_to_everything(): + assert find_nearest_timestamp_indices(np.array([5.0]), np.array([1.0, 9.0])).tolist() == [0, 0] + + +def test_clean_drops_duplicates_and_reports_too_few_samples(): + t = np.array([0.0, 0.0, 1.0, 2.0]) + v = np.array([0.0, 99.0, 1.0, 2.0]) + out = interpolate_field("linear", t, v, np.array([0.5])) + assert out.tolist() == [0.5] + with pytest.raises(InsufficientSamplesError, match="1 remain"): + interpolate_field("linear", np.array([1.0, 1.0]), np.array([0.0, 1.0]), np.array([1.0])) + + +def test_interpolate_pose_stack(): + t = np.array([0.0, 1.0]) + values = np.stack([np.eye(4), np.eye(4)]) + values[1, :3, 3] = [2.0, 0.0, 0.0] + out = interpolate_field("pose", t, values, np.array([0.5])) + assert out.shape == (1, 4, 4) + assert np.allclose(out[0, :3, 3], [1.0, 0.0, 0.0]) + assert out[0, 3].tolist() == [0.0, 0.0, 0.0, 1.0] + + +def test_pose_with_other_shape_falls_back_to_elementwise(): + t = np.array([0.0, 1.0]) + values = np.array([[0.0, 0.0], [2.0, 4.0]]) + assert interpolate_field("pose", t, values, np.array([0.5])).tolist() == [[1.0, 2.0]] + + +def test_linear_on_scalars_vectors_and_hand_stacks(): + t = np.array([0.0, 2.0]) + targets = np.array([1.0]) + assert interpolate_field("linear", t, np.array([0.0, 4.0]), targets).tolist() == [2.0] + assert interpolate_field("linear", t, np.array([[0.0, 1.0], [2.0, 3.0]]), targets).tolist() == [[1.0, 2.0]] + hands = np.stack([np.zeros((25, 4, 4)), np.full((25, 4, 4), 2.0)]) + assert np.allclose(interpolate_field("linear", t, hands, targets), np.ones((1, 25, 4, 4))) + + +def test_nearest_rule_takes_the_nearest_sample_value(): + out = interpolate_field("nearest", np.array([0.0, 1.0, 2.0]), np.array([10, 11, 12]), np.array([0.9, 2.0])) + assert out.tolist() == [11, 12] + + +def test_raw_rule_returns_none(): + assert interpolate_field("raw", np.array([0.0, 1.0]), np.array([1, 2]), np.array([0.5])) is None + + +def test_unknown_rule_is_an_error(): + with pytest.raises(AlignmentError, match="unknown field rule"): + interpolate_field("cubic", np.array([0.0, 1.0]), np.array([1.0, 2.0]), np.array([0.5])) + + +def simple(index=None, other=None, downsample=1, rules=None, fields=None, **kwargs): + times = { + "cam": np.arange(10.0) if index is None else index, + "arm": np.arange(-2.0, 12.0, 0.5) if other is None else other, + } + arm_fields = {"x": times["arm"] * 2.0} + return align_episode( + "cam", + times, + ["cam"], + {"arm": arm_fields} if fields is None else fields, + {"arm": {"x": "linear"}} if rules is None else rules, + downsample=downsample, + **kwargs, + ) + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"index": None, "downsample": 0}, "at least 1"), + ({"other": np.array([])}, "non-empty"), + ({"other": np.array([0.0, np.nan])}, "finite"), + ({"other": np.array([0.0, np.inf])}, "finite"), + ({"other": np.zeros((2, 2))}, "non-empty"), + ({"other": np.array([2.0, 1.0])}, "backwards"), + ], +) +def test_bad_input_is_refused(kwargs, message): + with pytest.raises(AlignmentError, match=message): + simple(**kwargs) + + +def test_missing_index_source_is_refused(): + with pytest.raises(AlignmentError, match="no times"): + align_episode("cam", {"arm": np.arange(3.0)}, [], {}, {}) + + +def test_empty_valid_range_is_an_error_not_an_empty_success(): + with pytest.raises(AlignmentError, match="common range"): + simple(other=np.arange(20.0, 25.0)) + + +def test_downsample_counts_from_the_first_frame(): + # Frames 0 and 1 are outside the arm's range, so the stride keeps 4, 6, 8 (even positions), not 2, 4, 6. + result = simple(other=np.arange(3.5, 20.0), downsample=2) + assert result.valid_timestamps.tolist() == [4.0, 6.0, 8.0] + + +def test_unrecorded_and_raw_rules_add_nothing(): + result = simple(rules={"arm": {"x": "raw", "missing": "linear"}, "ghost": {"y": "linear"}}) + assert result.sources["arm"].interpolated == {} + + +def test_source_without_recorded_fields_adds_nothing(): + result = simple(fields={}, rules={"arm": {"x": "linear"}}) + assert result.sources["arm"].interpolated == {} + + +def test_index_source_fields_are_only_filtered(): + times = {"cam": np.arange(10.0), "arm": np.arange(2.0, 8.0)} + values = np.arange(10) * 3 + result = align_episode("cam", times, ["cam"], {"cam": {"v": values}}, {"cam": {"v": "linear"}}) + assert result.sources["cam"].interpolated["v"].tolist() == [6, 9, 12, 15, 18, 21] + + +def test_camera_index_source_frames_are_the_valid_positions(): + result = simple() + assert np.array_equal(result.sources["cam"].aligned_frame_indices, np.flatnonzero(result.valid_indices)) + assert result.sources["arm"].aligned_frame_indices is None + + +def test_valid_range_is_first_and_last_common_time(): + assert simple().valid_range == (0.0, 9.0) + + +def event_time(**entries): + """A timing attribute shaped like the backend's, with per-source event-time entries.""" + return {"event_time": entries} + + +# What the backend records today: a sensor-capture camera qualified, a frame-ready camera and both tracking +# sources not. +REALISTIC = event_time( + cam_capture={"meaning": "sensor_capture", "qualified": True}, + cam_ready={"meaning": "frame_ready", "qualified": False}, + pico_controller={"meaning": "pose_evaluation_time", "qualified": False}, + manus_glove={"meaning": "host_push_time", "qualified": False}, +) + + +def test_only_sources_recorded_as_qualified_are_qualified(): + sources = ["cam_capture", "cam_ready", "pico_controller", "manus_glove", "unlisted"] + assert qualified_sources(REALISTIC, sources) == { + "cam_capture": True, + "cam_ready": False, + "pico_controller": False, + "manus_glove": False, + "unlisted": False, + } + + +def test_the_result_has_exactly_the_requested_sources(): + assert qualified_sources(REALISTIC, []) == {} + assert list(qualified_sources(REALISTIC, ["manus_glove", "cam_capture"])) == ["manus_glove", "cam_capture"] + + +@pytest.mark.parametrize("qualified", [False, None, "true", "True", 1, 1.0, [True], {}]) +def test_a_qualified_value_that_is_not_literally_true_is_false(qualified): + assert qualified_sources(event_time(cam={"qualified": qualified}), ["cam"]) == {"cam": False} + + +def test_an_entry_without_a_qualified_key_is_false(): + assert qualified_sources(event_time(cam={"meaning": "sensor_capture"}), ["cam"]) == {"cam": False} + + +@pytest.mark.parametrize("entry", [None, True, "cam", 1, [("qualified", True)]]) +def test_an_entry_that_is_not_a_mapping_is_false(entry): + assert qualified_sources(event_time(cam=entry), ["cam"]) == {"cam": False} + + +@pytest.mark.parametrize("value", [None, [], "cam", 7, [{"qualified": True}], True]) +def test_an_event_time_that_is_not_a_mapping_gives_false(value): + assert qualified_sources({"event_time": value}, ["cam"]) == {"cam": False} + + +@pytest.mark.parametrize("t", [None, {}, {"clock_mapping": {"streams": [{"name": "cam"}]}}]) +def test_timing_without_event_time_gives_false(t): + assert qualified_sources(t, ["cam"]) == {"cam": False} + + +def test_a_clock_mapping_or_physical_alignment_text_is_never_qualification(): + t = { + "clock_mapping": {"streams": [{"name": "cam", "time_meaning": "sensor_capture"}]}, + "tracking_semantics": {"physical_alignment": "available: measurement time per sample"}, + } + assert qualified_sources(t, ["cam", "pico_controller"]) == {"cam": False, "pico_controller": False} + + +def test_controller_poses_are_slerped_and_buttons_stay_raw(): + rules = FRAME_FIELD_RULES["PicoTrackingFrame"] + assert {rules[f] for f in ("head_pose", "left_pose", "right_pose")} == {"pose"} + assert {rules[f] for f in ("left_trigger", "right_trigger")} == {"linear"} + assert not {"a_button", "b_button", "x_button", "y_button"} & set(rules) + + +def test_glove_poses_are_elementwise_linear_never_slerped(): + rules = FRAME_FIELD_RULES["ManusFrame"] + assert rules["left_hand_pose"] == rules["right_hand_pose"] == "linear" + assert "pose" not in rules.values() + + +@pytest.mark.parametrize("side", ["left", "right"]) +def test_glove_sensors_are_linear_and_their_counts_nearest(side): + rules = FRAME_FIELD_RULES["ManusFrame"] + assert [rules[f"{side}_sensor_{kind}"] for kind in ("position", "orientation", "rotation")] == ["linear"] * 3 + assert rules[f"{side}_sensor_count"] == "nearest" + + +def test_camera_frames_resample_a_pose_and_generic_numeric_frames_nothing(): + assert FRAME_FIELD_RULES["RGBCameraFrame"] == {"pose": "pose"} + assert FRAME_FIELD_RULES["NumericFrame"] == {} + + +def test_source_rules_takes_each_sources_declared_frame_type(): + rules = source_rules(CONFIG) + assert set(rules) == {"head_camera", "wrist_camera", "pico_controller", "manus_glove"} + assert rules["head_camera"] == {"pose": "pose"} + assert rules["pico_controller"] == FRAME_FIELD_RULES["PicoTrackingFrame"] + assert rules["manus_glove"] == FRAME_FIELD_RULES["ManusFrame"] + + +def test_source_rules_returns_copies_so_a_caller_cannot_alter_the_table(): + source_rules(CONFIG)["pico_controller"]["head_pose"] = "raw" + assert FRAME_FIELD_RULES["PicoTrackingFrame"]["head_pose"] == "pose" + + +def test_source_without_a_frame_type_takes_the_sessions_default_for_its_kind(): + config = { + "recording": {"video_frame_type": "NumericFrame", "numeric_frame_type": "ManusFrame"}, + "video_sources": [{"name": "cam"}], + "numeric_sources": [{"name": "glove"}, {"name": "tracker", "frame_type": "NumericFrame"}], + } + assert source_rules(config) == { + "cam": {}, + "glove": FRAME_FIELD_RULES["ManusFrame"], + "tracker": {}, + } + + +@pytest.mark.parametrize("recording", [None, "x", ["RGBCameraFrame"], {}]) +def test_missing_or_malformed_recording_defaults_to_camera_and_numeric_frames(recording): + config = {"video_sources": [{"name": "cam"}], "numeric_sources": [{"name": "arm"}]} + if recording is not None: + config["recording"] = recording + assert source_rules(config) == {"cam": {"pose": "pose"}, "arm": {}} + + +def test_a_recording_section_naming_one_default_leaves_the_other(): + config = { + "recording": {"numeric_frame_type": "ManusFrame"}, + "video_sources": [{"name": "cam"}], + "numeric_sources": [{"name": "glove"}], + } + assert source_rules(config) == {"cam": {"pose": "pose"}, "glove": FRAME_FIELD_RULES["ManusFrame"]} + + +@pytest.mark.parametrize( + "config", [{}, {"video_sources": None, "numeric_sources": "arm"}, {"numeric_sources": {}}] +) +def test_absent_or_malformed_sections_declare_nothing(config): + assert source_rules(config) == {} + + +def test_sections_may_be_tuples_or_lists(): + as_tuples = {"video_sources": ({"name": "cam"},), "numeric_sources": ({"name": "arm"},)} + as_lists = {"video_sources": [{"name": "cam"}], "numeric_sources": [{"name": "arm"}]} + assert source_rules(as_tuples) == source_rules(as_lists) == {"cam": {"pose": "pose"}, "arm": {}} + + +def test_unknown_frame_type_is_refused_naming_the_source(): + config = {"numeric_sources": [{"name": "arm", "frame_type": "Bogus"}]} + with pytest.raises(AlignmentError, match="arm.*Bogus"): + source_rules(config) + + +def test_unknown_default_frame_type_is_refused(): + config = {"recording": {"video_frame_type": "Bogus"}, "video_sources": [{"name": "cam"}]} + with pytest.raises(AlignmentError, match="Bogus"): + source_rules(config) + + +@pytest.mark.parametrize("entry", ["cam", None, {"frame_type": "NumericFrame"}, {"name": 3}, ["name"]]) +def test_an_entry_that_is_not_a_named_mapping_is_refused(entry): + with pytest.raises(AlignmentError, match="video_sources: every source needs a name"): + source_rules({"video_sources": [entry]}) + + +def clean(scope="clock_mapping", **changes): + return {"scope": scope, "identity": 1, "breaks": [], "known": True, **changes} + + +def test_only_clock_mapping_and_host_clock_scopes_are_accepted_as_continuity_evidence(): + assert alignment.ACCEPTED_CONTINUITY_SCOPES == ("clock_mapping", "host_clock") + + +def test_clean_host_clock_evidence_is_accepted(): + assert check_continuity({"arm": clean("host_clock")}, ["arm"]) == [] + + +def test_clean_clock_mapping_evidence_for_every_source_is_accepted(): + assert check_continuity({"cam": clean(), "arm": clean()}, ["cam", "arm"]) == [] + + +def test_evidence_for_sources_that_are_not_asked_about_is_ignored(): + assert check_continuity({"cam": clean(), "other": clean(known=False)}, ["cam"]) == [] + + +def test_no_evidence_at_all_is_one_problem_not_a_clean_pass(): + assert check_continuity(None, ["cam", "arm"]) == ["no continuity evidence was recorded for this episode"] + + +def test_a_source_without_an_entry_is_refused(): + assert check_continuity({"cam": clean()}, ["cam", "arm"]) == ["arm: no continuity evidence"] + + +def test_an_entry_that_is_not_a_mapping_is_refused(): + assert check_continuity({"cam": [1, 2]}, ["cam"]) == ["cam: no continuity evidence"] + + +@pytest.mark.parametrize("known", [False, None, "yes", 1]) +def test_continuity_that_is_not_exactly_known_true_is_unknown(known): + assert check_continuity({"cam": clean(known=known)}, ["cam"]) == ["cam: continuity is unknown"] + + +def test_a_missing_known_flag_is_unknown(): + entry = clean() + del entry["known"] + assert check_continuity({"cam": entry}, ["cam"]) == ["cam: continuity is unknown"] + + +@pytest.mark.parametrize("scope", ["capture_session", None, "other"]) +def test_a_scope_that_does_not_cover_the_source_is_refused(scope): + # Transport continuity says the connection stayed up, not that the device behind it did not restart. + (problem,) = check_continuity({"cam": clean(scope)}, ["cam"]) + assert problem.startswith("cam: its continuity evidence (") and f"{scope!r}" in problem + + +def test_a_break_is_reported_with_its_rows(): + problems = check_continuity({"cam": clean(breaks=[4, 9])}, ["cam"]) + assert problems == ["cam: has a restart inside the episode (rows [4, 9])"] + + +def test_every_refused_source_is_reported_at_once(): + evidence = { + "a": clean(breaks=[2]), + "b": clean("capture_session"), + "c": clean(known=False), + "ok": clean(), + } + problems = check_continuity(evidence, ["a", "b", "c", "ok", "missing"]) + assert [p.split(":")[0] for p in problems] == ["a", "b", "c", "missing"] + + +def test_missing_identity_is_incomplete(): + entry = clean() + del entry["identity"] + assert check_continuity({"cam": entry}, ["cam"]) == ["cam: continuity evidence is incomplete or malformed"] + + +@pytest.mark.parametrize("identity", [None, True, False, 1.5, [1]]) +def test_an_identity_that_is_not_a_str_or_int_is_incomplete(identity): + assert check_continuity({"cam": clean(identity=identity)}, ["cam"]) == [ + "cam: continuity evidence is incomplete or malformed" + ] + + +@pytest.mark.parametrize("identity", ["epoch-a", 0, 7]) +def test_str_and_int_identities_are_complete(identity): + assert check_continuity({"cam": clean(identity=identity)}, ["cam"]) == [] + + +def test_missing_breaks_is_incomplete(): + entry = clean() + del entry["breaks"] + assert check_continuity({"cam": entry}, ["cam"]) == ["cam: continuity evidence is incomplete or malformed"] + + +@pytest.mark.parametrize("breaks", [None, 0, (), (3,), "3", [True], [-1], [1.0], [2, "3"], [3, False]]) +def test_breaks_that_are_not_a_list_of_non_negative_ints_are_incomplete(breaks): + assert check_continuity({"cam": clean(breaks=breaks)}, ["cam"]) == [ + "cam: continuity evidence is incomplete or malformed" + ] + + +def test_a_break_beyond_the_written_rows_is_malformed(): + problems = check_continuity({"cam": clean(breaks=[10])}, ["cam"], rows={"cam": 10}) + assert problems == ["cam: continuity evidence is incomplete or malformed"] + + +def test_a_break_at_the_last_row_is_a_restart_not_malformed(): + problems = check_continuity({"cam": clean(breaks=[9])}, ["cam"], rows={"cam": 10}) + assert problems == ["cam: has a restart inside the episode (rows [9])"] + + +def test_rows_for_other_sources_do_not_bound_this_one(): + assert check_continuity({"cam": clean(breaks=[])}, ["cam"], rows={"other": 1}) == [] + problems = check_continuity({"cam": clean(breaks=[50])}, ["cam"], rows={"other": 1}) + assert problems == ["cam: has a restart inside the episode (rows [50])"] diff --git a/examples/egocap/tests/recording/unit/test_count_daily_trajectories.py b/examples/egocap/tests/recording/unit/test_count_daily_trajectories.py index 0bed1e0bf4..0bf0b4634b 100644 --- a/examples/egocap/tests/recording/unit/test_count_daily_trajectories.py +++ b/examples/egocap/tests/recording/unit/test_count_daily_trajectories.py @@ -6,6 +6,8 @@ import time from pathlib import Path +import pytest + from egocap.analytics.count_daily_trajectories import ( _episode_index, _episode_recorded_date, @@ -260,11 +262,12 @@ def test_compute_station_data_breakdown_skips_non_matching_station(tmp_path): assert breakdown["total_seconds"] == 0.0 -def test_compute_station_data_breakdown_skips_aligned_and_non_dirs(tmp_path): +@pytest.mark.parametrize("session_name", ["2026-01-01-10-00-00_aligned", ".2026-01-01-10-00-00"]) +def test_compute_station_data_breakdown_skips_aligned_hidden_and_non_dirs(tmp_path, session_name): _build_session( tmp_path, "Kitchen", - "2026-01-01-10-00-00_aligned", + session_name, "station_01", episodes={0: {"episode_duration": 5.0, "recorded_at": "2026-01-01; 10-00-00"}}, ) @@ -363,11 +366,12 @@ def test_count_trajectories_date_mismatch_produces_no_session(tmp_path): assert count_trajectories(tmp_path, "2026-01-01") == {} -def test_count_trajectories_skips_aligned_and_non_dir_entries(tmp_path): +@pytest.mark.parametrize("session_name", ["2026-01-01-10-00-00_aligned", ".2026-01-01-10-00-00"]) +def test_count_trajectories_skips_aligned_hidden_and_non_dir_entries(tmp_path, session_name): _build_session( tmp_path, "Kitchen", - "2026-01-01-10-00-00_aligned", + session_name, "station_01", episodes={0: {"episode_duration": 5.0, "recorded_at": "2026-01-01; 10-00-00"}}, ) diff --git a/examples/egocap/tests/recording/unit/test_eod_studio_session_move.py b/examples/egocap/tests/recording/unit/test_eod_studio_session_move.py index c4ef78b8e3..a5e28ad163 100644 --- a/examples/egocap/tests/recording/unit/test_eod_studio_session_move.py +++ b/examples/egocap/tests/recording/unit/test_eod_studio_session_move.py @@ -87,6 +87,26 @@ def test_relocate_destination_collision_raises(tmp_path): relocate_session_to_canonical_studio(session, "Kitchen", KNOWN) +def test_relocate_moves_the_aligned_sibling_with_its_session(tmp_path): + session = _session_dir(tmp_path, "_unmatched") + aligned = session.with_name(f"{session.name}_aligned") + (aligned / "episode_0").mkdir(parents=True) + result = relocate_session_to_canonical_studio(session, "Kitchen", KNOWN) + moved = result.with_name(f"{result.name}_aligned") + assert (moved / "episode_0").is_dir() and not aligned.exists() and not session.exists() + + +def test_relocate_refuses_before_moving_anything_when_the_aligned_destination_exists(tmp_path): + session = _session_dir(tmp_path, "_unmatched") + aligned = session.with_name(f"{session.name}_aligned") + aligned.mkdir() + taken = tmp_path / "Kitchen" / f"{session.name}_aligned" + taken.mkdir(parents=True) + with pytest.raises(OSError, match="refuse overwrite"): + relocate_session_to_canonical_studio(session, "Kitchen", KNOWN) + assert session.is_dir() and aligned.is_dir() and not (tmp_path / "Kitchen" / session.name).exists() + + def test_relocate_have_equals_want_but_different_case_is_noop(tmp_path): # session folder is "kitchen" (lowercase); canonicalizing both "have" and # "want" against KNOWN resolves them to the same "Kitchen" spelling, even diff --git a/examples/egocap/tests/recording/unit/test_episode_store.py b/examples/egocap/tests/recording/unit/test_episode_store.py index ede3345fe2..853cff462a 100644 --- a/examples/egocap/tests/recording/unit/test_episode_store.py +++ b/examples/egocap/tests/recording/unit/test_episode_store.py @@ -26,6 +26,7 @@ FINALIZATION_RETAIN, IDENTITY_ATTR, MANIFEST_SCHEMA_VERSION, + READABLE_SCHEMA_VERSIONS, EpisodeManifest, episode_dir_names, finalize_episode_dir, @@ -464,3 +465,46 @@ def test_an_interrupted_older_camera_rejection_cleanup_is_still_recoverable(tmp_ def test_current_records_are_validated_strictly(tmp_path: Path, changes) -> None: with pytest.raises((ValueError, TypeError)): read_manifest(_checked_record(tmp_path, **changes)) + + +def test_schema_version_four_is_written_and_readable_versions_include_it(tmp_path: Path) -> None: + write_manifest(tmp_path, _manifest(0)) + assert json.loads(manifest_path(tmp_path, 0).read_text())["schema_version"] == 4 == MANIFEST_SCHEMA_VERSION + assert set(READABLE_SCHEMA_VERSIONS) == {1, 2, 3, 4} + + +def test_a_version_three_manifest_without_continuity_reads_as_no_evidence(tmp_path: Path) -> None: + """Absence is not a claim of continuity: the field decodes to None, never to an empty or clean record.""" + write_manifest(tmp_path, _manifest(0)) + file = manifest_path(tmp_path, 0) + data = json.loads(file.read_text()) + del data["continuity"] + data["schema_version"] = 3 + file.write_text(json.dumps(data)) + decoded = read_manifest(file) + assert decoded.continuity is None and decoded.schema_version == MANIFEST_SCHEMA_VERSION + + +def test_a_manifest_with_no_evidence_round_trips_as_none(tmp_path: Path) -> None: + write_manifest(tmp_path, _manifest(0)) + assert read_manifest(manifest_path(tmp_path, 0)).continuity is None + + +def test_continuity_evidence_round_trips_through_json_with_int_and_str_identities(tmp_path: Path) -> None: + manifest = _manifest(0) + manifest.continuity = { + "cam": {"scope": "clock_mapping", "identity": 7, "breaks": [3, 8], "known": True}, + "arm": {"scope": "capture_session", "identity": "session-a", "breaks": [], "known": True}, + "idle": {"scope": None, "identity": None, "breaks": [], "known": False}, + } + write_manifest(tmp_path, manifest) + assert read_manifest(manifest_path(tmp_path, 0)).continuity == manifest.continuity + + +def test_a_version_newer_than_this_reader_is_still_refused(tmp_path: Path) -> None: + write_manifest(tmp_path, _manifest(0)) + data = json.loads(manifest_path(tmp_path, 0).read_text()) + data["schema_version"] = MANIFEST_SCHEMA_VERSION + 1 + manifest_path(tmp_path, 1).write_text(json.dumps(data)) + found, problems = scan_manifests(tmp_path) + assert [m.episode_index for m in found] == [0] and "schema_version" in problems[0] diff --git a/examples/egocap/tests/recording/unit/test_publish_aligned_episode.py b/examples/egocap/tests/recording/unit/test_publish_aligned_episode.py new file mode 100644 index 0000000000..4f1a6d92a5 --- /dev/null +++ b/examples/egocap/tests/recording/unit/test_publish_aligned_episode.py @@ -0,0 +1,377 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +import json +import os +import shutil +from pathlib import Path + +import numpy as np +import pytest +import zarr + +from egocap.recording import aligned_writer +from egocap.recording.aligned_writer import ( + AlignedWriteError, + publish_aligned_episode, + validate_aligned_episode, + write_aligned_episode, +) + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "aligned_writer" +META = json.loads((FIXTURES / "scenarios.json").read_text()) +INDEX = META["index_source"] +QUALIFIED = dict.fromkeys(("head_camera", "wrist_camera", "pico_controller", "glove"), True) +CAMERAS = ("head_camera", "wrist_camera") + + +def clean_continuity(sources=tuple(QUALIFIED)): + """Continuity evidence that vouches for every source: the clock mapping, no restart, known.""" + return {name: {"scope": "clock_mapping", "identity": 0, "breaks": [], "known": True} for name in sources} + + +def video_bytes(source, episode): + return (f"dummy video {source} episode {episode} ".encode()) * 16 + + +def build_raw(root, scenario, drop_sources=()): + """The raw session a scenario starts from, in the layout the recording service writes.""" + root.mkdir(parents=True) + group = zarr.open_group(str(root), mode="w") + for key, value in scenario["raw_attrs"].items(): + group.attrs[key] = value + with np.load(FIXTURES / "raw_inputs.npz") as inputs: + arrays = {k: inputs[k] for k in inputs.files} + for episode in (0, 1): + episode_group = group.require_group(f"episode_{episode}") + for key, value in META["episode_attrs"].items(): + episode_group.attrs[key] = value + for key, array in arrays.items(): + tag, source, name = key.split("__") + if tag != f"e{episode}" or source in drop_sources: + continue + source_group = episode_group.require_group(source) + for attr, value in META["source_attrs"].get(source, {}).items(): + source_group.attrs[attr] = value + source_group[name] = array + for camera in CAMERAS: + (root / f"episode_{episode}" / f"{camera}.mp4").write_bytes(video_bytes(camera, episode)) + shutil.copy2(FIXTURES / "config.yaml", root / "config.yaml") + for name in scenario["sidecars"]: + shutil.copy2(FIXTURES / name, root / name) + + +def snapshot(root): + """Everything a reader of an aligned session sees: files, zarr nodes and their contents.""" + entries = {} + for path in sorted(root.rglob("*")): + rel = path.relative_to(root) + # Chunk files and node metadata belong to the zarr nodes, which are compared through zarr below. + if path.is_file() and path.name != "zarr.json" and "c" not in rel.parts[:-1]: + entries[rel.as_posix()] = ("file", path.stat().st_size, hashlib.sha256(path.read_bytes()).hexdigest()) + root_json = json.loads((root / "zarr.json").read_text()) + entries["zarr.json"] = ("root_json", root_json["attributes"]) + + def walk(group, prefix): + for name in sorted(group.group_keys()): + entries[f"{prefix}{name}"] = ("group", dict(group[name].attrs)) + walk(group[name], f"{prefix}{name}/") + for name in sorted(group.array_keys()): + array = group[name] + entries[f"{prefix}{name}"] = ( + "array", + str(array.dtype), + tuple(array.shape), + dict(array.attrs), + array[:], + ) + + walk(zarr.open_group(str(root), mode="r"), "") + return entries + + +def publish(raw, aligned, episode=0, downsample=1, qualified=QUALIFIED, continuity="clean"): + return publish_aligned_episode( + raw, + aligned, + episode, + INDEX, + downsample=downsample, + qualified=qualified, + continuity=clean_continuity() if continuity == "clean" else continuity, + ) + + +def write_control(raw, target, episode=0, downsample=1): + write_aligned_episode( + raw, target, episode, INDEX, downsample=downsample, qualified=QUALIFIED, continuity=clean_continuity() + ) + + +def listing(root): + """Every path under ``root`` (hidden ones included) with its size, to compare a tree before and after.""" + if not root.exists(): + return None + return {p.relative_to(root).as_posix(): (p.stat().st_size if p.is_file() else None) for p in root.rglob("*")} + + +def raw_state(root): + """Recursive listing plus a content hash of every file, for proving the raw session was not touched.""" + state = {} + for p in sorted(root.rglob("*")): + digest = hashlib.sha256(p.read_bytes()).hexdigest() if p.is_file() else None + state[p.relative_to(root).as_posix()] = digest + return state + + +def staging_dirs(aligned): + return sorted(aligned.parent.glob(f".{aligned.name}.staging-*")) + + +def assert_no_leftovers(aligned): + assert staging_dirs(aligned) == [] + + +def assert_snapshots_equal(got, want): + assert set(got) == set(want) + for path, expected in want.items(): + actual = got[path] + assert actual[0] == expected[0], path + if expected[0] != "array": + assert actual[1:] == expected[1:], path + else: + assert actual[1:4] == expected[1:4], path + assert np.array_equal(actual[4], expected[4]), path + + +@pytest.fixture +def raw(tmp_path): + build_raw(tmp_path / "raw", META["scenarios"]["basic_ds1"]) + return tmp_path / "raw" + + +@pytest.mark.parametrize("episode", [0, 1]) +def test_published_tree_equals_what_write_aligned_episode_produces(tmp_path, raw, episode): + write_control(raw, tmp_path / "ref", episode, downsample=2) + before = raw_state(raw) + publish(raw, tmp_path / "aligned", episode, downsample=2) + + assert_snapshots_equal(snapshot(tmp_path / "aligned"), snapshot(tmp_path / "ref")) + assert_no_leftovers(tmp_path / "aligned") + assert validate_aligned_episode(tmp_path / "aligned", episode, INDEX) == [] + assert raw_state(raw) == before + + +def test_publish_returns_the_alignment(tmp_path, raw): + aligned = publish(raw, tmp_path / "aligned") + again = write_aligned_episode( + raw, tmp_path / "ref", 0, INDEX, qualified=QUALIFIED, continuity=clean_continuity() + ) + assert np.array_equal(aligned.valid_indices, again.valid_indices) + assert np.array_equal(aligned.valid_timestamps, again.valid_timestamps) + + +def test_published_videos_are_hard_links_to_the_raw_ones(tmp_path, raw): + publish(raw, tmp_path / "aligned") + for camera in CAMERAS: + raw_video = raw / "episode_0" / f"{camera}.mp4" + aligned_video = tmp_path / "aligned" / "episode_0" / f"{camera}.mp4" + assert os.stat(raw_video).st_ino == os.stat(aligned_video).st_ino + assert aligned_video.read_bytes() == video_bytes(camera, 0) + + +def test_published_videos_are_copies_when_linking_is_not_possible(tmp_path, raw, monkeypatch): + real_link = os.link + + def no_link(source, target, *args, **kwargs): + if str(source).endswith(".mp4"): + raise OSError("cross-device link") + return real_link(source, target, *args, **kwargs) + + monkeypatch.setattr(aligned_writer.os, "link", no_link) + publish(raw, tmp_path / "aligned") + for camera in CAMERAS: + raw_video = raw / "episode_0" / f"{camera}.mp4" + aligned_video = tmp_path / "aligned" / "episode_0" / f"{camera}.mp4" + assert os.stat(raw_video).st_ino != os.stat(aligned_video).st_ino + assert aligned_video.read_bytes() == video_bytes(camera, 0) + + +def test_an_existing_episode_is_refused_and_left_untouched(tmp_path, raw): + aligned = tmp_path / "aligned" + publish(raw, aligned, 0) + before = snapshot(aligned) + files = listing(aligned) + + with pytest.raises(AlignedWriteError, match="already exists"): + publish(raw, aligned, 0, downsample=2) + + assert listing(aligned) == files + assert_snapshots_equal(snapshot(aligned), before) + assert_no_leftovers(aligned) + + +def test_a_different_episode_publishes_beside_the_first(tmp_path, raw): + aligned = tmp_path / "aligned" + publish(raw, aligned, 0, downsample=2) + publish(raw, aligned, 1, downsample=2) + + write_control(raw, tmp_path / "ref", 0, downsample=2) + write_control(raw, tmp_path / "ref", 1, downsample=2) + assert_snapshots_equal(snapshot(aligned), snapshot(tmp_path / "ref")) + assert_no_leftovers(aligned) + + +def test_an_episode_on_a_different_index_source_is_refused_and_leaves_the_aligned_session_unchanged(tmp_path, raw): + aligned = tmp_path / "aligned" + publish(raw, aligned, 0) + before, snap = listing(aligned), snapshot(aligned) + other = next(name for name in CAMERAS if name != INDEX) + + with pytest.raises(AlignedWriteError, match="the aligned session uses index source"): + publish_aligned_episode(raw, aligned, 1, other, qualified=QUALIFIED, continuity=clean_continuity()) + + assert listing(aligned) == before + assert_snapshots_equal(snapshot(aligned), snap) + assert zarr.open_group(str(aligned), mode="r").attrs["index_source"] == INDEX + assert_no_leftovers(aligned) + + +def with_entry(source, **changes): + evidence = clean_continuity() + evidence[source] = {**evidence[source], **changes} + return evidence + + +REFUSALS = { + "unqualified": {"qualified": {**QUALIFIED, "glove": False}}, + "unqualified_index": {"qualified": {**QUALIFIED, INDEX: False}}, + "continuity_none": {"continuity": None}, + "break_in_evidence": {"continuity": with_entry("glove", breaks=[3])}, +} + + +@pytest.mark.parametrize("case", sorted(REFUSALS)) +@pytest.mark.parametrize("existing", [False, True]) +def test_a_refused_publication_leaves_the_aligned_session_unchanged(tmp_path, raw, case, existing): + aligned = tmp_path / "aligned" + if existing: + publish(raw, aligned, 1) + before, raw_before = listing(aligned), raw_state(raw) + snap = snapshot(aligned) if existing else None + + with pytest.raises(AlignedWriteError): + publish(raw, aligned, 0, **REFUSALS[case]) + + assert listing(aligned) == before + if existing: + assert_snapshots_equal(snapshot(aligned), snap) + else: + assert not aligned.exists() + assert_no_leftovers(aligned) + assert raw_state(raw) == raw_before + + +def test_an_aligned_path_aliasing_the_raw_session_is_refused(tmp_path, raw): + (tmp_path / "alias").symlink_to(raw, target_is_directory=True) + before = raw_state(raw) + for target in (raw, tmp_path / "alias", raw / "inside"): + with pytest.raises(AlignedWriteError, match="separate"): + publish(raw, target) + assert raw_state(raw) == before + + +def test_a_failed_rename_removes_the_staging_tree(tmp_path, raw, monkeypatch): + aligned = tmp_path / "aligned" + publish(raw, aligned, 1) + before = listing(aligned) + + def failing_rename(source, target): + raise OSError("simulated failure") + + monkeypatch.setattr(aligned_writer.os, "rename", failing_rename) + with pytest.raises(OSError, match="simulated failure"): + publish(raw, aligned, 0) + monkeypatch.undo() + + assert listing(aligned) == before + assert_no_leftovers(aligned) + publish(raw, aligned, 0) + assert validate_aligned_episode(aligned, 0, INDEX) == [] + + +def test_an_unrelated_staging_directory_is_left_alone(tmp_path, raw): + aligned = tmp_path / "aligned" + other = tmp_path / ".aligned.staging-x" + (other / "episode_0").mkdir(parents=True) + (other / "episode_0" / "partial.bin").write_bytes(b"another run's work") + + publish(raw, aligned, 0) + + assert (other / "episode_0" / "partial.bin").read_bytes() == b"another run's work" + assert staging_dirs(aligned) == [other] + assert validate_aligned_episode(aligned, 0, INDEX) == [] + + +def test_each_attempt_stages_in_its_own_directory(tmp_path, raw, monkeypatch): + aligned = tmp_path / "aligned" + seen = [] + real_validate = aligned_writer.validate_aligned_episode + + def record(session, episode, index_source): + seen.append(session) + return real_validate(session, episode, index_source) + + monkeypatch.setattr(aligned_writer, "validate_aligned_episode", record) + publish(raw, aligned, 0) + publish(raw, aligned, 1) + + assert seen[0] != seen[1] + assert all(path.parent == tmp_path and path.name.startswith(".aligned.staging-") for path in seen) + assert_no_leftovers(aligned) + + +def test_a_missing_parent_of_the_aligned_session_is_created(tmp_path, raw): + aligned = tmp_path / "deeper" / "aligned" + publish(raw, aligned, 0) + assert validate_aligned_episode(aligned, 0, INDEX) == [] + assert_no_leftovers(aligned) + + +@pytest.mark.parametrize("error", [RuntimeError, KeyboardInterrupt]) +def test_a_failure_after_staging_started_removes_the_staging_tree_and_propagates( + tmp_path, raw, monkeypatch, error +): + aligned = tmp_path / "aligned" + publish(raw, aligned, 1) + before = listing(aligned) + seen = {} + + def fail_in_validation(session, episode, index_source): + seen["staged"] = session.exists() + raise error("boom") + + monkeypatch.setattr(aligned_writer, "validate_aligned_episode", fail_in_validation) + with pytest.raises(error, match="boom"): + publish(raw, aligned, 0) + + # Staging existed when the failure hit, and nothing of it is left or reached the aligned session. + assert seen["staged"] + assert_no_leftovers(aligned) + assert listing(aligned) == before + + +def test_a_refused_run_creates_nothing_inside_the_aligned_session(tmp_path, raw): + aligned = tmp_path / "aligned" + publish(raw, aligned, 1) + before = listing(aligned) + with pytest.raises(AlignedWriteError): + publish(raw, aligned, 0, continuity=None) + assert listing(aligned) == before + # A never-created session stays absent. + with pytest.raises(AlignedWriteError): + publish(raw, tmp_path / "fresh", 0, continuity=None) + assert not (tmp_path / "fresh").exists() + assert_no_leftovers(tmp_path / "fresh") diff --git a/examples/egocap/tests/recording/unit/test_update_episode_task_description.py b/examples/egocap/tests/recording/unit/test_update_episode_task_description.py index 45defd3624..824729f6fe 100644 --- a/examples/egocap/tests/recording/unit/test_update_episode_task_description.py +++ b/examples/egocap/tests/recording/unit/test_update_episode_task_description.py @@ -170,6 +170,40 @@ def test_update_episode_task_description_updates_config_yaml_sidecar(tmp_path): assert data["meta"]["task_description"] == "pour cereal" +def _make_aligned_copy(session: Path) -> Path: + aligned = session.with_name(f"{session.name}_aligned") + store = zarr.open(str(aligned), mode="a") + store.attrs["index_source"] = "head" + store.create_group("episode_0").attrs["episode_length"] = 5 + (aligned / "config.yaml").write_text("meta: {}\n", encoding="utf-8") + return aligned + + +def test_update_episode_task_description_updates_the_aligned_copy_beside_the_session(tmp_path): + session = tmp_path / "session" + _make_session(session, 2) + aligned = _make_aligned_copy(session) + update_episode_task_description( + session, "0-1", task_description="pour cereal", task_name="pour", studio="Kitchen" + ) + store = zarr.open(str(aligned), mode="r") + for attrs in (store.attrs, store["episode_0"].attrs): + assert attrs["task_description"] == "pour cereal" + assert attrs["task_name"] == "pour" and attrs["studio"] == "Kitchen" + assert "episode_1" not in store and store.attrs["index_source"] == "head" + meta = yaml.safe_load((aligned / "config.yaml").read_text(encoding="utf-8"))["meta"] + assert meta["task_description"] == "pour cereal" + + +def test_update_episode_task_description_leaves_the_aligned_name_and_studio_when_not_given(tmp_path): + session = tmp_path / "session" + _make_session(session, 1) + aligned = _make_aligned_copy(session) + update_episode_task_description(session, "0", task_description="x") + attrs = zarr.open(str(aligned), mode="r").attrs + assert attrs["task_description"] == "x" and "task_name" not in attrs and "studio" not in attrs + + def test_update_episode_task_description_bad_range_with_relocate_raises_without_moving(tmp_path): # Regression: episodes must be parsed before relocating, so a malformed # range raises and leaves the session at its original location rather diff --git a/examples/egocap/tests/recording_service/test_continuity.py b/examples/egocap/tests/recording_service/test_continuity.py new file mode 100644 index 0000000000..34dd5122e4 --- /dev/null +++ b/examples/egocap/tests/recording_service/test_continuity.py @@ -0,0 +1,320 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Per-source continuity evidence of a take: noted per accepted row, stored in the manifest, read by alignment.""" + +import asyncio +import json +from pathlib import Path + +import numpy as np +import pytest +import zarr + +import recording_service.recording_service as impl +from egocap.recording.aligned_writer import AlignedWriteError, write_aligned_episode +from egocap.recording.episode_layout import manifest_path, read_manifest +from recording_service import ( + Disposition, + FieldSpec, + NumericSample, + NumericSourceSpec, + PixelFormat, + QcConfig, + RecordingService, + ServiceConfig, + SubmissionStatus, + VideoSample, + VideoSourceSpec, +) + +MAPPING = "clock_mapping" + + +def _config(tmp_path: Path, **overrides) -> ServiceConfig: + return ServiceConfig( + output_root=tmp_path / "out", + numeric_sources=( + NumericSourceSpec("pose", (FieldSpec("v", "float64"),)), + NumericSourceSpec("idle", (FieldSpec("v", "float64"),)), + ), + qc=QcConfig(required_sources=("pose",)), + **overrides, + ) + + +def _pose(i: int, continuity): + return NumericSample("pose", float(i), float(i), {"v": float(i)}, continuity) + + +async def _record(tmp_path: Path, own, continuities, **overrides): + """Record one take of ``pose`` rows carrying the given continuity values; returns (session, result).""" + service = own(RecordingService(_config(tmp_path, **overrides))) + session = await service.open_session() + episode = await service.start_episode("t") + for i, continuity in enumerate(continuities): + assert service.submit_sample(episode.episode_id, _pose(i, continuity)).status is SubmissionStatus.ACCEPTED + result = await service.stop_episode(episode.episode_id) + return session, result + + +def _stored(session, index: int = 0): + return read_manifest(manifest_path(session.session_path, index)) + + +async def _evidence(tmp_path: Path, own, continuities): + session, _ = await _record(tmp_path, own, continuities) + return _stored(session).continuity["pose"] + + +async def test_a_constant_identity_is_known_with_no_breaks(tmp_path, own): + entry = await _evidence(tmp_path, own, [(MAPPING, 1)] * 4) + assert entry == {"scope": MAPPING, "identity": 1, "breaks": [], "known": True} + + +async def test_a_change_of_identity_is_a_break_at_the_accepted_row(tmp_path, own): + entry = await _evidence(tmp_path, own, [(MAPPING, 1)] * 3 + [(MAPPING, 2)] * 2) + assert entry["breaks"] == [3] and entry["known"] is True and entry["identity"] == 1 + + +async def test_two_changes_are_two_breaks_and_a_return_to_an_old_identity_is_one_too(tmp_path, own): + ids = [1, 1, 2, 2, 1, 1] + entry = await _evidence(tmp_path, own, [(MAPPING, i) for i in ids]) + assert entry["breaks"] == [2, 4] and entry["known"] is True + + +async def test_str_and_int_identities_are_stored_as_given(tmp_path, own): + entry = await _evidence(tmp_path, own, [("capture_session", "a"), ("capture_session", "b")]) + assert entry == {"scope": "capture_session", "identity": "a", "breaks": [1], "known": True} + + +async def test_a_row_without_evidence_makes_the_source_unknown(tmp_path, own): + entry = await _evidence(tmp_path, own, [(MAPPING, 1), None, (MAPPING, 1)]) + assert entry["known"] is False and entry["breaks"] == [] + + +async def test_a_first_row_without_evidence_makes_the_source_unknown(tmp_path, own): + entry = await _evidence(tmp_path, own, [None, (MAPPING, 1)]) + assert entry["known"] is False + + +async def test_a_change_of_scope_makes_the_source_unknown_not_a_break(tmp_path, own): + entry = await _evidence(tmp_path, own, [(MAPPING, 1), ("capture_session", 2)]) + assert entry["known"] is False and entry["breaks"] == [] and entry["scope"] == MAPPING + + +async def test_a_source_that_accepted_nothing_is_recorded_unknown(tmp_path, own): + session, _ = await _record(tmp_path, own, [(MAPPING, 1)] * 2) + assert _stored(session).continuity["idle"] == {"scope": None, "identity": None, "breaks": [], "known": False} + + +async def test_the_private_last_identity_never_reaches_the_manifest(tmp_path, own): + session, _ = await _record(tmp_path, own, [(MAPPING, 1), (MAPPING, 2)]) + raw = json.loads(manifest_path(session.session_path, 0).read_text()) + assert all("last" not in entry for entry in raw["continuity"].values()) + + +async def test_a_full_result_notes_nothing_and_the_retried_sample_is_noted_once(tmp_path, own): + service = own(RecordingService(_config(tmp_path, queue_capacity=2))) + await service.open_session() + episode = await service.start_episode("t") + eid = episode.episode_id + for i in range(2): + assert service.submit_sample(eid, _pose(i, (MAPPING, 1))).status is SubmissionStatus.ACCEPTED + changed = _pose(2, (MAPPING, 2)) + for _ in range(3): # a refused offer, repeated, leaves no trace + assert service.submit_sample(eid, changed).status is SubmissionStatus.FULL + assert service._take.continuity_record()["pose"]["breaks"] == [] + + while (result := service.submit_sample(eid, changed)).status is SubmissionStatus.FULL: + await asyncio.sleep(0) + assert result.admission_index == 2 + assert service._take.continuity_record()["pose"]["breaks"] == [2] + stopped = await service.stop_episode(eid) + assert {c.source_name: c.accepted for c in stopped.counts}["pose"] == 3 + + +async def test_stored_breaks_are_indices_of_accepted_rows(tmp_path, own): + session, result = await _record(tmp_path, own, [(MAPPING, i // 2) for i in range(8)]) + entry = _stored(session).continuity["pose"] + accepted = {c.source_name: c.accepted for c in result.counts}["pose"] + assert entry["breaks"] == [2, 4, 6] and all(b < accepted for b in entry["breaks"]) + + +async def test_evidence_survives_a_retained_take_and_its_recovery(tmp_path, own, monkeypatch): + """A take stopped before the final rename keeps its sealed evidence; recovery finishes without changing it.""" + real_finalize = impl.finalize_episode_dir + + def refuse(*_a, **_k): + raise OSError("interrupted") + + monkeypatch.setattr(impl, "finalize_episode_dir", refuse) + first = own(RecordingService(_config(tmp_path)), may_stay_open=True) + session = await first.open_session() + episode = await first.start_episode("t") + for i, identity in enumerate([1, 1, 2]): + first.submit_sample(episode.episode_id, _pose(i, (MAPPING, identity))) + result = await first.stop_episode(episode.episode_id) + assert result.disposition is Disposition.RECOVERY_REQUIRED + await first.close() # releases the session lock, as a restart would + before = _stored(session).continuity + assert before["pose"]["breaks"] == [2] + + monkeypatch.setattr(impl, "finalize_episode_dir", real_finalize) + service = own(RecordingService(_config(tmp_path))) + await service.open_session(session.session_id) + (pending,) = service.pending_episodes() + recovered = await service.recover_episode(pending.episode_id) + assert recovered.disposition is Disposition.KEPT + assert _stored(session).continuity == before + + +async def test_a_manifest_that_recorded_no_evidence_stays_none(tmp_path, own): + session, _ = await _record(tmp_path, own, [(MAPPING, 1)]) + file = manifest_path(session.session_path, 0) + data = json.loads(file.read_text()) + del data["continuity"] + data["schema_version"] = 3 + file.write_text(json.dumps(data)) + assert read_manifest(file).continuity is None + + +# --- a recorded episode, aligned only when its stored evidence allows it ------------------------ + + +def _alignable_config(tmp_path: Path) -> ServiceConfig: + return ServiceConfig( + output_root=tmp_path / "out", + numeric_sources=( + NumericSourceSpec("pico_controller", (FieldSpec("left_trigger", "float64"),), "PicoTrackingFrame"), + ), + video_sources=(VideoSourceSpec("cam", 64, 48, 30.0),), + qc=QcConfig(required_sources=("pico_controller", "cam")), + ) + + +async def _record_alignable(tmp_path: Path, own, camera_identities): + """A tiny episode: camera frames (one identity per frame) bracketed by tracking rows, all clock_mapping.""" + service = own(RecordingService(_alignable_config(tmp_path))) + session = await service.open_session() + episode = await service.start_episode("t") + frame = np.zeros((48, 64, 3), dtype=np.uint8) + items = [] + for i, identity in enumerate(camera_identities): + t = 1.0 + i / 30 + items.append(VideoSample("cam", frame, PixelFormat.RGB8, t, t, np.eye(3), (MAPPING, identity))) + for i in range(-2, 4 * len(camera_identities)): + t = 1.0 + i / 60 + items.append(NumericSample("pico_controller", t, t, {"left_trigger": t}, (MAPPING, 1))) + for item in sorted(items, key=lambda x: x.capture_time): + while service.submit_sample(episode.episode_id, item).status is SubmissionStatus.FULL: + await asyncio.sleep(0) + result = await service.stop_episode(episode.episode_id) + assert result.disposition is Disposition.KEPT + return session + + +def _align(session, tmp_path: Path): + manifest = _stored(session) + # The service writes each episode as its own store; the aligner reads episodes as groups of one session root. + zarr.open_group(str(session.session_path), mode="a") + qualified = dict.fromkeys(("cam", "pico_controller"), True) + return write_aligned_episode( + session.session_path, + tmp_path / "aligned", + 0, + "cam", + qualified=qualified, + continuity=manifest.continuity, + ) + + +async def test_an_episode_with_constant_evidence_aligns_from_its_stored_manifest(tmp_path, own): + session = await _record_alignable(tmp_path, own, [1] * 5) + manifest = _stored(session) + assert all(e["known"] and not e["breaks"] for e in manifest.continuity.values()) + aligned = _align(session, tmp_path) + assert aligned.valid_indices.any() + assert zarr.open_group(str(tmp_path / "aligned"), mode="r")["episode_0"]["cam"] is not None + + +async def test_an_episode_with_a_recorded_break_is_refused_and_writes_nothing(tmp_path, own): + session = await _record_alignable(tmp_path, own, [1, 1, 1, 2, 2]) + assert _stored(session).continuity["cam"]["breaks"] == [3] + with pytest.raises(AlignedWriteError, match=r"cam: has a restart inside the episode \(rows \[3\]\)"): + _align(session, tmp_path) + assert not (tmp_path / "aligned").exists() + + +# --- malformed evidence is refused before any bookkeeping -------------------------------------------------- + + +def _video(i: int, continuity): + frame = np.zeros((48, 64, 3), dtype=np.uint8) + return VideoSample("cam", frame, PixelFormat.RGB8, float(i), float(i), np.eye(3), continuity) + + +MALFORMED = [ + [MAPPING, 1], + (MAPPING, 1, 2), + (MAPPING,), + (3, 1), + (MAPPING, True), + (MAPPING, 1.5), + (MAPPING, None), + "clock_mapping", +] + + +@pytest.mark.parametrize("bad", MALFORMED, ids=repr) +async def test_malformed_continuity_is_refused_and_changes_nothing(tmp_path, own, bad): + service = own(RecordingService(_config(tmp_path))) + session = await service.open_session() + episode = await service.start_episode("t") + eid = episode.episode_id + assert service.submit_sample(eid, _pose(0, (MAPPING, 1))).admission_index == 0 + take = service._take + accepted, queued = dict(take.accepted), list(take.queue) + evidence = json.dumps(take.continuity_record(), sort_keys=True) + + with pytest.raises(impl.ValidationError, match="continuity"): + service.submit_sample(eid, _pose(1, bad)) + + assert take.accepted == accepted and list(take.queue) == queued + assert json.dumps(take.continuity_record(), sort_keys=True) == evidence + # The next valid sample takes the index the rejected one would have taken. + assert service.submit_sample(eid, _pose(1, (MAPPING, 1))).admission_index == 1 + await service.stop_episode(eid) + assert _stored(session).continuity["pose"] == {"scope": MAPPING, "identity": 1, "breaks": [], "known": True} + + +async def test_malformed_continuity_on_the_first_row_leaves_the_source_unaccepted(tmp_path, own): + service = own(RecordingService(_config(tmp_path))) + await service.open_session() + episode = await service.start_episode("t") + with pytest.raises(impl.ValidationError, match="continuity"): + service.submit_sample(episode.episode_id, _pose(0, (MAPPING, 2.0))) + assert service._take.accepted["pose"] == 0 and not service._take.queue + assert service._take.continuity_record()["pose"]["known"] is False + assert service.submit_sample(episode.episode_id, _pose(0, (MAPPING, 2))).admission_index == 0 + + +def test_copy_keeps_the_continuity_of_a_numeric_sample(): + copied = RecordingService._copy(_pose(0, (MAPPING, "epoch-1"))) + assert isinstance(copied, NumericSample) and copied.continuity == (MAPPING, "epoch-1") + + +def test_copy_keeps_the_continuity_of_a_video_sample(): + copied = RecordingService._copy(_video(0, (MAPPING, 5))) + assert isinstance(copied, VideoSample) and copied.continuity == (MAPPING, 5) + + +async def test_the_queued_copy_carries_the_submitted_continuity(tmp_path, own): + service = own(RecordingService(_config(tmp_path))) + await service.open_session() + episode = await service.start_episode("t") + service.submit_sample(episode.episode_id, _pose(0, (MAPPING, 9))) + # No await since the submission, so the worker has not drained the queue yet. + (queued,) = service._take.queue + assert queued.continuity == (MAPPING, 9) diff --git a/examples/egocap/tests/recording_service/test_episode_timing.py b/examples/egocap/tests/recording_service/test_episode_timing.py index 378afe0f50..cc6c38ee99 100644 --- a/examples/egocap/tests/recording_service/test_episode_timing.py +++ b/examples/egocap/tests/recording_service/test_episode_timing.py @@ -30,7 +30,6 @@ ROWS = 5 TIMING = { "clock_mapping": {"method": "m", "mappings": [{"epoch": 1, "offset_ns": -7}]}, - "tracking_time": "host_poll", } diff --git a/examples/egocap/tests/recording_service/test_session_config.py b/examples/egocap/tests/recording_service/test_session_config.py new file mode 100644 index 0000000000..f1f71b67c5 --- /dev/null +++ b/examples/egocap/tests/recording_service/test_session_config.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""The session config.yaml's source declarations and the digest's treatment of a numeric frame type.""" + +import asyncio +import dataclasses +import hashlib +import json +from pathlib import Path + +import numpy as np +import yaml +import zarr + +import recording_service.recording_service as impl +from egocap.recording.alignment import align_episode, source_rules +from recording_service import ( + CaptureQcConfig, + Disposition, + FieldSpec, + NumericSample, + NumericSourceSpec, + PixelFormat, + QcConfig, + RecordingService, + RoutingConfig, + ServiceConfig, + VideoSample, + VideoSourceSpec, +) + +FIELDS = (FieldSpec("v", "float64"),) +DEFAULTS = {"video_frame_type": "RGBCameraFrame", "numeric_frame_type": "NumericFrame"} + + +def _config(output_root: Path, **overrides) -> ServiceConfig: + base = ServiceConfig( + output_root=output_root, + numeric_sources=( + NumericSourceSpec("tracker", FIELDS, "PicoTrackingFrame"), + NumericSourceSpec("plain", FIELDS), + ), + video_sources=(VideoSourceSpec("cam", 64, 48, 30.0), VideoSourceSpec("wrist", 64, 48, 30.0)), + qc=QcConfig(required_sources=("tracker",)), + ) + return dataclasses.replace(base, **overrides) + + +async def _written(config: ServiceConfig, own) -> dict: + """Open a new session and return its config.yaml as the standard loader reads it.""" + session = await own(RecordingService(config)).open_session() + return yaml.safe_load((session.session_path / "config.yaml").read_text()) + + +async def test_declares_every_source_with_the_defaults_and_only_the_known_frame_types(tmp_path, own): + written = await _written(_config(tmp_path / "out"), own) + assert written == { + "recording": DEFAULTS, + "video_sources": [{"name": "cam"}, {"name": "wrist"}], + "numeric_sources": [{"name": "tracker", "frame_type": "PicoTrackingFrame"}, {"name": "plain"}], + } + + +async def test_station_meta_is_written_only_with_a_label(tmp_path, own): + labelled = await _written(_config(tmp_path / "a", station_label="Station 01"), own) + assert labelled["meta"] == {"egocap_station_name": "Station 01"} and labelled["recording"] == DEFAULTS + assert "meta" not in await _written(_config(tmp_path / "b"), own) + assert "meta" not in await _written(_config(tmp_path / "c", station_label=""), own) + + +async def test_latency_is_written_only_for_sources_that_have_one_and_never_as_zero(tmp_path, own): + config = _config(tmp_path / "out", latency={"tracker": 0.25, "cam": 0.0}) + written = await _written(config, own) + assert written["numeric_sources"] == [ + {"name": "tracker", "frame_type": "PicoTrackingFrame", "latency": 0.25}, + {"name": "plain"}, + ] + assert written["video_sources"] == [{"name": "cam", "latency": 0.0}, {"name": "wrist"}] # explicit 0 is kept + + +async def test_no_latency_configured_writes_none(tmp_path, own): + written = await _written(_config(tmp_path / "out", latency=None), own) + assert not any("latency" in e for e in written["numeric_sources"] + written["video_sources"]) + + +async def test_a_numeric_only_session_declares_an_empty_video_list(tmp_path, own): + config = _config(tmp_path / "out", video_sources=()) + assert (await _written(config, own))["video_sources"] == [] + + +async def test_reopening_a_session_keeps_its_config_yaml_and_still_works(tmp_path, own): + config = _config(tmp_path / "out", latency={"tracker": 0.1}) + service = own(RecordingService(config)) + session = await service.open_session() + path = session.session_path / "config.yaml" + first = path.read_text() + await service.close() + + changed = dataclasses.replace(config, latency={"tracker": 0.9}, station_label="Station 07") + again = own(RecordingService(changed)) + reopened = await again.open_session(session.session_id) + assert reopened.session_path == session.session_path and path.read_text() == first + + +async def test_a_reopened_session_records_its_next_take_under_the_frozen_config_yaml(tmp_path, own): + config = _config(tmp_path / "out", video_sources=()) + service = own(RecordingService(config)) + session = await service.open_session() + first = await service.start_episode("t") + service.submit_sample(first.episode_id, NumericSample("tracker", 0.0, 0.0, {"v": 1.0})) + await service.stop_episode(first.episode_id) + await service.close() + + again = own(RecordingService(config)) + await again.open_session(session.session_id) + assert not again.pending_episodes() + second = await again.start_episode("t") + assert second.episode_index == 1 and (session.session_path / "config.yaml").is_file() + await again.discard_episode(second.episode_id) + + +async def test_a_routed_group_copy_carries_the_declarations(tmp_path, own): + config = _config( + tmp_path / "out", + routing=RoutingConfig(), + video_sources=(), + numeric_sources=(NumericSourceSpec("tracker", FIELDS, "PicoTrackingFrame"),), + latency={"tracker": 0.5}, + station_label=None, + ) + service = own(RecordingService(config)) + session = await service.open_session() + episode = await service.start_episode("t") + for i in range(3): + service.submit_sample(episode.episode_id, NumericSample("tracker", float(i), float(i), {"v": 1.0})) + result = await service.stop_episode(episode.episode_id) + assert result.disposition is Disposition.KEPT + copied = yaml.safe_load((Path(result.final_path).parent / "config.yaml").read_text()) + assert copied == yaml.safe_load((session.session_path / "config.yaml").read_text()) + assert copied["numeric_sources"] == [{"name": "tracker", "frame_type": "PicoTrackingFrame", "latency": 0.5}] + + +def _pre_frame_type_digest(config: ServiceConfig) -> str: + qc = dataclasses.asdict(config.qc) + if config.qc.capture is None: + del qc["capture"] + else: + qc["capture_algorithm_version"] = impl.ALGORITHM_VERSION + canonical = { + **({"routing": True} if config.routing is not None else {}), + "numeric": [ + {"name": s.name, "fields": [dataclasses.asdict(f) for f in s.fields]} for s in config.numeric_sources + ], + "video": [dataclasses.asdict(s) for s in config.video_sources], + "qc": qc, + "latency": config.latency, + "calibration": [a.name for a in config.calibration_assets], + } + return hashlib.sha256(json.dumps(canonical, sort_keys=True).encode()).hexdigest() + + +def test_a_take_recorded_before_frame_types_existed_keeps_its_digest(): + for extra in ( + {}, + {"routing": RoutingConfig()}, + {"qc": QcConfig(capture=CaptureQcConfig())}, + {"latency": {"a": 1.0}}, + ): + config = ServiceConfig( + Path("out"), + (NumericSourceSpec("a", FIELDS),), + (VideoSourceSpec("cam", 64, 48, 30.0),), + **extra, + ) + assert impl._digest(config) == _pre_frame_type_digest(config) + assert impl._layout_digest(config) == _pre_frame_type_digest( + dataclasses.replace(config, qc=dataclasses.replace(config.qc, capture=None)) + ) + + +def test_the_digest_is_the_same_with_a_frame_type_unset_set_or_changed(): + def digest(frame_type): + return impl._digest(ServiceConfig(Path("out"), (NumericSourceSpec("a", FIELDS, frame_type),))) + + assert digest(None) == digest("PicoTrackingFrame") == digest("ManusFrame") + + +def test_a_frame_type_does_not_change_the_layout_digest_but_a_field_does(): + def layout(fields, frame_type): + return impl._layout_digest(ServiceConfig(Path("out"), (NumericSourceSpec("a", fields, frame_type),))) + + other = (FieldSpec("extra", "float64"),) + assert layout(FIELDS, None) == layout(FIELDS, "ManusFrame") + assert layout(FIELDS, None) != layout(other, None) + + +def test_other_source_fields_still_change_the_digest(): + def digest(name, fields): + return impl._digest(ServiceConfig(Path("out"), (NumericSourceSpec(name, fields, "ManusFrame"),))) + + assert digest("a", FIELDS) != digest("b", FIELDS) + assert digest("a", FIELDS) != digest("a", (FieldSpec("extra", "float64"),)) + + +async def test_a_recorded_episode_aligns_from_its_own_config_yaml_and_arrays(tmp_path, own): + config = ServiceConfig( + output_root=tmp_path / "out", + numeric_sources=( + NumericSourceSpec( + "pico_controller", + (FieldSpec("head_pose", "float64", (4, 4)), FieldSpec("left_trigger", "float64")), + "PicoTrackingFrame", + ), + NumericSourceSpec("manus_glove", (FieldSpec("left_hand_pose", "float64", (25, 4, 4)),), "ManusFrame"), + ), + video_sources=(VideoSourceSpec("cam", 64, 48, 30.0),), + qc=QcConfig(required_sources=("pico_controller", "cam")), + ) + service = own(RecordingService(config)) + session = await service.open_session() + episode = await service.start_episode("t") + frame = np.zeros((48, 64, 3), dtype=np.uint8) + items = [] + for i in range(6): # six camera frames at 30 Hz, bracketed by 60 Hz tracking samples + t = 1.0 + i / 30 + items.append(VideoSample("cam", frame, PixelFormat.RGB8, t, t, np.eye(3))) + for i in range(-2, 16): + t = 1.0 + i / 60 + head = np.eye(4) + head[0, 3] = t + items.append(NumericSample("pico_controller", t, t, {"head_pose": head, "left_trigger": t})) + items.append(NumericSample("manus_glove", t, t, {"left_hand_pose": np.full((25, 4, 4), t)})) + for item in sorted(items, key=lambda x: x.capture_time): + while service.submit_sample(episode.episode_id, item).status.name == "FULL": + await asyncio.sleep(0) + result = await service.stop_episode(episode.episode_id) + assert result.disposition is Disposition.KEPT + + declared = yaml.safe_load((session.session_path / "config.yaml").read_text()) + root = zarr.open_group(str(result.final_path), mode="r") + times = {name: root[name]["capture_time"][:] for name in ("cam", "pico_controller", "manus_glove")} + fields = { + "pico_controller": {k: root["pico_controller"][k][:] for k in ("head_pose", "left_trigger")}, + "manus_glove": {"left_hand_pose": root["manus_glove"]["left_hand_pose"][:]}, + } + rules = source_rules(declared) + aligned = align_episode("cam", times, ["cam"], fields, rules) + + count = int(aligned.valid_indices.sum()) + assert aligned.valid_indices.dtype == bool and aligned.valid_indices.shape == (6,) and count == 6 + assert aligned.sources["cam"].aligned_frame_indices.tolist() == list(range(6)) + pico = aligned.sources["pico_controller"].interpolated + assert pico["head_pose"].shape == (count, 4, 4) and pico["left_trigger"].shape == (count,) + assert np.allclose(pico["head_pose"][:, 0, 3], aligned.valid_timestamps) # poses follow the linear ramp + glove = aligned.sources["manus_glove"].interpolated["left_hand_pose"] + assert glove.shape == (count, 25, 4, 4) and np.allclose(glove[:, 0, 0, 0], aligned.valid_timestamps) diff --git a/examples/egocap/tests/recording_service/test_station_config.py b/examples/egocap/tests/recording_service/test_station_config.py index 498b7fe7b2..7cd468d84e 100644 --- a/examples/egocap/tests/recording_service/test_station_config.py +++ b/examples/egocap/tests/recording_service/test_station_config.py @@ -52,20 +52,21 @@ async def test_station_label_writes_config_yaml_once_and_freezes_on_reopen(tmp_p service = own(RecordingService(base)) session = await service.open_session() cfg_path = session.session_path / "config.yaml" - assert yaml.safe_load(cfg_path.read_text()) == {"meta": {"egocap_station_name": "Station 01"}} + assert yaml.safe_load(cfg_path.read_text())["meta"] == {"egocap_station_name": "Station 01"} await service.close() relabeled = dataclasses.replace(base, station_label="Station 02") service2 = own(RecordingService(relabeled)) await service2.open_session(session.session_id) # reopen the same session - assert yaml.safe_load(cfg_path.read_text()) == {"meta": {"egocap_station_name": "Station 01"}} # unchanged + assert yaml.safe_load(cfg_path.read_text())["meta"] == {"egocap_station_name": "Station 01"} # unchanged -async def test_no_station_label_writes_no_config_yaml(tmp_path, own): - """An unset station_label (the ServiceConfig default) writes no config.yaml at all.""" +async def test_no_station_label_writes_a_config_yaml_without_station_meta(tmp_path, own): + """An unset station_label (the ServiceConfig default) still writes the source declarations, but no meta.""" service = own(RecordingService(_config(tmp_path / "out"))) session = await service.open_session() - assert not (session.session_path / "config.yaml").exists() + written = yaml.safe_load((session.session_path / "config.yaml").read_text()) + assert "meta" not in written and written["numeric_sources"] == [{"name": "s"}] async def test_routed_group_gets_a_copy_of_the_session_config_yaml(tmp_path, own):