From 76d57e8e15916eaf3f5a75bffecaa2bc3bf63033 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 14:31:22 -0800 Subject: [PATCH 01/16] sensing: deploy the camera producer to an Orin with one script scripts/deploy_orin.py copies a snapshot of the tracked files to the Orin, builds only the producer there, packages the producer install component with checksums, and installs it as a versioned release run by one systemd service. It checks the ssh connection first, takes the password from ORIN_PASSWORD and the user from ORIN_USER, refuses to replace an unmanaged producer unless told to, and waits for the producer's own join and publish log lines before reporting ready. A failed build or transfer leaves the running producer alone. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .github/workflows/pre-commit.yaml | 2 + src/plugins/sensing/README.md | 1 + src/plugins/sensing/app/CMakeLists.txt | 6 + src/plugins/sensing/scripts/deploy_orin.py | 801 ++++++++++++++++++ .../scripts/isaaccapture-producer.service.in | 22 + tests/python/ci/test_deploy_orin.py | 721 ++++++++++++++++ 6 files changed, 1553 insertions(+) create mode 100755 src/plugins/sensing/scripts/deploy_orin.py create mode 100644 src/plugins/sensing/scripts/isaaccapture-producer.service.in create mode 100644 tests/python/ci/test_deploy_orin.py diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 17e6e4c1fa..5395ed50c0 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -39,6 +39,8 @@ jobs: python-version: "3.12" - name: Test preview selection run: python3 -m unittest discover -s tests/python/ci -p test_docs_preview.py -v + - name: Test the Orin deployment script + run: python3 -m unittest discover -s tests/python/ci -p test_deploy_orin.py -v synthetic-xr-example-tests: name: Test Synthetic XR task contracts diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 147e65c950..6752c72fbc 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -15,6 +15,7 @@ That guide owns the operating instructions; this README maps the source tree. - [`scripts/`](scripts/): host setup, validation, diagnostics, the Bash compatibility Docker launcher, and `setup_orin_crosscompile.py` for building on a PC. Each entry point exposes `--help`. + `scripts/deploy_orin.py` builds the producer on an Orin from this checkout and runs it as a systemd service. - [`docker/`](docker/): example image, image-side group configuration, and host-side container setup. - [`configs/`](configs/): the platform configuration and its provenance. diff --git a/src/plugins/sensing/app/CMakeLists.txt b/src/plugins/sensing/app/CMakeLists.txt index 613d396602..d225b6d68c 100644 --- a/src/plugins/sensing/app/CMakeLists.txt +++ b/src/plugins/sensing/app/CMakeLists.txt @@ -17,11 +17,15 @@ add_custom_command(TARGET camera_plugin_sensing POST_BUILD COMMENT "Staging SIPL platform configs beside camera_plugin_sensing" ) +# Everything below is the `producer` install component: `cmake --install --component producer` stages +# exactly what scripts/deploy-orin.sh packages for an Orin. install(TARGETS camera_plugin_sensing RUNTIME DESTINATION plugins/sensing_camera + COMPONENT producer ) install(DIRECTORY "${SENSING_CONFIGS_DIR}/" DESTINATION plugins/sensing_camera/configs + COMPONENT producer ) # The plugin links the LiveKit runtime, whose RUNPATH points into the build tree; ship it beside the @@ -30,9 +34,11 @@ install(FILES "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit.so" "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit_ffi.so" DESTINATION plugins/sensing_camera + COMPONENT producer ) install(FILES "${SENSING_LIVEKIT_LICENSE_FILE}" "${SENSING_LIVEKIT_NOTICE_FILE}" DESTINATION plugins/sensing_camera/licenses/livekit-sdk + COMPONENT producer ) # The vendored encoder classes are compiled into the binary, so their license text ships with it. install(FILES "${SENSING_MMAPI_VENDORED_DIR}/LICENSE" diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py new file mode 100755 index 0000000000..60765fb246 --- /dev/null +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -0,0 +1,801 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Build the camera producer on an Orin from this checkout, package it, and run it as a service. + +Run on the desktop: + + deploy_orin.py user@orin --backend sipl --config orin.json + +The config is JSON with the producer's own options, nothing else: + + {"livekit_url": "wss://livekit.example.org", + "token_file": "/etc/isaaccapture/join.token", + "args": ["--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} + +The token file stays on the Orin, outside the release; only its path is passed. Nothing is built, stopped or +replaced until the target passes its checks. A build or transfer failure leaves the running producer alone. +The same file runs on the target (`deploy_orin.py remote ...`) from the source snapshot it copied there. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pwd +import re +import shlex +import shutil +import signal +import subprocess +import sys +import tarfile +import tempfile +import time +from pathlib import Path + +SERVICE = "isaaccapture-producer.service" +BIN_REL = "plugins/sensing_camera/camera_plugin_sensing" +SCRIPT_REL = "src/plugins/sensing/scripts/deploy_orin.py" +UNIT_TEMPLATE = Path(__file__).with_name("isaaccapture-producer.service.in") +CACHE = "~/.cache/isaaccapture-deploy" +CONFIG_KEYS = {"livekit_url", "token_file", "args"} +# The producer's connection options come from the config's own fields, not from `args`. +RESERVED_ARGS = ("--livekit-url", "--livekit-token-file") + + +STEPS = 5 + + +def step(number: int, text: str) -> None: + print(f"==> [{number}/{STEPS}] {text}", flush=True) + + +class Progress: + """A text progress bar: redrawn in place on a terminal, one line per 10% otherwise.""" + + def __init__(self, label: str, total: int) -> None: + self.label, self.total = label, max(total, 1) + self._last = -1 + + def update(self, done: int) -> None: + percent = min(100, done * 100 // self.total) + if sys.stdout.isatty(): + filled = percent * 30 // 100 + bar = "#" * filled + "-" * (30 - filled) + print( + f"\r {self.label} [{bar}] {percent:3d}% ({done}/{self.total})", + end="", + flush=True, + ) + elif percent // 10 != self._last // 10: + print(f" {self.label} {percent}%", flush=True) + self._last = percent + + def finish(self) -> None: + self.update(self.total) + if sys.stdout.isatty(): + print() + + +class DeployError(Exception): + """A failure with a message worth showing as is.""" + + +def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: + result = subprocess.run(cmd, check=False, **kwargs) + if result.returncode != 0: + raise DeployError( + f"{shlex.join(cmd[:3])} ... failed with exit code {result.returncode}" + ) + return result + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +# --------------------------------------------------------------------------- +# Config and service unit +# --------------------------------------------------------------------------- + + +def load_config(path: Path) -> dict: + try: + config = json.loads(path.read_text()) + except (OSError, ValueError) as exc: + raise DeployError(f"cannot read the config {path}: {exc}") from exc + if ( + not isinstance(config, dict) + or set(config) - CONFIG_KEYS + or not {"livekit_url", "token_file"} <= set(config) + ): + raise DeployError( + f"{path}: expected an object with livekit_url, token_file and optional args only" + ) + args = config.get("args", []) + if not isinstance(args, list) or not all(isinstance(a, str) for a in args): + raise DeployError(f"{path}: args must be a list of strings") + if any(a.split("=", 1)[0] in RESERVED_ARGS for a in args): + raise DeployError( + f"{path}: give the URL and token file as livekit_url and token_file, not in args" + ) + if not any(a.startswith("--add-stream=") for a in args): + raise DeployError( + f"{path}: args needs at least one --add-stream=sensor=N,livekit=NAME" + ) + if not all( + isinstance(config[k], str) and config[k] for k in ("livekit_url", "token_file") + ): + raise DeployError( + f"{path}: livekit_url and token_file must be non-empty strings" + ) + if not config["token_file"].startswith("/"): + raise DeployError(f"{path}: token_file must be an absolute path on the Orin") + return config + + +def systemd_quote(arg: str) -> str: + """One ExecStart word: systemd expands % and $ and splits on whitespace, so escape and quote them.""" + arg = ( + arg.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("%", "%%") + .replace("$", "$$") + ) + return f'"{arg}"' if not arg or re.search(r"\s|[\"']", arg) else arg + + +def render_unit(config: dict, prefix: str, user: str, groups: str) -> str: + words = [ + f"{prefix}/current/{BIN_REL}", + f"--livekit-url={config['livekit_url']}", + f"--livekit-token-file={config['token_file']}", + *config.get("args", []), + ] + exec_start = " ".join(systemd_quote(w) for w in words) + fields = { + "SERVICE_USER": user, + "SERVICE_GROUPS": groups, + "PREFIX": prefix, + "EXEC_START": exec_start, + } + text = UNIT_TEMPLATE.read_text() + for key, value in fields.items(): + text = text.replace(f"@{key}@", value) + return text + + +# --------------------------------------------------------------------------- +# Desktop side +# --------------------------------------------------------------------------- + + +def repo_root() -> Path: + out = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=Path(__file__).parent, + capture_output=True, + text=True, + check=True, + ) + return Path(out.stdout.strip()) + + +def make_snapshot(root: Path, dest: Path) -> dict: + """Archive the tracked files as they are on disk, including tracked local edits; nothing untracked.""" + listing = subprocess.run( + ["git", "ls-files", "-z", "--recurse-submodules"], + cwd=root, + capture_output=True, + check=True, + ).stdout + files = sorted( + f for f in listing.decode().split("\0") if f and (root / f).is_file() + ) + bar = Progress("packing", len(files)) + with tarfile.open(dest, "w:gz") as tar: + for done, name in enumerate(files, 1): + tar.add(root / name, arcname=name, recursive=False) + if done % 200 == 0: + bar.update(done) + bar.finish() + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + capture_output=True, + text=True, + check=True, + ) + dirty = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=no"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + return { + "base_revision": head.stdout.strip(), + "dirty": bool(dirty), + "sha256": sha256_file(dest), + } + + +PASSWORD_ENV = "ORIN_PASSWORD" +USER_ENV = "ORIN_USER" + + +def target_host(target: str) -> str: + """`host` becomes `$ORIN_USER@host` when ORIN_USER is set; `user@host` is used as given.""" + user = os.environ.get(USER_ENV) + return f"{user}@{target}" if user and "@" not in target else target + + +class Target: + """The Orin over ssh/scp. + + With ORIN_PASSWORD set, ssh gets it through SSH_ASKPASS and sudo reads it from stdin (`sudo -S`), so it never + appears in an argument list or a log; without it, ssh and sudo prompt as usual. One ssh connection is reused for + every step. + """ + + def __init__(self, host: str) -> None: + self.host = host + self.password = os.environ.get(PASSWORD_ENV) + self._dir = Path(tempfile.mkdtemp(prefix="deploy-orin-")) + self._env = dict(os.environ) + if self.password: + askpass = self._dir / "askpass" + askpass.write_text(f'#!/bin/sh\nprintf "%s\\n" "${PASSWORD_ENV}"\n') + askpass.chmod(0o700) + self._env.update(SSH_ASKPASS=str(askpass), SSH_ASKPASS_REQUIRE="force") + self._opts = [ + "-o", + "ControlMaster=auto", + "-o", + f"ControlPath={self._dir}/cm", + "-o", + "ControlPersist=300", + ] + + def _run(self, tool: str, args: list[str], **kwargs) -> subprocess.CompletedProcess: + return subprocess.run( + [tool, *self._opts, *args], env=self._env, check=False, **kwargs + ) + + def ssh(self, command: str, *, tty: bool = False) -> subprocess.CompletedProcess: + result = self._run("ssh", [*(["-t"] if tty else []), self.host, command]) + if result.returncode != 0: + raise DeployError( + f"ssh {self.host} failed with exit code {result.returncode}" + ) + return result + + def sudo(self, command: str) -> None: + """Run a command as root: the password from the environment on stdin, else an interactive prompt.""" + if self.password: + result = self._run( + "ssh", + [self.host, f"sudo -S -p '' {command}"], + input=self.password + "\n", + text=True, + ) + if result.returncode != 0: + raise DeployError( + f"sudo on {self.host} failed with exit code {result.returncode}" + ) + else: + self.ssh(f"sudo {command}", tty=True) + + def validate_connection(self) -> None: + """Fail early, with the reason, when the Orin cannot be reached or lacks what the deployment runs on it.""" + probe = self._run( + "ssh", + [ + "-o", + "ConnectTimeout=10", + self.host, + "command -v python3 >/dev/null && uname -m", + ], + capture_output=True, + text=True, + ) + if probe.returncode != 0: + reason = probe.stderr.strip() or "python3 is not installed on the target" + raise DeployError(f"cannot use {self.host} over ssh: {reason}") + if probe.stdout.strip() != "aarch64": + raise DeployError( + f"{self.host} is {probe.stdout.strip() or 'unknown'}, not an aarch64 Jetson" + ) + + def capture(self, command: str) -> str: + out = self._run("ssh", [self.host, command], capture_output=True, text=True) + if out.returncode != 0: + raise DeployError( + f"ssh {self.host} {command!r} failed: {out.stderr.strip()}" + ) + return out.stdout.strip() + + def readable(self, path: str) -> bool: + return self._run("ssh", [self.host, "test", "-r", path]).returncode == 0 + + @staticmethod + def _scp_quiet() -> list[str]: + """scp shows its own transfer meter on a terminal; stay quiet when output is piped.""" + return [] if sys.stdout.isatty() else ["-q"] + + def put(self, local: Path, remote: str) -> None: + if ( + self._run( + "scp", [*self._scp_quiet(), str(local), f"{self.host}:{remote}"] + ).returncode + != 0 + ): + raise DeployError(f"copying {local.name} to {self.host} failed") + + def get(self, remote: str, local: Path) -> None: + if ( + self._run( + "scp", [*self._scp_quiet(), f"{self.host}:{remote}", str(local)] + ).returncode + != 0 + ): + raise DeployError(f"copying {remote} from {self.host} failed") + + def close(self) -> None: + self._run("ssh", ["-O", "exit", self.host], capture_output=True) + shutil.rmtree(self._dir, ignore_errors=True) + + +def deploy(args: argparse.Namespace) -> None: + config = load_config(Path(args.config)) + target = Target(target_host(args.target)) + try: + run_deploy(args, config, target) + finally: + target.close() + + +def run_deploy(args: argparse.Namespace, config: dict, target: Target) -> None: + root = repo_root() + step(1, f"Connecting to {target.host}") + target.validate_connection() + user = args.service_user or target.capture("id -un") + unit = render_unit(config, args.prefix, user, args.service_groups) + + step(2, f"Preflight on {target.host}") + cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") + if not target.readable(config["token_file"]): + raise DeployError( + f"the token file {config['token_file']} is not readable on the Orin" + ) + + with tempfile.TemporaryDirectory() as tmp: + snapshot_path = Path(tmp) / "snapshot.tar.gz" + step(3, "Snapshot and upload") + meta = make_snapshot(root, snapshot_path) + release = f"{time.strftime('%Y%m%d-%H%M%S')}-{meta['base_revision'][:9]}{'-dirty' if meta['dirty'] else ''}" + print( + f" base {meta['base_revision'][:12]}, dirty={meta['dirty']}, sha256 {meta['sha256'][:16]}" + ) + src = f"{cache}/src-{meta['sha256'][:16]}" + target.put(snapshot_path, f"{cache}/snapshot.tar.gz") + target.ssh( + f"rm -rf {src} && mkdir -p {src} && tar -xzf {cache}/snapshot.tar.gz -C {src}" + ) + remote = f"python3 {src}/{SCRIPT_REL} remote" + target.ssh( + f"{remote} preflight --backend {args.backend} --sdk-root {shlex.quote(args.sdk_root)}" + ) + + step(4, "Build and package on the Orin (the running producer is untouched)") + stage, archive = ( + f"{cache}/stage-{release}", + f"{cache}/producer-{release}.tar.gz", + ) + target.ssh( + f"{remote} build --src {src} --backend {args.backend} --stage {stage} --jobs {args.jobs} " + f"--sdk-root {shlex.quote(args.sdk_root)}" + ) + manifest = { + **meta, + "backend": args.backend, + "release": release, + "arch": "aarch64", + "config": args.config, + } + manifest_path = Path(tmp) / "MANIFEST.json" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + target.put(manifest_path, f"{stage}/MANIFEST.json") + target.ssh(f"{remote} package --stage {stage} --archive {archive}") + if args.fetch_archive: + Path(args.fetch_archive).mkdir(parents=True, exist_ok=True) + target.get(archive, Path(args.fetch_archive) / Path(archive).name) + + unit_path = Path(tmp) / SERVICE + unit_path.write_text(unit) + target.put(unit_path, f"{cache}/{SERVICE}") + + if args.no_activate: + print(f"==> Built release {release}; not activated (--no-activate)") + return + step(5, "Install and replace the running producer (needs sudo on the Orin)") + activate = ( + f"{remote} activate --archive {archive} --prefix {args.prefix} --release {release} " + f"--unit {cache}/{SERVICE} --owner {user} --ready-timeout {args.ready_timeout}" + + (" --replace-unmanaged" if args.replace_unmanaged else "") + ) + target.sudo(activate) + print(f"==> Deployed {release}; logs: ssh {target.host} journalctl -u {SERVICE} -f") + + +# --------------------------------------------------------------------------- +# Target side +# --------------------------------------------------------------------------- + + +def tail(cmd: list[str]) -> str: + out = subprocess.run(cmd, capture_output=True, text=True, check=False) + return (out.stdout + out.stderr).strip() + + +def sdk_dirs(sdk_root: str) -> tuple[Path, Path]: + """The Multimedia API and SIPL header trees: under --sdk-root when given, else the system locations.""" + base = Path(sdk_root) / "usr/src" if sdk_root else Path("/usr/src") + return base / "jetson_multimedia_api", base / "jetson_sipl_api" + + +def remote_preflight(args: argparse.Namespace) -> None: + if os.uname().machine != "aarch64": + raise DeployError(f"the target is {os.uname().machine}, not aarch64") + release_file = Path("/etc/nv_tegra_release") + if not release_file.exists(): + raise DeployError("no /etc/nv_tegra_release: this is not a Jetson L4T system") + print(f" L4T: {release_file.read_text().splitlines()[0]}") + for tool in ("cmake", "g++", "systemctl"): + if ( + subprocess.run(["which", tool], capture_output=True, check=False).returncode + != 0 + ): + raise DeployError(f"{tool} is not installed") + if ( + not Path("/usr/local/cuda/bin/nvcc").exists() + and subprocess.run( + ["which", "nvcc"], capture_output=True, check=False + ).returncode + ): + raise DeployError("nvcc not found (CUDA toolkit)") + stat = os.statvfs(Path.home()) + if stat.f_bavail * stat.f_frsize < 4 << 30: + raise DeployError("less than 4 GB free in the home directory") + mmapi, sipl_api = sdk_dirs(args.sdk_root) + if not mmapi.is_dir(): + raise DeployError( + f"the Jetson Multimedia API is missing at {mmapi}; run src/plugins/sensing/scripts/setup.sh " + "or pass --sdk-root" + ) + if args.backend == "sipl" and not sipl_api.is_dir(): + raise DeployError( + f"SIPL headers not found at {sipl_api}; run src/plugins/sensing/scripts/setup.sh or pass --sdk-root" + ) + if args.backend == "argus" and not (mmapi / "argus").is_dir(): + raise DeployError(f"Argus headers not found under {mmapi}/argus") + print(f" preflight ok for backend {args.backend}") + + +def remote_build(args: argparse.Namespace) -> None: + """Configure and build only the producer, then stage its install component. + + The build directory persists between deployments and is discarded when the toolchain, L4T release or backend + changes. + """ + cache = Path(os.path.expanduser(CACHE)) + build_dir = cache / f"build-{args.backend}" + stamp = "|".join( + [ + Path("/etc/nv_tegra_release").read_text().splitlines()[0], + tail(["cmake", "--version"]).splitlines()[0], + tail(["g++", "-dumpfullversion"]), + ] + ) + stamp_file = build_dir / ".stamp" + if not stamp_file.exists() or stamp_file.read_text() != stamp: + print(" new toolchain, release or backend: clean build directory") + run(["rm", "-rf", str(build_dir)]) + build_dir.mkdir(parents=True) + stamp_file.write_text(stamp) + sipl, argus = ("ON", "OFF") if args.backend == "sipl" else ("OFF", "ON") + mmapi, sipl_api = sdk_dirs(args.sdk_root) + env = {**os.environ, "PATH": f"/usr/local/cuda/bin:{os.environ.get('PATH', '')}"} + sdk_flags = [f"-DJETSON_MMAPI_DIR={mmapi}"] + if args.backend == "sipl": + sdk_flags.append(f"-DSIPL_API_DIR={sipl_api}") + # The CUDA driver library is not on the default link path on L4T, and is dropped by --as-needed. + gpu_libcuda = Path("/opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so.1") + link_flags = ( + [f"-DCMAKE_EXE_LINKER_FLAGS=-Wl,--no-as-needed {gpu_libcuda}"] + if gpu_libcuda.exists() + else [] + ) + run( + [ + "cmake", + "-S", + args.src, + "-B", + str(build_dir), + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CUDA_ARCHITECTURES=87", + "-DBUILD_PLUGIN_SENSING=ON", + f"-DSENSING_WITH_SIPL={sipl}", + f"-DSENSING_WITH_ARGUS={argus}", + "-DBUILD_TESTING=OFF", + "-DBUILD_EXAMPLES=OFF", + "-DBUILD_PYTHON_BINDINGS=OFF", + "-DBUILD_VIZ=OFF", + "-DENABLE_CLANG_FORMAT_CHECK=OFF", + "-DCMAKE_CXX_FLAGS=-I/usr/local/cuda/include", + *sdk_flags, + *link_flags, + ], + env=env, + ) + run( + [ + "cmake", + "--build", + str(build_dir), + "--target", + "camera_plugin_sensing", + "--parallel", + str(args.jobs), + ], + env=env, + ) + run(["rm", "-rf", args.stage]) + run( + [ + "cmake", + "--install", + str(build_dir), + "--component", + "producer", + "--prefix", + args.stage, + ] + ) + stage = Path(args.stage) + binary = stage / BIN_REL + if not binary.exists(): + raise DeployError(f"the staged tree has no {BIN_REL}") + missing = [ + ln for ln in tail(["ldd", str(binary)]).splitlines() if "not found" in ln + ] + if missing: + raise DeployError( + "the producer has unresolved shared libraries on this target:\n" + + "\n".join(missing) + ) + print(f" staged {sum(1 for p in stage.rglob('*') if p.is_file())} files") + + +def remote_package(args: argparse.Namespace) -> None: + stage = Path(args.stage) + sums = [] + for path in sorted( + p for p in stage.rglob("*") if p.is_file() and p.name != "SHA256SUMS" + ): + sums.append(f"{sha256_file(path)} ./{path.relative_to(stage)}") + (stage / "SHA256SUMS").write_text("\n".join(sums) + "\n") + with tarfile.open(args.archive, "w:gz") as tar: + tar.add(stage, arcname=".") + print(f" packaged {args.archive}") + + +def unmanaged_producers() -> list[tuple[int, str]]: + """Running camera_plugin_sensing processes the service does not own, as (pid, owner).""" + main_pid = tail(["systemctl", "show", "-p", "MainPID", "--value", SERVICE]) + found = [] + for proc in Path("/proc").iterdir(): + if not proc.name.isdigit() or proc.name == main_pid: + continue + try: + exe = os.readlink(proc / "exe") + owner = proc.stat().st_uid + except OSError: + continue + if Path(exe).name == "camera_plugin_sensing": + found.append((int(proc.name), pwd.getpwuid(owner).pw_name)) + return sorted(found) + + +def stop_unmanaged(procs: list[tuple[int, str]], timeout_s: int = 20) -> None: + """Interrupt exactly these PIDs (the producer stops cleanly on SIGINT) and wait for them to exit.""" + for pid, _ in procs: + os.kill(pid, signal.SIGINT) + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if not any(Path(f"/proc/{pid}").exists() for pid, _ in procs): + return + time.sleep(0.5) + raise DeployError( + f"unmanaged producer PIDs did not exit after SIGINT: {[p for p, _ in procs]}" + ) + + +def remote_activate(args: argparse.Namespace) -> None: + """Install the release, then stop the service, switch the active link and start it. Runs as root.""" + prefix = Path(args.prefix) + unmanaged = unmanaged_producers() + if unmanaged and not args.replace_unmanaged: + listing = ", ".join(f"pid {pid} (user {owner})" for pid, owner in unmanaged) + raise DeployError( + f"a camera_plugin_sensing is already running outside the service: {listing}. " + "Stop it, or pass --replace-unmanaged to stop exactly these processes." + ) + dest = prefix / "releases" / args.release + if dest.exists(): + raise DeployError( + f"release {args.release} already exists under {prefix}/releases" + ) + dest.mkdir(parents=True) + try: + with tarfile.open(args.archive) as tar: + tar.extractall(dest, filter="data") + for line in (dest / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + if sha256_file(dest / name) != digest: + raise DeployError( + f"checksum mismatch for {name} in release {args.release}" + ) + except BaseException: + shutil.rmtree(dest, ignore_errors=True) + raise + run(["chown", "-R", f"{args.owner}:", str(prefix)]) + print(f" installed release {args.release}") + + unit_target = Path("/etc/systemd/system") / SERVICE + unit_text = Path(args.unit).read_text() + if not unit_target.exists() or unit_target.read_text() != unit_text: + unit_target.write_text(unit_text) + run(["systemctl", "daemon-reload"]) + run(["systemctl", "enable", SERVICE], capture_output=True) + print(" service unit updated") + run(["systemctl", "stop", SERVICE]) + if unmanaged: + print(f" stopping unmanaged producer(s): {[pid for pid, _ in unmanaged]}") + stop_unmanaged(unmanaged) + link, staging_link = prefix / "current", prefix / "current.new" + if staging_link.is_symlink(): + staging_link.unlink() + staging_link.symlink_to(Path("releases") / args.release) + os.replace(staging_link, link) + since = time.strftime("%Y-%m-%d %H:%M:%S") + run(["systemctl", "start", SERVICE]) + wait_ready(since, args.ready_timeout) + print(f" producer is publishing (release {args.release})") + + +def wait_ready(since: str, timeout: int) -> None: + """Ready means the producer joined the room and has published frames, as its own log says. + + The two patterns come from the LiveKit session's join message and the periodic stats line in + sensing_camera.cpp; a pid alone is not readiness. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if ( + subprocess.run( + ["systemctl", "is-active", "--quiet", SERVICE], check=False + ).returncode + != 0 + ): + break + log = tail( + ["journalctl", "-u", SERVICE, "--since", since, "--no-pager", "-o", "cat"] + ) + if "Joined LiveKit room" in log and re.search( + r"livekit [1-9][0-9]* published", log + ): + return + time.sleep(2) + print( + tail(["journalctl", "-u", SERVICE, "--since", since, "--no-pager", "-n", "40"]), + file=sys.stderr, + ) + raise DeployError( + f"the producer did not become ready; logs: journalctl -u {SERVICE}" + ) + + +# --------------------------------------------------------------------------- + + +def parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument( + "target", help=f"host or user@host of the Orin (a bare host uses ${USER_ENV})" + ) + ap.add_argument("--backend", choices=("sipl", "argus"), required=True) + ap.add_argument("--config", required=True, help="producer options (JSON)") + ap.add_argument("--prefix", default="/opt/isaaccapture") + ap.add_argument( + "--jobs", type=int, default=2, help="parallel compile jobs on the Orin" + ) + ap.add_argument( + "--service-user", help="account the service runs as (default: the ssh user)" + ) + ap.add_argument( + "--service-groups", + default="video", + help="supplementary groups for device access", + ) + ap.add_argument("--ready-timeout", type=int, default=60) + ap.add_argument( + "--sdk-root", + default="", + help="directory on the Orin holding usr/src/jetson_multimedia_api and jetson_sipl_api (default: /)", + ) + ap.add_argument( + "--replace-unmanaged", + action="store_true", + help="stop a camera_plugin_sensing the service does not own (its exact PIDs are shown) instead of refusing", + ) + ap.add_argument("--no-activate", action="store_true", help="build and package only") + ap.add_argument( + "--fetch-archive", + metavar="DIR", + help="also copy the package back to this directory", + ) + return ap + + +def remote_parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser(prog="deploy_orin.py remote") + sub = ap.add_subparsers(dest="step", required=True) + pre = sub.add_parser("preflight") + pre.add_argument("--backend", required=True) + pre.add_argument("--sdk-root", default="") + build = sub.add_parser("build") + build.add_argument("--src", required=True) + build.add_argument("--backend", required=True) + build.add_argument("--stage", required=True) + build.add_argument("--jobs", type=int, default=2) + build.add_argument("--sdk-root", default="") + pack = sub.add_parser("package") + pack.add_argument("--stage", required=True) + pack.add_argument("--archive", required=True) + act = sub.add_parser("activate") + for name in ("archive", "prefix", "release", "unit", "owner"): + act.add_argument(f"--{name}", required=True) + act.add_argument("--ready-timeout", type=int, default=60) + act.add_argument("--replace-unmanaged", action="store_true") + return ap + + +def main(argv: list[str]) -> int: + try: + if argv and argv[0] == "remote": + args = remote_parser().parse_args(argv[1:]) + { + "preflight": remote_preflight, + "build": remote_build, + "package": remote_package, + "activate": remote_activate, + }[args.step](args) + else: + deploy(parser().parse_args(argv)) + except DeployError as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/plugins/sensing/scripts/isaaccapture-producer.service.in b/src/plugins/sensing/scripts/isaaccapture-producer.service.in new file mode 100644 index 0000000000..f9eea4d3fe --- /dev/null +++ b/src/plugins/sensing/scripts/isaaccapture-producer.service.in @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Template for the Orin camera producer service; scripts/deploy_orin.py fills the @...@ fields. +[Unit] +Description=Isaac Capture camera producer +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=@SERVICE_USER@ +SupplementaryGroups=@SERVICE_GROUPS@ +WorkingDirectory=@PREFIX@/current/plugins/sensing_camera +ExecStart=@EXEC_START@ +# The producer stops on SIGINT; systemd ends anything left in this cgroup after the timeout. +KillSignal=SIGINT +KillMode=control-group +TimeoutStopSec=20 + +[Install] +WantedBy=multi-user.target diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py new file mode 100644 index 0000000000..174f83f02c --- /dev/null +++ b/tests/python/ci/test_deploy_orin.py @@ -0,0 +1,721 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the Orin deploy script; every subprocess call is mocked or runs against a temp git repo.""" + +import argparse +import importlib.util +import io +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from repo_paths import repo_root + +spec = importlib.util.spec_from_file_location( + "deploy_orin", repo_root() / "src/plugins/sensing/scripts/deploy_orin.py" +) +deploy_orin = importlib.util.module_from_spec(spec) +spec.loader.exec_module(deploy_orin) + +DeployError = deploy_orin.DeployError +STREAM = "--add-stream=sensor=0,livekit=left" + + +def completed(returncode=0, stdout="", stderr=""): + return subprocess.CompletedProcess([], returncode, stdout, stderr) + + +class TempDirTestCase(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = Path(tmp.name) + + +class LoadConfigTests(TempDirTestCase): + def load(self, config): + path = self.tmp / "orin.json" + path.write_text(json.dumps(config)) + return deploy_orin.load_config(path) + + def valid(self, **override): + config = { + "livekit_url": "wss://lk.example.org", + "token_file": "/etc/isaaccapture/join.token", + "args": [STREAM], + } + config.update(override) + return config + + def test_accepts_valid_config(self): + config = self.valid() + self.assertEqual(self.load(config), config) + + def test_rejects_unknown_key(self): + with self.assertRaises(DeployError): + self.load(self.valid(extra=1)) + + def test_rejects_missing_required_keys(self): + for key in ("livekit_url", "token_file"): + config = self.valid() + del config[key] + with self.subTest(key=key), self.assertRaises(DeployError): + self.load(config) + + def test_rejects_relative_token_file(self): + with self.assertRaisesRegex(DeployError, "absolute"): + self.load(self.valid(token_file="join.token")) + + def test_rejects_missing_add_stream(self): + for args in ([], ["--other=1"]): + with ( + self.subTest(args=args), + self.assertRaisesRegex(DeployError, "add-stream"), + ): + self.load(self.valid(args=args)) + + def test_rejects_url_and_token_flags_in_args(self): + for flag in ("--livekit-url=wss://x", "--livekit-token-file=/t"): + with ( + self.subTest(flag=flag), + self.assertRaisesRegex(DeployError, "not in args"), + ): + self.load(self.valid(args=[STREAM, flag])) + + def test_rejects_non_string_url_and_token_file(self): + for key in ("livekit_url", "token_file"): + for value in (5, None, ["/t"], ""): + with ( + self.subTest(key=key, value=value), + self.assertRaisesRegex(DeployError, "non-empty strings"), + ): + self.load(self.valid(**{key: value})) + + def test_unreadable_or_invalid_json_is_a_deploy_error(self): + with self.assertRaisesRegex(DeployError, "cannot read the config"): + deploy_orin.load_config(self.tmp / "missing.json") + bad = self.tmp / "bad.json" + bad.write_text("{not json") + with self.assertRaisesRegex(DeployError, "cannot read the config"): + deploy_orin.load_config(bad) + + def test_rejects_non_object_and_bad_args(self): + path = self.tmp / "list.json" + path.write_text("[]") + with self.assertRaises(DeployError): + deploy_orin.load_config(path) + with self.assertRaisesRegex(DeployError, "list of strings"): + self.load(self.valid(args=[STREAM, 3])) + + +class UnitRenderTests(unittest.TestCase): + def test_quote_escapes_systemd_specials(self): + self.assertEqual(deploy_orin.systemd_quote("a%b"), "a%%b") + self.assertEqual(deploy_orin.systemd_quote("$HOME"), "$$HOME") + self.assertEqual(deploy_orin.systemd_quote("a\\b"), "a\\\\b") + + def test_quote_wraps_whitespace_and_quotes(self): + self.assertEqual(deploy_orin.systemd_quote("a b"), '"a b"') + self.assertEqual(deploy_orin.systemd_quote('a"b'), '"a\\"b"') + self.assertEqual(deploy_orin.systemd_quote("plain"), "plain") + + def test_quote_wraps_empty_and_single_quote_args(self): + self.assertEqual(deploy_orin.systemd_quote(""), '""') + self.assertEqual(deploy_orin.systemd_quote("a'b"), '"a\'b"') + + def test_render_unit(self): + config = { + "livekit_url": "wss://lk.example.org", + "token_file": "/etc/t oken", + "args": [STREAM, "--name=100%", "--x=$Y"], + } + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + exec_start = next(ln for ln in text.splitlines() if ln.startswith("ExecStart=")) + self.assertEqual( + exec_start, + "ExecStart=/opt/ic/current/plugins/sensing_camera/camera_plugin_sensing " + "--livekit-url=wss://lk.example.org " + '"--livekit-token-file=/etc/t oken" ' + f"{STREAM} --name=100%% --x=$$Y", + ) + self.assertIn("User=bob", text) + self.assertIn("SupplementaryGroups=video", text) + self.assertIn("WorkingDirectory=/opt/ic/current/plugins/sensing_camera", text) + self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + + +def make_target(test, password=None): + """A Target with its temp dir removed at cleanup; ORIN_PASSWORD is set or cleared explicitly.""" + env = {k: v for k, v in os.environ.items() if k != deploy_orin.PASSWORD_ENV} + if password: + env[deploy_orin.PASSWORD_ENV] = password + with mock.patch.dict(os.environ, env, clear=True): + target = deploy_orin.Target("u@orin") + test.addCleanup(shutil.rmtree, target._dir, True) + return target + + +class ValidateConnectionTests(unittest.TestCase): + def validate(self, result): + target = make_target(self) + with mock.patch.object(deploy_orin.subprocess, "run", return_value=result): + target.validate_connection() + + def test_ssh_failure_surfaces_stderr(self): + with self.assertRaisesRegex(DeployError, "Permission denied"): + self.validate(completed(255, stderr="Permission denied (publickey)\n")) + + def test_missing_python3(self): + with self.assertRaisesRegex(DeployError, "python3 is not installed"): + self.validate(completed(1)) + + def test_non_aarch64(self): + with self.assertRaisesRegex(DeployError, "x86_64"): + self.validate(completed(0, stdout="x86_64\n")) + + def test_success(self): + self.validate(completed(0, stdout="aarch64\n")) + + +class PasswordTests(unittest.TestCase): + PASSWORD = "s3cr3t-pw" + + def test_password_never_in_argv(self): + target = make_target(self, self.PASSWORD) + calls = [] + + def fake_run(cmd, **kwargs): + calls.append((list(cmd), kwargs)) + return completed(0, stdout="aarch64\n") + + with mock.patch.object(deploy_orin.subprocess, "run", fake_run): + target.validate_connection() + target.capture("id -un") + target.readable("/t") + target.ssh("true") + target.sudo("python3 x remote activate --owner bob") + target.put(Path("/tmp/a"), "/r/a") + target.get("/r/a", Path("/tmp/a")) + askpass_text = Path(target._env["SSH_ASKPASS"]).read_text() + target.close() + self.assertGreater(len(calls), 7) + for cmd, _ in calls: + self.assertFalse([w for w in cmd if self.PASSWORD in w], cmd) + _, sudo_kwargs = next( + (c, k) for c, k in calls if "sudo -S -p '' python3" in c[-1] + ) + self.assertEqual(sudo_kwargs["input"], self.PASSWORD + "\n") + self.assertNotIn(self.PASSWORD, askpass_text) + + def test_without_password_sudo_uses_tty_and_no_askpass(self): + target = make_target(self) + with mock.patch.object( + deploy_orin.subprocess, "run", return_value=completed(0) + ) as run: + target.sudo("true") + self.assertIn("-t", run.call_args.args[0]) + self.assertNotIn("SSH_ASKPASS", target._env) + self.assertNotIn("input", run.call_args.kwargs) + + def test_sudo_failure_raises(self): + target = make_target(self, self.PASSWORD) + with mock.patch.object( + deploy_orin.subprocess, "run", return_value=completed(1) + ): + with self.assertRaisesRegex(DeployError, "sudo"): + target.sudo("true") + + +class DeployOrderTests(TempDirTestCase): + def run_deploy(self, validate_effect=None): + config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} + args = deploy_orin.parser().parse_args( + ["u@orin", "--backend", "sipl", "--config", "orin.json", "--no-activate"] + ) + manager = mock.Mock() + manager.make_snapshot.return_value = { + "base_revision": "a" * 40, + "dirty": False, + "sha256": "b" * 64, + } + manager.target.validate_connection.side_effect = validate_effect + manager.target.capture.return_value = "x" + manager.target.readable.return_value = True + with ( + mock.patch.object(deploy_orin, "repo_root", return_value=self.tmp), + mock.patch.object(deploy_orin, "make_snapshot", manager.make_snapshot), + ): + try: + deploy_orin.run_deploy(args, config, manager.target) + except DeployError: + pass + return [c[0] for c in manager.mock_calls] + + def test_validates_connection_first(self): + names = self.run_deploy() + self.assertEqual(names[0], "target.validate_connection") + self.assertIn("make_snapshot", names) + self.assertIn("target.put", names) + + def test_failed_validation_blocks_everything_else(self): + names = self.run_deploy(validate_effect=DeployError("no route")) + self.assertEqual(names, ["target.validate_connection"]) + + def test_deploy_closes_target_on_failure(self): + config = self.tmp / "orin.json" + config.write_text( + json.dumps({"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]}) + ) + args = deploy_orin.parser().parse_args( + ["u@orin", "--backend", "sipl", "--config", str(config)] + ) + target = mock.Mock() + with ( + mock.patch.object(deploy_orin, "Target", return_value=target), + mock.patch.object(deploy_orin, "run_deploy", side_effect=DeployError("x")), + ): + with self.assertRaises(DeployError): + deploy_orin.deploy(args) + target.close.assert_called_once() + + +class SnapshotTests(TempDirTestCase): + def git(self, *args): + return subprocess.run( + [ + "git", + "-c", + "user.name=t", + "-c", + "user.email=t@example.org", + "-c", + "commit.gpgsign=false", + *args, + ], + cwd=self.repo, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + def setUp(self): + super().setUp() + self.repo = self.tmp / "repo" + (self.repo / "sub").mkdir(parents=True) + self.git("init", "-q") + (self.repo / "a.txt").write_text("one\n") + (self.repo / "sub/b.txt").write_text("two\n") + self.git("add", "-A") + self.git("commit", "-q", "-m", "init") + self.dest = self.tmp / "snap.tar.gz" + + def members(self): + with tarfile.open(self.dest) as tar: + return { + m.name: tar.extractfile(m).read().decode() + for m in tar.getmembers() + if m.isfile() + } + + def test_clean_tree_ignores_untracked(self): + (self.repo / "untracked.txt").write_text("x") + meta = deploy_orin.make_snapshot(self.repo, self.dest) + self.assertEqual(self.members(), {"a.txt": "one\n", "sub/b.txt": "two\n"}) + self.assertFalse(meta["dirty"]) + self.assertEqual(meta["base_revision"], self.git("rev-parse", "HEAD")) + self.assertEqual(meta["sha256"], deploy_orin.sha256_file(self.dest)) + + def test_tracked_edit_is_included_and_marks_dirty(self): + (self.repo / "a.txt").write_text("edited\n") + (self.repo / "untracked.txt").write_text("x") + meta = deploy_orin.make_snapshot(self.repo, self.dest) + self.assertEqual(self.members()["a.txt"], "edited\n") + self.assertNotIn("untracked.txt", self.members()) + self.assertTrue(meta["dirty"]) + + +class ActivateTests(TempDirTestCase): + """remote_activate with subprocess faked; the unit path is redirected into the temp dir.""" + + def setUp(self): + super().setUp() + self.prefix = self.tmp / "opt" + self.prefix.mkdir() + self.unit = self.tmp / "in.service" + self.unit.write_text("[Unit]\n") + # Path("/etc/systemd/system") / resolves to the absolute path. + self.service = str(self.tmp / "isaaccapture-producer.service") + self.calls = [] + self.events = [] + self.journal = "" + self.unmanaged = [] + for p in ( + mock.patch.object( + deploy_orin, "unmanaged_producers", lambda: list(self.unmanaged) + ), + mock.patch.object(deploy_orin, "SERVICE", self.service), + mock.patch.object(deploy_orin.subprocess, "run", self.fake_run), + mock.patch.object(deploy_orin.time, "sleep"), + ): + p.start() + self.addCleanup(p.stop) + + def link_target(self): + link = self.prefix / "current" + return os.readlink(link) if link.is_symlink() else None + + def fake_run(self, cmd, **kwargs): + self.calls.append(list(cmd)) + if cmd[:2] == ["systemctl", "stop"]: + self.events.append(("stop", self.link_target())) + elif cmd[:2] == ["systemctl", "start"]: + self.events.append(("start", self.link_target())) + elif cmd[0] == "journalctl": + return completed(0, stdout=self.journal) + return completed(0) + + def make_archive(self, good=True): + stage = self.tmp / "stage" + (stage / "bin").mkdir(parents=True) + (stage / "bin/producer").write_text("binary") + digest = deploy_orin.sha256_file(stage / "bin/producer") if good else "0" * 64 + (stage / "SHA256SUMS").write_text(f"{digest} ./bin/producer\n") + archive = self.tmp / "producer.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(stage, arcname=".") + return archive + + def activate(self, archive, timeout=60, replace_unmanaged=False): + deploy_orin.remote_activate( + argparse.Namespace( + archive=str(archive), + prefix=str(self.prefix), + release="r1", + unit=str(self.unit), + owner="bob", + ready_timeout=timeout, + replace_unmanaged=replace_unmanaged, + ) + ) + + def test_checksum_mismatch_aborts_before_stop(self): + with self.assertRaisesRegex(DeployError, "checksum mismatch"): + self.activate(self.make_archive(good=False)) + self.assertFalse([c for c in self.calls if c[0] == "systemctl"]) + self.assertIsNone(self.link_target()) + + def test_checksum_mismatch_removes_the_new_release_dir(self): + with self.assertRaisesRegex(DeployError, "checksum mismatch"): + self.activate(self.make_archive(good=False)) + self.assertFalse((self.prefix / "releases" / "r1").exists()) + self.assertFalse([c for c in self.calls if c[0] == "chown"]) + + def test_unmanaged_producer_refuses_before_installing_anything(self): + self.unmanaged = [(4242, "alice"), (4243, "bob")] + with mock.patch.object(deploy_orin.os, "kill") as kill: + with self.assertRaises(DeployError) as ctx: + self.activate(self.make_archive()) + message = str(ctx.exception) + self.assertIn("pid 4242 (user alice)", message) + self.assertIn("pid 4243 (user bob)", message) + self.assertIn("--replace-unmanaged", message) + kill.assert_not_called() + self.assertEqual(self.calls, []) + self.assertFalse((self.prefix / "releases").exists()) + self.assertIsNone(self.link_target()) + self.assertFalse(Path(self.service).exists()) + + def test_replace_unmanaged_interrupts_only_listed_pids_between_stop_and_start( + self, + ): + self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" + self.unmanaged = [(4242, "alice")] + order = [] + self.events = order + + def fake_kill(pid, sig): + order.append(("kill", pid, sig)) + + def fake_stop(procs, timeout_s=20): + real_kill = deploy_orin.os.kill + for pid, _ in procs: + real_kill(pid, deploy_orin.signal.SIGINT) + + with mock.patch.object(deploy_orin.os, "kill", fake_kill): + with mock.patch.object( + deploy_orin, "stop_unmanaged", side_effect=fake_stop + ): + self.activate(self.make_archive(), replace_unmanaged=True) + self.assertEqual([e[0] for e in order], ["stop", "kill", "start"], order) + self.assertEqual(order[1], ("kill", 4242, signal.SIGINT)) + self.assertEqual(order[2], ("start", "releases/r1")) + + def test_without_unmanaged_nothing_is_killed(self): + self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" + with mock.patch.object(deploy_orin.os, "kill") as kill: + self.activate(self.make_archive(), replace_unmanaged=True) + kill.assert_not_called() + + def test_success_switches_link_between_stop_and_start(self): + self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" + replaced = [] + real_replace = os.replace + + def spy(src, dst): + replaced.append((Path(src).name, Path(dst).name)) + real_replace(src, dst) + + with mock.patch.object(deploy_orin.os, "replace", spy): + self.activate(self.make_archive()) + self.assertEqual(replaced, [("current.new", "current")]) + self.assertEqual(self.events, [("stop", None), ("start", "releases/r1")]) + self.assertFalse((self.prefix / "current.new").exists()) + + def test_readiness_times_out_without_markers(self): + self.journal = "Joined LiveKit room x\nno stats yet\n" + ticks = iter(range(0, 1000, 10)) + with mock.patch.object(deploy_orin.time, "monotonic", lambda: next(ticks)): + with mock.patch.object(deploy_orin, "print", create=True): + with self.assertRaisesRegex(DeployError, "did not become ready"): + self.activate(self.make_archive(), timeout=25) + self.assertEqual(self.events[-1], ("start", "releases/r1")) + + +class FakeStdout(io.StringIO): + def __init__(self, tty): + super().__init__() + self.tty = tty + + def isatty(self): + return self.tty + + +class ProgressTests(unittest.TestCase): + def drawn(self, tty, total, updates): + out = FakeStdout(tty) + with mock.patch.object(deploy_orin.sys, "stdout", out): + bar = deploy_orin.Progress("pack", total) + for done in updates: + bar.update(done) + bar.finish() + return out.getvalue() + + def test_non_tty_prints_at_most_one_line_per_ten_percent(self): + text = self.drawn(False, 1000, range(0, 1001, 7)) + lines = text.splitlines() + self.assertLessEqual(len(lines), 11) + self.assertNotIn("\r", text) + self.assertIn("100%", lines[-1]) + self.assertEqual(len(lines), len(set(lines))) + + def test_non_tty_finish_reports_100_once(self): + lines = self.drawn(False, 4, [4]).splitlines() + self.assertEqual(lines, [" pack 100%"]) + + def test_tty_draws_a_30_char_bar_redrawn_in_place(self): + text = self.drawn(True, 10, [5]) + self.assertIn("\r pack [" + "#" * 15 + "-" * 15 + "] 50% (5/10)", text) + self.assertIn("[" + "#" * 30 + "] 100% (10/10)", text) + self.assertTrue(text.endswith("\n")) + + def test_scp_is_quiet_only_when_stdout_is_not_a_tty(self): + for tty, expected in ((False, ["-q"]), (True, [])): + with ( + self.subTest(tty=tty), + mock.patch.object(deploy_orin.sys, "stdout", FakeStdout(tty)), + ): + self.assertEqual(deploy_orin.Target._scp_quiet(), expected) + + def test_put_and_get_pass_quiet_flag_to_scp(self): + target = make_target(self) + with ( + mock.patch.object(deploy_orin.sys, "stdout", FakeStdout(False)), + mock.patch.object( + deploy_orin.subprocess, "run", return_value=completed(0) + ) as run, + ): + target.put(Path("/tmp/a"), "/r/a") + target.get("/r/a", Path("/tmp/a")) + for call in run.call_args_list: + self.assertEqual(call.args[0][0], "scp") + self.assertIn("-q", call.args[0]) + + def test_step_heading_format(self): + out = FakeStdout(False) + with mock.patch.object(deploy_orin.sys, "stdout", out): + deploy_orin.step(2, "Preflight") + self.assertEqual(out.getvalue(), f"==> [2/{deploy_orin.STEPS}] Preflight\n") + + +class StopUnmanagedTests(unittest.TestCase): + def test_sigints_exactly_the_listed_pids_and_returns_when_gone(self): + with ( + mock.patch.object(deploy_orin.os, "kill") as kill, + mock.patch.object(Path, "exists", return_value=False), + ): + deploy_orin.stop_unmanaged([(11, "a"), (12, "b")]) + self.assertEqual( + kill.call_args_list, + [mock.call(11, signal.SIGINT), mock.call(12, signal.SIGINT)], + ) + + def test_raises_when_a_pid_survives(self): + ticks = iter(range(0, 1000, 10)) + with ( + mock.patch.object(deploy_orin.os, "kill"), + mock.patch.object(Path, "exists", return_value=True), + mock.patch.object(deploy_orin.time, "sleep"), + mock.patch.object(deploy_orin.time, "monotonic", lambda: next(ticks)), + ): + with self.assertRaisesRegex(DeployError, r"\[11\]"): + deploy_orin.stop_unmanaged([(11, "a")], timeout_s=25) + + +class SdkDirsTests(unittest.TestCase): + def test_default_uses_system_locations(self): + self.assertEqual( + deploy_orin.sdk_dirs(""), + ( + Path("/usr/src/jetson_multimedia_api"), + Path("/usr/src/jetson_sipl_api"), + ), + ) + + def test_sdk_root_prefixes_usr_src(self): + self.assertEqual( + deploy_orin.sdk_dirs("/mnt/sdk"), + ( + Path("/mnt/sdk/usr/src/jetson_multimedia_api"), + Path("/mnt/sdk/usr/src/jetson_sipl_api"), + ), + ) + + def test_cli_parsers_accept_sdk_root_and_replace_unmanaged(self): + args = deploy_orin.parser().parse_args( + ["u@orin", "--backend", "sipl", "--config", "c", "--sdk-root", "/s"] + ) + self.assertEqual(args.sdk_root, "/s") + self.assertFalse(args.replace_unmanaged) + self.assertEqual( + deploy_orin.parser() + .parse_args(["h", "--backend", "argus", "--config", "c"]) + .sdk_root, + "", + ) + remote = deploy_orin.remote_parser() + for step, extra in ( + ("preflight", []), + ("build", ["--src", "s", "--stage", "t"]), + ): + parsed = remote.parse_args( + [step, "--backend", "sipl", "--sdk-root", "/s", *extra] + ) + self.assertEqual(parsed.sdk_root, "/s") + + def test_preflight_names_the_sdk_root_path_when_headers_are_missing(self): + args = argparse.Namespace(backend="sipl", sdk_root="/nonexistent-sdk") + with ( + mock.patch.object(deploy_orin.os, "uname") as uname, + mock.patch.object(Path, "exists", return_value=True), + mock.patch.object(Path, "read_text", return_value="# R36\n"), + mock.patch.object(deploy_orin.os, "statvfs") as statvfs, + mock.patch.object(deploy_orin.subprocess, "run", return_value=completed(0)), + ): + uname.return_value.machine = "aarch64" + statvfs.return_value.f_bavail = 1 << 30 + statvfs.return_value.f_frsize = 4096 + with self.assertRaisesRegex(DeployError, "/nonexistent-sdk/usr/src"): + deploy_orin.remote_preflight(args) + + +class RemoteBuildTests(TempDirTestCase): + LIBCUDA = "/opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so.1" + + def build(self, backend="sipl", sdk_root="", libcuda=False, path="/usr/bin"): + stage = self.tmp / "stage" + (stage / deploy_orin.BIN_REL).parent.mkdir(parents=True, exist_ok=True) + (stage / deploy_orin.BIN_REL).write_text("bin") + args = argparse.Namespace( + src="/src", + backend=backend, + stage=str(stage), + jobs=3, + sdk_root=sdk_root, + ) + real_exists, real_read_text = Path.exists, Path.read_text + + def fake_exists(p, *a, **k): + return libcuda if str(p) == self.LIBCUDA else real_exists(p, *a, **k) + + def fake_read_text(p, *a, **k): + if str(p) == "/etc/nv_tegra_release": + return "# R36 (release)\n" + return real_read_text(p, *a, **k) + + calls = [] + + def fake_run(cmd, **kwargs): + calls.append((list(cmd), kwargs)) + return completed(0) + + with ( + mock.patch.object(deploy_orin, "CACHE", str(self.tmp / "cache")), + mock.patch.object(deploy_orin, "run", fake_run), + mock.patch.object(deploy_orin, "tail", return_value="x"), + mock.patch.object(Path, "exists", fake_exists), + mock.patch.object(Path, "read_text", fake_read_text), + mock.patch.dict(os.environ, {"PATH": path}), + ): + deploy_orin.remote_build(args) + return calls + + def configure(self, calls): + return next(c for c in calls if c[0][:2] == ["cmake", "-S"]) + + def test_sipl_configure_flags_and_cuda_path(self): + cmd, kwargs = self.configure(self.build(sdk_root="/mnt/sdk")) + for flag in ( + "-DSENSING_WITH_SIPL=ON", + "-DSENSING_WITH_ARGUS=OFF", + "-DJETSON_MMAPI_DIR=/mnt/sdk/usr/src/jetson_multimedia_api", + "-DSIPL_API_DIR=/mnt/sdk/usr/src/jetson_sipl_api", + "-DENABLE_CLANG_FORMAT_CHECK=OFF", + "-DCMAKE_CXX_FLAGS=-I/usr/local/cuda/include", + ): + self.assertIn(flag, cmd) + self.assertEqual(kwargs["env"]["PATH"], "/usr/local/cuda/bin:/usr/bin") + + def test_build_step_also_gets_the_cuda_path(self): + calls = self.build() + cmd, kwargs = next(c for c in calls if c[0][:2] == ["cmake", "--build"]) + self.assertEqual(kwargs["env"]["PATH"], "/usr/local/cuda/bin:/usr/bin") + self.assertIn("camera_plugin_sensing", cmd) + + def test_argus_has_no_sipl_dir(self): + cmd, _ = self.configure(self.build(backend="argus")) + self.assertIn("-DSENSING_WITH_ARGUS=ON", cmd) + self.assertIn("-DJETSON_MMAPI_DIR=/usr/src/jetson_multimedia_api", cmd) + self.assertFalse([a for a in cmd if a.startswith("-DSIPL_API_DIR")]) + + def test_linker_flag_only_when_gpu_libcuda_exists(self): + absent, _ = self.configure(self.build(libcuda=False)) + self.assertFalse([a for a in absent if "LINKER_FLAGS" in a]) + present, _ = self.configure(self.build(libcuda=True)) + self.assertIn( + f"-DCMAKE_EXE_LINKER_FLAGS=-Wl,--no-as-needed {self.LIBCUDA}", present + ) + + +if __name__ == "__main__": + unittest.main() From 3a45748424b6b1ce9d4b0eaceaaf6460377f46df Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 14:58:57 -0800 Subject: [PATCH 02/16] sensing: send only changed source to the Orin; find uv and CUDA on the build PATH The deploy compares per-file hashes of the tracked source, grouped into files that change often and files that rarely do, with the Orin's record and uploads only the difference, keeping one source directory so the persistent build stays incremental. The helper script is uploaded fresh and preflight runs before any source transfer. The build PATH now includes ~/.local/bin and /usr/local/cuda/bin, and preflight requires uv, which the build's Python setup needs. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/scripts/deploy_orin.py | 238 +++++++++-- tests/python/ci/test_deploy_orin.py | 443 +++++++++++++++++---- 2 files changed, 560 insertions(+), 121 deletions(-) diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index 60765fb246..4e91762c32 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -22,6 +22,7 @@ import argparse import hashlib +import io import json import os import pwd @@ -38,7 +39,6 @@ SERVICE = "isaaccapture-producer.service" BIN_REL = "plugins/sensing_camera/camera_plugin_sensing" -SCRIPT_REL = "src/plugins/sensing/scripts/deploy_orin.py" UNIT_TEMPLATE = Path(__file__).with_name("isaaccapture-producer.service.in") CACHE = "~/.cache/isaaccapture-deploy" CONFIG_KEYS = {"livekit_url", "token_file", "args"} @@ -188,31 +188,90 @@ def repo_root() -> Path: return Path(out.stdout.strip()) -def make_snapshot(root: Path, dest: Path) -> dict: - """Archive the tracked files as they are on disk, including tracked local edits; nothing untracked.""" +# Files that change while developing the producer, and everything else, which rarely does. A group whose digest +# matches the Orin's is not compared file by file. +VOLATILE_PREFIXES = ("src/", "cmake/", "tests/", "examples/egocap/") +GROUPS = ("source", "bulk") +DEPLOY_DIR = ".deploy" + + +def file_group(path: str) -> str: + return ( + "source" + if path == "CMakeLists.txt" or path.startswith(VOLATILE_PREFIXES) + else "bulk" + ) + + +def tracked_files(root: Path) -> list[str]: + """The tracked files that exist on disk, so tracked local edits are included and untracked files never are.""" listing = subprocess.run( ["git", "ls-files", "-z", "--recurse-submodules"], cwd=root, capture_output=True, check=True, ).stdout - files = sorted( - f for f in listing.decode().split("\0") if f and (root / f).is_file() - ) - bar = Progress("packing", len(files)) - with tarfile.open(dest, "w:gz") as tar: - for done, name in enumerate(files, 1): - tar.add(root / name, arcname=name, recursive=False) - if done % 200 == 0: - bar.update(done) + return sorted(f for f in listing.decode().split("\0") if f and (root / f).is_file()) + + +def hash_files(root: Path, files: list[str]) -> dict[str, str]: + """Content hash per file, with the executable bit, which the Orin's copy must also have.""" + bar = Progress("hashing", len(files)) + entries = {} + for done, name in enumerate(files, 1): + path = root / name + executable = "x" if os.access(path, os.X_OK) else "-" + entries[name] = f"{sha256_file(path)}{executable}" + if done % 200 == 0: + bar.update(done) bar.finish() + return entries + + +def group_digest(entries: dict[str, str], group: str) -> str: + lines = ( + f"{name}\0{entries[name]}" + for name in sorted(entries) + if file_group(name) == group + ) + return hashlib.sha256("\n".join(lines).encode()).hexdigest() + + +def diff_manifests( + local: dict[str, str], remote: dict[str, str] +) -> tuple[list[str], list[str]]: + """(files to send, files to remove on the Orin). A group with an equal digest is skipped without comparing.""" + send, removed = [], [] + for group in GROUPS: + mine = {n: h for n, h in local.items() if file_group(n) == group} + theirs = {n: h for n, h in remote.items() if file_group(n) == group} + if group_digest(local, group) == group_digest(remote, group): + continue + send += [n for n, h in mine.items() if theirs.get(n) != h] + removed += [n for n in theirs if n not in mine] + return sorted(send), sorted(removed) + + +def read_remote_manifest(target: "Target", cache: str) -> dict[str, str]: + """What the Orin's source copy holds, or nothing when it has no copy or the record is unreadable.""" + text = target.capture( + f"test -d {cache}/src && cat {cache}/src.manifest.json || true" + ) + try: + entries = json.loads(text) if text else {} + except ValueError: + return {} + return entries if isinstance(entries, dict) else {} + + +def git_state(root: Path) -> dict: head = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, check=True, - ) + ).stdout.strip() dirty = subprocess.run( ["git", "status", "--porcelain", "--untracked-files=no"], cwd=root, @@ -220,11 +279,30 @@ def make_snapshot(root: Path, dest: Path) -> dict: text=True, check=True, ).stdout.strip() - return { - "base_revision": head.stdout.strip(), - "dirty": bool(dirty), - "sha256": sha256_file(dest), - } + return {"base_revision": head, "dirty": bool(dirty)} + + +def build_delta( + root: Path, + send: list[str], + removed: list[str], + manifest: dict[str, str], + dest: Path, +) -> None: + """Archive only the files to send, with the new full manifest and the list of files to remove.""" + bar = Progress("packing", len(send)) + with tarfile.open(dest, "w:gz") as tar: + for done, name in enumerate(send, 1): + tar.add(root / name, arcname=name, recursive=False) + if done % 200 == 0: + bar.update(done) + for name, payload in (("manifest.json", manifest), ("removed.json", removed)): + data = json.dumps(payload).encode() + info = tarfile.TarInfo(f"{DEPLOY_DIR}/{name}") + info.size = len(data) + info.mtime = int(time.time()) + tar.addfile(info, io.BytesIO(data)) + bar.finish() PASSWORD_ENV = "ORIN_PASSWORD" @@ -375,25 +453,49 @@ def run_deploy(args: argparse.Namespace, config: dict, target: Target) -> None: raise DeployError( f"the token file {config['token_file']} is not readable on the Orin" ) + # The helper always travels alone and fresh, so the Orin-side steps never depend on an older source copy. + helper = f"{cache}/deploy_orin.py" + target.put(Path(__file__).resolve(), helper) + remote = f"python3 {helper} remote" + target.ssh( + f"{remote} preflight --backend {args.backend} --sdk-root {shlex.quote(args.sdk_root)}" + ) with tempfile.TemporaryDirectory() as tmp: - snapshot_path = Path(tmp) / "snapshot.tar.gz" - step(3, "Snapshot and upload") - meta = make_snapshot(root, snapshot_path) + step(3, "Compare the source with the Orin's and upload what differs") + meta = git_state(root) + files = tracked_files(root) + local = hash_files(root, files) + meta["manifest_sha256"] = hashlib.sha256( + json.dumps(local, sort_keys=True).encode() + ).hexdigest() release = f"{time.strftime('%Y%m%d-%H%M%S')}-{meta['base_revision'][:9]}{'-dirty' if meta['dirty'] else ''}" print( - f" base {meta['base_revision'][:12]}, dirty={meta['dirty']}, sha256 {meta['sha256'][:16]}" - ) - src = f"{cache}/src-{meta['sha256'][:16]}" - target.put(snapshot_path, f"{cache}/snapshot.tar.gz") - target.ssh( - f"rm -rf {src} && mkdir -p {src} && tar -xzf {cache}/snapshot.tar.gz -C {src}" + f" base {meta['base_revision'][:12]}, dirty={meta['dirty']}, {len(files)} tracked files" ) - remote = f"python3 {src}/{SCRIPT_REL} remote" - target.ssh( - f"{remote} preflight --backend {args.backend} --sdk-root {shlex.quote(args.sdk_root)}" - ) - + # One fixed source path, so the persistent build directory's CMake cache keeps matching it; files that are + # unchanged keep their modification times and are not rebuilt. + src = f"{cache}/src" + remote_manifest = read_remote_manifest(target, cache) + send, removed = diff_manifests(local, remote_manifest) + for group in GROUPS: + names = [n for n in local if file_group(n) == group] + to_send = [n for n in send if file_group(n) == group] + size = sum((root / n).stat().st_size for n in to_send) + print( + f" {group}: {len(names) - len(to_send)} identical, {len(to_send)} to send " + f"({size / 1e6:.1f} MB), {sum(1 for n in removed if file_group(n) == group)} to remove" + ) + if send or removed: + delta = Path(tmp) / "delta.tar.gz" + build_delta(root, send, removed, local, delta) + target.put(delta, f"{cache}/delta.tar.gz") + target.ssh( + f"{remote} apply --archive {cache}/delta.tar.gz --src {src} --manifest {cache}/src.manifest.json" + + ("" if remote_manifest else " --fresh") + ) + else: + print(" the Orin's source is identical; nothing to send") step(4, "Build and package on the Orin (the running producer is untouched)") stage, archive = ( f"{cache}/stage-{release}", @@ -451,6 +553,12 @@ def sdk_dirs(sdk_root: str) -> tuple[Path, Path]: return base / "jetson_multimedia_api", base / "jetson_sipl_api" +def build_path() -> str: + """PATH for the build: an ssh command runs without the login profile, which is where uv and CUDA are added.""" + extra = [str(Path.home() / ".local/bin"), "/usr/local/cuda/bin"] + return ":".join([*extra, os.environ.get("PATH", "")]) + + def remote_preflight(args: argparse.Namespace) -> None: if os.uname().machine != "aarch64": raise DeployError(f"the target is {os.uname().machine}, not aarch64") @@ -458,12 +566,18 @@ def remote_preflight(args: argparse.Namespace) -> None: if not release_file.exists(): raise DeployError("no /etc/nv_tegra_release: this is not a Jetson L4T system") print(f" L4T: {release_file.read_text().splitlines()[0]}") - for tool in ("cmake", "g++", "systemctl"): - if ( - subprocess.run(["which", tool], capture_output=True, check=False).returncode - != 0 - ): - raise DeployError(f"{tool} is not installed") + on_path = {**os.environ, "PATH": build_path()} + for tool in ("cmake", "g++", "systemctl", "uv"): + found = subprocess.run( + ["which", tool], capture_output=True, check=False, env=on_path + ) + if found.returncode != 0: + hint = ( + " (the build's Python setup needs it: curl -LsSf https://astral.sh/uv/install.sh | sh)" + if tool == "uv" + else "" + ) + raise DeployError(f"{tool} is not installed{hint}") if ( not Path("/usr/local/cuda/bin/nvcc").exists() and subprocess.run( @@ -502,6 +616,7 @@ def remote_build(args: argparse.Namespace) -> None: Path("/etc/nv_tegra_release").read_text().splitlines()[0], tail(["cmake", "--version"]).splitlines()[0], tail(["g++", "-dumpfullversion"]), + args.src, ] ) stamp_file = build_dir / ".stamp" @@ -512,7 +627,7 @@ def remote_build(args: argparse.Namespace) -> None: stamp_file.write_text(stamp) sipl, argus = ("ON", "OFF") if args.backend == "sipl" else ("OFF", "ON") mmapi, sipl_api = sdk_dirs(args.sdk_root) - env = {**os.environ, "PATH": f"/usr/local/cuda/bin:{os.environ.get('PATH', '')}"} + env = {**os.environ, "PATH": build_path()} sdk_flags = [f"-DJETSON_MMAPI_DIR={mmapi}"] if args.backend == "sipl": sdk_flags.append(f"-DSIPL_API_DIR={sipl_api}") @@ -585,6 +700,46 @@ def remote_build(args: argparse.Namespace) -> None: print(f" staged {sum(1 for p in stage.rglob('*') if p.is_file())} files") +def remote_apply(args: argparse.Namespace) -> None: + """Apply a source delta: extract the sent files, delete the removed ones, verify, record the new manifest.""" + src = Path(args.src) + if args.fresh: + # No trustworthy record of what is there: start from nothing so no stale file can reach the build. + shutil.rmtree(src, ignore_errors=True) + src.mkdir(parents=True, exist_ok=True) + with tarfile.open(args.archive) as tar: + members = tar.getmembers() + meta = {m.name: m for m in members if m.name.startswith(f"{DEPLOY_DIR}/")} + manifest = json.loads( + tar.extractfile(meta[f"{DEPLOY_DIR}/manifest.json"]).read() + ) + removed = json.loads(tar.extractfile(meta[f"{DEPLOY_DIR}/removed.json"]).read()) + sent = [m for m in members if m.name not in meta] + tar.extractall(src, members=sent, filter="data") + for name in removed: + target = (src / name).resolve() + if src.resolve() not in target.parents: + raise DeployError( + f"refusing to remove {name}: outside the source directory" + ) + target.unlink(missing_ok=True) + for parent in target.parents: + if parent == src.resolve() or any(parent.iterdir()): + break + parent.rmdir() + for member in sent: + expected = manifest[member.name] + if ( + f"{sha256_file(src / member.name)}{'x' if os.access(src / member.name, os.X_OK) else '-'}" + != expected + ): + raise DeployError( + f"{member.name} does not match its recorded hash after extraction" + ) + Path(args.manifest).write_text(json.dumps(manifest)) + print(f" applied {len(sent)} files, removed {len(removed)}") + + def remote_package(args: argparse.Namespace) -> None: stage = Path(args.stage) sums = [] @@ -768,6 +923,10 @@ def remote_parser() -> argparse.ArgumentParser: build.add_argument("--stage", required=True) build.add_argument("--jobs", type=int, default=2) build.add_argument("--sdk-root", default="") + apply_ = sub.add_parser("apply") + for name in ("archive", "src", "manifest"): + apply_.add_argument(f"--{name}", required=True) + apply_.add_argument("--fresh", action="store_true") pack = sub.add_parser("package") pack.add_argument("--stage", required=True) pack.add_argument("--archive", required=True) @@ -787,6 +946,7 @@ def main(argv: list[str]) -> int: "preflight": remote_preflight, "build": remote_build, "package": remote_package, + "apply": remote_apply, "activate": remote_activate, }[args.step](args) else: diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index 174f83f02c..476948591c 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -237,40 +237,130 @@ def test_sudo_failure_raises(self): target.sudo("true") -class DeployOrderTests(TempDirTestCase): - def run_deploy(self, validate_effect=None): - config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} +CACHE_DIR = "/home/u/.cache/isaaccapture-deploy" + + +def git_repo(test, files): + """A temp git repo holding the given {path: text} tracked files; returns (root, git runner).""" + root = Path(tempfile.mkdtemp(dir=test.tmp)) + + def git(*args): + return subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@example.org", + "-c", "commit.gpgsign=false", *args], + cwd=root, capture_output=True, text=True, check=True, + ).stdout.strip() # fmt: skip + + git("init", "-q") + for name, text in files.items(): + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_text(text) + git("add", "-A") + git("commit", "-q", "-m", "init") + return root, git + + +class FakeOrin: + """A mock Target that records uploads and keeps the source manifest an `apply` would write.""" + + def __init__(self, manifest=None): + self.manifest = manifest + self.events = [] + self.deltas = [] + self.pending = None + self.target = mock.Mock() + self.target.host = "u@orin" + self.target.readable.return_value = True + self.target.capture.side_effect = self.capture + self.target.put.side_effect = self.put + self.target.ssh.side_effect = self.ssh + + def capture(self, cmd): + if "pwd" in cmd: + return CACHE_DIR + if "src.manifest.json" in cmd: + return json.dumps(self.manifest) if self.manifest is not None else "" + return "u" + + def put(self, local, remote): + self.events.append(("put", remote)) + if remote.endswith("delta.tar.gz"): + with tarfile.open(local) as tar: + names = tar.getnames() + self.pending = json.loads( + tar.extractfile(f"{deploy_orin.DEPLOY_DIR}/manifest.json").read() + ) + self.deltas.append(sorted(n for n in names if not n.startswith(".deploy/"))) + + def ssh(self, cmd): + self.events.append(("ssh", cmd)) + if " apply " in cmd: + self.manifest = self.pending + + def run(self, root): args = deploy_orin.parser().parse_args( ["u@orin", "--backend", "sipl", "--config", "orin.json", "--no-activate"] ) - manager = mock.Mock() - manager.make_snapshot.return_value = { - "base_revision": "a" * 40, - "dirty": False, - "sha256": "b" * 64, - } - manager.target.validate_connection.side_effect = validate_effect - manager.target.capture.return_value = "x" - manager.target.readable.return_value = True - with ( - mock.patch.object(deploy_orin, "repo_root", return_value=self.tmp), - mock.patch.object(deploy_orin, "make_snapshot", manager.make_snapshot), - ): - try: - deploy_orin.run_deploy(args, config, manager.target) - except DeployError: - pass - return [c[0] for c in manager.mock_calls] + config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} + with mock.patch.object(deploy_orin, "repo_root", return_value=root): + deploy_orin.run_deploy(args, config, self.target) - def test_validates_connection_first(self): - names = self.run_deploy() - self.assertEqual(names[0], "target.validate_connection") - self.assertIn("make_snapshot", names) - self.assertIn("target.put", names) +class DeployOrderTests(TempDirTestCase): def test_failed_validation_blocks_everything_else(self): - names = self.run_deploy(validate_effect=DeployError("no route")) - self.assertEqual(names, ["target.validate_connection"]) + orin = FakeOrin() + orin.target.validate_connection.side_effect = DeployError("no route") + with self.assertRaises(DeployError): + orin.run(self.tmp) + self.assertEqual(orin.target.mock_calls[0][0], "validate_connection") + self.assertEqual(len(orin.target.mock_calls), 1) + + def test_helper_uploaded_and_preflight_run_before_any_delta(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n", "README.md": "r\n"}) + orin = FakeOrin() + orin.run(root) + kinds = [] + for kind, value in orin.events: + if kind == "put" and value.endswith("/deploy_orin.py"): + kinds.append("helper") + elif kind == "ssh" and " preflight " in value: + kinds.append("preflight") + elif kind == "put" and value.endswith("delta.tar.gz"): + kinds.append("delta") + self.assertEqual(kinds, ["helper", "preflight", "delta"]) + apply_cmd = next(v for k, v in orin.events if k == "ssh" and " apply " in v) + self.assertIn("--fresh", apply_cmd) + + def test_unchanged_second_run_sends_nothing_and_one_change_sends_only_it(self): + root, _ = git_repo( + self, {"src/a.cpp": "a\n", "src/b.cpp": "b\n", "README.md": "r\n"} + ) + orin = FakeOrin() + orin.run(root) + self.assertEqual(orin.deltas, [["README.md", "src/a.cpp", "src/b.cpp"]]) + + orin.run(root) + self.assertEqual(len(orin.deltas), 1) + + (root / "src/b.cpp").write_text("changed\n") + orin.run(root) + self.assertEqual(orin.deltas[1:], [["src/b.cpp"]]) + applies = [v for k, v in orin.events if k == "ssh" and " apply " in v] + self.assertIn("--fresh", applies[0]) + self.assertNotIn("--fresh", applies[-1]) + + def test_same_src_path_for_different_edits(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.run(root) + (root / "src/a.cpp").write_text("z\n") + orin.run(root) + srcs = { + m.group(1) + for k, v in orin.events + if k == "ssh" and (m := re.search(r"--src (\S+)", v)) + } + self.assertEqual(srcs, {f"{CACHE_DIR}/src"}) def test_deploy_closes_target_on_failure(self): config = self.tmp / "orin.json" @@ -290,59 +380,187 @@ def test_deploy_closes_target_on_failure(self): target.close.assert_called_once() -class SnapshotTests(TempDirTestCase): - def git(self, *args): - return subprocess.run( - [ - "git", - "-c", - "user.name=t", - "-c", - "user.email=t@example.org", - "-c", - "commit.gpgsign=false", - *args, - ], - cwd=self.repo, - capture_output=True, - text=True, - check=True, - ).stdout.strip() +class SyncTests(TempDirTestCase): + def test_file_group(self): + for name in ("CMakeLists.txt", "src/a.cpp", "cmake/x.cmake", "tests/t.py", "examples/egocap/x"): # fmt: skip + self.assertEqual(deploy_orin.file_group(name), "source", name) + for name in ("deps/lib/x.c", "README.md", "examples/other/x", "docs/CMakeLists.txt"): # fmt: skip + self.assertEqual(deploy_orin.file_group(name), "bulk", name) + + def test_tracked_files_include_edits_not_untracked(self): + root, _ = git_repo(self, {"a.txt": "1\n", "sub/b.txt": "2\n"}) + (root / "a.txt").write_text("edited\n") + (root / "untracked.txt").write_text("x") + self.assertEqual(deploy_orin.tracked_files(root), ["a.txt", "sub/b.txt"]) + + def test_tracked_files_skip_deleted(self): + root, _ = git_repo(self, {"a.txt": "1\n", "b.txt": "2\n"}) + (root / "b.txt").unlink() + self.assertEqual(deploy_orin.tracked_files(root), ["a.txt"]) + + def test_hash_files_records_exec_bit(self): + root, _ = git_repo(self, {"a.txt": "1\n", "run.sh": "#!/bin/sh\n"}) + (root / "run.sh").chmod(0o755) + entries = deploy_orin.hash_files(root, ["a.txt", "run.sh"]) + self.assertEqual( + entries["a.txt"], deploy_orin.sha256_file(root / "a.txt") + "-" + ) + self.assertEqual( + entries["run.sh"], deploy_orin.sha256_file(root / "run.sh") + "x" + ) + + def test_group_digest_is_stable_and_tracks_changes(self): + entries = {"src/a": "h1-", "src/b": "h2-", "README.md": "h3-"} + reordered = {"README.md": "h3-", "src/b": "h2-", "src/a": "h1-"} + digest = deploy_orin.group_digest + self.assertEqual(digest(entries, "source"), digest(reordered, "source")) + for changed in ({"src/a": "h9-"}, {"src/a": "h1x"}, {"src/c": "h4-"}): + self.assertNotEqual( + digest(entries, "source"), digest({**entries, **changed}, "source") + ) + self.assertEqual( + digest(entries, "bulk"), digest({**entries, "src/a": "h9-"}, "bulk") + ) + + def test_diff_identical_sends_nothing(self): + entries = {"src/a": "1-", "README.md": "2-"} + self.assertEqual(deploy_orin.diff_manifests(entries, dict(entries)), ([], [])) + def test_diff_changed_new_and_removed_per_group(self): + local = {"src/a": "1-", "src/new": "n-", "src/same": "s-", "README.md": "r2-", "doc": "d-"} # fmt: skip + remote = {"src/a": "0-", "src/same": "s-", "src/gone": "g-", "README.md": "r1-", "doc": "d-", "old.txt": "o-"} # fmt: skip + send, removed = deploy_orin.diff_manifests(local, remote) + self.assertEqual(send, ["README.md", "src/a", "src/new"]) + self.assertEqual(removed, ["old.txt", "src/gone"]) + + def test_diff_skips_a_group_with_equal_digest_without_comparing_files(self): + local = {"src/a": "1-", "README.md": "r2-"} + remote = {"src/a": "0-", "README.md": "r1-"} + + def fake_digest(entries, group): + return "same" if group == "source" else repr(sorted(entries.items())) + + with mock.patch.object(deploy_orin, "group_digest", fake_digest): + send, removed = deploy_orin.diff_manifests(local, remote) + self.assertEqual((send, removed), (["README.md"], [])) + + def test_diff_against_empty_remote_sends_everything(self): + local = {"src/a": "1-", "README.md": "2-"} + self.assertEqual( + deploy_orin.diff_manifests(local, {}), (["README.md", "src/a"], []) + ) + + def test_build_delta_has_only_sent_files_and_deploy_members(self): + root, _ = git_repo(self, {"src/a": "a\n", "src/b": "b\n", "README.md": "r\n"}) + manifest = deploy_orin.hash_files(root, deploy_orin.tracked_files(root)) + dest = self.tmp / "delta.tar.gz" + deploy_orin.build_delta(root, ["src/b"], ["old/x"], manifest, dest) + with tarfile.open(dest) as tar: + self.assertEqual( + sorted(tar.getnames()), + [".deploy/manifest.json", ".deploy/removed.json", "src/b"], + ) + self.assertEqual( + json.loads(tar.extractfile(".deploy/manifest.json").read()), manifest + ) + self.assertEqual( + json.loads(tar.extractfile(".deploy/removed.json").read()), ["old/x"] + ) + + def test_read_remote_manifest_tolerates_missing_or_garbage(self): + def read(text): + target = mock.Mock() + target.capture.return_value = text + return deploy_orin.read_remote_manifest(target, "/c") + + self.assertEqual(read('{"a": "h-"}'), {"a": "h-"}) + for text in ("", "{oops", "[1]"): + self.assertEqual(read(text), {}) + + +class ApplyTests(TempDirTestCase): def setUp(self): super().setUp() - self.repo = self.tmp / "repo" - (self.repo / "sub").mkdir(parents=True) - self.git("init", "-q") - (self.repo / "a.txt").write_text("one\n") - (self.repo / "sub/b.txt").write_text("two\n") - self.git("add", "-A") - self.git("commit", "-q", "-m", "init") - self.dest = self.tmp / "snap.tar.gz" - - def members(self): - with tarfile.open(self.dest) as tar: - return { - m.name: tar.extractfile(m).read().decode() - for m in tar.getmembers() - if m.isfile() - } - - def test_clean_tree_ignores_untracked(self): - (self.repo / "untracked.txt").write_text("x") - meta = deploy_orin.make_snapshot(self.repo, self.dest) - self.assertEqual(self.members(), {"a.txt": "one\n", "sub/b.txt": "two\n"}) - self.assertFalse(meta["dirty"]) - self.assertEqual(meta["base_revision"], self.git("rev-parse", "HEAD")) - self.assertEqual(meta["sha256"], deploy_orin.sha256_file(self.dest)) - - def test_tracked_edit_is_included_and_marks_dirty(self): - (self.repo / "a.txt").write_text("edited\n") - (self.repo / "untracked.txt").write_text("x") - meta = deploy_orin.make_snapshot(self.repo, self.dest) - self.assertEqual(self.members()["a.txt"], "edited\n") - self.assertNotIn("untracked.txt", self.members()) - self.assertTrue(meta["dirty"]) + self.src = self.tmp / "orin" / "src" + self.manifest_path = self.tmp / "orin" / "src.manifest.json" + self.manifest_path.parent.mkdir() + + def delta(self, files, removed=(), manifest=None): + root = self.tmp / "local" + root.mkdir(exist_ok=True) + for name, text in files.items(): + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_text(text) + if manifest is None: + manifest = deploy_orin.hash_files(root, sorted(files)) + dest = self.tmp / "delta.tar.gz" + deploy_orin.build_delta(root, sorted(files), list(removed), manifest, dest) + return dest + + def apply(self, archive, fresh=False): + deploy_orin.remote_apply( + argparse.Namespace( + archive=str(archive), + src=str(self.src), + manifest=str(self.manifest_path), + fresh=fresh, + ) + ) + + def test_extracts_and_writes_manifest(self): + self.apply(self.delta({"src/a.cpp": "a\n", "README.md": "r\n"})) + self.assertEqual((self.src / "src/a.cpp").read_text(), "a\n") + self.assertFalse((self.src / ".deploy").exists()) + manifest = json.loads(self.manifest_path.read_text()) + self.assertEqual(sorted(manifest), ["README.md", "src/a.cpp"]) + + def test_removes_files_and_prunes_empty_dirs(self): + (self.src / "gone/deep").mkdir(parents=True) + (self.src / "gone/deep/x").write_text("x") + (self.src / "keep").mkdir() + (self.src / "keep/a").write_text("a") + (self.src / "keep/b").write_text("b") + self.apply(self.delta({"new": "n\n"}, removed=["gone/deep/x", "keep/b"])) + self.assertFalse((self.src / "gone").exists()) + self.assertTrue((self.src / "keep/a").exists()) + self.assertFalse((self.src / "keep/b").exists()) + self.assertTrue(self.src.is_dir()) + + def test_hash_mismatch_raises_and_leaves_manifest_unwritten(self): + bad = {"a": "0" * 64 + "-"} + with self.assertRaisesRegex(DeployError, "does not match"): + self.apply(self.delta({"a": "a\n"}, manifest=bad)) + self.assertFalse(self.manifest_path.exists()) + + def test_fresh_wipes_stale_files(self): + self.src.mkdir() + (self.src / "stale").write_text("old") + self.apply(self.delta({"a": "a\n"}), fresh=True) + self.assertFalse((self.src / "stale").exists()) + self.assertTrue((self.src / "a").exists()) + + def test_without_fresh_keeps_untouched_files(self): + self.src.mkdir() + (self.src / "stale").write_text("old") + self.apply(self.delta({"a": "a\n"})) + self.assertTrue((self.src / "stale").exists()) + + def test_refuses_removal_outside_src(self): + victim = self.tmp / "orin" / "victim" + victim.write_text("keep") + for name in ("../victim", "/etc/hostname"): + with ( + self.subTest(name=name), + self.assertRaisesRegex(DeployError, "outside"), + ): + self.apply(self.delta({"a": "a\n"}, removed=[name])) + self.assertTrue(victim.exists()) + + def test_remote_parser_apply_arguments(self): + parsed = deploy_orin.remote_parser().parse_args( + ["apply", "--archive", "a", "--src", "s", "--manifest", "m", "--fresh"] + ) + self.assertTrue(parsed.fresh) class ActivateTests(TempDirTestCase): @@ -639,15 +857,65 @@ def test_preflight_names_the_sdk_root_path_when_headers_are_missing(self): deploy_orin.remote_preflight(args) +class BuildPathTests(unittest.TestCase): + def test_build_path_prefixes_uv_then_cuda_before_the_inherited_path(self): + with mock.patch.dict(os.environ, {"PATH": "/usr/bin:/bin"}): + self.assertEqual( + deploy_orin.build_path(), + f"{Path.home()}/.local/bin:/usr/local/cuda/bin:/usr/bin:/bin", + ) + + def preflight(self, missing=()): + seen = [] + + def fake_run(cmd, **kwargs): + seen.append((cmd, kwargs)) + return completed(1 if cmd[0] == "which" and cmd[1] in missing else 0) + + args = argparse.Namespace(backend="sipl", sdk_root="/nonexistent-sdk") + with ( + mock.patch.object(deploy_orin.os, "uname") as uname, + mock.patch.object(Path, "exists", return_value=True), + mock.patch.object(Path, "read_text", return_value="# R36\n"), + mock.patch.object(Path, "is_dir", return_value=True), + mock.patch.object(deploy_orin.os, "statvfs") as statvfs, + mock.patch.object(deploy_orin.subprocess, "run", fake_run), + mock.patch.dict(os.environ, {"PATH": "/usr/bin"}), + ): + uname.return_value.machine = "aarch64" + statvfs.return_value.f_bavail = 1 << 30 + statvfs.return_value.f_frsize = 4096 + try: + deploy_orin.remote_preflight(args) + finally: + self.seen = seen + + def test_preflight_requires_uv_using_the_build_path(self): + with self.assertRaisesRegex(DeployError, "uv is not installed.*astral.sh"): + self.preflight(missing=("uv",)) + uv_call = next(k for c, k in self.seen if c == ["which", "uv"]) + self.assertEqual( + uv_call["env"]["PATH"], + f"{Path.home()}/.local/bin:/usr/local/cuda/bin:/usr/bin", + ) + + def test_preflight_succeeds_when_uv_present(self): + self.preflight() + self.assertIn(["which", "uv"], [c for c, _ in self.seen]) + + class RemoteBuildTests(TempDirTestCase): LIBCUDA = "/opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so.1" + EXPECTED_PATH = f"{Path.home()}/.local/bin:/usr/local/cuda/bin:/usr/bin" - def build(self, backend="sipl", sdk_root="", libcuda=False, path="/usr/bin"): + def build( + self, backend="sipl", sdk_root="", libcuda=False, path="/usr/bin", src="/src" + ): stage = self.tmp / "stage" (stage / deploy_orin.BIN_REL).parent.mkdir(parents=True, exist_ok=True) (stage / deploy_orin.BIN_REL).write_text("bin") args = argparse.Namespace( - src="/src", + src=src, backend=backend, stage=str(stage), jobs=3, @@ -667,6 +935,8 @@ def fake_read_text(p, *a, **k): def fake_run(cmd, **kwargs): calls.append((list(cmd), kwargs)) + if cmd[:2] == ["rm", "-rf"] and "build-" in cmd[2]: + shutil.rmtree(cmd[2], ignore_errors=True) return completed(0) with ( @@ -694,12 +964,21 @@ def test_sipl_configure_flags_and_cuda_path(self): "-DCMAKE_CXX_FLAGS=-I/usr/local/cuda/include", ): self.assertIn(flag, cmd) - self.assertEqual(kwargs["env"]["PATH"], "/usr/local/cuda/bin:/usr/bin") + self.assertEqual(kwargs["env"]["PATH"], self.EXPECTED_PATH) + + def cleaned(self, calls): + return [c for c, _ in calls if c[:2] == ["rm", "-rf"] and "build-sipl" in c[2]] + + def test_changed_source_path_cleans_build_dir_but_same_path_keeps_it(self): + self.assertTrue(self.cleaned(self.build(src="/cache/src"))) + self.assertFalse(self.cleaned(self.build(src="/cache/src"))) + self.assertTrue(self.cleaned(self.build(src="/cache/src-abc123"))) + self.assertFalse(self.cleaned(self.build(src="/cache/src-abc123"))) def test_build_step_also_gets_the_cuda_path(self): calls = self.build() cmd, kwargs = next(c for c in calls if c[0][:2] == ["cmake", "--build"]) - self.assertEqual(kwargs["env"]["PATH"], "/usr/local/cuda/bin:/usr/bin") + self.assertEqual(kwargs["env"]["PATH"], self.EXPECTED_PATH) self.assertIn("camera_plugin_sensing", cmd) def test_argus_has_no_sipl_dir(self): From e711069ec78c5923362bdcc5a7364e3b78084584 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 15:01:30 -0800 Subject: [PATCH 03/16] sensing: install the LiveKit licence files from the right FetchContent directory FetchContent publishes _SOURCE_DIR, so the uppercase LICENSE and NOTICE lookups were empty and the install step looked for //LICENSE. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index 0a93bee669..cdfe43f317 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -202,7 +202,9 @@ foreach(_notice LICENSE NOTICE) DOWNLOAD_NO_EXTRACT TRUE ) FetchContent_MakeAvailable(livekit_sdk_${_notice}) - set(_found "${livekit_sdk_${_notice}_SOURCE_DIR}/${_notice}") + # FetchContent publishes _SOURCE_DIR. + string(TOLOWER "${_notice}" _notice_lower) + set(_found "${livekit_sdk_${_notice_lower}_SOURCE_DIR}/${_notice}") endif() set(SENSING_LIVEKIT_${_notice}_FILE "${_found}" CACHE INTERNAL "") endforeach() From 91a9626399e58bb97af182354dd861924348e5ee Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 15:15:11 -0800 Subject: [PATCH 04/16] sensing: document running the Orin deploy script Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 6752c72fbc..72c1eee211 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -47,3 +47,43 @@ SIPL drivers. hardware encoder and publishes one LiveKit video track per camera. The [SENSING user guide](https://nvidia.github.io/IsaacCapture/main/device/camera/sensing.html) has the build and run steps. + +## Orin: deploy from the desktop with one script + +`scripts/deploy_orin.py` builds the producer on an Orin from the checkout you run it in and runs it as the +`isaaccapture-producer.service` systemd service, so you do not build by hand on the Orin. It needs `ssh`, `scp` and +`git` on the desktop, and `python3`, CMake, g++, CUDA, `uv` and the SDK headers (see `scripts/setup.sh`) on the Orin. + +Write a config naming the producer's own options (the LiveKit token file stays on the Orin; only its path is used): + +```json +{"livekit_url": "ws://127.0.0.1:7880", + "token_file": "/home/ido/livekit/join.token", + "args": ["--camera-backend=sipl", "--link-masks=0x0000 0x1100", + "--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} +``` + +Then run it from the repository root. The ssh user and password can come from the environment, which keeps the +password out of the command line and the output: + +```bash +export ORIN_USER=ido +read -rs ORIN_PASSWORD && export ORIN_PASSWORD # optional; without it ssh and sudo prompt +python3 src/plugins/sensing/scripts/deploy_orin.py 192.168.1.176 --backend sipl --config orin.json \ + --sdk-root /home/ido/sensing-sdk/root --replace-unmanaged +``` + +- `--backend sipl|argus` selects which capture SDK is built in. `--sdk-root DIR` is the Orin directory holding + `usr/src/jetson_multimedia_api` and `usr/src/jetson_sipl_api` when they are not under `/usr/src`. +- `--replace-unmanaged` stops a `camera_plugin_sensing` that someone started by hand (its exact PIDs are shown); + without it the deployment refuses and changes nothing. `--no-activate` builds and packages only, `--fetch-archive + DIR` copies the package back, `--jobs N` bounds compile jobs, `--prefix DIR` changes the install root + (default `/opt/isaaccapture`). +- The script checks the ssh connection first, then preflights the Orin, uploads only the source files that differ + from what the Orin already holds, builds, installs a versioned release, replaces the running producer and waits for + it to join the room and publish frames. Steps are numbered and the upload and packing show progress. The sudo steps + prompt for the Orin password. +- A failed build or transfer leaves the running producer untouched. There is no automatic rollback; earlier releases + stay under `/opt/isaaccapture/releases/`. Logs: `journalctl -u isaaccapture-producer.service -f` on the Orin. Stop + it with `sudo systemctl stop isaaccapture-producer.service` before starting a producer by hand: the cameras are + exclusive. From 90b62b78d72eb8ebdfd85c31fce047aaad646065 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 16:14:39 -0800 Subject: [PATCH 05/16] sensing: deploy another checkout or a branch's committed files to the Orin --source DIR deploys a different git checkout (a worktree, say) and --branch NAME deploys that branch's committed files from it, exported with git archive so the working tree is ignored. The default is unchanged: the script's own checkout, local edits included. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 3 + src/plugins/sensing/scripts/deploy_orin.py | 80 ++++++++++++++- tests/python/ci/test_deploy_orin.py | 107 ++++++++++++++++++++- 3 files changed, 184 insertions(+), 6 deletions(-) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 72c1eee211..43305e89d4 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -75,6 +75,9 @@ python3 src/plugins/sensing/scripts/deploy_orin.py 192.168.1.176 --backend sipl - `--backend sipl|argus` selects which capture SDK is built in. `--sdk-root DIR` is the Orin directory holding `usr/src/jetson_multimedia_api` and `usr/src/jetson_sipl_api` when they are not under `/usr/src`. +- `--source DIR` deploys another checkout (for example a git worktree of a feature branch) instead of the one holding + the script; `--branch NAME` deploys that branch's committed files from it, ignoring the working tree. Without them + the script deploys its own checkout, local edits included. - `--replace-unmanaged` stops a `camera_plugin_sensing` that someone started by hand (its exact PIDs are shown); without it the deployment refuses and changes nothing. `--no-activate` builds and packages only, `--fetch-archive DIR` copies the package back, `--jobs N` bounds compile jobs, `--prefix DIR` changes the install root diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index 4e91762c32..0a444b90de 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -21,6 +21,7 @@ from __future__ import annotations import argparse +import contextlib import hashlib import io import json @@ -440,8 +441,72 @@ def deploy(args: argparse.Namespace) -> None: target.close() +def git_toplevel(folder: Path) -> Path: + found = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=folder, + capture_output=True, + text=True, + check=False, + ) + if found.returncode != 0: + raise DeployError(f"{folder} is not inside a git checkout") + return Path(found.stdout.strip()) + + +@contextlib.contextmanager +def open_source(args: argparse.Namespace): + """The source to deploy as (root, files, git state). + + By default the tracked files of the working tree of the checkout holding this script, with local edits. `--source` + names another checkout (a worktree, say) to deploy instead. `--branch` deploys that branch's committed files from + that checkout, ignoring the working tree, exported to a temporary directory. + """ + repo = git_toplevel(Path(args.source)) if args.source else repo_root() + if not args.branch: + yield repo, tracked_files(repo), git_state(repo) + return + found = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{args.branch}^{{commit}}"], + cwd=repo, + capture_output=True, + text=True, + check=False, + ) + if found.returncode != 0: + raise DeployError(f"branch {args.branch!r} not found in {repo}") + revision = found.stdout.strip() + with tempfile.TemporaryDirectory() as tmp: + export = Path(tmp) / "tree" + export.mkdir() + archive = subprocess.Popen( + ["git", "archive", "--format=tar", revision], + cwd=repo, + stdout=subprocess.PIPE, + ) + with tarfile.open(fileobj=archive.stdout, mode="r|") as tar: + tar.extractall(export, filter="data") + if archive.wait() != 0: + raise DeployError(f"git archive of {args.branch!r} failed") + files = sorted( + p.relative_to(export).as_posix() for p in export.rglob("*") if p.is_file() + ) + yield export, files, {"base_revision": revision, "dirty": False} + + def run_deploy(args: argparse.Namespace, config: dict, target: Target) -> None: - root = repo_root() + with open_source(args) as (root, files, state): + deploy_source(args, config, target, root, files, state) + + +def deploy_source( + args: argparse.Namespace, + config: dict, + target: Target, + root: Path, + files: list[str], + state: dict, +) -> None: step(1, f"Connecting to {target.host}") target.validate_connection() user = args.service_user or target.capture("id -un") @@ -463,8 +528,7 @@ def run_deploy(args: argparse.Namespace, config: dict, target: Target) -> None: with tempfile.TemporaryDirectory() as tmp: step(3, "Compare the source with the Orin's and upload what differs") - meta = git_state(root) - files = tracked_files(root) + meta = dict(state) local = hash_files(root, files) meta["manifest_sha256"] = hashlib.sha256( json.dumps(local, sort_keys=True).encode() @@ -902,6 +966,16 @@ def parser() -> argparse.ArgumentParser: action="store_true", help="stop a camera_plugin_sensing the service does not own (its exact PIDs are shown) instead of refusing", ) + ap.add_argument( + "--source", + metavar="DIR", + help="deploy this git checkout (a worktree, say) instead of the one holding this script", + ) + ap.add_argument( + "--branch", + metavar="NAME", + help="deploy this branch's committed files (from --source, or this checkout), ignoring the working tree", + ) ap.add_argument("--no-activate", action="store_true", help="build and package only") ap.add_argument( "--fetch-archive", diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index 476948591c..4b093e17b5 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -4,6 +4,7 @@ """Tests for the Orin deploy script; every subprocess call is mocked or runs against a temp git repo.""" import argparse +import hashlib import importlib.util import io import json @@ -297,9 +298,17 @@ def ssh(self, cmd): if " apply " in cmd: self.manifest = self.pending - def run(self, root): + def run(self, root, *extra): args = deploy_orin.parser().parse_args( - ["u@orin", "--backend", "sipl", "--config", "orin.json", "--no-activate"] + [ + "u@orin", + "--backend", + "sipl", + "--config", + "orin.json", + "--no-activate", + *extra, + ] ) config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} with mock.patch.object(deploy_orin, "repo_root", return_value=root): @@ -308,10 +317,11 @@ def run(self, root): class DeployOrderTests(TempDirTestCase): def test_failed_validation_blocks_everything_else(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) orin = FakeOrin() orin.target.validate_connection.side_effect = DeployError("no route") with self.assertRaises(DeployError): - orin.run(self.tmp) + orin.run(root) self.assertEqual(orin.target.mock_calls[0][0], "validate_connection") self.assertEqual(len(orin.target.mock_calls), 1) @@ -380,6 +390,97 @@ def test_deploy_closes_target_on_failure(self): target.close.assert_called_once() +class OpenSourceTests(TempDirTestCase): + def args(self, *extra): + return deploy_orin.parser().parse_args( + ["u@orin", "--backend", "sipl", "--config", "c", *extra] + ) + + def repo_with_branch(self): + root, git = git_repo( + self, {"src/a.cpp": "committed\n", "run.sh": "#!/bin/sh\n"} + ) + (root / "run.sh").chmod(0o755) + git("add", "run.sh") + git("commit", "-q", "-m", "exec") + git("branch", "feature") + (root / "src/a.cpp").write_text("edited later\n") + (root / "untracked.txt").write_text("x") + return root, git + + def test_cli_parses_source_and_branch(self): + args = self.args("--source", "/w", "--branch", "feat") + self.assertEqual((args.source, args.branch), ("/w", "feat")) + args = self.args() + self.assertFalse(args.source) + self.assertFalse(args.branch) + + def test_default_yields_script_checkout_with_local_edits(self): + root, _ = git_repo(self, {"a.txt": "1\n", "b.txt": "2\n"}) + (root / "a.txt").write_text("edited\n") + (root / "untracked.txt").write_text("x") + with mock.patch.object(deploy_orin, "repo_root", return_value=root): + with deploy_orin.open_source(self.args()) as (src, files, state): + self.assertEqual(src, root) + self.assertEqual(files, ["a.txt", "b.txt"]) + self.assertTrue(state["dirty"]) + + def test_source_option_uses_another_checkout(self): + root, git = git_repo(self, {"a.txt": "1\n"}) + sub = root / "sub" + sub.mkdir() + with mock.patch.object( + deploy_orin, "repo_root", side_effect=AssertionError("own checkout used") + ): + with deploy_orin.open_source(self.args("--source", str(sub))) as ( + src, + files, + state, + ): + self.assertEqual(src.resolve(), root.resolve()) + self.assertEqual(files, ["a.txt"]) + self.assertEqual(state["base_revision"], git("rev-parse", "HEAD")) + self.assertFalse(state["dirty"]) + + def test_branch_yields_committed_content_only(self): + root, git = self.repo_with_branch() + with deploy_orin.open_source( + self.args("--source", str(root), "--branch", "feature") + ) as (export, files, state): + self.assertEqual(files, ["run.sh", "src/a.cpp"]) + self.assertEqual((export / "src/a.cpp").read_text(), "committed\n") + self.assertFalse((export / "untracked.txt").exists()) + self.assertTrue(os.access(export / "run.sh", os.X_OK)) + self.assertFalse(os.access(export / "src/a.cpp", os.X_OK)) + self.assertEqual( + state, {"base_revision": git("rev-parse", "feature"), "dirty": False} + ) + kept = export + self.assertFalse(kept.exists()) + + def test_unknown_branch_and_non_git_source_raise(self): + root, _ = git_repo(self, {"a.txt": "1\n"}) + with self.assertRaisesRegex(DeployError, "not found"): + with deploy_orin.open_source( + self.args("--source", str(root), "--branch", "nope") + ): + pass + plain = self.tmp / "plain" + plain.mkdir() + with self.assertRaisesRegex(DeployError, "not inside a git"): + with deploy_orin.open_source(self.args("--source", str(plain))): + pass + + def test_run_deploy_with_branch_uploads_the_branch_files(self): + root, _ = self.repo_with_branch() + orin = FakeOrin() + orin.run(root, "--source", str(root), "--branch", "feature") + self.assertEqual(orin.deltas, [["run.sh", "src/a.cpp"]]) + self.assertEqual( + orin.pending["src/a.cpp"], hashlib.sha256(b"committed\n").hexdigest() + "-" + ) + + class SyncTests(TempDirTestCase): def test_file_group(self): for name in ("CMakeLists.txt", "src/a.cpp", "cmake/x.cmake", "tests/t.py", "examples/egocap/x"): # fmt: skip From 38f7cea01a4823752fdef88bbeb9b57c9c49b756 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 6 Nov 2026 08:41:43 -0800 Subject: [PATCH 06/16] sensing: deploy script no longer depends on the capture backend or branch Drop --backend and the SENSING_WITH_* flags: which capture SDK is built is the deployed source's own CMake decision, so the same script deploys any branch. Add a repeatable --cmake-arg for options only some trees define, and pass -DSIPL_API_DIR only when that header tree exists. Remove the camera-backend flag from the README example and fix a stale script name in a comment. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 10 ++- src/plugins/sensing/app/CMakeLists.txt | 2 +- src/plugins/sensing/scripts/deploy_orin.py | 54 ++++++------ tests/python/ci/test_deploy_orin.py | 95 ++++++++++++++++------ 4 files changed, 103 insertions(+), 58 deletions(-) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 43305e89d4..ee87dd5fad 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -59,7 +59,7 @@ Write a config naming the producer's own options (the LiveKit token file stays o ```json {"livekit_url": "ws://127.0.0.1:7880", "token_file": "/home/ido/livekit/join.token", - "args": ["--camera-backend=sipl", "--link-masks=0x0000 0x1100", + "args": ["--link-masks=0x0000 0x1100", "--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} ``` @@ -69,12 +69,14 @@ password out of the command line and the output: ```bash export ORIN_USER=ido read -rs ORIN_PASSWORD && export ORIN_PASSWORD # optional; without it ssh and sudo prompt -python3 src/plugins/sensing/scripts/deploy_orin.py 192.168.1.176 --backend sipl --config orin.json \ +python3 src/plugins/sensing/scripts/deploy_orin.py 192.168.1.176 --config orin.json \ --sdk-root /home/ido/sensing-sdk/root --replace-unmanaged ``` -- `--backend sipl|argus` selects which capture SDK is built in. `--sdk-root DIR` is the Orin directory holding - `usr/src/jetson_multimedia_api` and `usr/src/jetson_sipl_api` when they are not under `/usr/src`. +- The script has no knowledge of which capture SDK the deployed source uses: that comes from the source's own CMake. + `--cmake-arg=ARG` (repeatable, `=` form) passes options only some source trees define, such as a capture backend + switch. `--sdk-root DIR` is the Orin directory holding `usr/src/jetson_multimedia_api` and + `usr/src/jetson_sipl_api` when they are not under `/usr/src`. - `--source DIR` deploys another checkout (for example a git worktree of a feature branch) instead of the one holding the script; `--branch NAME` deploys that branch's committed files from it, ignoring the working tree. Without them the script deploys its own checkout, local edits included. diff --git a/src/plugins/sensing/app/CMakeLists.txt b/src/plugins/sensing/app/CMakeLists.txt index d225b6d68c..edb7c51a74 100644 --- a/src/plugins/sensing/app/CMakeLists.txt +++ b/src/plugins/sensing/app/CMakeLists.txt @@ -18,7 +18,7 @@ add_custom_command(TARGET camera_plugin_sensing POST_BUILD ) # Everything below is the `producer` install component: `cmake --install --component producer` stages -# exactly what scripts/deploy-orin.sh packages for an Orin. +# exactly what scripts/deploy_orin.py packages for an Orin. install(TARGETS camera_plugin_sensing RUNTIME DESTINATION plugins/sensing_camera COMPONENT producer diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index 0a444b90de..9c896e641b 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -5,7 +5,7 @@ Run on the desktop: - deploy_orin.py user@orin --backend sipl --config orin.json + deploy_orin.py user@orin --config orin.json The config is JSON with the producer's own options, nothing else: @@ -522,9 +522,7 @@ def deploy_source( helper = f"{cache}/deploy_orin.py" target.put(Path(__file__).resolve(), helper) remote = f"python3 {helper} remote" - target.ssh( - f"{remote} preflight --backend {args.backend} --sdk-root {shlex.quote(args.sdk_root)}" - ) + target.ssh(f"{remote} preflight --sdk-root {shlex.quote(args.sdk_root)}") with tempfile.TemporaryDirectory() as tmp: step(3, "Compare the source with the Orin's and upload what differs") @@ -566,12 +564,12 @@ def deploy_source( f"{cache}/producer-{release}.tar.gz", ) target.ssh( - f"{remote} build --src {src} --backend {args.backend} --stage {stage} --jobs {args.jobs} " + f"{remote} build --src {src} --stage {stage} --jobs {args.jobs} " f"--sdk-root {shlex.quote(args.sdk_root)}" + + "".join(f" --cmake-arg={shlex.quote(a)}" for a in args.cmake_arg) ) manifest = { **meta, - "backend": args.backend, "release": release, "arch": "aarch64", "config": args.config, @@ -612,7 +610,10 @@ def tail(cmd: list[str]) -> str: def sdk_dirs(sdk_root: str) -> tuple[Path, Path]: - """The Multimedia API and SIPL header trees: under --sdk-root when given, else the system locations.""" + """The Multimedia API and SIPL header trees: under --sdk-root when given, else the system locations. + + Only where the headers live is decided here; whether a given source tree needs them is its own CMake's concern. + """ base = Path(sdk_root) / "usr/src" if sdk_root else Path("/usr/src") return base / "jetson_multimedia_api", base / "jetson_sipl_api" @@ -652,48 +653,44 @@ def remote_preflight(args: argparse.Namespace) -> None: stat = os.statvfs(Path.home()) if stat.f_bavail * stat.f_frsize < 4 << 30: raise DeployError("less than 4 GB free in the home directory") - mmapi, sipl_api = sdk_dirs(args.sdk_root) + mmapi, _ = sdk_dirs(args.sdk_root) if not mmapi.is_dir(): raise DeployError( f"the Jetson Multimedia API is missing at {mmapi}; run src/plugins/sensing/scripts/setup.sh " "or pass --sdk-root" ) - if args.backend == "sipl" and not sipl_api.is_dir(): - raise DeployError( - f"SIPL headers not found at {sipl_api}; run src/plugins/sensing/scripts/setup.sh or pass --sdk-root" - ) - if args.backend == "argus" and not (mmapi / "argus").is_dir(): - raise DeployError(f"Argus headers not found under {mmapi}/argus") - print(f" preflight ok for backend {args.backend}") + print(" preflight ok") def remote_build(args: argparse.Namespace) -> None: """Configure and build only the producer, then stage its install component. - The build directory persists between deployments and is discarded when the toolchain, L4T release or backend - changes. + The build directory persists between deployments and is discarded when the toolchain, L4T release, source + path or CMake arguments change. """ cache = Path(os.path.expanduser(CACHE)) - build_dir = cache / f"build-{args.backend}" + build_dir = cache / "build" stamp = "|".join( [ Path("/etc/nv_tegra_release").read_text().splitlines()[0], tail(["cmake", "--version"]).splitlines()[0], tail(["g++", "-dumpfullversion"]), args.src, + *args.cmake_arg, ] ) stamp_file = build_dir / ".stamp" if not stamp_file.exists() or stamp_file.read_text() != stamp: - print(" new toolchain, release or backend: clean build directory") + print( + " new toolchain, release, source or CMake arguments: clean build directory" + ) run(["rm", "-rf", str(build_dir)]) build_dir.mkdir(parents=True) stamp_file.write_text(stamp) - sipl, argus = ("ON", "OFF") if args.backend == "sipl" else ("OFF", "ON") mmapi, sipl_api = sdk_dirs(args.sdk_root) env = {**os.environ, "PATH": build_path()} sdk_flags = [f"-DJETSON_MMAPI_DIR={mmapi}"] - if args.backend == "sipl": + if sipl_api.is_dir(): sdk_flags.append(f"-DSIPL_API_DIR={sipl_api}") # The CUDA driver library is not on the default link path on L4T, and is dropped by --as-needed. gpu_libcuda = Path("/opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so.1") @@ -712,8 +709,6 @@ def remote_build(args: argparse.Namespace) -> None: "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_CUDA_ARCHITECTURES=87", "-DBUILD_PLUGIN_SENSING=ON", - f"-DSENSING_WITH_SIPL={sipl}", - f"-DSENSING_WITH_ARGUS={argus}", "-DBUILD_TESTING=OFF", "-DBUILD_EXAMPLES=OFF", "-DBUILD_PYTHON_BINDINGS=OFF", @@ -722,6 +717,8 @@ def remote_build(args: argparse.Namespace) -> None: "-DCMAKE_CXX_FLAGS=-I/usr/local/cuda/include", *sdk_flags, *link_flags, + # Options only some source trees define (a capture backend, say) go last so they can override. + *args.cmake_arg, ], env=env, ) @@ -941,7 +938,6 @@ def parser() -> argparse.ArgumentParser: ap.add_argument( "target", help=f"host or user@host of the Orin (a bare host uses ${USER_ENV})" ) - ap.add_argument("--backend", choices=("sipl", "argus"), required=True) ap.add_argument("--config", required=True, help="producer options (JSON)") ap.add_argument("--prefix", default="/opt/isaaccapture") ap.add_argument( @@ -976,6 +972,13 @@ def parser() -> argparse.ArgumentParser: metavar="NAME", help="deploy this branch's committed files (from --source, or this checkout), ignoring the working tree", ) + ap.add_argument( + "--cmake-arg", + action="append", + default=[], + metavar="ARG", + help="extra CMake argument for the producer build, repeatable (for options only some source trees have)", + ) ap.add_argument("--no-activate", action="store_true", help="build and package only") ap.add_argument( "--fetch-archive", @@ -989,11 +992,10 @@ def remote_parser() -> argparse.ArgumentParser: ap = argparse.ArgumentParser(prog="deploy_orin.py remote") sub = ap.add_subparsers(dest="step", required=True) pre = sub.add_parser("preflight") - pre.add_argument("--backend", required=True) pre.add_argument("--sdk-root", default="") build = sub.add_parser("build") build.add_argument("--src", required=True) - build.add_argument("--backend", required=True) + build.add_argument("--cmake-arg", action="append", default=[]) build.add_argument("--stage", required=True) build.add_argument("--jobs", type=int, default=2) build.add_argument("--sdk-root", default="") diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index 4b093e17b5..133cec1c55 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -4,6 +4,7 @@ """Tests for the Orin deploy script; every subprocess call is mocked or runs against a temp git repo.""" import argparse +import contextlib import hashlib import importlib.util import io @@ -302,8 +303,6 @@ def run(self, root, *extra): args = deploy_orin.parser().parse_args( [ "u@orin", - "--backend", - "sipl", "--config", "orin.json", "--no-activate", @@ -372,14 +371,19 @@ def test_same_src_path_for_different_edits(self): } self.assertEqual(srcs, {f"{CACHE_DIR}/src"}) + def test_cmake_args_are_forwarded_to_the_remote_build_in_order(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.run(root, "--cmake-arg=-DA=1", "--cmake-arg=-DB=a b") + build = next(v for k, v in orin.events if k == "ssh" and " build " in v) + self.assertTrue(build.endswith(" --cmake-arg=-DA=1 --cmake-arg='-DB=a b'")) + def test_deploy_closes_target_on_failure(self): config = self.tmp / "orin.json" config.write_text( json.dumps({"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]}) ) - args = deploy_orin.parser().parse_args( - ["u@orin", "--backend", "sipl", "--config", str(config)] - ) + args = deploy_orin.parser().parse_args(["u@orin", "--config", str(config)]) target = mock.Mock() with ( mock.patch.object(deploy_orin, "Target", return_value=target), @@ -392,9 +396,7 @@ def test_deploy_closes_target_on_failure(self): class OpenSourceTests(TempDirTestCase): def args(self, *extra): - return deploy_orin.parser().parse_args( - ["u@orin", "--backend", "sipl", "--config", "c", *extra] - ) + return deploy_orin.parser().parse_args(["u@orin", "--config", "c", *extra]) def repo_with_branch(self): root, git = git_repo( @@ -922,14 +924,12 @@ def test_sdk_root_prefixes_usr_src(self): def test_cli_parsers_accept_sdk_root_and_replace_unmanaged(self): args = deploy_orin.parser().parse_args( - ["u@orin", "--backend", "sipl", "--config", "c", "--sdk-root", "/s"] + ["u@orin", "--config", "c", "--sdk-root", "/s"] ) self.assertEqual(args.sdk_root, "/s") self.assertFalse(args.replace_unmanaged) self.assertEqual( - deploy_orin.parser() - .parse_args(["h", "--backend", "argus", "--config", "c"]) - .sdk_root, + deploy_orin.parser().parse_args(["h", "--config", "c"]).sdk_root, "", ) remote = deploy_orin.remote_parser() @@ -937,13 +937,34 @@ def test_cli_parsers_accept_sdk_root_and_replace_unmanaged(self): ("preflight", []), ("build", ["--src", "s", "--stage", "t"]), ): - parsed = remote.parse_args( - [step, "--backend", "sipl", "--sdk-root", "/s", *extra] - ) + parsed = remote.parse_args([step, "--sdk-root", "/s", *extra]) self.assertEqual(parsed.sdk_root, "/s") + def test_backend_option_is_rejected_everywhere(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + deploy_orin.parser().parse_args( + ["u@orin", "--backend", "sipl", "--config", "c"] + ) + with self.assertRaises(SystemExit): + deploy_orin.remote_parser().parse_args( + ["preflight", "--backend", "sipl"] + ) + + def test_cmake_arg_is_repeatable_and_defaults_empty(self): + parser = deploy_orin.parser() + self.assertEqual(parser.parse_args(["h", "--config", "c"]).cmake_arg, []) + args = parser.parse_args( + ["h", "--config", "c", "--cmake-arg=-DA=1", "--cmake-arg=-DB=2"] + ) + self.assertEqual(args.cmake_arg, ["-DA=1", "-DB=2"]) + remote = deploy_orin.remote_parser().parse_args( + ["build", "--src", "s", "--stage", "t", "--cmake-arg=-DA=1"] + ) + self.assertEqual(remote.cmake_arg, ["-DA=1"]) + def test_preflight_names_the_sdk_root_path_when_headers_are_missing(self): - args = argparse.Namespace(backend="sipl", sdk_root="/nonexistent-sdk") + args = argparse.Namespace(sdk_root="/nonexistent-sdk") with ( mock.patch.object(deploy_orin.os, "uname") as uname, mock.patch.object(Path, "exists", return_value=True), @@ -973,7 +994,7 @@ def fake_run(cmd, **kwargs): seen.append((cmd, kwargs)) return completed(1 if cmd[0] == "which" and cmd[1] in missing else 0) - args = argparse.Namespace(backend="sipl", sdk_root="/nonexistent-sdk") + args = argparse.Namespace(sdk_root="/nonexistent-sdk") with ( mock.patch.object(deploy_orin.os, "uname") as uname, mock.patch.object(Path, "exists", return_value=True), @@ -1010,14 +1031,20 @@ class RemoteBuildTests(TempDirTestCase): EXPECTED_PATH = f"{Path.home()}/.local/bin:/usr/local/cuda/bin:/usr/bin" def build( - self, backend="sipl", sdk_root="", libcuda=False, path="/usr/bin", src="/src" + self, + sdk_root="", + libcuda=False, + path="/usr/bin", + src="/src", + cmake_arg=(), + sipl=True, ): stage = self.tmp / "stage" (stage / deploy_orin.BIN_REL).parent.mkdir(parents=True, exist_ok=True) (stage / deploy_orin.BIN_REL).write_text("bin") args = argparse.Namespace( src=src, - backend=backend, + cmake_arg=list(cmake_arg), stage=str(stage), jobs=3, sdk_root=sdk_root, @@ -1036,7 +1063,7 @@ def fake_read_text(p, *a, **k): def fake_run(cmd, **kwargs): calls.append((list(cmd), kwargs)) - if cmd[:2] == ["rm", "-rf"] and "build-" in cmd[2]: + if cmd[:2] == ["rm", "-rf"] and cmd[2].endswith("/build"): shutil.rmtree(cmd[2], ignore_errors=True) return completed(0) @@ -1045,6 +1072,7 @@ def fake_run(cmd, **kwargs): mock.patch.object(deploy_orin, "run", fake_run), mock.patch.object(deploy_orin, "tail", return_value="x"), mock.patch.object(Path, "exists", fake_exists), + mock.patch.object(Path, "is_dir", return_value=sipl), mock.patch.object(Path, "read_text", fake_read_text), mock.patch.dict(os.environ, {"PATH": path}), ): @@ -1054,11 +1082,9 @@ def fake_run(cmd, **kwargs): def configure(self, calls): return next(c for c in calls if c[0][:2] == ["cmake", "-S"]) - def test_sipl_configure_flags_and_cuda_path(self): + def test_configure_flags_and_cuda_path(self): cmd, kwargs = self.configure(self.build(sdk_root="/mnt/sdk")) for flag in ( - "-DSENSING_WITH_SIPL=ON", - "-DSENSING_WITH_ARGUS=OFF", "-DJETSON_MMAPI_DIR=/mnt/sdk/usr/src/jetson_multimedia_api", "-DSIPL_API_DIR=/mnt/sdk/usr/src/jetson_sipl_api", "-DENABLE_CLANG_FORMAT_CHECK=OFF", @@ -1068,7 +1094,9 @@ def test_sipl_configure_flags_and_cuda_path(self): self.assertEqual(kwargs["env"]["PATH"], self.EXPECTED_PATH) def cleaned(self, calls): - return [c for c, _ in calls if c[:2] == ["rm", "-rf"] and "build-sipl" in c[2]] + return [ + c for c, _ in calls if c[:2] == ["rm", "-rf"] and c[2].endswith("/build") + ] def test_changed_source_path_cleans_build_dir_but_same_path_keeps_it(self): self.assertTrue(self.cleaned(self.build(src="/cache/src"))) @@ -1082,12 +1110,25 @@ def test_build_step_also_gets_the_cuda_path(self): self.assertEqual(kwargs["env"]["PATH"], self.EXPECTED_PATH) self.assertIn("camera_plugin_sensing", cmd) - def test_argus_has_no_sipl_dir(self): - cmd, _ = self.configure(self.build(backend="argus")) - self.assertIn("-DSENSING_WITH_ARGUS=ON", cmd) + def test_sipl_dir_omitted_when_absent(self): + cmd, _ = self.configure(self.build(sipl=False)) self.assertIn("-DJETSON_MMAPI_DIR=/usr/src/jetson_multimedia_api", cmd) self.assertFalse([a for a in cmd if a.startswith("-DSIPL_API_DIR")]) + def test_cmake_args_are_appended_last_in_order(self): + extra = ["-DENABLE_CLANG_FORMAT_CHECK=ON", "-DX=1"] + cmd, _ = self.configure(self.build(cmake_arg=extra, libcuda=True)) + self.assertEqual(cmd[-2:], extra) + self.assertGreater( + cmd.index(extra[0]), + max(i for i, a in enumerate(cmd) if a.startswith("-DCMAKE_EXE_LINKER")), + ) + + def test_changed_cmake_args_clean_build_dir(self): + self.assertTrue(self.cleaned(self.build(cmake_arg=["-DX=1"]))) + self.assertFalse(self.cleaned(self.build(cmake_arg=["-DX=1"]))) + self.assertTrue(self.cleaned(self.build(cmake_arg=["-DX=2"]))) + def test_linker_flag_only_when_gpu_libcuda_exists(self): absent, _ = self.configure(self.build(libcuda=False)) self.assertFalse([a for a in absent if "LINKER_FLAGS" in a]) From 1a4657684e6c796dec1bb6618531d0020cc35ffa Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 6 Nov 2026 08:59:37 -0800 Subject: [PATCH 07/16] sensing: stage the Orin release from the build tree and drop fixed deploy options The release is assembled from the build (binary, platform configs, the LiveKit libraries the binary resolves, LiveKit licence files) instead of the source's install rules, so any branch deploys. The service puts the release directory first on LD_LIBRARY_PATH. Remove --prefix, --service-groups, --ready-timeout and --fetch-archive; their values are constants. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CMakeLists.txt | 4 +- src/plugins/sensing/README.md | 5 +- src/plugins/sensing/app/CMakeLists.txt | 6 - src/plugins/sensing/scripts/deploy_orin.py | 135 ++++++---- .../scripts/isaaccapture-producer.service.in | 2 + tests/python/ci/test_deploy_orin.py | 232 +++++++++++++++++- 6 files changed, 315 insertions(+), 69 deletions(-) diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index cdfe43f317..0a93bee669 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -202,9 +202,7 @@ foreach(_notice LICENSE NOTICE) DOWNLOAD_NO_EXTRACT TRUE ) FetchContent_MakeAvailable(livekit_sdk_${_notice}) - # FetchContent publishes _SOURCE_DIR. - string(TOLOWER "${_notice}" _notice_lower) - set(_found "${livekit_sdk_${_notice_lower}_SOURCE_DIR}/${_notice}") + set(_found "${livekit_sdk_${_notice}_SOURCE_DIR}/${_notice}") endif() set(SENSING_LIVEKIT_${_notice}_FILE "${_found}" CACHE INTERNAL "") endforeach() diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index ee87dd5fad..ce269a20d0 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -81,9 +81,8 @@ python3 src/plugins/sensing/scripts/deploy_orin.py 192.168.1.176 --config orin.j the script; `--branch NAME` deploys that branch's committed files from it, ignoring the working tree. Without them the script deploys its own checkout, local edits included. - `--replace-unmanaged` stops a `camera_plugin_sensing` that someone started by hand (its exact PIDs are shown); - without it the deployment refuses and changes nothing. `--no-activate` builds and packages only, `--fetch-archive - DIR` copies the package back, `--jobs N` bounds compile jobs, `--prefix DIR` changes the install root - (default `/opt/isaaccapture`). + without it the deployment refuses and changes nothing. `--no-activate` builds and packages only, and `--jobs N` + bounds compile jobs. The release is installed under `/opt/isaaccapture`. - The script checks the ssh connection first, then preflights the Orin, uploads only the source files that differ from what the Orin already holds, builds, installs a versioned release, replaces the running producer and waits for it to join the room and publish frames. Steps are numbered and the upload and packing show progress. The sudo steps diff --git a/src/plugins/sensing/app/CMakeLists.txt b/src/plugins/sensing/app/CMakeLists.txt index edb7c51a74..613d396602 100644 --- a/src/plugins/sensing/app/CMakeLists.txt +++ b/src/plugins/sensing/app/CMakeLists.txt @@ -17,15 +17,11 @@ add_custom_command(TARGET camera_plugin_sensing POST_BUILD COMMENT "Staging SIPL platform configs beside camera_plugin_sensing" ) -# Everything below is the `producer` install component: `cmake --install --component producer` stages -# exactly what scripts/deploy_orin.py packages for an Orin. install(TARGETS camera_plugin_sensing RUNTIME DESTINATION plugins/sensing_camera - COMPONENT producer ) install(DIRECTORY "${SENSING_CONFIGS_DIR}/" DESTINATION plugins/sensing_camera/configs - COMPONENT producer ) # The plugin links the LiveKit runtime, whose RUNPATH points into the build tree; ship it beside the @@ -34,11 +30,9 @@ install(FILES "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit.so" "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit_ffi.so" DESTINATION plugins/sensing_camera - COMPONENT producer ) install(FILES "${SENSING_LIVEKIT_LICENSE_FILE}" "${SENSING_LIVEKIT_NOTICE_FILE}" DESTINATION plugins/sensing_camera/licenses/livekit-sdk - COMPONENT producer ) # The vendored encoder classes are compiled into the binary, so their license text ships with it. install(FILES "${SENSING_MMAPI_VENDORED_DIR}/LICENSE" diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index 9c896e641b..71248cfc5e 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -44,6 +44,9 @@ CACHE = "~/.cache/isaaccapture-deploy" CONFIG_KEYS = {"livekit_url", "token_file", "args"} # The producer's connection options come from the config's own fields, not from `args`. +PREFIX = "/opt/isaaccapture" +SERVICE_GROUPS = "video" # device access for the cameras +READY_TIMEOUT_S = 60 RESERVED_ARGS = ("--livekit-url", "--livekit-token-file") @@ -418,15 +421,6 @@ def put(self, local: Path, remote: str) -> None: ): raise DeployError(f"copying {local.name} to {self.host} failed") - def get(self, remote: str, local: Path) -> None: - if ( - self._run( - "scp", [*self._scp_quiet(), f"{self.host}:{remote}", str(local)] - ).returncode - != 0 - ): - raise DeployError(f"copying {remote} from {self.host} failed") - def close(self) -> None: self._run("ssh", ["-O", "exit", self.host], capture_output=True) shutil.rmtree(self._dir, ignore_errors=True) @@ -510,7 +504,7 @@ def deploy_source( step(1, f"Connecting to {target.host}") target.validate_connection() user = args.service_user or target.capture("id -un") - unit = render_unit(config, args.prefix, user, args.service_groups) + unit = render_unit(config, PREFIX, user, SERVICE_GROUPS) step(2, f"Preflight on {target.host}") cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") @@ -578,10 +572,6 @@ def deploy_source( manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") target.put(manifest_path, f"{stage}/MANIFEST.json") target.ssh(f"{remote} package --stage {stage} --archive {archive}") - if args.fetch_archive: - Path(args.fetch_archive).mkdir(parents=True, exist_ok=True) - target.get(archive, Path(args.fetch_archive) / Path(archive).name) - unit_path = Path(tmp) / SERVICE unit_path.write_text(unit) target.put(unit_path, f"{cache}/{SERVICE}") @@ -591,8 +581,8 @@ def deploy_source( return step(5, "Install and replace the running producer (needs sudo on the Orin)") activate = ( - f"{remote} activate --archive {archive} --prefix {args.prefix} --release {release} " - f"--unit {cache}/{SERVICE} --owner {user} --ready-timeout {args.ready_timeout}" + f"{remote} activate --archive {archive} --prefix {PREFIX} --release {release} " + f"--unit {cache}/{SERVICE} --owner {user} --ready-timeout {READY_TIMEOUT_S}" + (" --replace-unmanaged" if args.replace_unmanaged else "") ) target.sudo(activate) @@ -662,8 +652,84 @@ def remote_preflight(args: argparse.Namespace) -> None: print(" preflight ok") +def cache_value(build_dir: Path, name: str) -> str: + """One entry of the build's CMakeCache.txt, or empty.""" + for line in (build_dir / "CMakeCache.txt").read_text().splitlines(): + if line.startswith(f"{name}:"): + return line.split("=", 1)[1] + return "" + + +def livekit_notice(build_dir: Path, name: str) -> Path: + """The LiveKit SDK's LICENSE or NOTICE: where the build recorded it, else where the fetch or a local SDK put it.""" + sdk = cache_value(build_dir, "LIVEKIT_SDK_DIR") + candidates = [ + cache_value(build_dir, f"SENSING_LIVEKIT_{name}_FILE"), + str(build_dir / "_deps" / f"livekit_sdk_{name.lower()}-src" / name), + f"{sdk}/{name}" if sdk else "", + f"{sdk}/../{name}" if sdk else "", + ] + for candidate in candidates: + if candidate and Path(candidate).is_file(): + return Path(candidate) + raise DeployError(f"the LiveKit SDK's {name} file was not found for the release") + + +def stage_producer(build_dir: Path, stage: Path) -> None: + """Assemble the release from the build tree itself, so it does not depend on the source's install rules. + + The release is the binary, the SIPL platform configs the build puts beside it, the LiveKit runtime libraries + the binary resolves, and the LiveKit licence files. The service sets LD_LIBRARY_PATH to the release directory, + so the copied libraries win over the build tree's RUNPATH. + """ + binaries = [p for p in build_dir.rglob("camera_plugin_sensing") if p.is_file()] + if len(binaries) != 1: + raise DeployError( + f"expected one camera_plugin_sensing in the build, found {len(binaries)}" + ) + built = binaries[0] + out = stage / Path(BIN_REL).parent + out.mkdir(parents=True) + binary = stage / BIN_REL + shutil.copy2(built, binary) + configs = built.parent / "configs" + if not configs.is_dir(): + raise DeployError(f"the build staged no platform configs beside {built}") + shutil.copytree(configs, out / "configs") + # The libraries are found by name among what the binary resolves in the build, not by the SDK's layout. + for line in tail(["ldd", str(built)]).splitlines(): + name, _, rest = line.strip().partition(" => ") + if name.startswith("liblivekit") and rest and not rest.startswith("not found"): + shutil.copy2(rest.split(" (")[0], out / name) + if not list(out.glob("liblivekit*.so*")): + raise DeployError("no LiveKit runtime library was resolved for the producer") + notices = out / "licenses" / "livekit-sdk" + notices.mkdir(parents=True) + for name in ("LICENSE", "NOTICE"): + shutil.copy2(livekit_notice(build_dir, name), notices / name) + # Everything must resolve from the release or the system: a library found only in the build tree would vanish + # with the next clean build. + env = {**os.environ, "LD_LIBRARY_PATH": str(out)} + resolved = subprocess.run( + ["ldd", str(binary)], capture_output=True, text=True, check=False, env=env + ) + listing = (resolved.stdout + resolved.stderr).splitlines() + missing = [ln for ln in listing if "not found" in ln] + if missing: + raise DeployError( + "the producer has unresolved shared libraries on this target:\n" + + "\n".join(missing) + ) + in_build = [ln for ln in listing if str(build_dir) in ln] + if in_build: + raise DeployError( + "the producer still resolves libraries from the build tree:\n" + + "\n".join(in_build) + ) + + def remote_build(args: argparse.Namespace) -> None: - """Configure and build only the producer, then stage its install component. + """Configure and build only the producer, then stage the release from the build tree. The build directory persists between deployments and is discarded when the toolchain, L4T release, source path or CMake arguments change. @@ -735,29 +801,8 @@ def remote_build(args: argparse.Namespace) -> None: env=env, ) run(["rm", "-rf", args.stage]) - run( - [ - "cmake", - "--install", - str(build_dir), - "--component", - "producer", - "--prefix", - args.stage, - ] - ) stage = Path(args.stage) - binary = stage / BIN_REL - if not binary.exists(): - raise DeployError(f"the staged tree has no {BIN_REL}") - missing = [ - ln for ln in tail(["ldd", str(binary)]).splitlines() if "not found" in ln - ] - if missing: - raise DeployError( - "the producer has unresolved shared libraries on this target:\n" - + "\n".join(missing) - ) + stage_producer(build_dir, stage) print(f" staged {sum(1 for p in stage.rglob('*') if p.is_file())} files") @@ -939,19 +984,12 @@ def parser() -> argparse.ArgumentParser: "target", help=f"host or user@host of the Orin (a bare host uses ${USER_ENV})" ) ap.add_argument("--config", required=True, help="producer options (JSON)") - ap.add_argument("--prefix", default="/opt/isaaccapture") ap.add_argument( "--jobs", type=int, default=2, help="parallel compile jobs on the Orin" ) ap.add_argument( "--service-user", help="account the service runs as (default: the ssh user)" ) - ap.add_argument( - "--service-groups", - default="video", - help="supplementary groups for device access", - ) - ap.add_argument("--ready-timeout", type=int, default=60) ap.add_argument( "--sdk-root", default="", @@ -980,11 +1018,6 @@ def parser() -> argparse.ArgumentParser: help="extra CMake argument for the producer build, repeatable (for options only some source trees have)", ) ap.add_argument("--no-activate", action="store_true", help="build and package only") - ap.add_argument( - "--fetch-archive", - metavar="DIR", - help="also copy the package back to this directory", - ) return ap diff --git a/src/plugins/sensing/scripts/isaaccapture-producer.service.in b/src/plugins/sensing/scripts/isaaccapture-producer.service.in index f9eea4d3fe..c149c54649 100644 --- a/src/plugins/sensing/scripts/isaaccapture-producer.service.in +++ b/src/plugins/sensing/scripts/isaaccapture-producer.service.in @@ -12,6 +12,8 @@ Type=simple User=@SERVICE_USER@ SupplementaryGroups=@SERVICE_GROUPS@ WorkingDirectory=@PREFIX@/current/plugins/sensing_camera +# The release carries its own LiveKit libraries; they must win over the build-tree path the binary was linked with. +Environment=LD_LIBRARY_PATH=@PREFIX@/current/plugins/sensing_camera ExecStart=@EXEC_START@ # The producer stops on SIGINT; systemd ends anything left in this cgroup after the timeout. KillSignal=SIGINT diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index 133cec1c55..b82bd59f2b 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -208,10 +208,9 @@ def fake_run(cmd, **kwargs): target.ssh("true") target.sudo("python3 x remote activate --owner bob") target.put(Path("/tmp/a"), "/r/a") - target.get("/r/a", Path("/tmp/a")) askpass_text = Path(target._env["SSH_ASKPASS"]).read_text() target.close() - self.assertGreater(len(calls), 7) + self.assertGreater(len(calls), 6) for cmd, _ in calls: self.assertFalse([w for w in cmd if self.PASSWORD in w], cmd) _, sudo_kwargs = next( @@ -378,6 +377,38 @@ def test_cmake_args_are_forwarded_to_the_remote_build_in_order(self): build = next(v for k, v in orin.events if k == "ssh" and " build " in v) self.assertTrue(build.endswith(" --cmake-arg=-DA=1 --cmake-arg='-DB=a b'")) + def test_activate_command_line_uses_the_module_constants(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + args = deploy_orin.parser().parse_args(["u@orin", "--config", "orin.json"]) + config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} + with mock.patch.object(deploy_orin, "repo_root", return_value=root): + deploy_orin.run_deploy(args, config, orin.target) + (activate,), _ = orin.target.sudo.call_args + self.assertIn(" activate ", activate) + self.assertIn("--prefix /opt/isaaccapture", activate) + self.assertIn("--ready-timeout 60", activate) + self.assertEqual( + ( + deploy_orin.PREFIX, + deploy_orin.SERVICE_GROUPS, + deploy_orin.READY_TIMEOUT_S, + ), + ("/opt/isaaccapture", "video", 60), + ) + + def test_unit_is_rendered_with_prefix_and_service_groups(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + with mock.patch.object( + deploy_orin, "render_unit", return_value="unit\n" + ) as render: + orin.run(root) + _, prefix, _, groups = render.call_args.args + self.assertEqual( + (prefix, groups), (deploy_orin.PREFIX, deploy_orin.SERVICE_GROUPS) + ) + def test_deploy_closes_target_on_failure(self): config = self.tmp / "orin.json" config.write_text( @@ -858,7 +889,7 @@ def test_scp_is_quiet_only_when_stdout_is_not_a_tty(self): ): self.assertEqual(deploy_orin.Target._scp_quiet(), expected) - def test_put_and_get_pass_quiet_flag_to_scp(self): + def test_put_passes_quiet_flag_to_scp(self): target = make_target(self) with ( mock.patch.object(deploy_orin.sys, "stdout", FakeStdout(False)), @@ -867,7 +898,7 @@ def test_put_and_get_pass_quiet_flag_to_scp(self): ) as run, ): target.put(Path("/tmp/a"), "/r/a") - target.get("/r/a", Path("/tmp/a")) + self.assertEqual(len(run.call_args_list), 1) for call in run.call_args_list: self.assertEqual(call.args[0][0], "scp") self.assertIn("-q", call.args[0]) @@ -951,6 +982,20 @@ def test_backend_option_is_rejected_everywhere(self): ["preflight", "--backend", "sipl"] ) + def test_removed_local_options_are_rejected(self): + for flag in ("--prefix", "--service-groups", "--ready-timeout"): + with self.subTest(flag=flag): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + deploy_orin.parser().parse_args( + ["u@orin", "--config", "c", flag, "1"] + ) + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + deploy_orin.parser().parse_args( + ["u@orin", "--config", "c", "--fetch-archive"] + ) + def test_cmake_arg_is_repeatable_and_defaults_empty(self): parser = deploy_orin.parser() self.assertEqual(parser.parse_args(["h", "--config", "c"]).cmake_arg, []) @@ -1040,8 +1085,6 @@ def build( sipl=True, ): stage = self.tmp / "stage" - (stage / deploy_orin.BIN_REL).parent.mkdir(parents=True, exist_ok=True) - (stage / deploy_orin.BIN_REL).write_text("bin") args = argparse.Namespace( src=src, cmake_arg=list(cmake_arg), @@ -1071,6 +1114,7 @@ def fake_run(cmd, **kwargs): mock.patch.object(deploy_orin, "CACHE", str(self.tmp / "cache")), mock.patch.object(deploy_orin, "run", fake_run), mock.patch.object(deploy_orin, "tail", return_value="x"), + mock.patch.object(deploy_orin, "stage_producer") as self.stage_producer, mock.patch.object(Path, "exists", fake_exists), mock.patch.object(Path, "is_dir", return_value=sipl), mock.patch.object(Path, "read_text", fake_read_text), @@ -1104,6 +1148,15 @@ def test_changed_source_path_cleans_build_dir_but_same_path_keeps_it(self): self.assertTrue(self.cleaned(self.build(src="/cache/src-abc123"))) self.assertFalse(self.cleaned(self.build(src="/cache/src-abc123"))) + def test_release_is_staged_from_the_build_tree_not_cmake_install(self): + calls = self.build() + self.assertEqual( + self.stage_producer.call_args.args, + (self.tmp / "cache/build", self.tmp / "stage"), + ) + self.assertFalse([c for c, _ in calls if "--install" in c]) + self.assertFalse([a for c, _ in calls for a in c if "component" in a]) + def test_build_step_also_gets_the_cuda_path(self): calls = self.build() cmd, kwargs = next(c for c in calls if c[0][:2] == ["cmake", "--build"]) @@ -1138,5 +1191,172 @@ def test_linker_flag_only_when_gpu_libcuda_exists(self): ) +LIB_LINE = "\tliblivekit_ffi.so => {} (0x0000ffff)" + + +class CacheValueTests(TempDirTestCase): + def test_reads_typed_entries_and_ignores_prefix_lookalikes(self): + (self.tmp / "CMakeCache.txt").write_text( + "// comment\nFOO_BAR:STRING=wrong\nFOO:PATH=/a=b\nEMPTY:STRING=\n" + ) + self.assertEqual(deploy_orin.cache_value(self.tmp, "FOO"), "/a=b") + self.assertEqual(deploy_orin.cache_value(self.tmp, "EMPTY"), "") + self.assertEqual(deploy_orin.cache_value(self.tmp, "ABSENT"), "") + + +class StageProducerTests(TempDirTestCase): + """stage_producer on a fabricated build tree, with both ldd runs faked.""" + + def setUp(self): + super().setUp() + self.build = self.tmp / "build" + self.stage = self.tmp / "stage" + self.plugin = self.build / "src/plugins/sensing/app" + (self.plugin / "configs").mkdir(parents=True) + (self.plugin / "configs/platform.json").write_text("{}") + (self.plugin / "camera_plugin_sensing").write_text("bin") + self.libdir = self.build / "_deps/sdk/lib" + self.libdir.mkdir(parents=True) + (self.libdir / "liblivekit_ffi.so").write_text("lib") + self.cache = "" + self.build_ldd = LIB_LINE.format(self.libdir / "liblivekit_ffi.so") + self.release_ldd = "\tlibc.so.6 => /lib/aarch64-linux-gnu/libc.so.6 (0x1)" + self.licenses = self.tmp / "lic" + self.licenses.mkdir() + (self.licenses / "LICENSE").write_text("license") + (self.licenses / "NOTICE").write_text("notice") + + def stage_it(self, cache=None): + """Run stage_producer; cache defaults to recording both licence files.""" + if cache is None: + cache = ( + f"SENSING_LIVEKIT_LICENSE_FILE:FILEPATH={self.licenses}/LICENSE\n" + f"SENSING_LIVEKIT_NOTICE_FILE:FILEPATH={self.licenses}/NOTICE\n" + ) + (self.build / "CMakeCache.txt").write_text(cache) + runs = [] + + def fake_run(cmd, **kwargs): + runs.append((list(cmd), kwargs)) + return completed(0, stdout=self.release_ldd) + + with ( + mock.patch.object(deploy_orin, "tail", return_value=self.build_ldd), + mock.patch.object(deploy_orin.subprocess, "run", fake_run), + ): + deploy_orin.stage_producer(self.build, self.stage) + return runs + + def test_happy_path_stages_binary_configs_libs_and_licences(self): + runs = self.stage_it() + out = self.stage / "plugins/sensing_camera" + self.assertEqual((self.stage / deploy_orin.BIN_REL).read_text(), "bin") + self.assertTrue((out / "configs/platform.json").is_file()) + self.assertEqual((out / "liblivekit_ffi.so").read_text(), "lib") + self.assertEqual((out / "licenses/livekit-sdk/LICENSE").read_text(), "license") + self.assertEqual((out / "licenses/livekit-sdk/NOTICE").read_text(), "notice") + cmd, kwargs = runs[-1] + self.assertEqual(cmd, ["ldd", str(self.stage / deploy_orin.BIN_REL)]) + self.assertEqual(kwargs["env"]["LD_LIBRARY_PATH"], str(out)) + + def test_ldd_not_found_for_a_livekit_library_is_not_copied(self): + self.build_ldd = "\tliblivekit_ffi.so => not found" + with self.assertRaisesRegex(DeployError, "no LiveKit runtime library"): + self.stage_it() + + def test_no_binary_is_an_error(self): + (self.plugin / "camera_plugin_sensing").unlink() + with self.assertRaisesRegex(DeployError, "found 0"): + self.stage_it() + + def test_a_directory_with_the_binary_name_is_not_a_binary(self): + (self.plugin / "camera_plugin_sensing").unlink() + (self.build / "camera_plugin_sensing").mkdir() + with self.assertRaisesRegex(DeployError, "found 0"): + self.stage_it() + + def test_two_binaries_is_an_error(self): + other = self.build / "other" + other.mkdir() + (other / "camera_plugin_sensing").write_text("bin2") + with self.assertRaisesRegex(DeployError, "found 2"): + self.stage_it() + + def test_missing_configs_dir_is_an_error(self): + shutil.rmtree(self.plugin / "configs") + with self.assertRaisesRegex(DeployError, "platform configs"): + self.stage_it() + + def test_no_livekit_library_resolved_is_an_error(self): + self.build_ldd = "\tlibc.so.6 => /lib/libc.so.6 (0x1)" + with self.assertRaisesRegex(DeployError, "no LiveKit runtime library"): + self.stage_it() + + def test_missing_licence_is_an_error(self): + for name in ("LICENSE", "NOTICE"): + (self.licenses / name).unlink() + with self.subTest(missing=name), self.assertRaisesRegex(DeployError, name): + self.stage_it() + (self.licenses / name).write_text("x") + shutil.rmtree(self.stage) + + def test_library_still_resolved_from_the_build_tree_is_an_error(self): + self.release_ldd = LIB_LINE.format(self.libdir / "liblivekit_ffi.so") + with self.assertRaisesRegex(DeployError, "from the build tree"): + self.stage_it() + + def test_unresolved_library_is_an_error(self): + self.release_ldd = "\tlibfoo.so.1 => not found" + with self.assertRaisesRegex(DeployError, "unresolved shared libraries"): + self.stage_it() + + def test_licence_found_via_the_cache_entry(self): + self.stage_it() + self.assertEqual( + deploy_orin.livekit_notice(self.build, "LICENSE"), self.licenses / "LICENSE" + ) + + def test_licence_found_in_the_fetchcontent_directory(self): + for name in ("LICENSE", "NOTICE"): + fetched = self.build / "_deps" / f"livekit_sdk_{name.lower()}-src" + fetched.mkdir(parents=True) + (fetched / name).write_text(f"fetched {name}") + self.stage_it(cache="") + notices = self.stage / "plugins/sensing_camera/licenses/livekit-sdk" + self.assertEqual((notices / "LICENSE").read_text(), "fetched LICENSE") + self.assertEqual((notices / "NOTICE").read_text(), "fetched NOTICE") + + def test_licence_found_in_or_beside_a_local_sdk_directory(self): + sdk = self.tmp / "sdk-root/install" + sdk.mkdir(parents=True) + cache = f"LIVEKIT_SDK_DIR:PATH={sdk}\n" + (sdk / "LICENSE").write_text("in sdk") + (sdk.parent / "NOTICE").write_text("beside sdk") + self.stage_it(cache=cache) + notices = self.stage / "plugins/sensing_camera/licenses/livekit-sdk" + self.assertEqual((notices / "LICENSE").read_text(), "in sdk") + self.assertEqual((notices / "NOTICE").read_text(), "beside sdk") + + def test_cache_entry_wins_over_the_other_locations(self): + fetched = self.build / "_deps/livekit_sdk_license-src" + fetched.mkdir(parents=True) + (fetched / "LICENSE").write_text("fetched") + self.stage_it() + self.assertEqual( + ( + self.stage / "plugins/sensing_camera/licenses/livekit-sdk/LICENSE" + ).read_text(), + "license", + ) + + def test_unit_sets_ld_library_path_to_the_release_directory(self): + config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + self.assertIn( + "Environment=LD_LIBRARY_PATH=/opt/ic/current/plugins/sensing_camera", + text.splitlines(), + ) + + if __name__ == "__main__": unittest.main() From 15c162ebbf7d31b2be7731f635856e547d6fbdd5 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 6 Nov 2026 10:12:13 -0800 Subject: [PATCH 08/16] sensing: deploy a producer without LiveKit; readiness per sensor; a real failure report The LiveKit settings are optional (both or neither). Readiness requires every configured sensor to be producing (frames without LiveKit, published with it). When the producer does not become ready the script prints the service state, the producer's fault lines and the log with the driver's debug lines removed, and saves the complete log on the Orin. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/scripts/deploy_orin.py | 185 ++++++++--- tests/python/ci/test_deploy_orin.py | 351 ++++++++++++++++++++- 2 files changed, 489 insertions(+), 47 deletions(-) diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index 71248cfc5e..c5da752f4e 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -9,9 +9,7 @@ The config is JSON with the producer's own options, nothing else: - {"livekit_url": "wss://livekit.example.org", - "token_file": "/etc/isaaccapture/join.token", - "args": ["--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} + {"args": ["--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} The token file stays on the Orin, outside the release; only its path is passed. Nothing is built, stopped or replaced until the target passes its checks. A build or transfer failure leaves the running producer alone. @@ -110,18 +108,30 @@ def sha256_file(path: Path) -> str: # --------------------------------------------------------------------------- +def configured_sensors(config: dict) -> list[int]: + """The sensor numbers the config asks the producer to publish, from its --add-stream=sensor=N,... arguments.""" + sensors = [] + for arg in config.get("args", []): + if arg.startswith("--add-stream="): + match = re.search(r"(?:^|,|=)sensor=(\d+)", arg) + if match: + sensors.append(int(match.group(1))) + return sensors + + def load_config(path: Path) -> dict: try: config = json.loads(path.read_text()) except (OSError, ValueError) as exc: raise DeployError(f"cannot read the config {path}: {exc}") from exc - if ( - not isinstance(config, dict) - or set(config) - CONFIG_KEYS - or not {"livekit_url", "token_file"} <= set(config) - ): + if not isinstance(config, dict) or set(config) - CONFIG_KEYS: raise DeployError( - f"{path}: expected an object with livekit_url, token_file and optional args only" + f"{path}: expected an object with args and, optionally, livekit_url and token_file" + ) + if ("livekit_url" in config) != ("token_file" in config): + raise DeployError( + f"{path}: livekit_url and token_file go together; leave both out for a producer that sends only over " + "its direct links" ) args = config.get("args", []) if not isinstance(args, list) or not all(isinstance(a, str) for a in args): @@ -134,14 +144,18 @@ def load_config(path: Path) -> dict: raise DeployError( f"{path}: args needs at least one --add-stream=sensor=N,livekit=NAME" ) - if not all( - isinstance(config[k], str) and config[k] for k in ("livekit_url", "token_file") - ): - raise DeployError( - f"{path}: livekit_url and token_file must be non-empty strings" - ) - if not config["token_file"].startswith("/"): - raise DeployError(f"{path}: token_file must be an absolute path on the Orin") + if "token_file" in config: + if not all( + isinstance(config[k], str) and config[k] + for k in ("livekit_url", "token_file") + ): + raise DeployError( + f"{path}: livekit_url and token_file must be non-empty strings" + ) + if not config["token_file"].startswith("/"): + raise DeployError( + f"{path}: token_file must be an absolute path on the Orin" + ) return config @@ -159,8 +173,14 @@ def systemd_quote(arg: str) -> str: def render_unit(config: dict, prefix: str, user: str, groups: str) -> str: words = [ f"{prefix}/current/{BIN_REL}", - f"--livekit-url={config['livekit_url']}", - f"--livekit-token-file={config['token_file']}", + *( + [ + f"--livekit-url={config['livekit_url']}", + f"--livekit-token-file={config['token_file']}", + ] + if "livekit_url" in config + else [] + ), *config.get("args", []), ] exec_start = " ".join(systemd_quote(w) for w in words) @@ -508,7 +528,7 @@ def deploy_source( step(2, f"Preflight on {target.host}") cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") - if not target.readable(config["token_file"]): + if "token_file" in config and not target.readable(config["token_file"]): raise DeployError( f"the token file {config['token_file']} is not readable on the Orin" ) @@ -582,7 +602,9 @@ def deploy_source( step(5, "Install and replace the running producer (needs sudo on the Orin)") activate = ( f"{remote} activate --archive {archive} --prefix {PREFIX} --release {release} " - f"--unit {cache}/{SERVICE} --owner {user} --ready-timeout {READY_TIMEOUT_S}" + f"--unit {cache}/{SERVICE} --owner {user} --ready-timeout {READY_TIMEOUT_S} " + f"--sensors {','.join(str(n) for n in configured_sensors(config))}" + + ("" if "livekit_url" in config else " --no-livekit") + (" --replace-unmanaged" if args.replace_unmanaged else "") ) target.sudo(activate) @@ -701,12 +723,13 @@ def stage_producer(build_dir: Path, stage: Path) -> None: name, _, rest = line.strip().partition(" => ") if name.startswith("liblivekit") and rest and not rest.startswith("not found"): shutil.copy2(rest.split(" (")[0], out / name) - if not list(out.glob("liblivekit*.so*")): - raise DeployError("no LiveKit runtime library was resolved for the producer") - notices = out / "licenses" / "livekit-sdk" - notices.mkdir(parents=True) - for name in ("LICENSE", "NOTICE"): - shutil.copy2(livekit_notice(build_dir, name), notices / name) + # A producer that sends only over its direct links links no LiveKit; one that does carries its libraries and the + # SDK's licence files. + if list(out.glob("liblivekit*.so*")): + notices = out / "licenses" / "livekit-sdk" + notices.mkdir(parents=True) + for name in ("LICENSE", "NOTICE"): + shutil.copy2(livekit_notice(build_dir, name), notices / name) # Everything must resolve from the release or the system: a library found only in the build tree would vanish # with the next clean build. env = {**os.environ, "LD_LIBRARY_PATH": str(out)} @@ -939,17 +962,94 @@ def remote_activate(args: argparse.Namespace) -> None: os.replace(staging_link, link) since = time.strftime("%Y-%m-%d %H:%M:%S") run(["systemctl", "start", SERVICE]) - wait_ready(since, args.ready_timeout) + wait_ready( + since, + args.ready_timeout, + [int(n) for n in args.sensors.split(",") if n], + livekit=not args.no_livekit, + ) print(f" producer is publishing (release {args.release})") -def wait_ready(since: str, timeout: int) -> None: - """Ready means the producer joined the room and has published frames, as its own log says. +# The capture driver logs its own INFO and DEBUG lines at severity 5 and 6; they bury the lines that explain a failure. +DRIVER_NOISE = re.compile(r"Severity [56] :") - The two patterns come from the LiveKit session's join message and the periodic stats line in - sensing_camera.cpp; a pid alone is not readiness. + +def failure_report(since: str, sensors: list[int]) -> str: + """What to read when the producer did not become ready. + + The service's state, how much of its log is driver chatter, the first and last real lines of what remains, the + producer's own fault lines, which sensors never published, and where the complete log was saved. + """ + state = tail( + [ + "systemctl", + "show", + SERVICE, + "-p", + "ActiveState", + "-p", + "SubState", + "-p", + "Result", + ] + + ["-p", "ExecMainStatus", "-p", "NRestarts"] + ).replace("\n", ", ") + log = tail( + ["journalctl", "-u", SERVICE, "--since", since, "--no-pager", "-o", "cat"] + ) + saved = Path(os.path.expanduser(CACHE)) / "last-start-failure.log" + saved.parent.mkdir(parents=True, exist_ok=True) + saved.write_text(log + "\n") + lines = log.splitlines() + quiet = [ln for ln in lines if not DRIVER_NOISE.search(ln)] + faults = [ + ln + for ln in quiet + if re.search( + r"camera_|startup_failed|stream_failed|publish_failed|check each camera|driver log:", + ln, + ) + ] + out = [ + f"service: {state}", + f"log: {len(lines)} lines since the start, {len(lines) - len(quiet)} of them driver INFO/DEBUG; " + f"complete log saved on the Orin at {saved}", + ] + if sensors: + out.append(f"sensors expected to publish: {', '.join(map(str, sensors))}") + if faults: + out.append("producer fault lines:") + out += [f" {ln}" for ln in faults[:8]] + out.append("first lines (driver chatter removed):") + first, rest = quiet[:15], quiet[15:] + out += [f" {ln}" for ln in first] + last = rest[-15:] + if last: + hidden = len(rest) - len(last) + out.append( + f"last lines ({hidden} not shown between):" + if hidden + else "remaining lines:" + ) + out += [f" {ln}" for ln in last] + return "\n".join(out) + + +def wait_ready( + since: str, timeout: int, sensors: list[int], livekit: bool = True +) -> None: + """Ready means the producer joined the room and every configured sensor has published frames. + + The patterns come from the LiveKit session's join message and the per-sensor stats line in + sensing_camera.cpp ("sensor N: ... | livekit M published"); a pid alone is not readiness, and one publishing + camera does not make a two-camera rig ready. With no sensors known, any publishing sensor counts. A producer + without LiveKit (direct links only) never joins a room or prints "published": there every configured sensor + must be capturing ("sensor N: M frames", M >= 1). """ + progress = r"livekit [1-9][0-9]* published" if livekit else r"[1-9][0-9]* frames" deadline = time.monotonic() + timeout + log = "" while time.monotonic() < deadline: if ( subprocess.run( @@ -961,17 +1061,22 @@ def wait_ready(since: str, timeout: int) -> None: log = tail( ["journalctl", "-u", SERVICE, "--since", since, "--no-pager", "-o", "cat"] ) - if "Joined LiveKit room" in log and re.search( - r"livekit [1-9][0-9]* published", log - ): + publishing = [ + re.search(rf"sensor {n}: [^\n]*{progress}", log) for n in sensors + ] or [re.search(progress, log)] + if (not livekit or "Joined LiveKit room" in log) and all(publishing): return time.sleep(2) - print( - tail(["journalctl", "-u", SERVICE, "--since", since, "--no-pager", "-n", "40"]), - file=sys.stderr, + print(failure_report(since, sensors), file=sys.stderr) + # Judged on the log the last check saw, so the error names what that check found missing. + missing = [ + n for n in sensors if not re.search(rf"sensor {n}: [^\n]*{progress}", log) + ] + detail = ( + f"; sensors not publishing: {', '.join(map(str, missing))}" if missing else "" ) raise DeployError( - f"the producer did not become ready; logs: journalctl -u {SERVICE}" + f"the producer did not become ready{detail}; logs: journalctl -u {SERVICE}" ) @@ -1043,6 +1148,8 @@ def remote_parser() -> argparse.ArgumentParser: for name in ("archive", "prefix", "release", "unit", "owner"): act.add_argument(f"--{name}", required=True) act.add_argument("--ready-timeout", type=int, default=60) + act.add_argument("--sensors", default="") + act.add_argument("--no-livekit", action="store_true") act.add_argument("--replace-unmanaged", action="store_true") return ap diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index b82bd59f2b..2f3f8ef713 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -68,13 +68,24 @@ def test_rejects_unknown_key(self): with self.assertRaises(DeployError): self.load(self.valid(extra=1)) - def test_rejects_missing_required_keys(self): + def test_rejects_only_one_of_url_and_token_file(self): for key in ("livekit_url", "token_file"): config = self.valid() del config[key] - with self.subTest(key=key), self.assertRaises(DeployError): + with ( + self.subTest(only_missing=key), + self.assertRaisesRegex(DeployError, "go together"), + ): self.load(config) + def test_accepts_config_with_only_args(self): + config = {"args": [STREAM]} + self.assertEqual(self.load(config), config) + + def test_args_are_still_required_without_livekit(self): + with self.assertRaisesRegex(DeployError, "add-stream"): + self.load({}) + def test_rejects_relative_token_file(self): with self.assertRaisesRegex(DeployError, "absolute"): self.load(self.valid(token_file="join.token")) @@ -156,6 +167,16 @@ def test_render_unit(self): self.assertIn("WorkingDirectory=/opt/ic/current/plugins/sensing_camera", text) self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + def test_render_unit_without_livekit_args(self): + text = deploy_orin.render_unit({"args": [STREAM]}, "/opt/ic", "bob", "video") + exec_start = next(ln for ln in text.splitlines() if ln.startswith("ExecStart=")) + self.assertEqual( + exec_start, + "ExecStart=/opt/ic/current/plugins/sensing_camera/camera_plugin_sensing " + + STREAM, + ) + self.assertNotIn("--livekit", text) + def make_target(test, password=None): """A Target with its temp dir removed at cleanup; ORIN_PASSWORD is set or cleared explicitly.""" @@ -377,6 +398,16 @@ def test_cmake_args_are_forwarded_to_the_remote_build_in_order(self): build = next(v for k, v in orin.events if k == "ssh" and " build " in v) self.assertTrue(build.endswith(" --cmake-arg=-DA=1 --cmake-arg='-DB=a b'")) + def test_activate_command_line_without_livekit_appends_no_livekit(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + args = deploy_orin.parser().parse_args(["u@orin", "--config", "orin.json"]) + config = {"args": [STREAM]} + with mock.patch.object(deploy_orin, "repo_root", return_value=root): + deploy_orin.run_deploy(args, config, orin.target) + (activate,), _ = orin.target.sudo.call_args + self.assertTrue((activate + " ").count(" --no-livekit ") == 1) + def test_activate_command_line_uses_the_module_constants(self): root, _ = git_repo(self, {"src/a.cpp": "a\n"}) orin = FakeOrin() @@ -388,6 +419,8 @@ def test_activate_command_line_uses_the_module_constants(self): self.assertIn(" activate ", activate) self.assertIn("--prefix /opt/isaaccapture", activate) self.assertIn("--ready-timeout 60", activate) + self.assertIn("--sensors 0 ", activate + " ") + self.assertNotIn("--no-livekit", activate) self.assertEqual( ( deploy_orin.PREFIX, @@ -696,6 +729,22 @@ def test_remote_parser_apply_arguments(self): ) self.assertTrue(parsed.fresh) + def test_remote_parser_activate_accepts_sensors(self): + base = ["activate"] + for name in ("archive", "prefix", "release", "unit", "owner"): + base += [f"--{name}", "x"] + self.assertEqual(deploy_orin.remote_parser().parse_args(base).sensors, "") + parsed = deploy_orin.remote_parser().parse_args(base + ["--sensors", "0,1"]) + self.assertEqual(parsed.sensors, "0,1") + + def test_remote_parser_activate_no_livekit_flag(self): + base = ["activate"] + for name in ("archive", "prefix", "release", "unit", "owner"): + base += [f"--{name}", "x"] + self.assertFalse(deploy_orin.remote_parser().parse_args(base).no_livekit) + parsed = deploy_orin.remote_parser().parse_args(base + ["--no-livekit"]) + self.assertTrue(parsed.no_livekit) + class ActivateTests(TempDirTestCase): """remote_activate with subprocess faked; the unit path is redirected into the temp dir.""" @@ -748,7 +797,14 @@ def make_archive(self, good=True): tar.add(stage, arcname=".") return archive - def activate(self, archive, timeout=60, replace_unmanaged=False): + def activate( + self, + archive, + timeout=60, + replace_unmanaged=False, + sensors="", + no_livekit=False, + ): deploy_orin.remote_activate( argparse.Namespace( archive=str(archive), @@ -757,6 +813,8 @@ def activate(self, archive, timeout=60, replace_unmanaged=False): unit=str(self.unit), owner="bob", ready_timeout=timeout, + sensors=sensors, + no_livekit=no_livekit, replace_unmanaged=replace_unmanaged, ) ) @@ -819,6 +877,15 @@ def test_without_unmanaged_nothing_is_killed(self): self.activate(self.make_archive(), replace_unmanaged=True) kill.assert_not_called() + def test_no_livekit_activation_is_ready_on_frames_without_a_join(self): + self.journal = "sensor 0: 150 frames | encoded 150, dropped 0\n" + self.activate(self.make_archive(), sensors="0", no_livekit=True) + + def test_livekit_activation_still_needs_the_join(self): + self.journal = "sensor 0: 150 frames | livekit 5 published\n" + with self.assertRaises(DeployError): + self.activate(self.make_archive(), sensors="0", timeout=1) + def test_success_switches_link_between_stop_and_start(self): self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" replaced = [] @@ -1261,8 +1328,10 @@ def test_happy_path_stages_binary_configs_libs_and_licences(self): def test_ldd_not_found_for_a_livekit_library_is_not_copied(self): self.build_ldd = "\tliblivekit_ffi.so => not found" - with self.assertRaisesRegex(DeployError, "no LiveKit runtime library"): - self.stage_it() + self.stage_it() + out = self.stage / "plugins/sensing_camera" + self.assertEqual(list(out.glob("liblivekit*")), []) + self.assertFalse((out / "licenses").exists()) def test_no_binary_is_an_error(self): (self.plugin / "camera_plugin_sensing").unlink() @@ -1287,10 +1356,21 @@ def test_missing_configs_dir_is_an_error(self): with self.assertRaisesRegex(DeployError, "platform configs"): self.stage_it() - def test_no_livekit_library_resolved_is_an_error(self): + def test_no_livekit_library_resolved_stages_without_libs_or_licences(self): self.build_ldd = "\tlibc.so.6 => /lib/libc.so.6 (0x1)" - with self.assertRaisesRegex(DeployError, "no LiveKit runtime library"): - self.stage_it() + # No licence cache entries either: nothing should look for them. + self.stage_it(cache="") + out = self.stage / "plugins/sensing_camera" + self.assertEqual((self.stage / deploy_orin.BIN_REL).read_text(), "bin") + self.assertTrue((out / "configs/platform.json").is_file()) + self.assertEqual(list(out.glob("liblivekit*")), []) + self.assertFalse((out / "licenses").exists()) + + def test_livekit_library_resolved_stages_libs_and_licences(self): + self.stage_it() + out = self.stage / "plugins/sensing_camera" + self.assertTrue((out / "liblivekit_ffi.so").is_file()) + self.assertTrue((out / "licenses/livekit-sdk/LICENSE").is_file()) def test_missing_licence_is_an_error(self): for name in ("LICENSE", "NOTICE"): @@ -1358,5 +1438,260 @@ def test_unit_sets_ld_library_path_to_the_release_directory(self): ) +class ConfiguredSensorsTests(unittest.TestCase): + def sensors(self, *args): + return deploy_orin.configured_sensors({"args": list(args)}) + + def test_single_and_multiple_streams_in_order(self): + self.assertEqual(self.sensors(STREAM), [0]) + self.assertEqual( + self.sensors( + "--add-stream=sensor=0,livekit=left", + "--add-stream=sensor=1,livekit=right", + ), + [0, 1], + ) + + def test_sensor_not_first_field_and_multi_digit(self): + self.assertEqual(self.sensors("--add-stream=livekit=a,sensor=12"), [12]) + + def test_ignores_other_arguments(self): + self.assertEqual(self.sensors("--verbose", "--other=sensor=3", STREAM), [0]) + + def test_no_match(self): + self.assertEqual(self.sensors("--add-stream=livekit=x"), []) + self.assertEqual(deploy_orin.configured_sensors({}), []) + + +class ReadinessTests(TempDirTestCase): + JOINED = "Joined LiveKit room r\n" + + def run_wait(self, journal, sensors, active=True, timeout=10, livekit=True): + def fake_run(cmd, **kwargs): + return completed(0 if active else 3) + + self.journal_calls = 0 + + def fake_tail(cmd): + if cmd[0] != "journalctl": + return "ActiveState=active" + self.journal_calls += 1 + return journal + + ticks = iter(range(0, 1000, 5)) + stderr = io.StringIO() + with ( + mock.patch.object(deploy_orin.subprocess, "run", fake_run), + mock.patch.object(deploy_orin, "tail", fake_tail), + mock.patch.object(deploy_orin.time, "sleep"), + mock.patch.object(deploy_orin.time, "monotonic", lambda: next(ticks)), + mock.patch.object(deploy_orin, "CACHE", str(self.tmp)), + contextlib.redirect_stderr(stderr), + ): + try: + deploy_orin.wait_ready( + "2026-01-01 00:00:00", timeout, sensors, livekit=livekit + ) + finally: + self.stderr = stderr.getvalue() + + def test_one_of_two_sensors_publishing_names_the_other(self): + journal = ( + self.JOINED + + "sensor 0: 30 fps | livekit 12 published\n" + + "sensor 1: 0 fps | livekit 0 published\n" + ) + with self.assertRaises(DeployError) as ctx: + self.run_wait(journal, [0, 1]) + message = str(ctx.exception) + self.assertIn("sensors not publishing: 1", message) + self.assertNotIn("publishing: 0", message) + + def test_both_sensors_publishing_returns(self): + journal = ( + self.JOINED + + "sensor 0: 30 fps | livekit 12 published\n" + + "sensor 1: 30 fps | livekit 3 published\n" + ) + self.run_wait(journal, [0, 1]) + + def test_sensor_number_does_not_match_a_longer_number(self): + journal = self.JOINED + "sensor 10: 30 fps | livekit 5 published\n" + with self.assertRaisesRegex(DeployError, "sensors not publishing: 1"): + self.run_wait(journal, [1]) + + def test_without_livekit_every_sensor_with_frames_is_ready(self): + journal = ( + "sensor 0: 150 frames | encoded 150, dropped 0\n" + "sensor 1: 10 frames | encoded 10, dropped 0\n" + ) + self.run_wait(journal, [0, 1], livekit=False) + + def test_without_livekit_a_sensor_with_zero_frames_is_named(self): + journal = "sensor 0: 150 frames | encoded 150\nsensor 1: 0 frames | encoded 0\n" + with self.assertRaises(DeployError) as ctx: + self.run_wait(journal, [0, 1], livekit=False) + self.assertIn("sensors not publishing: 1", str(ctx.exception)) + self.assertNotIn("publishing: 0", str(ctx.exception)) + + def test_without_livekit_sensor_number_is_not_confused_with_a_longer_one(self): + journal = "sensor 0: 5 frames\nsensor 10: 10 frames\n" + with self.assertRaisesRegex(DeployError, "sensors not publishing: 1;"): + self.run_wait(journal, [0, 1], livekit=False) + self.run_wait("sensor 1: 10 frames\n", [1], livekit=False) + + def test_without_livekit_a_zero_frames_line_with_a_later_count_digit(self): + with self.assertRaises(DeployError): + self.run_wait("sensor 0: 0 frames\n", [0], livekit=False) + + def test_with_livekit_frames_alone_are_not_readiness(self): + with self.assertRaises(DeployError): + self.run_wait(self.JOINED + "sensor 0: 150 frames\n", [0]) + + def test_requires_joining_the_room(self): + with self.assertRaises(DeployError): + self.run_wait("sensor 0: x | livekit 5 published\n", [0]) + + def test_no_sensors_any_publishing_counts(self): + self.run_wait(self.JOINED + "stats livekit 7 published\n", []) + + def test_no_sensors_nothing_publishing_fails_without_a_sensor_list(self): + with self.assertRaises(DeployError) as ctx: + self.run_wait(self.JOINED + "livekit 0 published\n", []) + self.assertNotIn("sensors not publishing", str(ctx.exception)) + + def test_inactive_service_stops_waiting_and_reports(self): + ticks_used = [] + with mock.patch.object(deploy_orin.time, "sleep", ticks_used.append): + with self.assertRaises(DeployError): + self.run_wait(self.JOINED + "livekit 5 published\n", [], active=False) + self.assertEqual(ticks_used, []) + + def test_inactive_service_lists_every_sensor_missing(self): + with self.assertRaisesRegex(DeployError, "sensors not publishing: 0, 1"): + self.run_wait("sensor 0: x | livekit 5 published\n", [0, 1], active=False) + + def test_inactive_service_never_fetches_for_the_loop(self): + with self.assertRaises(DeployError): + self.run_wait(self.JOINED, [0], active=False) + self.assertEqual(self.journal_calls, 1) # the report's fetch only + + def test_timeout_adds_no_journal_fetch_beyond_loop_and_report(self): + # Clock ticks 0 (deadline = 10), 5 (one loop fetch), 10 (expired); + # the failure report then makes the only other fetch. + with self.assertRaises(DeployError): + self.run_wait( + self.JOINED + "sensor 0: x | livekit 0 published\n", [0], timeout=10 + ) + self.assertEqual(self.journal_calls, 2) + + def test_failure_prints_the_report_to_stderr(self): + with self.assertRaises(DeployError): + self.run_wait("boom\n", [0], active=False) + self.assertIn("service: ActiveState=active", self.stderr) + self.assertIn("boom", self.stderr) + + +class FailureReportTests(TempDirTestCase): + def report(self, log, sensors=(0, 1)): + def fake_tail(cmd): + if cmd[0] == "systemctl": + return "ActiveState=failed\nSubState=failed\nNRestarts=2" + return log + + with ( + mock.patch.object(deploy_orin, "tail", fake_tail), + mock.patch.object(deploy_orin, "CACHE", str(self.tmp / "cache")), + ): + return deploy_orin.failure_report("since", list(sensors)) + + def test_noise_is_filtered_out_and_counted(self): + log = "\n".join( + [ + "Severity 5 : driver info", + "Severity 6 : driver debug", + "real line one", + "real line two", + ] + ) + out = self.report(log) + self.assertNotIn("driver info", out) + self.assertNotIn("driver debug", out) + self.assertIn("4 lines since the start, 2 of them driver INFO/DEBUG", out) + self.assertIn(" real line one", out) + + def test_service_state_and_sensors_listed(self): + out = self.report("x") + self.assertIn("service: ActiveState=failed, SubState=failed, NRestarts=2", out) + self.assertIn("sensors expected to publish: 0, 1", out) + self.assertNotIn("sensors expected", self.report("x", sensors=())) + + def test_fault_lines_listed_and_capped_at_eight(self): + log = "\n".join(f"camera_{i} startup_failed" for i in range(12)) + out = self.report(log) + self.assertIn("producer fault lines:", out) + faults = out.split("producer fault lines:")[1].split("first lines")[0] + self.assertEqual(len([ln for ln in faults.splitlines() if ln.strip()]), 8) + + def test_no_fault_section_without_fault_lines(self): + self.assertNotIn("producer fault lines", self.report("plain")) + + @staticmethod + def numbered(n): + return "\n".join(f"line {i:02d}" for i in range(n)) + + @staticmethod + def listed(out): + return [ln.strip() for ln in out.splitlines() if ln.startswith(" line ")] + + def test_first_and_last_lines_with_elision_count(self): + out = self.report(self.numbered(40)) + self.assertIn("last lines (10 not shown between):", out) + head, tail_part = out.split("last lines") + self.assertIn("line 00", head) + self.assertIn("line 14", head) + self.assertNotIn("line 15", head) + self.assertNotIn("line 24", tail_part) + self.assertIn("line 25", tail_part) + self.assertIn("line 39", tail_part) + + def test_16_to_29_quiet_lines_have_no_duplicates(self): + for n in (16, 22, 29, 30): + with self.subTest(n=n): + out = self.report(self.numbered(n)) + shown = self.listed(out) + self.assertEqual(shown, [f"line {i:02d}" for i in range(n)]) + self.assertIn("remaining lines:", out) + self.assertNotIn("not shown", out) + self.assertNotIn("last lines", out) + + def test_31_lines_hides_one(self): + out = self.report(self.numbered(31)) + self.assertIn("last lines (1 not shown between):", out) + self.assertEqual(len(self.listed(out)), 30) + + def test_15_or_fewer_lines_have_no_last_section(self): + for n in (1, 14, 15): + with self.subTest(n=n): + out = self.report(self.numbered(n)) + self.assertNotIn("last lines", out) + self.assertNotIn("remaining lines", out) + self.assertNotIn("not shown", out) + self.assertEqual(len(self.listed(out)), n) + + def test_complete_journal_saved_under_cache(self): + log = "Severity 5 : noise\nreal\n" + out = self.report(log) + saved = self.tmp / "cache" / "last-start-failure.log" + self.assertEqual(saved.read_text(), log + "\n") + self.assertIn(str(saved), out) + + def test_empty_log(self): + out = self.report("") + self.assertIn("log: 0 lines since the start, 0 of them", out) + self.assertTrue((self.tmp / "cache" / "last-start-failure.log").exists()) + self.assertNotIn("last lines", out) + + if __name__ == "__main__": unittest.main() From c85d76ec76c71409cd21ae94182da5b467744cdf Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 6 Nov 2026 12:52:57 -0800 Subject: [PATCH 09/16] sensing: judge deploy readiness on the current session; one manifest comparison The config names its mode (direct or livekit). Readiness is each configured sensor's latest status line since the producer's last start: frames captured and, for a stream with a direct destination, that link connected. Progress left over from an earlier attempt no longer counts. The file-grouping layer is removed: one comparison of per-file hashes decides what to upload. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 19 +- src/plugins/sensing/scripts/deploy_orin.py | 196 ++++--- tests/python/ci/test_deploy_orin.py | 562 +++++++++++++++++---- 3 files changed, 585 insertions(+), 192 deletions(-) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index ce269a20d0..5320d21241 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -54,10 +54,20 @@ has the build and run steps. `isaaccapture-producer.service` systemd service, so you do not build by hand on the Orin. It needs `ssh`, `scp` and `git` on the desktop, and `python3`, CMake, g++, CUDA, `uv` and the SDK headers (see `scripts/setup.sh`) on the Orin. -Write a config naming the producer's own options (the LiveKit token file stays on the Orin; only its path is used): +Write a config naming the producer's own options. `"mode"` says what the producer does and so how readiness is +judged: `"direct"` (the default) for a producer that sends only over TCP to the PC video stage, or `"livekit"` for one +that publishes to LiveKit itself, which also needs `livekit_url` and `token_file` (the token file stays on the Orin; +only its path is used). Everything else goes in `args`, spelled as the deployed source's producer expects. ```json -{"livekit_url": "ws://127.0.0.1:7880", +{"args": ["--link-masks=0x0000 0x1100", + "--add-stream=sensor=0,name=sensing-left,direct=PC_HOST:5600", + "--add-stream=sensor=1,name=sensing-right,direct=PC_HOST:5601"]} +``` + +```json +{"mode": "livekit", + "livekit_url": "ws://127.0.0.1:7880", "token_file": "/home/ido/livekit/join.token", "args": ["--link-masks=0x0000 0x1100", "--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} @@ -85,8 +95,9 @@ python3 src/plugins/sensing/scripts/deploy_orin.py 192.168.1.176 --config orin.j bounds compile jobs. The release is installed under `/opt/isaaccapture`. - The script checks the ssh connection first, then preflights the Orin, uploads only the source files that differ from what the Orin already holds, builds, installs a versioned release, replaces the running producer and waits for - it to join the room and publish frames. Steps are numbered and the upload and packing show progress. The sudo steps - prompt for the Orin password. + every configured camera to be up. "Up" is the producer's own latest output for its current session: frames captured + and, for a stream with a `direct=` destination, that link connected (a stream with none is reported as capture-only). + Steps are numbered and the upload and packing show progress. The sudo steps prompt for the Orin password. - A failed build or transfer leaves the running producer untouched. There is no automatic rollback; earlier releases stay under `/opt/isaaccapture/releases/`. Logs: `journalctl -u isaaccapture-producer.service -f` on the Orin. Stop it with `sudo systemctl stop isaaccapture-producer.service` before starting a producer by hand: the cameras are diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index c5da752f4e..e0d44ba48b 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -40,7 +40,8 @@ BIN_REL = "plugins/sensing_camera/camera_plugin_sensing" UNIT_TEMPLATE = Path(__file__).with_name("isaaccapture-producer.service.in") CACHE = "~/.cache/isaaccapture-deploy" -CONFIG_KEYS = {"livekit_url", "token_file", "args"} +MODES = ("direct", "livekit") +CONFIG_KEYS = {"mode", "livekit_url", "token_file", "args"} # The producer's connection options come from the config's own fields, not from `args`. PREFIX = "/opt/isaaccapture" SERVICE_GROUPS = "video" # device access for the cameras @@ -108,15 +109,26 @@ def sha256_file(path: Path) -> str: # --------------------------------------------------------------------------- -def configured_sensors(config: dict) -> list[int]: - """The sensor numbers the config asks the producer to publish, from its --add-stream=sensor=N,... arguments.""" - sensors = [] +def configured_streams(config: dict) -> list[tuple[int, bool]]: + """(sensor, has a direct destination) for each --add-stream=sensor=N,... argument of the config.""" + streams = [] for arg in config.get("args", []): - if arg.startswith("--add-stream="): - match = re.search(r"(?:^|,|=)sensor=(\d+)", arg) - if match: - sensors.append(int(match.group(1))) - return sensors + if not arg.startswith("--add-stream="): + continue + # The argument is a comma-separated list of key=value pairs, in any order. + pairs = dict( + item.partition("=")[::2] + for item in arg.split("=", 1)[1].split(",") + if "=" in item + ) + if pairs.get("sensor", "").isdigit(): + streams.append((int(pairs["sensor"]), bool(pairs.get("direct")))) + return streams + + +def configured_sensors(config: dict) -> list[int]: + """The sensor numbers the config asks the producer to publish.""" + return [sensor for sensor, _ in configured_streams(config)] def load_config(path: Path) -> dict: @@ -128,11 +140,16 @@ def load_config(path: Path) -> dict: raise DeployError( f"{path}: expected an object with args and, optionally, livekit_url and token_file" ) - if ("livekit_url" in config) != ("token_file" in config): + mode = config.get("mode", "direct") + if mode not in MODES: + raise DeployError(f'{path}: "mode" must be one of {", ".join(MODES)}') + has_livekit = "livekit_url" in config or "token_file" in config + if mode == "direct" and has_livekit: raise DeployError( - f"{path}: livekit_url and token_file go together; leave both out for a producer that sends only over " - "its direct links" + f'{path}: livekit_url and token_file belong to "mode": "livekit"; a direct producer has no LiveKit settings' ) + if mode == "livekit" and not ("livekit_url" in config and "token_file" in config): + raise DeployError(f'{path}: "mode": "livekit" needs livekit_url and token_file') args = config.get("args", []) if not isinstance(args, list) or not all(isinstance(a, str) for a in args): raise DeployError(f"{path}: args must be a list of strings") @@ -178,7 +195,7 @@ def render_unit(config: dict, prefix: str, user: str, groups: str) -> str: f"--livekit-url={config['livekit_url']}", f"--livekit-token-file={config['token_file']}", ] - if "livekit_url" in config + if config.get("mode") == "livekit" else [] ), *config.get("args", []), @@ -212,21 +229,9 @@ def repo_root() -> Path: return Path(out.stdout.strip()) -# Files that change while developing the producer, and everything else, which rarely does. A group whose digest -# matches the Orin's is not compared file by file. -VOLATILE_PREFIXES = ("src/", "cmake/", "tests/", "examples/egocap/") -GROUPS = ("source", "bulk") DEPLOY_DIR = ".deploy" -def file_group(path: str) -> str: - return ( - "source" - if path == "CMakeLists.txt" or path.startswith(VOLATILE_PREFIXES) - else "bulk" - ) - - def tracked_files(root: Path) -> list[str]: """The tracked files that exist on disk, so tracked local edits are included and untracked files never are.""" listing = subprocess.run( @@ -252,28 +257,13 @@ def hash_files(root: Path, files: list[str]) -> dict[str, str]: return entries -def group_digest(entries: dict[str, str], group: str) -> str: - lines = ( - f"{name}\0{entries[name]}" - for name in sorted(entries) - if file_group(name) == group - ) - return hashlib.sha256("\n".join(lines).encode()).hexdigest() - - def diff_manifests( local: dict[str, str], remote: dict[str, str] ) -> tuple[list[str], list[str]]: - """(files to send, files to remove on the Orin). A group with an equal digest is skipped without comparing.""" - send, removed = [], [] - for group in GROUPS: - mine = {n: h for n, h in local.items() if file_group(n) == group} - theirs = {n: h for n, h in remote.items() if file_group(n) == group} - if group_digest(local, group) == group_digest(remote, group): - continue - send += [n for n, h in mine.items() if theirs.get(n) != h] - removed += [n for n in theirs if n not in mine] - return sorted(send), sorted(removed) + """(files to send, files to remove on the Orin): one comparison of the two per-file manifests.""" + send = sorted(n for n, h in local.items() if remote.get(n) != h) + removed = sorted(n for n in remote if n not in local) + return send, removed def read_remote_manifest(target: "Target", cache: str) -> dict[str, str]: @@ -528,7 +518,7 @@ def deploy_source( step(2, f"Preflight on {target.host}") cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") - if "token_file" in config and not target.readable(config["token_file"]): + if config.get("mode") == "livekit" and not target.readable(config["token_file"]): raise DeployError( f"the token file {config['token_file']} is not readable on the Orin" ) @@ -554,14 +544,11 @@ def deploy_source( src = f"{cache}/src" remote_manifest = read_remote_manifest(target, cache) send, removed = diff_manifests(local, remote_manifest) - for group in GROUPS: - names = [n for n in local if file_group(n) == group] - to_send = [n for n in send if file_group(n) == group] - size = sum((root / n).stat().st_size for n in to_send) - print( - f" {group}: {len(names) - len(to_send)} identical, {len(to_send)} to send " - f"({size / 1e6:.1f} MB), {sum(1 for n in removed if file_group(n) == group)} to remove" - ) + size = sum((root / n).stat().st_size for n in send) + print( + f" {len(local) - len(send)} identical, {len(send)} to send ({size / 1e6:.1f} MB), " + f"{len(removed)} to remove" + ) if send or removed: delta = Path(tmp) / "delta.tar.gz" build_delta(root, send, removed, local, delta) @@ -603,8 +590,9 @@ def deploy_source( activate = ( f"{remote} activate --archive {archive} --prefix {PREFIX} --release {release} " f"--unit {cache}/{SERVICE} --owner {user} --ready-timeout {READY_TIMEOUT_S} " - f"--sensors {','.join(str(n) for n in configured_sensors(config))}" - + ("" if "livekit_url" in config else " --no-livekit") + f"--sensors={','.join(str(n) for n in configured_sensors(config))} " + f"--direct-sensors={','.join(str(n) for n, direct in configured_streams(config) if direct)} " + f"--mode={config.get('mode', 'direct')}" + (" --replace-unmanaged" if args.replace_unmanaged else "") ) target.sudo(activate) @@ -962,13 +950,16 @@ def remote_activate(args: argparse.Namespace) -> None: os.replace(staging_link, link) since = time.strftime("%Y-%m-%d %H:%M:%S") run(["systemctl", "start", SERVICE]) - wait_ready( + summary = wait_ready( since, args.ready_timeout, [int(n) for n in args.sensors.split(",") if n], - livekit=not args.no_livekit, + mode=args.mode, + direct_sensors=[int(n) for n in args.direct_sensors.split(",") if n], ) - print(f" producer is publishing (release {args.release})") + for line in summary: + print(f" {line}") + print(f" producer is up (release {args.release})") # The capture driver logs its own INFO and DEBUG lines at severity 5 and 6; they bury the lines that explain a failure. @@ -1036,18 +1027,68 @@ def failure_report(since: str, sensors: list[int]) -> str: return "\n".join(out) -def wait_ready( - since: str, timeout: int, sensors: list[int], livekit: bool = True -) -> None: - """Ready means the producer joined the room and every configured sensor has published frames. +def current_session(log: str, mode: str) -> str: + """The journal text of the producer's current session: what follows its last start marker. - The patterns come from the LiveKit session's join message and the per-sensor stats line in - sensing_camera.cpp ("sensor N: ... | livekit M published"); a pid alone is not readiness, and one publishing - camera does not make a two-camera rig ready. With no sensors known, any publishing sensor counts. A producer - without LiveKit (direct links only) never joins a room or prints "published": there every configured sensor - must be capturing ("sensor N: M frames", M >= 1). + A producer that restarts its cameras inside one process (it retries after a fault) logs the marker each time, + so progress from an earlier attempt never counts toward the current one. """ - progress = r"livekit [1-9][0-9]* published" if livekit else r"[1-9][0-9]* frames" + marker = "Capturing." if mode == "direct" else "Joined LiveKit room" + return log.rsplit(marker, 1)[1] if marker in log else "" + + +def readiness( + log: str, mode: str, sensors: list[int], direct_sensors: list[int] +) -> tuple[list[str], list[str]]: + """(what is ready, what is not) for every configured sensor, judged on the latest status line of the session. + + direct mode: a sensor is up when its latest line shows frames and, if the config gave it a direct destination, + that link connected; a stream with no destination is reported as capture-only. livekit mode (an older producer + that publishes to LiveKit itself): the latest line shows a positive "livekit N published". + """ + current = current_session(log, mode) + ready, problems = [], [] + if not current: + marker = "capturing" if mode == "direct" else "joined a LiveKit room" + return ready, [f"the producer has not reported {marker} yet"] + for n in sensors: + if mode == "livekit": + found = re.findall(rf"sensor {n}: [^\n]*livekit (\d+) published", current) + if found and int(found[-1]) > 0: + ready.append(f"sensor {n}: publishing to LiveKit") + else: + problems.append(f"sensor {n}: not publishing to LiveKit") + continue + lines = re.findall(rf"sensor {n}: (\d+) frames([^\n]*)", current) + if not lines or int(lines[-1][0]) == 0: + problems.append(f"sensor {n}: no frames captured") + continue + rest = lines[-1][1] + if n not in direct_sensors: + ready.append(f"sensor {n}: capturing (no direct destination configured)") + elif re.search(r"direct: \S+ connected", rest) and "NOT connected" not in rest: + ready.append(f"sensor {n}: capturing and connected to its destination") + else: + reason = re.search(r"direct: (\S+ NOT connected[^\n]*)", rest) + problems.append( + f"sensor {n}: capturing but {reason.group(1) if reason else 'its direct link is not connected'}" + ) + return ready, problems + + +def wait_ready( + since: str, + timeout: int, + sensors: list[int], + mode: str = "direct", + direct_sensors: list[int] | None = None, +) -> list[str]: + """Wait until the current session is up for every configured sensor; return one line per sensor. + + "Up" is the producer's own current output (see readiness()), never a pid and never progress left over from an + earlier attempt. A producer that is not up in time raises, with the sensors that were not. + """ + direct_sensors = direct_sensors or [] deadline = time.monotonic() + timeout log = "" while time.monotonic() < deadline: @@ -1061,20 +1102,14 @@ def wait_ready( log = tail( ["journalctl", "-u", SERVICE, "--since", since, "--no-pager", "-o", "cat"] ) - publishing = [ - re.search(rf"sensor {n}: [^\n]*{progress}", log) for n in sensors - ] or [re.search(progress, log)] - if (not livekit or "Joined LiveKit room" in log) and all(publishing): - return + ready, problems = readiness(log, mode, sensors, direct_sensors) + if not problems: + return ready time.sleep(2) print(failure_report(since, sensors), file=sys.stderr) # Judged on the log the last check saw, so the error names what that check found missing. - missing = [ - n for n in sensors if not re.search(rf"sensor {n}: [^\n]*{progress}", log) - ] - detail = ( - f"; sensors not publishing: {', '.join(map(str, missing))}" if missing else "" - ) + _, problems = readiness(log, mode, sensors, direct_sensors) + detail = f"; {'; '.join(problems)}" if problems else "" raise DeployError( f"the producer did not become ready{detail}; logs: journalctl -u {SERVICE}" ) @@ -1149,7 +1184,8 @@ def remote_parser() -> argparse.ArgumentParser: act.add_argument(f"--{name}", required=True) act.add_argument("--ready-timeout", type=int, default=60) act.add_argument("--sensors", default="") - act.add_argument("--no-livekit", action="store_true") + act.add_argument("--direct-sensors", default="") + act.add_argument("--mode", choices=MODES, default="direct") act.add_argument("--replace-unmanaged", action="store_true") return ap diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index 2f3f8ef713..d04d7c9627 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -53,6 +53,7 @@ def load(self, config): def valid(self, **override): config = { + "mode": "livekit", "livekit_url": "wss://lk.example.org", "token_file": "/etc/isaaccapture/join.token", "args": [STREAM], @@ -68,21 +69,55 @@ def test_rejects_unknown_key(self): with self.assertRaises(DeployError): self.load(self.valid(extra=1)) - def test_rejects_only_one_of_url_and_token_file(self): + def test_livekit_mode_needs_both_url_and_token_file(self): for key in ("livekit_url", "token_file"): config = self.valid() del config[key] with ( self.subTest(only_missing=key), - self.assertRaisesRegex(DeployError, "go together"), + self.assertRaisesRegex(DeployError, "needs livekit_url and token_file"), ): self.load(config) + def test_mode_defaults_to_direct_and_direct_is_accepted(self): + for config in ({"args": [STREAM]}, {"mode": "direct", "args": [STREAM]}): + with self.subTest(config=config): + self.assertEqual(self.load(config), config) + + def test_direct_mode_forbids_livekit_settings(self): + extras = ( + {"livekit_url": "wss://x"}, + {"token_file": "/t"}, + {"livekit_url": "wss://x", "token_file": "/t"}, + ) + for extra in extras: + for mode in ({}, {"mode": "direct"}): + with ( + self.subTest(extra=extra, mode=mode), + self.assertRaisesRegex(DeployError, "belong to"), + ): + self.load({"args": [STREAM], **mode, **extra}) + + def test_livekit_mode_without_any_livekit_settings_is_rejected(self): + with self.assertRaisesRegex(DeployError, "needs livekit_url"): + self.load({"mode": "livekit", "args": [STREAM]}) + + def test_mode_must_be_one_of_the_known_modes(self): + for mode in ("LiveKit", "", "both", None, 3): + with ( + self.subTest(mode=mode), + self.assertRaisesRegex(DeployError, "must be one of direct, livekit"), + ): + self.load({"mode": mode, "args": [STREAM]}) + + def test_mode_is_an_allowed_config_key(self): + self.assertIn("mode", deploy_orin.CONFIG_KEYS) + def test_accepts_config_with_only_args(self): config = {"args": [STREAM]} self.assertEqual(self.load(config), config) - def test_args_are_still_required_without_livekit(self): + def test_args_are_still_required_in_direct_mode(self): with self.assertRaisesRegex(DeployError, "add-stream"): self.load({}) @@ -149,6 +184,7 @@ def test_quote_wraps_empty_and_single_quote_args(self): def test_render_unit(self): config = { + "mode": "livekit", "livekit_url": "wss://lk.example.org", "token_file": "/etc/t oken", "args": [STREAM, "--name=100%", "--x=$Y"], @@ -167,8 +203,13 @@ def test_render_unit(self): self.assertIn("WorkingDirectory=/opt/ic/current/plugins/sensing_camera", text) self.assertIsNone(re.search(r"@[A-Z_]+@", text)) - def test_render_unit_without_livekit_args(self): - text = deploy_orin.render_unit({"args": [STREAM]}, "/opt/ic", "bob", "video") + def test_render_unit_direct_mode_has_no_livekit_args(self): + for config in ({"args": [STREAM]}, {"mode": "direct", "args": [STREAM]}): + with self.subTest(config=config): + self.assert_direct_unit(config) + + def assert_direct_unit(self, config): + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") exec_start = next(ln for ln in text.splitlines() if ln.startswith("ExecStart=")) self.assertEqual( exec_start, @@ -329,7 +370,12 @@ def run(self, root, *extra): *extra, ] ) - config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} + config = { + "mode": "livekit", + "livekit_url": "wss://x", + "token_file": "/t", + "args": [STREAM], + } with mock.patch.object(deploy_orin, "repo_root", return_value=root): deploy_orin.run_deploy(args, config, self.target) @@ -398,28 +444,65 @@ def test_cmake_args_are_forwarded_to_the_remote_build_in_order(self): build = next(v for k, v in orin.events if k == "ssh" and " build " in v) self.assertTrue(build.endswith(" --cmake-arg=-DA=1 --cmake-arg='-DB=a b'")) - def test_activate_command_line_without_livekit_appends_no_livekit(self): + def activate_command(self, config): root, _ = git_repo(self, {"src/a.cpp": "a\n"}) orin = FakeOrin() args = deploy_orin.parser().parse_args(["u@orin", "--config", "orin.json"]) - config = {"args": [STREAM]} with mock.patch.object(deploy_orin, "repo_root", return_value=root): deploy_orin.run_deploy(args, config, orin.target) (activate,), _ = orin.target.sudo.call_args - self.assertTrue((activate + " ").count(" --no-livekit ") == 1) + return activate + + def activate_options(self, config): + activate = self.activate_command(config) + return activate[activate.index(" --sensors=") :] + + def test_activate_command_direct_two_sensors(self): + config = { + "args": [ + "--add-stream=sensor=0,direct=10.0.0.1:5000", + "--add-stream=sensor=1,direct=10.0.0.1:5001", + ] + } + self.assertEqual( + self.activate_options(config), + " --sensors=0,1 --direct-sensors=0,1 --mode=direct", + ) + + def test_activate_command_only_streams_with_a_direct_key_are_direct_sensors(self): + config = { + "args": [ + "--add-stream=sensor=0,direct=10.0.0.1:5000", + "--add-stream=sensor=1,livekit=right", + ] + } + self.assertEqual( + self.activate_options(config), + " --sensors=0,1 --direct-sensors=0 --mode=direct", + ) + + def test_activate_command_with_no_direct_stream_has_an_empty_direct_list(self): + self.assertEqual( + self.activate_options({"args": [STREAM]}), + " --sensors=0 --direct-sensors= --mode=direct", + ) + + def test_activate_command_livekit_mode(self): + config = { + "mode": "livekit", + "livekit_url": "wss://x", + "token_file": "/t", + "args": [STREAM], + } + activate = self.activate_command(config) + self.assertIn(" --sensors=0 --direct-sensors= --mode=livekit", activate) + self.assertNotIn("--no-livekit", activate) def test_activate_command_line_uses_the_module_constants(self): - root, _ = git_repo(self, {"src/a.cpp": "a\n"}) - orin = FakeOrin() - args = deploy_orin.parser().parse_args(["u@orin", "--config", "orin.json"]) - config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} - with mock.patch.object(deploy_orin, "repo_root", return_value=root): - deploy_orin.run_deploy(args, config, orin.target) - (activate,), _ = orin.target.sudo.call_args + activate = self.activate_command({"args": [STREAM]}) self.assertIn(" activate ", activate) self.assertIn("--prefix /opt/isaaccapture", activate) self.assertIn("--ready-timeout 60", activate) - self.assertIn("--sensors 0 ", activate + " ") self.assertNotIn("--no-livekit", activate) self.assertEqual( ( @@ -445,7 +528,14 @@ def test_unit_is_rendered_with_prefix_and_service_groups(self): def test_deploy_closes_target_on_failure(self): config = self.tmp / "orin.json" config.write_text( - json.dumps({"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]}) + json.dumps( + { + "mode": "livekit", + "livekit_url": "wss://x", + "token_file": "/t", + "args": [STREAM], + } + ) ) args = deploy_orin.parser().parse_args(["u@orin", "--config", str(config)]) target = mock.Mock() @@ -548,12 +638,6 @@ def test_run_deploy_with_branch_uploads_the_branch_files(self): class SyncTests(TempDirTestCase): - def test_file_group(self): - for name in ("CMakeLists.txt", "src/a.cpp", "cmake/x.cmake", "tests/t.py", "examples/egocap/x"): # fmt: skip - self.assertEqual(deploy_orin.file_group(name), "source", name) - for name in ("deps/lib/x.c", "README.md", "examples/other/x", "docs/CMakeLists.txt"): # fmt: skip - self.assertEqual(deploy_orin.file_group(name), "bulk", name) - def test_tracked_files_include_edits_not_untracked(self): root, _ = git_repo(self, {"a.txt": "1\n", "sub/b.txt": "2\n"}) (root / "a.txt").write_text("edited\n") @@ -576,40 +660,28 @@ def test_hash_files_records_exec_bit(self): entries["run.sh"], deploy_orin.sha256_file(root / "run.sh") + "x" ) - def test_group_digest_is_stable_and_tracks_changes(self): - entries = {"src/a": "h1-", "src/b": "h2-", "README.md": "h3-"} - reordered = {"README.md": "h3-", "src/b": "h2-", "src/a": "h1-"} - digest = deploy_orin.group_digest - self.assertEqual(digest(entries, "source"), digest(reordered, "source")) - for changed in ({"src/a": "h9-"}, {"src/a": "h1x"}, {"src/c": "h4-"}): - self.assertNotEqual( - digest(entries, "source"), digest({**entries, **changed}, "source") - ) - self.assertEqual( - digest(entries, "bulk"), digest({**entries, "src/a": "h9-"}, "bulk") - ) - def test_diff_identical_sends_nothing(self): entries = {"src/a": "1-", "README.md": "2-"} self.assertEqual(deploy_orin.diff_manifests(entries, dict(entries)), ([], [])) - def test_diff_changed_new_and_removed_per_group(self): + def test_diff_changed_new_and_removed(self): local = {"src/a": "1-", "src/new": "n-", "src/same": "s-", "README.md": "r2-", "doc": "d-"} # fmt: skip remote = {"src/a": "0-", "src/same": "s-", "src/gone": "g-", "README.md": "r1-", "doc": "d-", "old.txt": "o-"} # fmt: skip send, removed = deploy_orin.diff_manifests(local, remote) self.assertEqual(send, ["README.md", "src/a", "src/new"]) self.assertEqual(removed, ["old.txt", "src/gone"]) - def test_diff_skips_a_group_with_equal_digest_without_comparing_files(self): - local = {"src/a": "1-", "README.md": "r2-"} - remote = {"src/a": "0-", "README.md": "r1-"} - - def fake_digest(entries, group): - return "same" if group == "source" else repr(sorted(entries.items())) + def test_diff_is_per_file_whatever_the_path(self): + local = {"src/a": "1x", "src/b": "2-", "deps/x": "3-", "z": "4-"} + remote = {"src/a": "1-", "src/b": "2-", "deps/x": "9-", "gone/y": "5-"} + self.assertEqual( + deploy_orin.diff_manifests(local, remote), + (["deps/x", "src/a", "z"], ["gone/y"]), + ) - with mock.patch.object(deploy_orin, "group_digest", fake_digest): - send, removed = deploy_orin.diff_manifests(local, remote) - self.assertEqual((send, removed), (["README.md"], [])) + def test_grouping_helpers_are_gone(self): + for name in ("GROUPS", "VOLATILE_PREFIXES", "file_group", "group_digest"): + self.assertFalse(hasattr(deploy_orin, name), name) def test_diff_against_empty_remote_sends_everything(self): local = {"src/a": "1-", "README.md": "2-"} @@ -737,13 +809,20 @@ def test_remote_parser_activate_accepts_sensors(self): parsed = deploy_orin.remote_parser().parse_args(base + ["--sensors", "0,1"]) self.assertEqual(parsed.sensors, "0,1") - def test_remote_parser_activate_no_livekit_flag(self): + def test_remote_parser_activate_mode_and_direct_sensors(self): base = ["activate"] for name in ("archive", "prefix", "release", "unit", "owner"): base += [f"--{name}", "x"] - self.assertFalse(deploy_orin.remote_parser().parse_args(base).no_livekit) - parsed = deploy_orin.remote_parser().parse_args(base + ["--no-livekit"]) - self.assertTrue(parsed.no_livekit) + parsed = deploy_orin.remote_parser().parse_args(base) + self.assertEqual((parsed.mode, parsed.direct_sensors), ("direct", "")) + parsed = deploy_orin.remote_parser().parse_args( + base + ["--mode=livekit", "--direct-sensors=0,1"] + ) + self.assertEqual((parsed.mode, parsed.direct_sensors), ("livekit", "0,1")) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + deploy_orin.remote_parser().parse_args(base + ["--mode=both"]) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + deploy_orin.remote_parser().parse_args(base + ["--no-livekit"]) class ActivateTests(TempDirTestCase): @@ -803,7 +882,8 @@ def activate( timeout=60, replace_unmanaged=False, sensors="", - no_livekit=False, + mode="livekit", + direct_sensors="", ): deploy_orin.remote_activate( argparse.Namespace( @@ -814,7 +894,8 @@ def activate( owner="bob", ready_timeout=timeout, sensors=sensors, - no_livekit=no_livekit, + mode=mode, + direct_sensors=direct_sensors, replace_unmanaged=replace_unmanaged, ) ) @@ -877,9 +958,39 @@ def test_without_unmanaged_nothing_is_killed(self): self.activate(self.make_archive(), replace_unmanaged=True) kill.assert_not_called() - def test_no_livekit_activation_is_ready_on_frames_without_a_join(self): - self.journal = "sensor 0: 150 frames | encoded 150, dropped 0\n" - self.activate(self.make_archive(), sensors="0", no_livekit=True) + def test_direct_activation_prints_ready_lines_and_up(self): + self.journal = ( + "Capturing.\nsensor 0: 150 frames | direct: h:1 connected\n" + "sensor 1: 9 frames | encoded 9\n" + ) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.activate( + self.make_archive(), + sensors="0,1", + mode="direct", + direct_sensors="0", + ) + text = out.getvalue() + self.assertIn("sensor 0: capturing and connected to its destination", text) + self.assertIn("sensor 1: capturing (no direct destination configured)", text) + self.assertIn("producer is up (release r1)", text) + + def test_direct_activation_not_connected_fails(self): + self.journal = ( + "Capturing.\nsensor 0: 5 frames | direct: h:1 NOT connected (x)\n" + ) + with ( + contextlib.redirect_stderr(io.StringIO()), + self.assertRaisesRegex(DeployError, "NOT connected"), + ): + self.activate( + self.make_archive(), + sensors="0", + mode="direct", + direct_sensors="0", + timeout=1, + ) def test_livekit_activation_still_needs_the_join(self): self.journal = "sensor 0: 150 frames | livekit 5 published\n" @@ -907,7 +1018,7 @@ def test_readiness_times_out_without_markers(self): with mock.patch.object(deploy_orin.time, "monotonic", lambda: next(ticks)): with mock.patch.object(deploy_orin, "print", create=True): with self.assertRaisesRegex(DeployError, "did not become ready"): - self.activate(self.make_archive(), timeout=25) + self.activate(self.make_archive(), sensors="0", timeout=25) self.assertEqual(self.events[-1], ("start", "releases/r1")) @@ -1430,7 +1541,12 @@ def test_cache_entry_wins_over_the_other_locations(self): ) def test_unit_sets_ld_library_path_to_the_release_directory(self): - config = {"livekit_url": "wss://x", "token_file": "/t", "args": [STREAM]} + config = { + "mode": "livekit", + "livekit_url": "wss://x", + "token_file": "/t", + "args": [STREAM], + } text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") self.assertIn( "Environment=LD_LIBRARY_PATH=/opt/ic/current/plugins/sensing_camera", @@ -1463,10 +1579,203 @@ def test_no_match(self): self.assertEqual(deploy_orin.configured_sensors({}), []) +class ConfiguredStreamsTests(unittest.TestCase): + def streams(self, *args): + return deploy_orin.configured_streams({"args": list(args)}) + + def test_direct_key_with_a_value_marks_a_direct_stream(self): + self.assertEqual( + self.streams( + "--add-stream=sensor=0,direct=10.0.0.1:5000", + "--add-stream=sensor=1,livekit=right", + "--add-stream=livekit=a,direct=h:1,sensor=2", + ), + [(0, True), (1, False), (2, True)], + ) + + def test_direct_as_the_first_key_is_a_direct_stream(self): + self.assertEqual(self.streams("--add-stream=direct=h:1,sensor=2"), [(2, True)]) + + def test_empty_direct_value_before_another_key_is_not_a_destination(self): + self.assertEqual( + self.streams("--add-stream=sensor=1,direct=,livekit=a"), [(1, False)] + ) + + def test_empty_direct_value_at_the_end_is_not_a_destination(self): + self.assertEqual(self.streams("--add-stream=sensor=0,direct="), [(0, False)]) + + def test_direct_inside_another_key_is_not_a_direct_key(self): + self.assertEqual( + self.streams("--add-stream=sensor=0,indirect=x,livekit=direct=y"), + [(0, False)], + ) + + def test_ignores_other_arguments_and_streams_without_a_sensor(self): + self.assertEqual( + self.streams("--verbose", "--add-stream=direct=h:1", "--other=sensor=3"), + [], + ) + self.assertEqual(deploy_orin.configured_streams({}), []) + + def test_configured_sensors_derives_from_the_streams(self): + config = {"args": ["--add-stream=sensor=4,direct=h:1", STREAM]} + self.assertEqual(deploy_orin.configured_sensors(config), [4, 0]) + + +DIRECT_OK = "direct: 10.0.0.1:5000 connected" + + +class CurrentSessionTests(unittest.TestCase): + def test_text_after_the_last_marker(self): + log = "a Capturing. b Capturing. c" + self.assertEqual(deploy_orin.current_session(log, "direct"), " c") + log = "x Joined LiveKit room r1 y Joined LiveKit room r2 z" + self.assertEqual(deploy_orin.current_session(log, "livekit"), " r2 z") + + def test_absent_marker_is_empty_and_markers_are_per_mode(self): + self.assertEqual( + deploy_orin.current_session("sensor 0: 5 frames", "direct"), "" + ) + self.assertEqual(deploy_orin.current_session("Capturing.\n", "livekit"), "") + self.assertEqual( + deploy_orin.current_session("Joined LiveKit room r\n", "direct"), "" + ) + + +class ReadinessFunctionTests(unittest.TestCase): + def check(self, log, sensors, direct=(), mode="direct"): + return deploy_orin.readiness(log, mode, sensors, list(direct)) + + def test_no_marker_is_one_problem_per_mode(self): + ready, problems = self.check("sensor 0: 5 frames\n", [0]) + self.assertEqual(ready, []) + self.assertEqual(problems, ["the producer has not reported capturing yet"]) + _, problems = self.check("livekit 5 published\n", [0], mode="livekit") + self.assertEqual( + problems, ["the producer has not reported joined a LiveKit room yet"] + ) + + def test_connected_direct_sensor_is_ready(self): + log = f"Capturing.\nsensor 0: 30 frames | {DIRECT_OK}\n" + self.assertEqual( + self.check(log, [0], [0]), + (["sensor 0: capturing and connected to its destination"], []), + ) + + def test_restart_inside_the_unit_discards_the_earlier_attempt(self): + log = ( + f"Capturing.\nsensor 0: 30 frames | {DIRECT_OK}\n" + "Capturing.\nsensor 1: 30 frames | " + DIRECT_OK + "\n" + ) + ready, problems = self.check(log, [0, 1], [0, 1]) + self.assertEqual( + ready, ["sensor 1: capturing and connected to its destination"] + ) + self.assertEqual(problems, ["sensor 0: no frames captured"]) + + def test_not_connected_is_a_problem_even_with_frames(self): + log = ( + "Capturing.\nsensor 0: 30 frames | direct: 10.0.0.1:5000 NOT connected " + "(cannot connect to 10.0.0.1:5000)\n" + ) + ready, problems = self.check(log, [0], [0]) + self.assertEqual(ready, []) + self.assertEqual(len(problems), 1) + self.assertIn("sensor 0", problems[0]) + self.assertIn("NOT connected (cannot connect to 10.0.0.1:5000)", problems[0]) + + def test_configured_direct_sensor_with_no_direct_text_is_a_problem(self): + _, problems = self.check( + "Capturing.\nsensor 0: 30 frames | encoded 30\n", [0], [0] + ) + self.assertEqual( + problems, ["sensor 0: capturing but its direct link is not connected"] + ) + + def test_sensor_without_a_destination_is_capture_only(self): + log = "Capturing.\nsensor 0: 30 frames | encoded 30\n" + self.assertEqual( + self.check(log, [0], []), + (["sensor 0: capturing (no direct destination configured)"], []), + ) + + def test_destinationless_sensor_ignores_a_not_connected_text(self): + log = "Capturing.\nsensor 0: 30 frames | direct: h:1 NOT connected (x)\n" + ready, problems = self.check(log, [0], []) + self.assertEqual(problems, []) + self.assertEqual(len(ready), 1) + + def test_latest_line_wins_connected_then_not_connected(self): + log = ( + f"Capturing.\nsensor 0: 10 frames | {DIRECT_OK}\n" + "sensor 0: 20 frames | direct: 10.0.0.1:5000 NOT connected (link lost)\n" + ) + ready, problems = self.check(log, [0], [0]) + self.assertEqual(ready, []) + self.assertIn("link lost", problems[0]) + + def test_latest_line_wins_not_connected_then_connected(self): + log = ( + "Capturing.\nsensor 0: 10 frames | direct: h:1 NOT connected (x)\n" + f"sensor 0: 20 frames | {DIRECT_OK}\n" + ) + ready, problems = self.check(log, [0], [0]) + self.assertEqual(problems, []) + self.assertEqual(len(ready), 1) + + def test_latest_zero_frames_line_is_a_problem(self): + log = "Capturing.\nsensor 0: 10 frames\nsensor 0: 0 frames\n" + self.assertEqual(self.check(log, [0])[1], ["sensor 0: no frames captured"]) + + def test_zero_frames_with_a_later_digit_on_the_line(self): + log = "Capturing.\nsensor 0: 0 frames | encoded 5, dropped 7\n" + self.assertEqual(self.check(log, [0])[1], ["sensor 0: no frames captured"]) + + def test_sensor_one_is_not_confused_with_sensor_ten(self): + log = f"Capturing.\nsensor 10: 30 frames | {DIRECT_OK}\n" + ready, problems = self.check(log, [1, 10], [1, 10]) + self.assertEqual(problems, ["sensor 1: no frames captured"]) + self.assertEqual(len(ready), 1) + self.assertIn("sensor 10", ready[0]) + + def test_livekit_mode_latest_published_count_decides(self): + log = ( + "Joined LiveKit room r\n" + "sensor 0: 30 fps | livekit 12 published\n" + "sensor 1: 30 fps | livekit 0 published\n" + "sensor 10: 30 fps | livekit 5 published\n" + ) + ready, problems = self.check(log, [0, 1], mode="livekit") + self.assertEqual(ready, ["sensor 0: publishing to LiveKit"]) + self.assertEqual(problems, ["sensor 1: not publishing to LiveKit"]) + + def test_livekit_mode_latest_line_wins_and_restart_resets(self): + log = ( + "Joined LiveKit room r\nsensor 0: x | livekit 5 published\n" + "sensor 0: x | livekit 0 published\n" + ) + self.assertEqual(len(self.check(log, [0], mode="livekit")[1]), 1) + log = ( + "Joined LiveKit room r\nsensor 0: x | livekit 5 published\n" + "Joined LiveKit room r2\n" + ) + self.assertEqual(len(self.check(log, [0], mode="livekit")[1]), 1) + + def test_livekit_mode_ignores_direct_markers(self): + log = "Capturing.\nsensor 0: 5 frames | " + DIRECT_OK + "\n" + self.assertEqual( + self.check(log, [0], [0], mode="livekit")[1], + ["the producer has not reported joined a LiveKit room yet"], + ) + + class ReadinessTests(TempDirTestCase): JOINED = "Joined LiveKit room r\n" + CAPTURING = "Capturing.\n" - def run_wait(self, journal, sensors, active=True, timeout=10, livekit=True): + def run_wait( + self, journal, sensors, active=True, timeout=10, mode="livekit", direct=None + ): def fake_run(cmd, **kwargs): return completed(0 if active else 3) @@ -1489,86 +1798,123 @@ def fake_tail(cmd): contextlib.redirect_stderr(stderr), ): try: - deploy_orin.wait_ready( - "2026-01-01 00:00:00", timeout, sensors, livekit=livekit + return deploy_orin.wait_ready( + "2026-01-01 00:00:00", + timeout, + sensors, + mode=mode, + direct_sensors=direct, ) finally: self.stderr = stderr.getvalue() - def test_one_of_two_sensors_publishing_names_the_other(self): + def test_direct_is_the_default_mode(self): + journal = f"Capturing.\nsensor 0: 3 frames | {DIRECT_OK}\n" + self.assertEqual(deploy_orin.wait_ready.__defaults__[0:1], ("direct",)) + with ( + mock.patch.object(deploy_orin, "tail", lambda cmd: journal), + mock.patch.object( + deploy_orin.subprocess, "run", lambda *a, **k: completed(0) + ), + ): + ready = deploy_orin.wait_ready("t", 10, [0], direct_sensors=[0]) + self.assertEqual( + ready, ["sensor 0: capturing and connected to its destination"] + ) + + def test_direct_returns_one_line_per_sensor(self): journal = ( - self.JOINED - + "sensor 0: 30 fps | livekit 12 published\n" - + "sensor 1: 0 fps | livekit 0 published\n" + self.CAPTURING + + f"sensor 0: 150 frames | {DIRECT_OK}\n" + + "sensor 1: 10 frames | encoded 10\n" + ) + self.assertEqual( + self.run_wait(journal, [0, 1], mode="direct", direct=[0]), + [ + "sensor 0: capturing and connected to its destination", + "sensor 1: capturing (no direct destination configured)", + ], + ) + + def test_direct_restart_inside_the_unit_is_not_ready_for_sensor_zero(self): + journal = ( + f"Capturing.\nsensor 0: 30 frames | {DIRECT_OK}\n" + f"Capturing.\nsensor 1: 30 frames | {DIRECT_OK}\n" ) with self.assertRaises(DeployError) as ctx: - self.run_wait(journal, [0, 1]) + self.run_wait(journal, [0, 1], mode="direct", direct=[0, 1]) message = str(ctx.exception) - self.assertIn("sensors not publishing: 1", message) - self.assertNotIn("publishing: 0", message) + self.assertIn("sensor 0: no frames captured", message) + self.assertNotIn("sensor 1:", message) - def test_both_sensors_publishing_returns(self): + def test_direct_not_connected_fails_naming_the_reason(self): + journal = ( + self.CAPTURING + + "sensor 0: 30 frames | direct: h:1 NOT connected (cannot connect to h:1)\n" + ) + with self.assertRaisesRegex( + DeployError, + r"did not become ready; sensor 0: capturing but h:1 NOT connected", + ): + self.run_wait(journal, [0], mode="direct", direct=[0]) + self.assertIn("sensor 0", self.stderr) + + def test_direct_no_marker_names_the_missing_capture_report(self): + with self.assertRaisesRegex(DeployError, "has not reported capturing yet"): + self.run_wait("sensor 0: 5 frames\n", [0], mode="direct") + + def test_direct_sensor_with_zero_frames_is_named(self): + journal = self.CAPTURING + "sensor 0: 150 frames\nsensor 1: 0 frames\n" + with self.assertRaises(DeployError) as ctx: + self.run_wait(journal, [0, 1], mode="direct") + self.assertIn("sensor 1: no frames captured", str(ctx.exception)) + self.assertNotIn("sensor 0:", str(ctx.exception)) + + def test_livekit_both_sensors_publishing_returns_lines(self): journal = ( self.JOINED + "sensor 0: 30 fps | livekit 12 published\n" + "sensor 1: 30 fps | livekit 3 published\n" ) - self.run_wait(journal, [0, 1]) - - def test_sensor_number_does_not_match_a_longer_number(self): - journal = self.JOINED + "sensor 10: 30 fps | livekit 5 published\n" - with self.assertRaisesRegex(DeployError, "sensors not publishing: 1"): - self.run_wait(journal, [1]) + self.assertEqual( + self.run_wait(journal, [0, 1]), + ["sensor 0: publishing to LiveKit", "sensor 1: publishing to LiveKit"], + ) - def test_without_livekit_every_sensor_with_frames_is_ready(self): + def test_livekit_one_of_two_sensors_names_the_other(self): journal = ( - "sensor 0: 150 frames | encoded 150, dropped 0\n" - "sensor 1: 10 frames | encoded 10, dropped 0\n" + self.JOINED + + "sensor 0: 30 fps | livekit 12 published\n" + + "sensor 1: 0 fps | livekit 0 published\n" ) - self.run_wait(journal, [0, 1], livekit=False) - - def test_without_livekit_a_sensor_with_zero_frames_is_named(self): - journal = "sensor 0: 150 frames | encoded 150\nsensor 1: 0 frames | encoded 0\n" with self.assertRaises(DeployError) as ctx: - self.run_wait(journal, [0, 1], livekit=False) - self.assertIn("sensors not publishing: 1", str(ctx.exception)) - self.assertNotIn("publishing: 0", str(ctx.exception)) - - def test_without_livekit_sensor_number_is_not_confused_with_a_longer_one(self): - journal = "sensor 0: 5 frames\nsensor 10: 10 frames\n" - with self.assertRaisesRegex(DeployError, "sensors not publishing: 1;"): - self.run_wait(journal, [0, 1], livekit=False) - self.run_wait("sensor 1: 10 frames\n", [1], livekit=False) + self.run_wait(journal, [0, 1]) + message = str(ctx.exception) + self.assertIn("sensor 1: not publishing to LiveKit", message) + self.assertNotIn("sensor 0:", message) - def test_without_livekit_a_zero_frames_line_with_a_later_count_digit(self): - with self.assertRaises(DeployError): - self.run_wait("sensor 0: 0 frames\n", [0], livekit=False) + def test_livekit_sensor_number_does_not_match_a_longer_number(self): + journal = self.JOINED + "sensor 10: 30 fps | livekit 5 published\n" + with self.assertRaisesRegex(DeployError, "sensor 1: not publishing"): + self.run_wait(journal, [1]) - def test_with_livekit_frames_alone_are_not_readiness(self): + def test_livekit_frames_alone_are_not_readiness(self): with self.assertRaises(DeployError): self.run_wait(self.JOINED + "sensor 0: 150 frames\n", [0]) - def test_requires_joining_the_room(self): - with self.assertRaises(DeployError): + def test_livekit_requires_joining_the_room(self): + with self.assertRaisesRegex(DeployError, "joined a LiveKit room"): self.run_wait("sensor 0: x | livekit 5 published\n", [0]) - def test_no_sensors_any_publishing_counts(self): - self.run_wait(self.JOINED + "stats livekit 7 published\n", []) - - def test_no_sensors_nothing_publishing_fails_without_a_sensor_list(self): - with self.assertRaises(DeployError) as ctx: - self.run_wait(self.JOINED + "livekit 0 published\n", []) - self.assertNotIn("sensors not publishing", str(ctx.exception)) - def test_inactive_service_stops_waiting_and_reports(self): ticks_used = [] with mock.patch.object(deploy_orin.time, "sleep", ticks_used.append): with self.assertRaises(DeployError): - self.run_wait(self.JOINED + "livekit 5 published\n", [], active=False) + self.run_wait(self.JOINED + "livekit 5 published\n", [0], active=False) self.assertEqual(ticks_used, []) - def test_inactive_service_lists_every_sensor_missing(self): - with self.assertRaisesRegex(DeployError, "sensors not publishing: 0, 1"): + def test_inactive_service_reports_the_producer_never_reported(self): + with self.assertRaisesRegex(DeployError, "has not reported"): self.run_wait("sensor 0: x | livekit 5 published\n", [0, 1], active=False) def test_inactive_service_never_fetches_for_the_loop(self): From e0d2c5f0aaf1eb055a63008f6f7d5210c4e83274 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 6 Nov 2026 14:07:44 -0800 Subject: [PATCH 10/16] deploy_orin: add an env option for extra unit environment variables Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/scripts/deploy_orin.py | 24 ++++++- .../scripts/isaaccapture-producer.service.in | 3 +- tests/python/ci/test_deploy_orin.py | 69 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index e0d44ba48b..62c3bb362e 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -41,7 +41,9 @@ UNIT_TEMPLATE = Path(__file__).with_name("isaaccapture-producer.service.in") CACHE = "~/.cache/isaaccapture-deploy" MODES = ("direct", "livekit") -CONFIG_KEYS = {"mode", "livekit_url", "token_file", "args"} +CONFIG_KEYS = {"mode", "livekit_url", "token_file", "args", "env"} +ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +RESERVED_ENV = ("LD_LIBRARY_PATH",) # The producer's connection options come from the config's own fields, not from `args`. PREFIX = "/opt/isaaccapture" SERVICE_GROUPS = "video" # device access for the cameras @@ -140,6 +142,21 @@ def load_config(path: Path) -> dict: raise DeployError( f"{path}: expected an object with args and, optionally, livekit_url and token_file" ) + env = config.get("env", {}) + if not isinstance(env, dict) or not all( + isinstance(k, str) + and ENV_NAME.fullmatch(k) + and isinstance(v, str) + and "\n" not in v + for k, v in env.items() + ): + raise DeployError( + f"{path}: env must map variable names to single-line string values" + ) + if any(k in RESERVED_ENV for k in env): + raise DeployError( + f"{path}: env may not set {', '.join(RESERVED_ENV)}; the release sets it" + ) mode = config.get("mode", "direct") if mode not in MODES: raise DeployError(f'{path}: "mode" must be one of {", ".join(MODES)}') @@ -206,6 +223,11 @@ def render_unit(config: dict, prefix: str, user: str, groups: str) -> str: "SERVICE_GROUPS": groups, "PREFIX": prefix, "EXEC_START": exec_start, + # The producer's own environment variables (for example a diagnostic switch), one Environment= line each. + "ENVIRONMENT": "\n".join( + f"Environment={systemd_quote(f'{name}={value}')}" + for name, value in config.get("env", {}).items() + ), } text = UNIT_TEMPLATE.read_text() for key, value in fields.items(): diff --git a/src/plugins/sensing/scripts/isaaccapture-producer.service.in b/src/plugins/sensing/scripts/isaaccapture-producer.service.in index c149c54649..ce940fbd6b 100644 --- a/src/plugins/sensing/scripts/isaaccapture-producer.service.in +++ b/src/plugins/sensing/scripts/isaaccapture-producer.service.in @@ -12,8 +12,9 @@ Type=simple User=@SERVICE_USER@ SupplementaryGroups=@SERVICE_GROUPS@ WorkingDirectory=@PREFIX@/current/plugins/sensing_camera -# The release carries its own LiveKit libraries; they must win over the build-tree path the binary was linked with. +# The release carries its own shared libraries; they must win over the build-tree paths the binary was linked with. Environment=LD_LIBRARY_PATH=@PREFIX@/current/plugins/sensing_camera +@ENVIRONMENT@ ExecStart=@EXEC_START@ # The producer stops on SIGINT; systemd ends anything left in this cgroup after the timeout. KillSignal=SIGINT diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index d04d7c9627..a3acf6bf08 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -113,6 +113,36 @@ def test_mode_must_be_one_of_the_known_modes(self): def test_mode_is_an_allowed_config_key(self): self.assertIn("mode", deploy_orin.CONFIG_KEYS) + def test_env_is_an_allowed_config_key(self): + self.assertIn("env", deploy_orin.CONFIG_KEYS) + + def test_accepts_env(self): + config = self.valid(env={"FOO": "bar baz", "_X1": ""}) + self.assertEqual(self.load(config), config) + + def test_rejects_bad_env(self): + bad = [ + {"1X": "v"}, + {"A-B": "v"}, + {"": "v"}, + {"FOO": 1}, + {"FOO": None}, + {"FOO": "a\nb"}, + {"LD_LIBRARY_PATH": "/x"}, + ["FOO=bar"], + "FOO=bar", + None, + ] + for env in bad: + with self.subTest(env=env), self.assertRaises(DeployError): + self.load(self.valid(env=env)) + + def test_bad_env_message(self): + with self.assertRaisesRegex( + DeployError, "env must map variable names to single-line string values" + ): + self.load(self.valid(env={"A-B": "v"})) + def test_accepts_config_with_only_args(self): config = {"args": [STREAM]} self.assertEqual(self.load(config), config) @@ -203,6 +233,45 @@ def test_render_unit(self): self.assertIn("WorkingDirectory=/opt/ic/current/plugins/sensing_camera", text) self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + def test_render_unit_env_lines_follow_ld_library_path_in_order(self): + config = {"args": [STREAM], "env": {"ZED": "1", "ALPHA": "two"}} + lines = deploy_orin.render_unit(config, "/opt/ic", "bob", "video").splitlines() + i = next( + n + for n, ln in enumerate(lines) + if ln.startswith("Environment=LD_LIBRARY_PATH=") + ) + self.assertEqual( + lines[i + 1 : i + 3], ["Environment=ZED=1", "Environment=ALPHA=two"] + ) + self.assertTrue(lines[i + 3].startswith("ExecStart=")) + + def test_render_unit_without_env_adds_no_environment_lines(self): + for config in ({"args": [STREAM]}, {"args": [STREAM], "env": {}}): + with self.subTest(config=config): + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + env_lines = [ + ln for ln in text.splitlines() if ln.startswith("Environment=") + ] + self.assertEqual(len(env_lines), 1) + self.assertTrue(env_lines[0].startswith("Environment=LD_LIBRARY_PATH=")) + self.assertNotIn("@ENVIRONMENT@", text) + self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + + def test_render_unit_env_values_are_quoted(self): + env = {"A": "x y", "B": 'q"q', "C": "100%", "D": "$HOME", "E": ""} + config = {"args": [STREAM], "env": env} + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + lines = text.splitlines() + for name, value in env.items(): + self.assertIn( + f"Environment={deploy_orin.systemd_quote(f'{name}={value}')}", lines + ) + self.assertIn('Environment="A=x y"', lines) + self.assertIn('Environment="B=q\\"q"', lines) + self.assertIn("Environment=C=100%%", lines) + self.assertIn("Environment=D=$$HOME", lines) + def test_render_unit_direct_mode_has_no_livekit_args(self): for config in ({"args": [STREAM]}, {"mode": "direct", "args": [STREAM]}): with self.subTest(config=config): From c02782dcdc2d2ecfece2459666e50d9a2680d94b Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Sun, 8 Nov 2026 15:20:17 -0800 Subject: [PATCH 11/16] deploy_orin: restart the producer after a fatal exit The producer ends its own process when a camera session will not stop, and relies on the service to start it again. Restart on failure after five seconds; a clean stop does not restart, and a start limit keeps a producer that cannot start from restarting forever. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../scripts/isaaccapture-producer.service.in | 6 ++++++ tests/python/ci/test_deploy_orin.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/plugins/sensing/scripts/isaaccapture-producer.service.in b/src/plugins/sensing/scripts/isaaccapture-producer.service.in index ce940fbd6b..3bb57feba1 100644 --- a/src/plugins/sensing/scripts/isaaccapture-producer.service.in +++ b/src/plugins/sensing/scripts/isaaccapture-producer.service.in @@ -16,6 +16,12 @@ WorkingDirectory=@PREFIX@/current/plugins/sensing_camera Environment=LD_LIBRARY_PATH=@PREFIX@/current/plugins/sensing_camera @ENVIRONMENT@ ExecStart=@EXEC_START@ +# A fatal exit (a camera session that will not stop ends the process) restarts the producer; a clean stop does not, +# and the start limit keeps a producer that cannot start from restarting forever. +Restart=on-failure +RestartSec=5 +StartLimitIntervalSec=120 +StartLimitBurst=5 # The producer stops on SIGINT; systemd ends anything left in this cgroup after the timeout. KillSignal=SIGINT KillMode=control-group diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index a3acf6bf08..c869020bdc 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -233,6 +233,20 @@ def test_render_unit(self): self.assertIn("WorkingDirectory=/opt/ic/current/plugins/sensing_camera", text) self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + def test_render_unit_restarts_on_failure_with_start_limit(self): + text = deploy_orin.render_unit({"args": [STREAM]}, "/opt/ic", "bob", "video") + lines = text.splitlines() + service = lines[lines.index("[Service]") + 1 :] + for line in ( + "Restart=on-failure", + "RestartSec=5", + "StartLimitIntervalSec=120", + "StartLimitBurst=5", + ): + self.assertEqual(service.count(line), 1, line) + self.assertNotIn("Restart=always", lines) + self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + def test_render_unit_env_lines_follow_ld_library_path_in_order(self): config = {"args": [STREAM], "env": {"ZED": "1", "ALPHA": "two"}} lines = deploy_orin.render_unit(config, "/opt/ic", "bob", "video").splitlines() From 80796b2ff323b9747389e94c45eb5b84902ebf93 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 9 Nov 2026 11:54:45 -0500 Subject: [PATCH 12/16] deploy_orin: put the start limit in [Unit], where systemd reads it StartLimitIntervalSec and StartLimitBurst are unit settings; in [Service] systemd ignores them. Move them, and make the unit test section-aware (plus a systemd-analyze verify check, which on its own does not flag a misplaced key). Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../scripts/isaaccapture-producer.service.in | 8 +-- tests/python/ci/test_deploy_orin.py | 50 +++++++++++++++---- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/plugins/sensing/scripts/isaaccapture-producer.service.in b/src/plugins/sensing/scripts/isaaccapture-producer.service.in index 3bb57feba1..9ba4113a89 100644 --- a/src/plugins/sensing/scripts/isaaccapture-producer.service.in +++ b/src/plugins/sensing/scripts/isaaccapture-producer.service.in @@ -6,6 +6,9 @@ Description=Isaac Capture camera producer After=network-online.target Wants=network-online.target +# Keeps a producer that cannot start from restarting forever (these are unit settings, not service settings). +StartLimitIntervalSec=120 +StartLimitBurst=5 [Service] Type=simple @@ -16,12 +19,9 @@ WorkingDirectory=@PREFIX@/current/plugins/sensing_camera Environment=LD_LIBRARY_PATH=@PREFIX@/current/plugins/sensing_camera @ENVIRONMENT@ ExecStart=@EXEC_START@ -# A fatal exit (a camera session that will not stop ends the process) restarts the producer; a clean stop does not, -# and the start limit keeps a producer that cannot start from restarting forever. +# A fatal exit (a camera session that will not stop ends the process) restarts the producer; a clean stop does not. Restart=on-failure RestartSec=5 -StartLimitIntervalSec=120 -StartLimitBurst=5 # The producer stops on SIGINT; systemd ends anything left in this cgroup after the timeout. KillSignal=SIGINT KillMode=control-group diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index c869020bdc..d20228ce20 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -233,20 +233,52 @@ def test_render_unit(self): self.assertIn("WorkingDirectory=/opt/ic/current/plugins/sensing_camera", text) self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + @staticmethod + def _unit_sections(text): + sections = {} + current = None + for line in text.splitlines(): + line = line.strip() + if line.startswith("[") and line.endswith("]"): + current = sections.setdefault(line, []) + elif current is not None and line and not line.startswith("#"): + current.append(line) + return sections + def test_render_unit_restarts_on_failure_with_start_limit(self): text = deploy_orin.render_unit({"args": [STREAM]}, "/opt/ic", "bob", "video") - lines = text.splitlines() - service = lines[lines.index("[Service]") + 1 :] - for line in ( - "Restart=on-failure", - "RestartSec=5", - "StartLimitIntervalSec=120", - "StartLimitBurst=5", - ): + sections = self._unit_sections(text) + unit, service = sections["[Unit]"], sections["[Service]"] + for line in ("Restart=on-failure", "RestartSec=5"): self.assertEqual(service.count(line), 1, line) - self.assertNotIn("Restart=always", lines) + # systemd reads the start limit from [Unit] and ignores it in [Service]. + for line in ("StartLimitIntervalSec=120", "StartLimitBurst=5"): + self.assertEqual(unit.count(line), 1, line) + self.assertNotIn(line, service) + self.assertNotIn("Restart=always", unit + service) self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + @unittest.skipUnless(shutil.which("systemd-analyze"), "systemd-analyze not found") + def test_render_unit_passes_systemd_analyze_verify(self): + import getpass + + text = deploy_orin.render_unit( + {"args": [STREAM]}, "/opt/ic", getpass.getuser(), "video" + ) + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) / "isaaccapture-producer.service" + unit.write_text(text) + result = subprocess.run( + ["systemd-analyze", "verify", str(unit)], + capture_output=True, + text=True, + check=False, + ) + output = result.stdout + result.stderr + # verify exits 0 even when it ignores a key, so read the diagnostics. + for diagnostic in ("Unknown key name", "ignoring"): + self.assertNotIn(diagnostic, output) + def test_render_unit_env_lines_follow_ld_library_path_in_order(self): config = {"args": [STREAM], "env": {"ZED": "1", "ALPHA": "two"}} lines = deploy_orin.render_unit(config, "/opt/ic", "bob", "video").splitlines() From e6d63f6b2005c7c47ce0df25f9ed66e4ad695fcc Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 9 Nov 2026 17:43:15 -0500 Subject: [PATCH 13/16] sensing: add orin_setup.sh to prepare an Orin and find its cameras orin_setup.sh installs what the producer build needs (toolchain, patchelf, JetPack packages, uv, the SIPL headers from the L4T archive), runs setup.sh for the SENSING drivers and overlay, then checks the cameras. setup.sh -y now selects the device-tree overlay through Jetson-IO's command line instead of its menu and schedules a reboot (exit 3). It does so whenever the deserializer's I2C bus is absent, and not again after a reboot that did not bring the bus up. orin_setup.sh --scan tries each of the carrier's 8 camera inputs alone. --detect DIR does the same, writes a platform config with the cameras it found, prints the link masks as JSON, and saves the inputs in known-cameras.json so the next run probes only those; all 8 inputs are scanned again when the file is missing or a saved camera does not answer. --sudo-password-stdin answers every sudo from a private tmpfs file so nothing prompts. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/scripts/orin_setup.sh | 340 ++++++++++++++++++++++ src/plugins/sensing/scripts/setup.sh | 38 ++- 2 files changed, 377 insertions(+), 1 deletion(-) create mode 100755 src/plugins/sensing/scripts/orin_setup.sh diff --git a/src/plugins/sensing/scripts/orin_setup.sh b/src/plugins/sensing/scripts/orin_setup.sh new file mode 100755 index 0000000000..428b032377 --- /dev/null +++ b/src/plugins/sensing/scripts/orin_setup.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Prepare an Orin for the camera producer, then check the cameras work. Run it on the +# Orin. It ships with deploy_orin.py, both ahead of the first build and in each release. +# +# 1. Prerequisites: the build toolchain, the JetPack CUDA and Multimedia API packages and +# uv, then the SENSING drivers and device-tree overlay (setup.sh), then a check of what +# the deploy preflight checks. The SIPL headers come from the L4T SIPL API package. +# 2. Camera check: verify.sh (read-only report), then validate.sh (streams both sensors). When the +# stream fails, or with --scan, all 8 camera inputs are tried one at a time to show where cameras are. +# +# Exit status: 0 if everything requested passed, 2 if a reboot or new login is needed before +# going on, 1 otherwise. +# +# Usage: +# ./orin_setup.sh [--prereqs-only | --check-only] [--no-apt] [--no-drivers] [--sipl-api FILE] [--sdk-root DIR] [--sudo-password-stdin] [--scan | --detect DIR [--rescan]] [-t SECONDS] +# +# Options: +# --prereqs-only install and check prerequisites; skip the camera check +# --check-only run the camera check only +# --no-apt skip apt and uv installation; only check what is present +# --no-drivers skip setup.sh (the SENSING drivers and overlay) +# --sipl-api FILE unpack this Jetson_SIPL_API_R_aarch64.tbz2 if the SIPL headers are missing +# --sdk-root DIR where usr/src/jetson_multimedia_api and jetson_sipl_api live (default /) +# --sudo-password-stdin read the sudo password from stdin, so nothing prompts +# --scan try all 8 camera inputs alone and report where cameras are (stops the producer service) +# --detect DIR the same probe, then write DIR/platform.json for the cameras found and print +# {"config", "masks", "count", "used", "positions"} as JSON on stdout. The inputs found are +# saved in DIR/known-cameras.json and tried first next time; all 8 are scanned only when that +# file is missing or a saved camera does not answer +# --rescan with --detect: ignore the saved cameras and scan all 8 inputs +# -t, --seconds N how long validate.sh streams (default 10) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./common.sh +source "$SCRIPT_DIR/common.sh" + +SERVICE=isaaccapture-producer.service +DO_PREREQS=1 +DO_CHECK=1 +DO_APT=1 +DO_DRIVERS=1 +SIPL_API= +SDK_ROOT= +REBOOT=0 +SUDO_PASSWORD_STDIN=0 +DO_SCAN=0 +DETECT_DIR= +RESCAN=0 +SECONDS_ARG=10 + +usage() { sed -n '4,35p' "${BASH_SOURCE[0]}" | sed 's/^# \?//'; } + +while (( $# )); do + case "$1" in + --prereqs-only) DO_CHECK=0; shift ;; + --check-only) DO_PREREQS=0; shift ;; + --no-apt) DO_APT=0; shift ;; + --no-drivers) DO_DRIVERS=0; shift ;; + --sipl-api) SIPL_API="${2:?--sipl-api needs a file}"; shift 2 ;; + --sdk-root) SDK_ROOT="${2:?--sdk-root needs a directory}"; shift 2 ;; + --sudo-password-stdin) SUDO_PASSWORD_STDIN=1; shift ;; + --scan) DO_SCAN=1; shift ;; + --detect) DETECT_DIR="${2:?--detect needs a directory}"; shift 2 ;; + --rescan) RESCAN=1; shift ;; + -t|--seconds) SECONDS_ARG="${2:?--seconds needs a value}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" "Run $0 --help" ;; + esac +done + +if [[ "$SUDO_PASSWORD_STDIN" -eq 1 ]]; then + # Every sudo below, including the ones in setup.sh and common.sh, answers from a private tmpfs file that is + # removed on exit: the exported function adds -A, and the askpass program prints the file. + umask 077 + SUDO_PWFILE="$(mktemp -p "${XDG_RUNTIME_DIR:-/dev/shm}" orin-sudo.XXXXXX)" + SUDO_ASKPASS="$(mktemp -p "${XDG_RUNTIME_DIR:-/dev/shm}" orin-askpass.XXXXXX)" + trap 'rm -f "$SUDO_PWFILE" "$SUDO_ASKPASS"' EXIT + cat > "$SUDO_PWFILE" + printf '#!/bin/sh\ncat "$SUDO_PWFILE"\n' > "$SUDO_ASKPASS" + chmod 700 "$SUDO_ASKPASS" + export SUDO_PWFILE SUDO_ASKPASS + sudo() { command sudo -A "$@"; } + export -f sudo +fi + +APT_PACKAGES=( + build-essential cmake pkg-config patchelf git curl rsync + python3 python3-dev python3-venv + nvidia-jetpack nvidia-l4t-jetson-multimedia-api +) + +FAIL=0 +check() { + local name="$1"; shift + if "$@" >/dev/null 2>&1; then ok "$name"; else warn "missing: $name"; FAIL=1; fi +} + +prerequisites() { + [[ "$(uname -m)" == aarch64 && -f /etc/nv_tegra_release ]] \ + || die "this is not a Jetson (aarch64 L4T) system." + + if [[ "$DO_APT" -eq 1 ]]; then + step "Installing apt packages" + sudo apt-get update + sudo apt-get install -y "${APT_PACKAGES[@]}" + if ! PATH="$HOME/.local/bin:$PATH" have uv; then + step "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + fi + + if [[ "$DO_DRIVERS" -eq 1 ]]; then + # Idempotent: it reinstalls only what differs. It exits nonzero while a reboot or a new login is still + # needed (the Jetson-IO overlay choice, group membership), so stop there and re-run afterwards. + step "Installing the SENSING drivers (setup.sh)" + # -y: no prompts. Exit 3: the overlay is selected and a reboot is scheduled. Exit 2: only a new login is + # needed for group membership, and every new ssh session has it. + local code=0 + "$SCRIPT_DIR/setup.sh" -y || code=$? + case "$code" in + 0|2) ;; + 3) warn "rebooting to load the device-tree overlay; run this again afterwards." + FAIL=1; REBOOT=1; return ;; + *) warn "setup.sh failed (exit $code)."; FAIL=1; return ;; + esac + fi + + if [[ ! -d "${SDK_ROOT%/}/usr/src/jetson_sipl_api" && -n "$SIPL_API" ]]; then + step "Unpacking the SIPL API headers" + sudo tar -xjf "$SIPL_API" -C "${SDK_ROOT:-/}" + fi + + step "Checking prerequisites" + export PATH="$HOME/.local/bin:/usr/local/cuda/bin:$PATH" + local tool + for tool in cmake g++ systemctl uv nvcc; do check "$tool" have "$tool"; done + check "patchelf" have patchelf + local sdk="${SDK_ROOT%/}/usr/src" + check "Jetson Multimedia API ($sdk/jetson_multimedia_api)" test -d "$sdk/jetson_multimedia_api" + check "SIPL headers ($sdk/jetson_sipl_api)" test -d "$sdk/jetson_sipl_api" + check "SENSING drivers (nvsipl_camera)" have nvsipl_camera + check "ISP tuning ($SIPL_NITO_DIR/$SENSING_NITO)" test -r "$SIPL_NITO_DIR/$SENSING_NITO" + check "device-tree overlay ($SIPL_BOOT_DIR/$SENSING_DTBO)" test -f "$SIPL_BOOT_DIR/$SENSING_DTBO" + check "overlay active: deserializer I2C bus (/dev/i2c-9)" test -e /dev/i2c-9 + check "4 GB free in $HOME" test "$(df --output=avail -B1 "$HOME" | tail -1)" -ge $((4 << 30)) + if [[ "$FAIL" -ne 0 ]]; then + warn "SIPL headers: unpack Jetson_SIPL_API_R_aarch64.tbz2 into / (it holds usr/src/jetson_sipl_api)." + warn "SENSING drivers and overlay: rerun without --no-drivers, then reboot if asked." + fi +} + +# Try every camera input alone: four links on each of the two deserializers. The platform config places its modules +# on links 2 and 3 of deserializer CD (transport 1), so each probe uses a temporary copy whose one module sits on the +# input under test. The vendor file itself is never changed. A link whose camera does not lock fails SIPL +# initialisation (status 10). Fills FOUND (transport:link) and FOUND_LABEL. +FOUND=() +FOUND_LABEL=() + +input_label() { + if [[ "$1" -eq 1 ]]; then + printf 'CN1 COAX%s (deserializer CD, link %s)' "$2" "$2" + else + printf 'CN2 COAX%s (deserializer AB, link %s)' "$((4 + $2))" "$2" + fi +} + +# Does a camera answer on this input (transport, link)? Records it in FOUND on success. +probe_input() { + local transport="$1" link="$2" label config mask slots + label="$(input_label "$transport" "$link")" + config="$PROBE_TMP/shw5g-$transport-$link.json" + python3 - "$SENSING_CONFIGS_DIR/shw5g.json" "$config" "$transport" "$link" <<'PY' +import json, sys +src, out, transport, link = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]) +data = json.load(open(src)) +module = data["cameraConfigs"][0]["cameraModules"][0] +module.update(linkIndex=link, transportIndex=transport) +data["cameraConfigs"][0]["cameraModules"] = [module] +json.dump(data, open(out, "w")) +PY + slots=("0x0000" "0x0000") + slots[transport]="$(printf '0x%04x' $((1 << (4 * link))))" + mask="${slots[0]} ${slots[1]}" + if nvsipl_camera -t "$config" -c "$SENSING_PLATFORM_CONFIG" -m "$mask" \ + --enable-camera-hal -s -Z -r 3 "$PROBE_TMP/log" 2>&1; then + ok "$label: camera streaming" + FOUND+=("$transport:$link") + FOUND_LABEL+=("$label") + return 0 + fi + info "$label: no camera" + return 1 +} + +probe_inputs() { + local transport link + PROBE_TMP="$(mktemp -d)" + step "Scanning all 8 camera inputs (one at a time, about 5 s each)" + for transport in 1 0; do + for link in 0 1 2 3; do + probe_input "$transport" "$link" || true + done + done + rm -rf "$PROBE_TMP" +} + +scan_links() { + probe_inputs + if [[ "${#FOUND[@]}" -eq 0 ]]; then + warn "no camera answers on any input. Check the carrier's 12 V supply and the power switching harness, that each" + warn "camera is clicked into its cable, and that the cable is clicked into its harness lead." + FAIL=1 + return + fi + ok "cameras found on ${#FOUND[@]} input(s); the deploy can use any of them (cameras: auto)" +} + +# Write DIR/platform.json for the cameras found (the SHW5G config with its modules moved onto those inputs, at most +# the two the config names) and print what the producer needs on stdout. Everything else goes to stderr. +detect_cameras() { + local dir="$1" known="$1/known-cameras.json" pair transport link stale=0 saved=() + mkdir -p "$dir" + if [[ "$RESCAN" -eq 0 ]]; then + # The cameras saved last time, as "transport:link". A missing or unreadable file means none. + mapfile -t saved < <(python3 - "$known" <<'PY' +import json, sys +try: + for camera in json.load(open(sys.argv[1]))["cameras"]: + print("%d:%d" % (camera["transport"], camera["link"])) +except (OSError, ValueError, KeyError, TypeError): + pass +PY +) + fi + if [[ "${#saved[@]}" -gt 0 ]]; then + # One short probe per saved camera instead of all 8 inputs. + PROBE_TMP="$(mktemp -d)" + step "Trying the cameras saved last time" >&2 + for pair in "${saved[@]}"; do + transport="${pair%%:*}" + link="${pair##*:}" + probe_input "$transport" "$link" >&2 || stale=1 + done + rm -rf "$PROBE_TMP" + if [[ "$stale" -eq 1 ]]; then + warn "a saved camera did not answer; scanning all inputs." + FOUND=() + FOUND_LABEL=() + fi + fi + [[ "${#FOUND[@]}" -gt 0 ]] || probe_inputs >&2 + [[ "${#FOUND[@]}" -gt 0 ]] || { warn "no camera answers on any input."; return 1; } + python3 - "$known" "${FOUND[@]}" -- "${FOUND_LABEL[@]}" <<'PY' +import json, sys +out, rest = sys.argv[1], sys.argv[2:] +split = rest.index("--") +cameras = [] +for item, label in zip(rest[:split], rest[split + 1:]): + transport, link = map(int, item.split(":")) + cameras.append({"transport": transport, "link": link, "label": label}) +json.dump({"cameras": cameras}, open(out, "w"), indent=1) +PY + python3 - "$SENSING_CONFIGS_DIR/shw5g.json" "$dir/platform.json" "${FOUND[@]}" -- "${FOUND_LABEL[@]}" <<'PY' +import json, sys +src, out = sys.argv[1], sys.argv[2] +rest = sys.argv[3:] +split = rest.index("--") +found = [tuple(map(int, item.split(":"))) for item in rest[:split]] +labels = rest[split + 1:] +data = json.load(open(src)) +config = data["cameraConfigs"][0] +templates = config["cameraModules"] # the two modules the vendor config names +used = found[: len(templates)] +modules, slots = [], [0, 0] +for template, (transport, link) in zip(templates, used): + modules.append(dict(template, linkIndex=link, transportIndex=transport)) + slots[transport] |= 1 << (4 * link) +config["cameraModules"] = modules +json.dump(data, open(out, "w"), indent=1) +print(json.dumps({ + "config": out, + "masks": " ".join("0x%04x" % s for s in slots), + "count": len(found), + "used": len(used), + "positions": labels, +})) +PY +} + +camera_check() { + # SIPL has one client at a time; a running producer would make the checks fail confusingly. + if systemctl is-active --quiet "$SERVICE" 2>/dev/null; then + if [[ "$DO_SCAN" -eq 0 ]]; then + warn "$SERVICE is running and holds the cameras." + warn "Stop it first: sudo systemctl stop $SERVICE" + FAIL=1 + return + fi + step "Stopping $SERVICE so the scan can open the cameras" + sudo systemctl stop "$SERVICE" + fi + if [[ "$DO_SCAN" -eq 1 ]]; then + scan_links + return + fi + step "Camera report (verify.sh)" + "$SCRIPT_DIR/verify.sh" || FAIL=1 + step "Camera stream (validate.sh)" + if ! "$SCRIPT_DIR/validate.sh" -t "$SECONDS_ARG"; then + FAIL=1 + scan_links + fi +} + +if [[ -n "$DETECT_DIR" ]]; then + have nvsipl_camera || die "nvsipl_camera is not on PATH." "Run orin_setup.sh --prereqs-only first." + if systemctl is-active --quiet "$SERVICE" 2>/dev/null; then + warn "$SERVICE is running and holds the cameras; stop it first (sudo systemctl stop $SERVICE)." >&2 + exit 1 + fi + detect_cameras "$DETECT_DIR" + exit $? +fi + +[[ "$DO_PREREQS" -eq 0 ]] || prerequisites +if [[ "$DO_CHECK" -eq 1 ]]; then + if [[ "$FAIL" -ne 0 ]]; then + warn "skipping the camera check until the prerequisites above are fixed" + else + camera_check + fi +fi + +[[ "$FAIL" -eq 0 ]] || exit $(( REBOOT ? 2 : 1 )) +ok "all requested checks passed" diff --git a/src/plugins/sensing/scripts/setup.sh b/src/plugins/sensing/scripts/setup.sh index 3b438ee0b1..8a7eca83d4 100755 --- a/src/plugins/sensing/scripts/setup.sh +++ b/src/plugins/sensing/scripts/setup.sh @@ -16,6 +16,8 @@ # Usage: # ./setup.sh [options] # +# Exit status: 0 done, 2 a new login or manual reboot is needed, 3 with -y: overlay selected and a reboot scheduled. +# # Options: # --pkg DIR vendor driver package dir (default: fetch from SENSING's repo) # --no-fetch do not download the vendor package; fail if it is not on disk @@ -40,7 +42,7 @@ REBOOT_REQUIRED=0 LOGIN_REQUIRED=0 ASSUME_YES=0 -usage() { sed -n '4,27p' "${BASH_SOURCE[0]}" | sed 's/^# \?//'; } +usage() { sed -n '4,29p' "${BASH_SOURCE[0]}" | sed 's/^# \?//'; } while (( $# )); do case "$1" in @@ -212,7 +214,41 @@ else info "nvpmodel not found" fi +# Select the SG8A overlay with Jetson-IO's command line, without its menu. +select_overlay() { + local tool=/opt/nvidia/jetson-io/config-by-hardware.py list header + [[ -x "$tool" ]] || die "$tool not found." "Install the JetPack Jetson-IO tool, or run without -y." + list="$(sudo "$tool" -l)" || die "Jetson-IO could not list its hardware modules." + header="$(awk -v name="$SENSING_OVERLAY_NAME" ' + /^Header [0-9]+/ { h = $2; sub(/[^0-9].*/, "", h) } + index($0, name) { print h; exit }' <<<"$list")" + [[ -n "$header" ]] || die "Jetson-IO lists no '$SENSING_OVERLAY_NAME'." "$list" + sudo "$tool" -n "$header=$SENSING_OVERLAY_NAME" || die "Jetson-IO could not select the overlay." +} + # Complete first-install setup before offering a capture command. +# The deserializer's I2C bus exists only once the overlay is selected and the Orin has rebooted. +OVERLAY_STAMP=/var/lib/sensing/overlay-selected-boot-id +boot_id() { cat /proc/sys/kernel/random/boot_id; } +if [[ "$ASSUME_YES" -eq 1 ]] && { [[ "$REBOOT_REQUIRED" -eq 1 ]] || [[ ! -e /dev/i2c-9 ]]; }; then + step "Activate the device-tree overlay" + previous="$(cat "$OVERLAY_STAMP" 2>/dev/null || true)" + # One selection per boot, and never a second reboot for the same fault: an Orin that has already rebooted + # after selecting and still lacks the bus needs a person. + if [[ "$REBOOT_REQUIRED" -eq 0 && -n "$previous" && "$previous" != "$(boot_id)" ]]; then + die "the overlay was selected and the Orin rebooted, but /dev/i2c-9 is still absent." \ + "Check the Jetson-IO selection, the carrier's 12 V supply and the cables." + fi + if [[ "$REBOOT_REQUIRED" -eq 1 || "$previous" != "$(boot_id)" ]]; then + select_overlay + sudo mkdir -p "$(dirname "$OVERLAY_STAMP")" + boot_id | sudo tee "$OVERLAY_STAMP" >/dev/null + fi + # Reboot after this script has exited, so a caller over ssh still gets the exit status. + sudo systemd-run --quiet --on-active=5 systemctl reboot + warn "overlay selected; rebooting in 5 s to load it." + exit 3 +fi if [[ "$REBOOT_REQUIRED" -eq 1 ]]; then step "Activate the device-tree overlay" hint "Configure Jetson AGX CSI Connector -> Configure for compatible hardware" From 17c03bf69f91f26bd97aa8089478bc5e9bf845a5 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 9 Nov 2026 17:43:15 -0500 Subject: [PATCH 14/16] sensing: deploy_orin.py installs prerequisites and finds the cameras itself A deploy now checks the Orin and, when something is missing, installs it through orin_setup.sh: packages, SIPL headers (the newest SDK Manager download, or --sipl-api), the SENSING drivers and overlay. It waits out the reboot the overlay needs and continues. With ORIN_PASSWORD set the Orin's sudo is answered from stdin and nothing prompts. "cameras": "auto" in the config finds the cameras on any of the carrier's 8 inputs, builds the producer's platform config and link masks for them, and keeps only the streams for sensors that were found. The inputs found are saved on the Orin and tried first next time; --rescan-cameras scans all 8. New options: --install-prereqs, --check-camera, --scan-cameras, --logs, --no-setup, --sipl-api. Step 5 shows progress for file verification and for the wait for the cameras. The failure report hides the camera driver's own log lines, names a camera link that did not lock, and lists the real driver errors. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/AGENTS.md | 4 + src/plugins/sensing/README.md | 12 + src/plugins/sensing/scripts/deploy_orin.py | 445 +++++++++++++++++- tests/python/ci/test_deploy_orin.py | 516 ++++++++++++++++++++- 4 files changed, 951 insertions(+), 26 deletions(-) diff --git a/src/plugins/sensing/AGENTS.md b/src/plugins/sensing/AGENTS.md index a8abfd9bc2..7fce3774c8 100644 --- a/src/plugins/sensing/AGENTS.md +++ b/src/plugins/sensing/AGENTS.md @@ -19,5 +19,9 @@ camera runtime checks must not require build SDKs or cached vendor packages. The quick-start camera command must open a live visual preview, not only print FPS. Container preview must carry the desktop display and authorization into the image. +`deploy_orin.py` is the single entry point for putting an Orin into a working state. +A prerequisite or setup step belongs in it, through `orin_setup.sh`, not in a document +the operator must follow by hand before running it. + Support SSH launch with a local DISPLAY: when XAUTHORITY is unset, check the current user's GDM authority file as well as ~/.Xauthority. diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 5320d21241..e98534865d 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -93,6 +93,18 @@ python3 src/plugins/sensing/scripts/deploy_orin.py 192.168.1.176 --config orin.j - `--replace-unmanaged` stops a `camera_plugin_sensing` that someone started by hand (its exact PIDs are shown); without it the deployment refuses and changes nothing. `--no-activate` builds and packages only, and `--jobs N` bounds compile jobs. The release is installed under `/opt/isaaccapture`. +- `scripts/orin_setup.sh` installs the Orin's prerequisites and the SENSING drivers (`setup.sh`), then checks the cameras (`verify.sh`, then a + `validate.sh` stream). The deploy uploads it, with those scripts, to `~/.cache/isaaccapture-deploy/tools/scripts/` + before the preflight, so a missing prerequisite can be fixed first, and each release carries a copy under + `plugins/sensing_camera/scripts/`. `--prereqs-only` and `--check-only` run one half; stop the producer service + before the camera check, since SIPL serves one client at a time. From the desktop, `deploy_orin.py USER@ORIN + --install-prereqs` and `--check-camera` run them over ssh without deploying (the sudo password is asked on the + Orin's terminal); together they run both halves. +- `"cameras": "auto"` in the deploy config makes the deploy find the cameras itself, on any of the carrier's 8 inputs, and + start the producer with them (at most the two the vendor config names; streams for sensors that were not found are + dropped). The inputs found are saved in `known-cameras.json` in the deploy's cache directory on the Orin and tried + first on the next deploy. All 8 are scanned again only when that file is missing or a saved camera does not answer, or + with `--rescan-cameras`. Leave `--link-masks`, `--platform-config` and `--config-name` out of `args` in this mode. - The script checks the ssh connection first, then preflights the Orin, uploads only the source files that differ from what the Orin already holds, builds, installs a versioned release, replaces the running producer and waits for every configured camera to be up. "Up" is the producer's own latest output for its current session: frames captured diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index 62c3bb362e..ff6591f823 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -7,6 +7,9 @@ deploy_orin.py user@orin --config orin.json +`--install-prereqs` and `--check-camera` run the Orin's prerequisite install and camera check instead of deploying +(neither needs a config). + The config is JSON with the producer's own options, nothing else: {"args": ["--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} @@ -41,14 +44,48 @@ UNIT_TEMPLATE = Path(__file__).with_name("isaaccapture-producer.service.in") CACHE = "~/.cache/isaaccapture-deploy" MODES = ("direct", "livekit") -CONFIG_KEYS = {"mode", "livekit_url", "token_file", "args", "env"} +CONFIG_KEYS = {"mode", "livekit_url", "token_file", "args", "env", "cameras"} +# With "cameras": "auto" the deploy finds the cameras and supplies these, so the config must not. +DETECTED_ARGS = ("--platform-config", "--config-name", "--link-masks") +PLATFORM_FILE = ( + "camera-platform.json" # under the install prefix; the detected platform config +) +PLATFORM_CONFIG_NAME = "SHW5G_2" ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") RESERVED_ENV = ("LD_LIBRARY_PATH",) # The producer's connection options come from the config's own fields, not from `args`. PREFIX = "/opt/isaaccapture" SERVICE_GROUPS = "video" # device access for the cameras READY_TIMEOUT_S = 60 +REBOOT_TIMEOUT_S = ( + 600 # a reboot to load the device-tree overlay, until ssh answers again +) +REBOOT_START_S = ( + 120 # how long to wait for the Orin to go down once setup asks for a reboot +) +NEEDS_REBOOT = 2 # orin_setup.sh exit status: reboot or log in again, then re-run +SDKM_DOWNLOADS = "~/Downloads/nvidia/sdkm_downloads" RESERVED_ARGS = ("--livekit-url", "--livekit-token-file") +# The Orin-side setup and camera-check scripts, and the platform config they stream with. +TOOL_SCRIPTS = ("orin_setup.sh", "setup.sh", "common.sh", "verify.sh", "validate.sh") +TOOL_CONFIG = "shw5g.json" +SETUP_HINT = f"bash {CACHE}/tools/scripts/orin_setup.sh" + + +def tool_files() -> list[tuple[Path, str]]: + """(local file, path under the tools directory) for the Orin-side scripts. + + They come from the checkout holding this helper, never from the deployed source, so any source can be deployed. + On the Orin the helper sits beside the `tools/` directory the desktop uploaded. + """ + here = Path(__file__).resolve().parent + if (here / "tools").is_dir(): + scripts, configs = here / "tools" / "scripts", here / "tools" / "configs" + else: + scripts, configs = here, here.parent / "configs" + return [(scripts / name, f"scripts/{name}") for name in TOOL_SCRIPTS] + [ + (configs / TOOL_CONFIG, f"configs/{TOOL_CONFIG}") + ] STEPS = 5 @@ -178,6 +215,13 @@ def load_config(path: Path) -> dict: raise DeployError( f"{path}: args needs at least one --add-stream=sensor=N,livekit=NAME" ) + if "cameras" in config: + if config["cameras"] != "auto": + raise DeployError(f'{path}: "cameras" must be "auto"') + if any(a.split("=", 1)[0] in DETECTED_ARGS for a in args): + raise DeployError( + f'{path}: with "cameras": "auto" the deploy supplies {", ".join(DETECTED_ARGS)}; remove them from args' + ) if "token_file" in config: if not all( isinstance(config[k], str) and config[k] @@ -391,6 +435,31 @@ def ssh(self, command: str, *, tty: bool = False) -> subprocess.CompletedProcess ) return result + def status( + self, + command: str, + *, + tty: bool = False, + timeout: int = 0, + stdin: str | None = None, + ) -> int: + """Run a command and return its exit status instead of raising (255 when ssh itself fails).""" + opts = ["-o", f"ConnectTimeout={timeout}"] if timeout else [] + extra: dict = {"capture_output": True} if timeout else {} + if stdin is not None: + extra.update(input=stdin, text=True) + return self._run( + "ssh", [*opts, *(["-t"] if tty else []), self.host, command], **extra + ).returncode + + def output(self, command: str, *, stdin: str | None = None) -> tuple[int, str]: + """Run a command; return (exit status, stdout). Its stderr still goes to this terminal.""" + extra: dict = ( + {"input": stdin, "text": True} if stdin is not None else {"text": True} + ) + result = self._run("ssh", [self.host, command], stdout=subprocess.PIPE, **extra) + return result.returncode, result.stdout + def sudo(self, command: str) -> None: """Run a command as root: the password from the environment on stdin, else an interactive prompt.""" if self.password: @@ -458,11 +527,234 @@ def close(self) -> None: shutil.rmtree(self._dir, ignore_errors=True) +def upload_tools(target: "Target", cache: str) -> None: + """Put the setup and camera-check scripts, with the config they stream with, under the Orin's cache.""" + target.capture(f"mkdir -p {cache}/tools/scripts {cache}/tools/configs") + for local_file, relative in tool_files(): + target.put(local_file, f"{cache}/tools/{relative}") + + +def show_logs(target: "Target") -> None: + """Print the producer service's state and recent journal; nothing on the Orin changes.""" + target.validate_connection() + target.sudo(f"systemctl status {SERVICE} --no-pager -l") + target.sudo(f"journalctl -u {SERVICE} -n 120 --no-pager") + + +def run_tools(args: argparse.Namespace, target: "Target") -> None: + """Install the Orin's prerequisites and/or check its cameras with the uploaded scripts; deploy nothing. + + The output shows live. Given ORIN_PASSWORD the Orin's sudo is answered for you; otherwise it prompts in the + terminal. + """ + if args.logs: + show_logs(target) + return + step(1, f"Connecting to {target.host}") + target.validate_connection() + cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") + step(2, "Upload the setup and camera-check scripts") + upload_tools(target, cache) + check = args.check_camera or args.scan_cameras + flags = [] + if args.install_prereqs and not check: + flags.append("--prereqs-only") + elif not args.install_prereqs: + flags.append("--check-only") + if args.scan_cameras: + flags.append("--scan") + label = " and ".join( + part + for part, wanted in ( + ("Install prerequisites", args.install_prereqs), + ( + "Scan the camera links" if args.scan_cameras else "Check the cameras", + check, + ), + ) + if wanted + ) + step(3, label) + code = run_orin_setup(target, cache, " ".join(flags)) + if code != 0: + raise DeployError(f"{label.lower()} failed on {target.host} (exit {code})") + + +def detect_cameras(target: "Target", cache: str, rescan: bool = False) -> dict: + """Find where cameras are connected, on any of the carrier's 8 inputs, and write the platform config for them. + + The inputs found are remembered on the Orin and tried first next time; all 8 are scanned again only when a + remembered camera does not answer, or with rescan. The Orin's producer service holds the cameras, so it is + stopped first. Returns the JSON orin_setup.sh prints: + the platform config's path on the Orin, the link masks, and how many cameras were found and used. + """ + print("==> Finding the cameras (any of the carrier's 8 inputs)", flush=True) + target.sudo(f"sh -c {shlex.quote(f'systemctl stop {SERVICE} || true')}") + code, text = target.output( + f"bash {cache}/tools/scripts/orin_setup.sh --detect {cache}/cameras" + + (" --rescan" if rescan else "") + ) + if code != 0: + raise DeployError( + "no camera answers on any of the 8 inputs; check the carrier's 12 V supply and each cable, " + "or run --scan-cameras" + ) + try: + found = json.loads(text.strip().splitlines()[-1]) + found["config"], found["masks"], found["used"] + except (ValueError, IndexError, KeyError) as exc: + raise DeployError( + f"camera detection printed no usable result: {text!r}" + ) from exc + for place in found.get("positions", []): + print(f" camera: {place}") + return found + + +def apply_detected(config: dict, found: dict) -> dict: + """The deploy config with the detected cameras: their platform config and masks, and streams for them only. + + Sensors are numbered from 0 in the order of the detected cameras, so a stream for a sensor that was not found + is dropped. All streams dropped is an error. + """ + used = found["used"] + kept, dropped = [], [] + for arg in config["args"]: + if arg.startswith("--add-stream="): + sensors = configured_sensors({"args": [arg]}) + if sensors and sensors[0] >= used: + dropped.append(arg) + continue + kept.append(arg) + if not configured_sensors({"args": kept}): + raise DeployError( + f"the config streams sensors the detected cameras do not have ({used} found)" + ) + if dropped: + print( + f" {used} camera(s) found; dropping {len(dropped)} stream(s) for sensors that are not there", + flush=True, + ) + return { + **config, + "args": [ + *kept, + f"--platform-config={PREFIX}/{PLATFORM_FILE}", + f"--config-name={PLATFORM_CONFIG_NAME}", + f"--link-masks={found['masks']}", + ], + } + + +def find_sipl_package(explicit: str | None) -> Path | None: + """The L4T SIPL API headers archive on this desktop: the named file, else the newest SDK Manager download.""" + if explicit: + path = Path(explicit).expanduser() + if not path.is_file(): + raise DeployError(f"--sipl-api {explicit}: no such file") + return path + found = sorted( + Path(SDKM_DOWNLOADS).expanduser().glob("Jetson_SIPL_API_R*_aarch64.tbz2"), + key=lambda p: p.stat().st_mtime, + ) + return found[-1] if found else None + + +def wait_for_reboot(target: "Target") -> None: + """Block until the Orin has gone down and answered ssh again; fail when it never reboots or never returns.""" + print("==> Waiting for the Orin to reboot", flush=True) + start = time.monotonic() + went_down = False + while time.monotonic() - start < REBOOT_TIMEOUT_S: + up = target.status("true", timeout=5) == 0 + if not up: + went_down = True + elif went_down: + return + elif time.monotonic() - start > REBOOT_START_S: + break + time.sleep(5) + raise DeployError( + "the Orin did not reboot and come back; reboot it yourself and run this command again" + if not went_down + else f"the Orin did not answer ssh within {REBOOT_TIMEOUT_S} s of rebooting" + ) + + +def prerequisites_ready(target: "Target", cache: str, sdk_root: str) -> bool: + """The Orin passes its own preflight and has the drivers, overlay and SIPL headers the cameras need.""" + remote = f"python3 {cache}/deploy_orin.py remote" + if target.status(f"{remote} preflight --sdk-root {shlex.quote(sdk_root)}") != 0: + return False + check = ( + f"bash {cache}/tools/scripts/orin_setup.sh --prereqs-only --no-apt --no-drivers" + ) + return target.status(f"{check} --sdk-root {shlex.quote(sdk_root or '/')}") == 0 + + +def run_orin_setup(target: "Target", cache: str, flags: str) -> int: + """Run orin_setup.sh on the Orin and return its exit status. + + With the password in the environment nothing prompts: it goes to the script on stdin, for its sudo calls. + Without one the script runs in a terminal and sudo prompts there. + """ + command = f"bash {cache}/tools/scripts/orin_setup.sh {flags}".rstrip() + if target.password: + return target.status( + f"{command} --sudo-password-stdin", stdin=target.password + "\n" + ) + return target.status(command, tty=True) + + +def install_prerequisites( + args: argparse.Namespace, target: "Target", cache: str +) -> None: + """Install what the preflight found missing: packages, uv, SIPL headers, the SENSING drivers and overlay. + + The drivers step selects the overlay through Jetson-IO's command line and reboots the Orin; the deploy waits out + the reboot and carries on. With no ORIN_PASSWORD it runs in a terminal and sudo prompts. + """ + print("==> Installing the Orin's missing prerequisites", flush=True) + flags = "--prereqs-only" + if args.sdk_root: + flags += f" --sdk-root {shlex.quote(args.sdk_root)}" + sipl = find_sipl_package(args.sipl_api) + if sipl: + target.put(sipl, f"{cache}/{sipl.name}") + flags += f" --sipl-api {cache}/{sipl.name}" + + def run_setup() -> int: + return run_orin_setup(target, cache, flags) + + code = run_setup() + if code in ( + NEEDS_REBOOT, + 255, + ): # 255: ssh dropped because Jetson-IO rebooted the Orin + wait_for_reboot(target) + target.validate_connection() + code = run_setup() + if code != 0: + raise DeployError( + f"prerequisite setup did not finish (exit {code}); fix what it reported and run this command again" + ) + if not prerequisites_ready(target, cache, args.sdk_root): + raise DeployError("the Orin still fails the preflight after setup") + + def deploy(args: argparse.Namespace) -> None: - config = load_config(Path(args.config)) + tools_only = ( + args.install_prereqs or args.check_camera or args.scan_cameras or args.logs + ) + if not tools_only and not args.config: + raise DeployError("--config is required to deploy") + config = None if tools_only else load_config(Path(args.config)) target = Target(target_host(args.target)) try: - run_deploy(args, config, target) + if tools_only: + run_tools(args, target) + else: + run_deploy(args, config, target) finally: target.close() @@ -536,7 +828,6 @@ def deploy_source( step(1, f"Connecting to {target.host}") target.validate_connection() user = args.service_user or target.capture("id -un") - unit = render_unit(config, PREFIX, user, SERVICE_GROUPS) step(2, f"Preflight on {target.host}") cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") @@ -547,8 +838,22 @@ def deploy_source( # The helper always travels alone and fresh, so the Orin-side steps never depend on an older source copy. helper = f"{cache}/deploy_orin.py" target.put(Path(__file__).resolve(), helper) + # The setup and camera-check scripts go up before the preflight, so a missing prerequisite can be fixed with them. + upload_tools(target, cache) remote = f"python3 {helper} remote" - target.ssh(f"{remote} preflight --sdk-root {shlex.quote(args.sdk_root)}") + if not prerequisites_ready(target, cache, args.sdk_root): + if args.no_setup: + raise DeployError( + f"the Orin fails the preflight; run {SETUP_HINT} --prereqs-only or drop --no-setup" + ) + install_prerequisites(args, target, cache) + + platform_file = "" + if config.get("cameras") == "auto": + found = detect_cameras(target, cache, args.rescan_cameras) + config = apply_detected(config, found) + platform_file = found["config"] + unit = render_unit(config, PREFIX, user, SERVICE_GROUPS) with tempfile.TemporaryDirectory() as tmp: step(3, "Compare the source with the Orin's and upload what differs") @@ -615,6 +920,7 @@ def deploy_source( f"--sensors={','.join(str(n) for n in configured_sensors(config))} " f"--direct-sensors={','.join(str(n) for n, direct in configured_streams(config) if direct)} " f"--mode={config.get('mode', 'direct')}" + + (f" --platform-file {shlex.quote(platform_file)}" if platform_file else "") + (" --replace-unmanaged" if args.replace_unmanaged else "") ) target.sudo(activate) @@ -659,26 +965,25 @@ def remote_preflight(args: argparse.Namespace) -> None: ["which", tool], capture_output=True, check=False, env=on_path ) if found.returncode != 0: - hint = ( - " (the build's Python setup needs it: curl -LsSf https://astral.sh/uv/install.sh | sh)" - if tool == "uv" - else "" + raise DeployError( + f"{tool} is not installed; run {SETUP_HINT} --prereqs-only" ) - raise DeployError(f"{tool} is not installed{hint}") if ( not Path("/usr/local/cuda/bin/nvcc").exists() and subprocess.run( ["which", "nvcc"], capture_output=True, check=False ).returncode ): - raise DeployError("nvcc not found (CUDA toolkit)") + raise DeployError( + f"nvcc not found (CUDA toolkit); run {SETUP_HINT} --prereqs-only" + ) stat = os.statvfs(Path.home()) if stat.f_bavail * stat.f_frsize < 4 << 30: raise DeployError("less than 4 GB free in the home directory") mmapi, _ = sdk_dirs(args.sdk_root) if not mmapi.is_dir(): raise DeployError( - f"the Jetson Multimedia API is missing at {mmapi}; run src/plugins/sensing/scripts/setup.sh " + f"the Jetson Multimedia API is missing at {mmapi}; run {SETUP_HINT} --prereqs-only " "or pass --sdk-root" ) print(" preflight ok") @@ -710,8 +1015,8 @@ def livekit_notice(build_dir: Path, name: str) -> Path: def stage_producer(build_dir: Path, stage: Path) -> None: """Assemble the release from the build tree itself, so it does not depend on the source's install rules. - The release is the binary, the SIPL platform configs the build puts beside it, the LiveKit runtime libraries - the binary resolves, and the LiveKit licence files. The service sets LD_LIBRARY_PATH to the release directory, + The release is the binary, the SIPL platform configs the build puts beside it, the setup and camera-check + scripts, the LiveKit runtime libraries the binary resolves, and the LiveKit licence files. The service sets LD_LIBRARY_PATH to the release directory, so the copied libraries win over the build tree's RUNPATH. """ binaries = [p for p in build_dir.rglob("camera_plugin_sensing") if p.is_file()] @@ -728,6 +1033,11 @@ def stage_producer(build_dir: Path, stage: Path) -> None: if not configs.is_dir(): raise DeployError(f"the build staged no platform configs beside {built}") shutil.copytree(configs, out / "configs") + # The setup and camera-check scripts resolve the platform config as ../configs from their own directory. + (out / "scripts").mkdir() + for local_file, relative in tool_files(): + if relative.startswith("scripts/"): + shutil.copy2(local_file, out / relative) # The libraries are found by name among what the binary resolves in the build, not by the SDK's layout. for line in tail(["ldd", str(built)]).splitlines(): name, _, rest = line.strip().partition(" => ") @@ -940,17 +1250,25 @@ def remote_activate(args: argparse.Namespace) -> None: ) dest.mkdir(parents=True) try: + print(" unpacking the release", flush=True) with tarfile.open(args.archive) as tar: tar.extractall(dest, filter="data") - for line in (dest / "SHA256SUMS").read_text().splitlines(): + sums = (dest / "SHA256SUMS").read_text().splitlines() + bar = Progress("verifying files", len(sums)) + for done, line in enumerate(sums, 1): digest, name = line.split(" ", 1) if sha256_file(dest / name) != digest: raise DeployError( f"checksum mismatch for {name} in release {args.release}" ) + bar.update(done) + bar.finish() except BaseException: shutil.rmtree(dest, ignore_errors=True) raise + if args.platform_file: + # The detected platform config outlives the release: the service reads it from a fixed path. + shutil.copy2(args.platform_file, prefix / PLATFORM_FILE) run(["chown", "-R", f"{args.owner}:", str(prefix)]) print(f" installed release {args.release}") @@ -961,6 +1279,7 @@ def remote_activate(args: argparse.Namespace) -> None: run(["systemctl", "daemon-reload"]) run(["systemctl", "enable", SERVICE], capture_output=True) print(" service unit updated") + print(" stopping the running producer", flush=True) run(["systemctl", "stop", SERVICE]) if unmanaged: print(f" stopping unmanaged producer(s): {[pid for pid, _ in unmanaged]}") @@ -971,6 +1290,7 @@ def remote_activate(args: argparse.Namespace) -> None: staging_link.symlink_to(Path("releases") / args.release) os.replace(staging_link, link) since = time.strftime("%Y-%m-%d %H:%M:%S") + print(" starting the new producer; waiting for the cameras", flush=True) run(["systemctl", "start", SERVICE]) summary = wait_ready( since, @@ -984,8 +1304,30 @@ def remote_activate(args: argparse.Namespace) -> None: print(f" producer is up (release {args.release})") -# The capture driver logs its own INFO and DEBUG lines at severity 5 and 6; they bury the lines that explain a failure. -DRIVER_NOISE = re.compile(r"Severity [56] :") +# Every line the capture driver logs carries "Severity N :"; at any severity they bury the lines that explain a +# failure. DRIVER_FAULT picks the few that do explain one, and BENIGN_DRIVER the ones that look like errors but are +# the driver noting a library or power driver it does not need. +DRIVER_NOISE = re.compile(r"Severity \d+ :") +DRIVER_FAULT = re.compile(r"Failed to|Error processing|HSL I2C error|status: \d+") +BENIGN_DRIVER = re.compile(r"driverCatalog|uddf_discover|No module power driver") + + +# The deserializer's own report that a link did not lock: SIPL names the link by letter, A to D for links 0 to 3. +DEAD_LINK = re.compile(r'Sequence: "enable_link_([A-D])"') + + +def dead_link_lines(lines: list[str]) -> list[str]: + """One plain line per camera link the deserializer could not lock, in the order they failed.""" + found = [] + for line in lines: + match = DEAD_LINK.search(line) + if match and match.group(1) not in found: + found.append(match.group(1)) + return [ + f"link {ord(letter) - ord('A')} (COAX{ord(letter) - ord('A')} on CN1) did not lock: no camera answers there. " + "Check that camera's cable and its harness lead." + for letter in found + ] def failure_report(since: str, sensors: list[int]) -> str: @@ -1024,11 +1366,25 @@ def failure_report(since: str, sensors: list[int]) -> str: ln, ) ] + driver_faults = [] + for line in lines: + if ( + DRIVER_NOISE.search(line) + and DRIVER_FAULT.search(line) + and not BENIGN_DRIVER.search(line) + ): + text = line.split(" : ", 1)[-1].strip() + if text not in driver_faults: + driver_faults.append(text) out = [ f"service: {state}", - f"log: {len(lines)} lines since the start, {len(lines) - len(quiet)} of them driver INFO/DEBUG; " + *dead_link_lines(lines), + f"log: {len(lines)} lines since the start, {len(lines) - len(quiet)} of them from the camera driver; " f"complete log saved on the Orin at {saved}", ] + if driver_faults: + out.append("camera driver errors:") + out += [f" {ln}" for ln in driver_faults[:4]] if sensors: out.append(f"sensors expected to publish: {', '.join(map(str, sensors))}") if faults: @@ -1111,7 +1467,9 @@ def wait_ready( earlier attempt. A producer that is not up in time raises, with the sensors that were not. """ direct_sensors = direct_sensors or [] - deadline = time.monotonic() + timeout + start = time.monotonic() + deadline = start + timeout + bar = Progress("waiting for cameras", timeout) log = "" while time.monotonic() < deadline: if ( @@ -1126,8 +1484,13 @@ def wait_ready( ) ready, problems = readiness(log, mode, sensors, direct_sensors) if not problems: + bar.finish() return ready + bar.label = f"waiting for cameras ({len(ready)}/{len(sensors)} up)" + bar.update(int(time.monotonic() - start)) time.sleep(2) + if sys.stdout.isatty(): + print() print(failure_report(since, sensors), file=sys.stderr) # Judged on the log the last check saw, so the error names what that check found missing. _, problems = readiness(log, mode, sensors, direct_sensors) @@ -1145,7 +1508,37 @@ def parser() -> argparse.ArgumentParser: ap.add_argument( "target", help=f"host or user@host of the Orin (a bare host uses ${USER_ENV})" ) - ap.add_argument("--config", required=True, help="producer options (JSON)") + ap.add_argument( + "--config", + help="producer options (JSON); required unless only setting up or checking", + ) + ap.add_argument( + "--install-prereqs", + action="store_true", + help="install the Orin's build prerequisites (asks for its sudo password), then stop without deploying", + ) + ap.add_argument( + "--logs", + action="store_true", + help="show the producer service's status and recent journal, then stop", + ) + ap.add_argument( + "--rescan-cameras", + action="store_true", + help='with "cameras": "auto", scan all 8 inputs instead of trying the cameras found last time first', + ) + ap.add_argument( + "--scan-cameras", + action="store_true", + help="try each camera link alone and report which has a camera (stops the producer service), then stop " + "without deploying", + ) + ap.add_argument( + "--check-camera", + action="store_true", + help="stream both cameras for a few seconds and report, then stop without deploying; " + "with --install-prereqs the prerequisites go first", + ) ap.add_argument( "--jobs", type=int, default=2, help="parallel compile jobs on the Orin" ) @@ -1180,6 +1573,17 @@ def parser() -> argparse.ArgumentParser: help="extra CMake argument for the producer build, repeatable (for options only some source trees have)", ) ap.add_argument("--no-activate", action="store_true", help="build and package only") + ap.add_argument( + "--no-setup", + action="store_true", + help="fail when the Orin lacks a prerequisite instead of installing it", + ) + ap.add_argument( + "--sipl-api", + metavar="FILE", + help="Jetson_SIPL_API_R_aarch64.tbz2 to unpack on the Orin when its SIPL headers are missing " + f"(default: the newest in {SDKM_DOWNLOADS})", + ) return ap @@ -1209,6 +1613,7 @@ def remote_parser() -> argparse.ArgumentParser: act.add_argument("--direct-sensors", default="") act.add_argument("--mode", choices=MODES, default="direct") act.add_argument("--replace-unmanaged", action="store_true") + act.add_argument("--platform-file", default="") return ap diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index d20228ce20..5d7f1abd0a 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -452,6 +452,14 @@ def __init__(self, manifest=None): self.target.capture.side_effect = self.capture self.target.put.side_effect = self.put self.target.ssh.side_effect = self.ssh + self.target.status.side_effect = self.status + self.target.password = None + self.target.output.side_effect = self.output + self.detected = None + self.status_calls = [] + self.preflight_codes = [] + self.rig_codes = [] + self.setup_codes = [] def capture(self, cmd): if "pwd" in cmd: @@ -470,12 +478,31 @@ def put(self, local, remote): ) self.deltas.append(sorted(n for n in names if not n.startswith(".deploy/"))) - def ssh(self, cmd): + def status(self, cmd, tty=False, timeout=0, stdin=None): + self.events.append(("status", cmd)) + self.status_calls.append((cmd, tty, stdin)) + if " preflight " in cmd: + queue = self.preflight_codes + elif "--no-apt" in cmd: # the read-only rig check + queue = self.rig_codes + elif "orin_setup.sh" in cmd: + queue = self.setup_codes + else: + queue = [] + return queue.pop(0) if queue else 0 + + def output(self, cmd, stdin=None): + self.events.append(("output", cmd)) + if self.detected is None: + return 1, "" + return 0, "noise\n" + json.dumps(self.detected) + "\n" + + def ssh(self, cmd, tty=False): self.events.append(("ssh", cmd)) if " apply " in cmd: self.manifest = self.pending - def run(self, root, *extra): + def run(self, root, *extra, config=None): args = deploy_orin.parser().parse_args( [ "u@orin", @@ -485,7 +512,7 @@ def run(self, root, *extra): *extra, ] ) - config = { + config = config or { "mode": "livekit", "livekit_url": "wss://x", "token_file": "/t", @@ -513,7 +540,7 @@ def test_helper_uploaded_and_preflight_run_before_any_delta(self): for kind, value in orin.events: if kind == "put" and value.endswith("/deploy_orin.py"): kinds.append("helper") - elif kind == "ssh" and " preflight " in value: + elif kind == "status" and " preflight " in value: kinds.append("preflight") elif kind == "put" and value.endswith("delta.tar.gz"): kinds.append("delta") @@ -521,6 +548,228 @@ def test_helper_uploaded_and_preflight_run_before_any_delta(self): apply_cmd = next(v for k, v in orin.events if k == "ssh" and " apply " in v) self.assertIn("--fresh", apply_cmd) + def test_setup_and_camera_check_scripts_are_uploaded_before_preflight(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.run(root) + wanted = [f"{CACHE_DIR}/tools/{rel}" for _, rel in deploy_orin.tool_files()] + uploads = [ + i for i, (k, v) in enumerate(orin.events) if k == "put" and v in wanted + ] + self.assertEqual(len(uploads), len(wanted)) + preflight = next( + i + for i, (k, v) in enumerate(orin.events) + if k == "status" and " preflight " in v + ) + self.assertTrue(all(i < preflight for i in uploads)) + names = {rel for _, rel in deploy_orin.tool_files()} + self.assertTrue( + { + "scripts/orin_setup.sh", + "scripts/setup.sh", + "scripts/validate.sh", + "scripts/verify.sh", + } + <= names + ) + + def tools_run(self, *flags): + args = deploy_orin.parser().parse_args(["u@orin", *flags]) + orin = FakeOrin() + deploy_orin.run_tools(args, orin.target) + return orin + + def tools_command(self, *flags): + orin = self.tools_run(*flags) + (call,) = [c for c in orin.status_calls if "orin_setup.sh" in c[0]] + return orin, call + + def test_install_prereqs_runs_only_the_prerequisite_half_in_a_terminal(self): + orin, call = self.tools_command("--install-prereqs") + self.assertTrue(call[0].endswith("orin_setup.sh --prereqs-only")) + self.assertTrue(call[1]) + self.assertFalse(any("deploy_orin.py" in v for _, v in orin.events)) + + def test_check_camera_runs_only_the_camera_half(self): + _, call = self.tools_command("--check-camera") + self.assertTrue(call[0].endswith("orin_setup.sh --check-only")) + + def test_both_flags_run_the_whole_script(self): + _, call = self.tools_command("--install-prereqs", "--check-camera") + self.assertTrue(call[0].endswith("orin_setup.sh")) + + def test_scan_cameras_runs_the_camera_half_with_the_scan(self): + _, call = self.tools_command("--scan-cameras") + self.assertTrue(call[0].endswith("orin_setup.sh --check-only --scan")) + + def test_install_and_scan_run_everything_with_the_scan(self): + _, call = self.tools_command("--install-prereqs", "--scan-cameras") + self.assertTrue(call[0].endswith("orin_setup.sh --scan")) + + def test_tools_modes_upload_the_scripts_first_and_need_no_config(self): + events = self.tools_run("--check-camera").events + self.assertEqual(events[-1][0], "status") + self.assertTrue(all(k == "put" for k, _ in events[:-1])) + + def test_a_failing_setup_script_is_a_deploy_error(self): + args = deploy_orin.parser().parse_args(["u@orin", "--scan-cameras"]) + orin = FakeOrin() + orin.setup_codes = [1] + with self.assertRaisesRegex(DeployError, "exit 1"): + deploy_orin.run_tools(args, orin.target) + + def test_tools_run_answers_sudo_from_the_environment_password(self): + args = deploy_orin.parser().parse_args(["u@orin", "--scan-cameras"]) + orin = FakeOrin() + orin.target.password = "s3cret" + deploy_orin.run_tools(args, orin.target) + (call,) = [c for c in orin.status_calls if "orin_setup.sh" in c[0]] + self.assertTrue(call[0].endswith("--sudo-password-stdin")) + self.assertEqual(call[2], "s3cret\n") + self.assertNotIn("s3cret", call[0]) + + def test_logs_only_reads_the_service_and_changes_nothing(self): + orin = self.tools_run("--logs") + commands = [c.args[0] for c in orin.target.sudo.call_args_list] + self.assertEqual(len(commands), 2) + self.assertIn(deploy_orin.SERVICE, commands[0]) + self.assertIn("journalctl", commands[1]) + self.assertEqual(orin.events, []) + + def test_deploying_without_a_config_is_an_error(self): + args = deploy_orin.parser().parse_args(["u@orin"]) + with self.assertRaisesRegex(DeployError, "--config is required"): + deploy_orin.deploy(args) + + def setup_events(self, orin): + return [ + v + for k, v in orin.events + if k == "status" and "orin_setup.sh" in v and "--no-apt" not in v + ] + + def test_passing_preflight_installs_nothing(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.run(root) + self.assertEqual(self.setup_events(orin), []) + + def test_failing_preflight_installs_prerequisites_then_rechecks(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=None): + orin.run(root) + (setup,) = self.setup_events(orin) + self.assertTrue(setup.endswith("orin_setup.sh --prereqs-only")) + kinds = [ + "preflight" if " preflight " in v else "setup" + for k, v in orin.events + if k == "status" + and (" preflight " in v or ("orin_setup.sh" in v and "--no-apt" not in v)) + ] + self.assertEqual(kinds, ["preflight", "setup", "preflight"]) + + def test_missing_drivers_or_overlay_alone_trigger_the_install(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.rig_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=None): + orin.run(root) + self.assertEqual(len(self.setup_events(orin)), 1) + + def test_install_without_a_password_runs_in_a_terminal(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=None): + orin.run(root) + (call,) = [ + c + for c in orin.status_calls + if "orin_setup.sh" in c[0] and "--no-apt" not in c[0] + ] + self.assertTrue(call[1]) + self.assertIsNone(call[2]) + self.assertNotIn("--sudo-password-stdin", call[0]) + + def test_install_with_a_password_sends_it_on_stdin_and_never_in_the_command(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.target.password = "s3cret" + orin.preflight_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=None): + orin.run(root) + (call,) = [ + c + for c in orin.status_calls + if "orin_setup.sh" in c[0] and "--no-apt" not in c[0] + ] + self.assertTrue(call[0].endswith("--sudo-password-stdin")) + self.assertEqual(call[2], "s3cret\n") + self.assertFalse(call[1]) + self.assertTrue(all("s3cret" not in c[0] for c in orin.status_calls)) + + def test_sipl_archive_is_uploaded_and_named_to_the_setup_script(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + archive = self.tmp / "Jetson_SIPL_API_R39.2.1_aarch64.tbz2" + archive.write_bytes(b"x") + orin = FakeOrin() + orin.preflight_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=archive): + orin.run(root) + self.assertIn(("put", f"{CACHE_DIR}/{archive.name}"), orin.events) + (setup,) = self.setup_events(orin) + self.assertIn(f"--sipl-api {CACHE_DIR}/{archive.name}", setup) + + def test_no_setup_refuses_instead_of_installing(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1] + with self.assertRaisesRegex(DeployError, "fails the preflight"): + orin.run(root, "--no-setup") + self.assertEqual(self.setup_events(orin), []) + + def test_reboot_request_waits_then_reruns_setup(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1, 0] + orin.setup_codes = [deploy_orin.NEEDS_REBOOT, 0] + with ( + mock.patch.object(deploy_orin, "find_sipl_package", return_value=None), + mock.patch.object(deploy_orin, "wait_for_reboot") as wait, + ): + orin.run(root) + wait.assert_called_once_with(orin.target) + self.assertEqual(len(self.setup_events(orin)), 2) + + def test_dropped_ssh_during_setup_is_treated_as_a_reboot(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1, 0] + orin.setup_codes = [255, 0] + with ( + mock.patch.object(deploy_orin, "find_sipl_package", return_value=None), + mock.patch.object(deploy_orin, "wait_for_reboot") as wait, + ): + orin.run(root) + wait.assert_called_once() + + def test_setup_failure_stops_the_deploy(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1] + orin.setup_codes = [1] + with ( + mock.patch.object(deploy_orin, "find_sipl_package", return_value=None), + self.assertRaisesRegex(DeployError, "prerequisite setup did not finish"), + ): + orin.run(root) + self.assertFalse( + any(k == "put" and v.endswith("delta.tar.gz") for k, v in orin.events) + ) + def test_unchanged_second_run_sends_nothing_and_one_change_sends_only_it(self): root, _ = git_repo( self, {"src/a.cpp": "a\n", "src/b.cpp": "b\n", "README.md": "r\n"} @@ -996,6 +1245,7 @@ def activate( archive, timeout=60, replace_unmanaged=False, + platform_file="", sensors="", mode="livekit", direct_sensors="", @@ -1012,9 +1262,17 @@ def activate( mode=mode, direct_sensors=direct_sensors, replace_unmanaged=replace_unmanaged, + platform_file=platform_file, ) ) + def test_platform_file_is_installed_under_the_prefix(self): + platform = self.tmp / "platform.json" + platform.write_text("{}") + self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" + self.activate(self.make_archive(), platform_file=str(platform)) + self.assertEqual((self.prefix / deploy_orin.PLATFORM_FILE).read_text(), "{}") + def test_checksum_mismatch_aborts_before_stop(self): with self.assertRaisesRegex(DeployError, "checksum mismatch"): self.activate(self.make_archive(good=False)) @@ -1351,7 +1609,7 @@ def fake_run(cmd, **kwargs): self.seen = seen def test_preflight_requires_uv_using_the_build_path(self): - with self.assertRaisesRegex(DeployError, "uv is not installed.*astral.sh"): + with self.assertRaisesRegex(DeployError, "uv is not installed.*orin_setup.sh"): self.preflight(missing=("uv",)) uv_call = next(k for c, k in self.seen if c == ["which", "uv"]) self.assertEqual( @@ -1552,6 +1810,12 @@ def test_happy_path_stages_binary_configs_libs_and_licences(self): self.assertEqual(cmd, ["ldd", str(self.stage / deploy_orin.BIN_REL)]) self.assertEqual(kwargs["env"]["LD_LIBRARY_PATH"], str(out)) + def test_release_carries_the_setup_and_camera_check_scripts(self): + self.stage_it() + scripts = self.stage / "plugins/sensing_camera/scripts" + for name in deploy_orin.TOOL_SCRIPTS: + self.assertTrue((scripts / name).is_file(), name) + def test_ldd_not_found_for_a_livekit_library_is_not_copied(self): self.build_ldd = "\tliblivekit_ffi.so => not found" self.stage_it() @@ -2078,9 +2342,54 @@ def test_noise_is_filtered_out_and_counted(self): out = self.report(log) self.assertNotIn("driver info", out) self.assertNotIn("driver debug", out) - self.assertIn("4 lines since the start, 2 of them driver INFO/DEBUG", out) + self.assertIn("4 lines since the start, 2 of them from the camera driver", out) self.assertIn(" real line one", out) + def test_a_link_that_did_not_lock_is_named_in_plain_words(self): + log = "\n".join( + [ + 'x [Deser_9] Sequence: "enable_link_C" (operation 4/7)', + 'x [Deser_9] Sequence: "enable_link_C" (operation 4/7)', + ] + ) + out = self.report(log) + self.assertEqual(out.count("link 2 (COAX2 on CN1) did not lock"), 1) + + def test_driver_chatter_at_every_severity_is_hidden(self): + log = "\n".join( + [ + "[TID=1] Module_id 30 Severity 2 : [PIPELINE (0,0)] CNvFusaRawBlock.cpp:2143: DBG GetOutputBuffer", + "[TID=1] Module_id 30 Severity 3 : [CameraHAL] WARNING: No module power driver found for GMSL transport 'Deser_9'", + "producer line", + ] + ) + out = self.report(log) + self.assertNotIn("GetOutputBuffer", out) + self.assertIn(" producer line", out) + + def test_only_real_driver_errors_are_listed_and_deduplicated(self): + log = "\n".join( + [ + "[TID=1] Module_id 30 Severity 2 : [CameraHAL] ERROR: CameraHalPortal.cpp:280: driverCatalog: path='/x'", + "[TID=1] Module_id 30 Severity 3 : dlsym failed for 'uddf_discover_drivers'", + "[TID=1] Module_id 30 Severity 2 : Failed to initialize camera module: 0", + "[TID=2] Module_id 30 Severity 2 : Failed to initialize camera module: 0", + "[TID=1] Module_id 30 Severity 2 : Master initialization failed. status: 10", + ] + ) + out = self.report(log) + errors = out.split("camera driver errors:")[1].split("first lines")[0] + self.assertEqual(errors.count("Failed to initialize camera module"), 1) + self.assertIn("Master initialization failed. status: 10", errors) + self.assertNotIn("driverCatalog", errors) + self.assertNotIn("uddf_discover", errors) + + def test_no_driver_error_section_without_driver_errors(self): + self.assertNotIn("camera driver errors", self.report("plain")) + + def test_no_link_failure_adds_no_link_line(self): + self.assertNotIn("did not lock", self.report("real line")) + def test_service_state_and_sensors_listed(self): out = self.report("x") self.assertIn("service: ActiveState=failed, SubState=failed, NRestarts=2", out) @@ -2156,3 +2465,198 @@ def test_empty_log(self): if __name__ == "__main__": unittest.main() + + +class SiplPackageTests(TempDirTestCase): + def test_explicit_file_wins_and_a_missing_one_is_an_error(self): + archive = self.tmp / "a.tbz2" + archive.write_bytes(b"x") + self.assertEqual(deploy_orin.find_sipl_package(str(archive)), archive) + with self.assertRaisesRegex(DeployError, "no such file"): + deploy_orin.find_sipl_package(str(self.tmp / "missing.tbz2")) + + def test_default_is_the_newest_sdk_manager_download(self): + old = self.tmp / "Jetson_SIPL_API_R39.2.0_aarch64.tbz2" + new = self.tmp / "Jetson_SIPL_API_R39.2.1_aarch64.tbz2" + for path, when in ((old, 100), (new, 200)): + path.write_bytes(b"x") + os.utime(path, (when, when)) + with mock.patch.object(deploy_orin, "SDKM_DOWNLOADS", str(self.tmp)): + self.assertEqual(deploy_orin.find_sipl_package(None), new) + + def test_no_download_means_none(self): + with mock.patch.object(deploy_orin, "SDKM_DOWNLOADS", str(self.tmp)): + self.assertIsNone(deploy_orin.find_sipl_package(None)) + + +class WaitForRebootTests(unittest.TestCase): + def wait(self, codes): + target = mock.Mock() + target.status.side_effect = codes + clock = iter(range(0, 100000, 5)) + with ( + mock.patch.object(deploy_orin.time, "sleep"), + mock.patch.object(deploy_orin.time, "monotonic", lambda: next(clock)), + ): + deploy_orin.wait_for_reboot(target) + + def test_returns_once_the_orin_has_gone_down_and_come_back(self): + self.wait([0, 255, 255, 0]) + + def test_an_orin_that_never_goes_down_is_an_error(self): + with self.assertRaisesRegex(DeployError, "did not reboot"): + self.wait([0] * 1000) + + def test_an_orin_that_never_returns_is_an_error(self): + with self.assertRaisesRegex(DeployError, "did not answer ssh"): + self.wait([255] * 1000) + + +FOUND_ONE = { + "config": f"{CACHE_DIR}/cameras/platform.json", + "masks": "0x0100 0x0000", + "count": 1, + "used": 1, + "positions": ["CN2 COAX6 (deserializer AB, link 2)"], +} +TWO_STREAMS = [ + "--add-stream=sensor=0,name=left,direct=10.0.0.1:5600", + "--add-stream=sensor=1,name=right,direct=10.0.0.1:5601", +] + + +class AutoCamerasConfigTests(TempDirTestCase): + def load(self, config): + path = self.tmp / "c.json" + path.write_text(json.dumps(config)) + return deploy_orin.load_config(path) + + def test_auto_is_accepted(self): + self.assertEqual( + self.load({"cameras": "auto", "args": TWO_STREAMS})["cameras"], "auto" + ) + + def test_only_auto_is_a_valid_value(self): + with self.assertRaisesRegex(DeployError, '"cameras" must be "auto"'): + self.load({"cameras": "manual", "args": TWO_STREAMS}) + + def test_auto_forbids_the_arguments_the_deploy_supplies(self): + for arg in ( + "--link-masks=0x0000 0x1100", + "--platform-config=/x", + "--config-name=Y", + ): + with self.subTest(arg), self.assertRaisesRegex(DeployError, "supplies"): + self.load({"cameras": "auto", "args": [*TWO_STREAMS, arg]}) + + +class ApplyDetectedTests(unittest.TestCase): + def test_streams_for_missing_sensors_are_dropped(self): + config = deploy_orin.apply_detected({"args": TWO_STREAMS}, FOUND_ONE) + self.assertEqual(deploy_orin.configured_sensors(config), [0]) + + def test_platform_arguments_are_appended(self): + config = deploy_orin.apply_detected({"args": TWO_STREAMS}, FOUND_ONE) + self.assertIn( + f"--platform-config={deploy_orin.PREFIX}/camera-platform.json", + config["args"], + ) + self.assertIn("--config-name=SHW5G_2", config["args"]) + self.assertIn("--link-masks=0x0100 0x0000", config["args"]) + + def test_other_keys_and_stream_order_are_kept(self): + config = deploy_orin.apply_detected( + {"mode": "direct", "env": {"A": "b"}, "args": ["--fps=30", TWO_STREAMS[0]]}, + {**FOUND_ONE, "used": 2}, + ) + self.assertEqual(config["env"], {"A": "b"}) + self.assertEqual(config["args"][:2], ["--fps=30", TWO_STREAMS[0]]) + + def test_two_cameras_keep_both_streams(self): + found = {**FOUND_ONE, "used": 2, "masks": "0x0000 0x1100"} + config = deploy_orin.apply_detected({"args": TWO_STREAMS}, found) + self.assertEqual(deploy_orin.configured_sensors(config), [0, 1]) + + def test_no_stream_left_is_an_error(self): + with self.assertRaisesRegex(DeployError, "do not have"): + deploy_orin.apply_detected({"args": [TWO_STREAMS[1]]}, FOUND_ONE) + + +class DetectCamerasTests(unittest.TestCase): + def target(self, code, text): + target = mock.Mock() + target.output.return_value = (code, text) + return target + + def test_returns_the_json_printed_last(self): + target = self.target(0, "junk\n" + json.dumps(FOUND_ONE) + "\n") + self.assertEqual(deploy_orin.detect_cameras(target, CACHE_DIR), FOUND_ONE) + + def test_stops_the_service_before_probing(self): + target = self.target(0, json.dumps(FOUND_ONE)) + deploy_orin.detect_cameras(target, CACHE_DIR) + self.assertIn(deploy_orin.SERVICE, target.sudo.call_args.args[0]) + self.assertIn("--detect", target.output.call_args.args[0]) + + def test_remembered_cameras_are_used_unless_a_rescan_is_asked_for(self): + target = self.target(0, json.dumps(FOUND_ONE)) + deploy_orin.detect_cameras(target, CACHE_DIR) + self.assertNotIn("--rescan", target.output.call_args.args[0]) + deploy_orin.detect_cameras(target, CACHE_DIR, rescan=True) + self.assertTrue(target.output.call_args.args[0].endswith("--rescan")) + + def test_no_camera_is_an_error_that_points_at_the_scan(self): + with self.assertRaisesRegex(DeployError, "--scan-cameras"): + deploy_orin.detect_cameras(self.target(1, ""), CACHE_DIR) + + def test_unusable_output_is_an_error(self): + with self.assertRaisesRegex(DeployError, "no usable result"): + deploy_orin.detect_cameras(self.target(0, "not json"), CACHE_DIR) + + +class AutoCamerasDeployTests(TempDirTestCase): + def deploy(self, detected=FOUND_ONE): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.detected = detected + written = {} + original = deploy_orin.render_unit + + def spy(config, *rest): + written["args"] = config["args"] + return original(config, *rest) + + with mock.patch.object(deploy_orin, "render_unit", spy): + orin.run( + root, config={"mode": "direct", "cameras": "auto", "args": TWO_STREAMS} + ) + return orin, written + + def test_the_unit_uses_the_detected_cameras(self): + _, written = self.deploy() + self.assertIn("--link-masks=0x0100 0x0000", written["args"]) + self.assertEqual(deploy_orin.configured_sensors({"args": written["args"]}), [0]) + + def test_detection_runs_after_the_prerequisites_and_before_the_upload(self): + orin, _ = self.deploy() + kinds = [ + "detect" + if k == "output" + else "upload" + if v.endswith("delta.tar.gz") + else None + for k, v in orin.events + if k == "output" or (k == "put" and v.endswith("delta.tar.gz")) + ] + self.assertEqual(kinds, ["detect", "upload"]) + + def test_no_detected_cameras_stops_before_anything_is_uploaded(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + with self.assertRaises(DeployError): + orin.run( + root, config={"mode": "direct", "cameras": "auto", "args": TWO_STREAMS} + ) + self.assertFalse( + any(k == "put" and v.endswith("delta.tar.gz") for k, v in orin.events) + ) From f4fcc57bbcde5e07a73f82e27f8f7236010ae2f1 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 11:43:14 -0500 Subject: [PATCH 15/16] sensing: let deploy_orin.py upload the LiveKit token A LiveKit-mode deploy needed the join token to already exist on the Orin, which meant a manual copy first. Add an optional "token_source" config key, a local file (relative to the config), that the deploy uploads to "token_file" before anything is built or stopped. It goes by scp into a staging file created with mode 0600 and is moved into place once complete, so it never appears in an argument list, the release or the output. A different --service-user is refused, since the file is readable only by the ssh user. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 3 +- src/plugins/sensing/scripts/deploy_orin.py | 60 ++++++++++++++-- tests/python/ci/test_deploy_orin.py | 82 ++++++++++++++++++++++ 3 files changed, 140 insertions(+), 5 deletions(-) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index e98534865d..6fe54c4228 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -57,7 +57,8 @@ has the build and run steps. Write a config naming the producer's own options. `"mode"` says what the producer does and so how readiness is judged: `"direct"` (the default) for a producer that sends only over TCP to the PC video stage, or `"livekit"` for one that publishes to LiveKit itself, which also needs `livekit_url` and `token_file` (the token file stays on the Orin; -only its path is used). Everything else goes in `args`, spelled as the deployed source's producer expects. +only its path is used). Add `"token_source": "join.token"` (a local file, relative to the config) and the deploy uploads +it to `token_file` with mode 0600 before anything is stopped, so no separate copy step is needed. Everything else goes in `args`, spelled as the deployed source's producer expects. ```json {"args": ["--link-masks=0x0000 0x1100", diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index ff6591f823..302c6c819a 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -14,7 +14,8 @@ {"args": ["--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} -The token file stays on the Orin, outside the release; only its path is passed. Nothing is built, stopped or +The token file stays on the Orin, outside the release; only its path is passed. With "token_source" (a local file) +the deploy first uploads it to "token_file" on the Orin with mode 0600. Nothing is built, stopped or replaced until the target passes its checks. A build or transfer failure leaves the running producer alone. The same file runs on the target (`deploy_orin.py remote ...`) from the source snapshot it copied there. """ @@ -27,6 +28,7 @@ import io import json import os +import posixpath import pwd import re import shlex @@ -44,7 +46,15 @@ UNIT_TEMPLATE = Path(__file__).with_name("isaaccapture-producer.service.in") CACHE = "~/.cache/isaaccapture-deploy" MODES = ("direct", "livekit") -CONFIG_KEYS = {"mode", "livekit_url", "token_file", "args", "env", "cameras"} +CONFIG_KEYS = { + "mode", + "livekit_url", + "token_file", + "token_source", + "args", + "env", + "cameras", +} # With "cameras": "auto" the deploy finds the cameras and supplies these, so the config must not. DETECTED_ARGS = ("--platform-config", "--config-name", "--link-masks") PLATFORM_FILE = ( @@ -197,10 +207,12 @@ def load_config(path: Path) -> dict: mode = config.get("mode", "direct") if mode not in MODES: raise DeployError(f'{path}: "mode" must be one of {", ".join(MODES)}') - has_livekit = "livekit_url" in config or "token_file" in config + has_livekit = any( + k in config for k in ("livekit_url", "token_file", "token_source") + ) if mode == "direct" and has_livekit: raise DeployError( - f'{path}: livekit_url and token_file belong to "mode": "livekit"; a direct producer has no LiveKit settings' + f'{path}: livekit_url, token_file and token_source belong to "mode": "livekit"; a direct producer has no LiveKit settings' ) if mode == "livekit" and not ("livekit_url" in config and "token_file" in config): raise DeployError(f'{path}: "mode": "livekit" needs livekit_url and token_file') @@ -234,9 +246,41 @@ def load_config(path: Path) -> dict: raise DeployError( f"{path}: token_file must be an absolute path on the Orin" ) + if "token_source" in config: + source = config["token_source"] + if not isinstance(source, str) or not source: + raise DeployError(f"{path}: token_source must be a non-empty string") + # A relative source is relative to the config file, so a config and its token can move together. + local = Path(source).expanduser() + if not local.is_absolute(): + local = path.parent / local + if not local.is_file(): + raise DeployError(f"{path}: token_source {local} is not a file") + if not local.read_text().strip(): + raise DeployError(f"{path}: token_source {local} is empty") + config["token_source"] = str(local.resolve()) return config +def install_token(target: "Target", local: Path, remote: str) -> None: + """Upload a local token file to ``remote`` on the Orin, readable only by the ssh user, replacing any older one. + + The token is never part of an argument list, the release or the output: it travels by scp into a staging file + created with mode 0600, and moves into place only once complete, so the service never reads half a token. + """ + staging = f"{remote}.new" + quoted_remote, quoted_staging = shlex.quote(remote), shlex.quote(staging) + # Create the staging file private before any bytes arrive; scp keeps an existing file's mode. + target.capture( + f"mkdir -p {shlex.quote(posixpath.dirname(remote))} && umask 077 && rm -f {quoted_staging} && : > {quoted_staging}" + ) + target.put(local, staging) + target.capture( + f"chmod 600 {quoted_staging} && mv -f {quoted_staging} {quoted_remote}" + ) + print(f" uploaded the LiveKit token to {remote} (mode 0600)") + + def systemd_quote(arg: str) -> str: """One ExecStart word: systemd expands % and $ and splits on whitespace, so escape and quote them.""" arg = ( @@ -831,6 +875,14 @@ def deploy_source( step(2, f"Preflight on {target.host}") cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") + if config.get("token_source"): + # The file is owned by the ssh user with mode 0600, so only a service running as that user can read it. + if args.service_user and args.service_user != target.capture("id -un"): + raise DeployError( + "token_source uploads the token readable by the ssh user only; " + "with a different --service-user, place the token file yourself" + ) + install_token(target, Path(config["token_source"]), config["token_file"]) if config.get("mode") == "livekit" and not target.readable(config["token_file"]): raise DeployError( f"the token file {config['token_file']} is not readable on the Orin" diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py index 5d7f1abd0a..e4f374633e 100644 --- a/tests/python/ci/test_deploy_orin.py +++ b/tests/python/ci/test_deploy_orin.py @@ -155,6 +155,33 @@ def test_rejects_relative_token_file(self): with self.assertRaisesRegex(DeployError, "absolute"): self.load(self.valid(token_file="join.token")) + def test_token_source_is_resolved_against_the_config_directory(self): + (self.tmp / "join.token").write_text("tok\n") + config = self.load(self.valid(token_source="join.token")) + self.assertEqual( + config["token_source"], str((self.tmp / "join.token").resolve()) + ) + + def test_token_source_must_be_an_existing_nonempty_file(self): + (self.tmp / "empty.token").write_text("\n") + for source, message in ( + ("missing.token", "is not a file"), + ("empty.token", "is empty"), + ("", "non-empty string"), + ): + with ( + self.subTest(source=source), + self.assertRaisesRegex(DeployError, message), + ): + self.load(self.valid(token_source=source)) + + def test_token_source_belongs_to_livekit_mode(self): + (self.tmp / "join.token").write_text("tok") + with self.assertRaisesRegex(DeployError, "belong to"): + self.load( + {"mode": "direct", "token_source": "join.token", "args": [STREAM]} + ) + def test_rejects_missing_add_stream(self): for args in ([], ["--other=1"]): with ( @@ -522,6 +549,61 @@ def run(self, root, *extra, config=None): deploy_orin.run_deploy(args, config, self.target) +class TokenSourceDeployTests(TempDirTestCase): + def deploy(self, **extra): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + token = self.tmp / "join.token" + token.write_text("SECRET-TOKEN-VALUE") + orin = FakeOrin() + config = { + "mode": "livekit", + "livekit_url": "ws://x", + "token_file": "/home/u/lk/join.token", + "token_source": str(token), + "args": [STREAM], + } + return orin, root, config + + def test_token_is_uploaded_private_and_moved_into_place_before_the_readable_check( + self, + ): + orin, root, config = self.deploy() + orin.run(root, config=config) + puts = [v for k, v in orin.events if k == "put" and v.startswith("/home/u/lk/")] + self.assertEqual(puts, ["/home/u/lk/join.token.new"]) + commands = [c.args[0] for c in orin.target.capture.call_args_list] + make = next(c for c in commands if "umask 077" in c) + move = next(c for c in commands if "mv -f" in c) + self.assertIn("mkdir -p /home/u/lk", make) + self.assertIn("chmod 600 /home/u/lk/join.token.new", move) + self.assertIn("/home/u/lk/join.token", move) + self.assertLess(commands.index(make), commands.index(move)) + + def test_the_token_value_never_appears_in_any_command(self): + orin, root, config = self.deploy() + orin.run(root, config=config) + everything = [str(c) for c in orin.target.mock_calls] + [ + str(e) for e in orin.events + ] + self.assertFalse([e for e in everything if "SECRET-TOKEN-VALUE" in e]) + + def test_a_different_service_user_is_refused_before_anything_is_uploaded(self): + orin, root, config = self.deploy() + with self.assertRaisesRegex(DeployError, "service-user"): + orin.run(root, "--service-user", "other", config=config) + self.assertEqual( + [e for e in orin.events if e[0] == "put" and "join.token" in e[1]], [] + ) + + def test_no_token_source_uploads_nothing(self): + orin, root, config = self.deploy() + del config["token_source"] + orin.run(root, config=config) + self.assertEqual( + [e for e in orin.events if e[0] == "put" and "join.token" in e[1]], [] + ) + + class DeployOrderTests(TempDirTestCase): def test_failed_validation_blocks_everything_else(self): root, _ = git_repo(self, {"src/a.cpp": "a\n"}) From 770d73b3b632d0bd097a6b3d9c7cb5ae07cab8b5 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 14:50:07 -0500 Subject: [PATCH 16/16] deploy_orin: count the staged files without failing when the stage directory is absent Path.rglob raises FileNotFoundError on a missing directory in Python 3.11 and returns nothing in 3.12, so the remote-build step and its unit tests failed under 3.11. os.walk behaves the same on both. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/scripts/deploy_orin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py index 302c6c819a..eab56fd499 100755 --- a/src/plugins/sensing/scripts/deploy_orin.py +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -1198,7 +1198,7 @@ def remote_build(args: argparse.Namespace) -> None: run(["rm", "-rf", args.stage]) stage = Path(args.stage) stage_producer(build_dir, stage) - print(f" staged {sum(1 for p in stage.rglob('*') if p.is_file())} files") + print(f" staged {sum(len(files) for _, _, files in os.walk(stage))} files") def remote_apply(args: argparse.Namespace) -> None: