diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 17e6e4c1fa..5395ed50c0 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -39,6 +39,8 @@ jobs: python-version: "3.12" - name: Test preview selection run: python3 -m unittest discover -s tests/python/ci -p test_docs_preview.py -v + - name: Test the Orin deployment script + run: python3 -m unittest discover -s tests/python/ci -p test_deploy_orin.py -v synthetic-xr-example-tests: name: Test Synthetic XR task contracts diff --git a/src/plugins/sensing/AGENTS.md b/src/plugins/sensing/AGENTS.md index a8abfd9bc2..7fce3774c8 100644 --- a/src/plugins/sensing/AGENTS.md +++ b/src/plugins/sensing/AGENTS.md @@ -19,5 +19,9 @@ camera runtime checks must not require build SDKs or cached vendor packages. The quick-start camera command must open a live visual preview, not only print FPS. Container preview must carry the desktop display and authorization into the image. +`deploy_orin.py` is the single entry point for putting an Orin into a working state. +A prerequisite or setup step belongs in it, through `orin_setup.sh`, not in a document +the operator must follow by hand before running it. + Support SSH launch with a local DISPLAY: when XAUTHORITY is unset, check the current user's GDM authority file as well as ~/.Xauthority. diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 147e65c950..6fe54c4228 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -15,6 +15,7 @@ That guide owns the operating instructions; this README maps the source tree. - [`scripts/`](scripts/): host setup, validation, diagnostics, the Bash compatibility Docker launcher, and `setup_orin_crosscompile.py` for building on a PC. Each entry point exposes `--help`. + `scripts/deploy_orin.py` builds the producer on an Orin from this checkout and runs it as a systemd service. - [`docker/`](docker/): example image, image-side group configuration, and host-side container setup. - [`configs/`](configs/): the platform configuration and its provenance. @@ -46,3 +47,71 @@ SIPL drivers. hardware encoder and publishes one LiveKit video track per camera. The [SENSING user guide](https://nvidia.github.io/IsaacCapture/main/device/camera/sensing.html) has the build and run steps. + +## Orin: deploy from the desktop with one script + +`scripts/deploy_orin.py` builds the producer on an Orin from the checkout you run it in and runs it as the +`isaaccapture-producer.service` systemd service, so you do not build by hand on the Orin. It needs `ssh`, `scp` and +`git` on the desktop, and `python3`, CMake, g++, CUDA, `uv` and the SDK headers (see `scripts/setup.sh`) on the Orin. + +Write a config naming the producer's own options. `"mode"` says what the producer does and so how readiness is +judged: `"direct"` (the default) for a producer that sends only over TCP to the PC video stage, or `"livekit"` for one +that publishes to LiveKit itself, which also needs `livekit_url` and `token_file` (the token file stays on the Orin; +only its path is used). Add `"token_source": "join.token"` (a local file, relative to the config) and the deploy uploads +it to `token_file` with mode 0600 before anything is stopped, so no separate copy step is needed. Everything else goes in `args`, spelled as the deployed source's producer expects. + +```json +{"args": ["--link-masks=0x0000 0x1100", + "--add-stream=sensor=0,name=sensing-left,direct=PC_HOST:5600", + "--add-stream=sensor=1,name=sensing-right,direct=PC_HOST:5601"]} +``` + +```json +{"mode": "livekit", + "livekit_url": "ws://127.0.0.1:7880", + "token_file": "/home/ido/livekit/join.token", + "args": ["--link-masks=0x0000 0x1100", + "--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} +``` + +Then run it from the repository root. The ssh user and password can come from the environment, which keeps the +password out of the command line and the output: + +```bash +export ORIN_USER=ido +read -rs ORIN_PASSWORD && export ORIN_PASSWORD # optional; without it ssh and sudo prompt +python3 src/plugins/sensing/scripts/deploy_orin.py 192.168.1.176 --config orin.json \ + --sdk-root /home/ido/sensing-sdk/root --replace-unmanaged +``` + +- The script has no knowledge of which capture SDK the deployed source uses: that comes from the source's own CMake. + `--cmake-arg=ARG` (repeatable, `=` form) passes options only some source trees define, such as a capture backend + switch. `--sdk-root DIR` is the Orin directory holding `usr/src/jetson_multimedia_api` and + `usr/src/jetson_sipl_api` when they are not under `/usr/src`. +- `--source DIR` deploys another checkout (for example a git worktree of a feature branch) instead of the one holding + the script; `--branch NAME` deploys that branch's committed files from it, ignoring the working tree. Without them + the script deploys its own checkout, local edits included. +- `--replace-unmanaged` stops a `camera_plugin_sensing` that someone started by hand (its exact PIDs are shown); + without it the deployment refuses and changes nothing. `--no-activate` builds and packages only, and `--jobs N` + bounds compile jobs. The release is installed under `/opt/isaaccapture`. +- `scripts/orin_setup.sh` installs the Orin's prerequisites and the SENSING drivers (`setup.sh`), then checks the cameras (`verify.sh`, then a + `validate.sh` stream). The deploy uploads it, with those scripts, to `~/.cache/isaaccapture-deploy/tools/scripts/` + before the preflight, so a missing prerequisite can be fixed first, and each release carries a copy under + `plugins/sensing_camera/scripts/`. `--prereqs-only` and `--check-only` run one half; stop the producer service + before the camera check, since SIPL serves one client at a time. From the desktop, `deploy_orin.py USER@ORIN + --install-prereqs` and `--check-camera` run them over ssh without deploying (the sudo password is asked on the + Orin's terminal); together they run both halves. +- `"cameras": "auto"` in the deploy config makes the deploy find the cameras itself, on any of the carrier's 8 inputs, and + start the producer with them (at most the two the vendor config names; streams for sensors that were not found are + dropped). The inputs found are saved in `known-cameras.json` in the deploy's cache directory on the Orin and tried + first on the next deploy. All 8 are scanned again only when that file is missing or a saved camera does not answer, or + with `--rescan-cameras`. Leave `--link-masks`, `--platform-config` and `--config-name` out of `args` in this mode. +- The script checks the ssh connection first, then preflights the Orin, uploads only the source files that differ + from what the Orin already holds, builds, installs a versioned release, replaces the running producer and waits for + every configured camera to be up. "Up" is the producer's own latest output for its current session: frames captured + and, for a stream with a `direct=` destination, that link connected (a stream with none is reported as capture-only). + Steps are numbered and the upload and packing show progress. The sudo steps prompt for the Orin password. +- A failed build or transfer leaves the running producer untouched. There is no automatic rollback; earlier releases + stay under `/opt/isaaccapture/releases/`. Logs: `journalctl -u isaaccapture-producer.service -f` on the Orin. Stop + it with `sudo systemctl stop isaaccapture-producer.service` before starting a producer by hand: the cameras are + exclusive. diff --git a/src/plugins/sensing/scripts/deploy_orin.py b/src/plugins/sensing/scripts/deploy_orin.py new file mode 100755 index 0000000000..eab56fd499 --- /dev/null +++ b/src/plugins/sensing/scripts/deploy_orin.py @@ -0,0 +1,1692 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Build the camera producer on an Orin from this checkout, package it, and run it as a service. + +Run on the desktop: + + deploy_orin.py user@orin --config orin.json + +`--install-prereqs` and `--check-camera` run the Orin's prerequisite install and camera check instead of deploying +(neither needs a config). + +The config is JSON with the producer's own options, nothing else: + + {"args": ["--add-stream=sensor=0,livekit=sensing-left", "--add-stream=sensor=1,livekit=sensing-right"]} + +The token file stays on the Orin, outside the release; only its path is passed. With "token_source" (a local file) +the deploy first uploads it to "token_file" on the Orin with mode 0600. Nothing is built, stopped or +replaced until the target passes its checks. A build or transfer failure leaves the running producer alone. +The same file runs on the target (`deploy_orin.py remote ...`) from the source snapshot it copied there. +""" + +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import io +import json +import os +import posixpath +import pwd +import re +import shlex +import shutil +import signal +import subprocess +import sys +import tarfile +import tempfile +import time +from pathlib import Path + +SERVICE = "isaaccapture-producer.service" +BIN_REL = "plugins/sensing_camera/camera_plugin_sensing" +UNIT_TEMPLATE = Path(__file__).with_name("isaaccapture-producer.service.in") +CACHE = "~/.cache/isaaccapture-deploy" +MODES = ("direct", "livekit") +CONFIG_KEYS = { + "mode", + "livekit_url", + "token_file", + "token_source", + "args", + "env", + "cameras", +} +# With "cameras": "auto" the deploy finds the cameras and supplies these, so the config must not. +DETECTED_ARGS = ("--platform-config", "--config-name", "--link-masks") +PLATFORM_FILE = ( + "camera-platform.json" # under the install prefix; the detected platform config +) +PLATFORM_CONFIG_NAME = "SHW5G_2" +ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +RESERVED_ENV = ("LD_LIBRARY_PATH",) +# The producer's connection options come from the config's own fields, not from `args`. +PREFIX = "/opt/isaaccapture" +SERVICE_GROUPS = "video" # device access for the cameras +READY_TIMEOUT_S = 60 +REBOOT_TIMEOUT_S = ( + 600 # a reboot to load the device-tree overlay, until ssh answers again +) +REBOOT_START_S = ( + 120 # how long to wait for the Orin to go down once setup asks for a reboot +) +NEEDS_REBOOT = 2 # orin_setup.sh exit status: reboot or log in again, then re-run +SDKM_DOWNLOADS = "~/Downloads/nvidia/sdkm_downloads" +RESERVED_ARGS = ("--livekit-url", "--livekit-token-file") +# The Orin-side setup and camera-check scripts, and the platform config they stream with. +TOOL_SCRIPTS = ("orin_setup.sh", "setup.sh", "common.sh", "verify.sh", "validate.sh") +TOOL_CONFIG = "shw5g.json" +SETUP_HINT = f"bash {CACHE}/tools/scripts/orin_setup.sh" + + +def tool_files() -> list[tuple[Path, str]]: + """(local file, path under the tools directory) for the Orin-side scripts. + + They come from the checkout holding this helper, never from the deployed source, so any source can be deployed. + On the Orin the helper sits beside the `tools/` directory the desktop uploaded. + """ + here = Path(__file__).resolve().parent + if (here / "tools").is_dir(): + scripts, configs = here / "tools" / "scripts", here / "tools" / "configs" + else: + scripts, configs = here, here.parent / "configs" + return [(scripts / name, f"scripts/{name}") for name in TOOL_SCRIPTS] + [ + (configs / TOOL_CONFIG, f"configs/{TOOL_CONFIG}") + ] + + +STEPS = 5 + + +def step(number: int, text: str) -> None: + print(f"==> [{number}/{STEPS}] {text}", flush=True) + + +class Progress: + """A text progress bar: redrawn in place on a terminal, one line per 10% otherwise.""" + + def __init__(self, label: str, total: int) -> None: + self.label, self.total = label, max(total, 1) + self._last = -1 + + def update(self, done: int) -> None: + percent = min(100, done * 100 // self.total) + if sys.stdout.isatty(): + filled = percent * 30 // 100 + bar = "#" * filled + "-" * (30 - filled) + print( + f"\r {self.label} [{bar}] {percent:3d}% ({done}/{self.total})", + end="", + flush=True, + ) + elif percent // 10 != self._last // 10: + print(f" {self.label} {percent}%", flush=True) + self._last = percent + + def finish(self) -> None: + self.update(self.total) + if sys.stdout.isatty(): + print() + + +class DeployError(Exception): + """A failure with a message worth showing as is.""" + + +def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: + result = subprocess.run(cmd, check=False, **kwargs) + if result.returncode != 0: + raise DeployError( + f"{shlex.join(cmd[:3])} ... failed with exit code {result.returncode}" + ) + return result + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +# --------------------------------------------------------------------------- +# Config and service unit +# --------------------------------------------------------------------------- + + +def configured_streams(config: dict) -> list[tuple[int, bool]]: + """(sensor, has a direct destination) for each --add-stream=sensor=N,... argument of the config.""" + streams = [] + for arg in config.get("args", []): + if not arg.startswith("--add-stream="): + continue + # The argument is a comma-separated list of key=value pairs, in any order. + pairs = dict( + item.partition("=")[::2] + for item in arg.split("=", 1)[1].split(",") + if "=" in item + ) + if pairs.get("sensor", "").isdigit(): + streams.append((int(pairs["sensor"]), bool(pairs.get("direct")))) + return streams + + +def configured_sensors(config: dict) -> list[int]: + """The sensor numbers the config asks the producer to publish.""" + return [sensor for sensor, _ in configured_streams(config)] + + +def load_config(path: Path) -> dict: + try: + config = json.loads(path.read_text()) + except (OSError, ValueError) as exc: + raise DeployError(f"cannot read the config {path}: {exc}") from exc + if not isinstance(config, dict) or set(config) - CONFIG_KEYS: + raise DeployError( + f"{path}: expected an object with args and, optionally, livekit_url and token_file" + ) + env = config.get("env", {}) + if not isinstance(env, dict) or not all( + isinstance(k, str) + and ENV_NAME.fullmatch(k) + and isinstance(v, str) + and "\n" not in v + for k, v in env.items() + ): + raise DeployError( + f"{path}: env must map variable names to single-line string values" + ) + if any(k in RESERVED_ENV for k in env): + raise DeployError( + f"{path}: env may not set {', '.join(RESERVED_ENV)}; the release sets it" + ) + mode = config.get("mode", "direct") + if mode not in MODES: + raise DeployError(f'{path}: "mode" must be one of {", ".join(MODES)}') + has_livekit = any( + k in config for k in ("livekit_url", "token_file", "token_source") + ) + if mode == "direct" and has_livekit: + raise DeployError( + f'{path}: livekit_url, token_file and token_source belong to "mode": "livekit"; a direct producer has no LiveKit settings' + ) + if mode == "livekit" and not ("livekit_url" in config and "token_file" in config): + raise DeployError(f'{path}: "mode": "livekit" needs livekit_url and token_file') + args = config.get("args", []) + if not isinstance(args, list) or not all(isinstance(a, str) for a in args): + raise DeployError(f"{path}: args must be a list of strings") + if any(a.split("=", 1)[0] in RESERVED_ARGS for a in args): + raise DeployError( + f"{path}: give the URL and token file as livekit_url and token_file, not in args" + ) + if not any(a.startswith("--add-stream=") for a in args): + raise DeployError( + f"{path}: args needs at least one --add-stream=sensor=N,livekit=NAME" + ) + if "cameras" in config: + if config["cameras"] != "auto": + raise DeployError(f'{path}: "cameras" must be "auto"') + if any(a.split("=", 1)[0] in DETECTED_ARGS for a in args): + raise DeployError( + f'{path}: with "cameras": "auto" the deploy supplies {", ".join(DETECTED_ARGS)}; remove them from args' + ) + if "token_file" in config: + if not all( + isinstance(config[k], str) and config[k] + for k in ("livekit_url", "token_file") + ): + raise DeployError( + f"{path}: livekit_url and token_file must be non-empty strings" + ) + if not config["token_file"].startswith("/"): + raise DeployError( + f"{path}: token_file must be an absolute path on the Orin" + ) + if "token_source" in config: + source = config["token_source"] + if not isinstance(source, str) or not source: + raise DeployError(f"{path}: token_source must be a non-empty string") + # A relative source is relative to the config file, so a config and its token can move together. + local = Path(source).expanduser() + if not local.is_absolute(): + local = path.parent / local + if not local.is_file(): + raise DeployError(f"{path}: token_source {local} is not a file") + if not local.read_text().strip(): + raise DeployError(f"{path}: token_source {local} is empty") + config["token_source"] = str(local.resolve()) + return config + + +def install_token(target: "Target", local: Path, remote: str) -> None: + """Upload a local token file to ``remote`` on the Orin, readable only by the ssh user, replacing any older one. + + The token is never part of an argument list, the release or the output: it travels by scp into a staging file + created with mode 0600, and moves into place only once complete, so the service never reads half a token. + """ + staging = f"{remote}.new" + quoted_remote, quoted_staging = shlex.quote(remote), shlex.quote(staging) + # Create the staging file private before any bytes arrive; scp keeps an existing file's mode. + target.capture( + f"mkdir -p {shlex.quote(posixpath.dirname(remote))} && umask 077 && rm -f {quoted_staging} && : > {quoted_staging}" + ) + target.put(local, staging) + target.capture( + f"chmod 600 {quoted_staging} && mv -f {quoted_staging} {quoted_remote}" + ) + print(f" uploaded the LiveKit token to {remote} (mode 0600)") + + +def systemd_quote(arg: str) -> str: + """One ExecStart word: systemd expands % and $ and splits on whitespace, so escape and quote them.""" + arg = ( + arg.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("%", "%%") + .replace("$", "$$") + ) + return f'"{arg}"' if not arg or re.search(r"\s|[\"']", arg) else arg + + +def render_unit(config: dict, prefix: str, user: str, groups: str) -> str: + words = [ + f"{prefix}/current/{BIN_REL}", + *( + [ + f"--livekit-url={config['livekit_url']}", + f"--livekit-token-file={config['token_file']}", + ] + if config.get("mode") == "livekit" + else [] + ), + *config.get("args", []), + ] + exec_start = " ".join(systemd_quote(w) for w in words) + fields = { + "SERVICE_USER": user, + "SERVICE_GROUPS": groups, + "PREFIX": prefix, + "EXEC_START": exec_start, + # The producer's own environment variables (for example a diagnostic switch), one Environment= line each. + "ENVIRONMENT": "\n".join( + f"Environment={systemd_quote(f'{name}={value}')}" + for name, value in config.get("env", {}).items() + ), + } + text = UNIT_TEMPLATE.read_text() + for key, value in fields.items(): + text = text.replace(f"@{key}@", value) + return text + + +# --------------------------------------------------------------------------- +# Desktop side +# --------------------------------------------------------------------------- + + +def repo_root() -> Path: + out = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=Path(__file__).parent, + capture_output=True, + text=True, + check=True, + ) + return Path(out.stdout.strip()) + + +DEPLOY_DIR = ".deploy" + + +def tracked_files(root: Path) -> list[str]: + """The tracked files that exist on disk, so tracked local edits are included and untracked files never are.""" + listing = subprocess.run( + ["git", "ls-files", "-z", "--recurse-submodules"], + cwd=root, + capture_output=True, + check=True, + ).stdout + return sorted(f for f in listing.decode().split("\0") if f and (root / f).is_file()) + + +def hash_files(root: Path, files: list[str]) -> dict[str, str]: + """Content hash per file, with the executable bit, which the Orin's copy must also have.""" + bar = Progress("hashing", len(files)) + entries = {} + for done, name in enumerate(files, 1): + path = root / name + executable = "x" if os.access(path, os.X_OK) else "-" + entries[name] = f"{sha256_file(path)}{executable}" + if done % 200 == 0: + bar.update(done) + bar.finish() + return entries + + +def diff_manifests( + local: dict[str, str], remote: dict[str, str] +) -> tuple[list[str], list[str]]: + """(files to send, files to remove on the Orin): one comparison of the two per-file manifests.""" + send = sorted(n for n, h in local.items() if remote.get(n) != h) + removed = sorted(n for n in remote if n not in local) + return send, removed + + +def read_remote_manifest(target: "Target", cache: str) -> dict[str, str]: + """What the Orin's source copy holds, or nothing when it has no copy or the record is unreadable.""" + text = target.capture( + f"test -d {cache}/src && cat {cache}/src.manifest.json || true" + ) + try: + entries = json.loads(text) if text else {} + except ValueError: + return {} + return entries if isinstance(entries, dict) else {} + + +def git_state(root: Path) -> dict: + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + dirty = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=no"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + return {"base_revision": head, "dirty": bool(dirty)} + + +def build_delta( + root: Path, + send: list[str], + removed: list[str], + manifest: dict[str, str], + dest: Path, +) -> None: + """Archive only the files to send, with the new full manifest and the list of files to remove.""" + bar = Progress("packing", len(send)) + with tarfile.open(dest, "w:gz") as tar: + for done, name in enumerate(send, 1): + tar.add(root / name, arcname=name, recursive=False) + if done % 200 == 0: + bar.update(done) + for name, payload in (("manifest.json", manifest), ("removed.json", removed)): + data = json.dumps(payload).encode() + info = tarfile.TarInfo(f"{DEPLOY_DIR}/{name}") + info.size = len(data) + info.mtime = int(time.time()) + tar.addfile(info, io.BytesIO(data)) + bar.finish() + + +PASSWORD_ENV = "ORIN_PASSWORD" +USER_ENV = "ORIN_USER" + + +def target_host(target: str) -> str: + """`host` becomes `$ORIN_USER@host` when ORIN_USER is set; `user@host` is used as given.""" + user = os.environ.get(USER_ENV) + return f"{user}@{target}" if user and "@" not in target else target + + +class Target: + """The Orin over ssh/scp. + + With ORIN_PASSWORD set, ssh gets it through SSH_ASKPASS and sudo reads it from stdin (`sudo -S`), so it never + appears in an argument list or a log; without it, ssh and sudo prompt as usual. One ssh connection is reused for + every step. + """ + + def __init__(self, host: str) -> None: + self.host = host + self.password = os.environ.get(PASSWORD_ENV) + self._dir = Path(tempfile.mkdtemp(prefix="deploy-orin-")) + self._env = dict(os.environ) + if self.password: + askpass = self._dir / "askpass" + askpass.write_text(f'#!/bin/sh\nprintf "%s\\n" "${PASSWORD_ENV}"\n') + askpass.chmod(0o700) + self._env.update(SSH_ASKPASS=str(askpass), SSH_ASKPASS_REQUIRE="force") + self._opts = [ + "-o", + "ControlMaster=auto", + "-o", + f"ControlPath={self._dir}/cm", + "-o", + "ControlPersist=300", + ] + + def _run(self, tool: str, args: list[str], **kwargs) -> subprocess.CompletedProcess: + return subprocess.run( + [tool, *self._opts, *args], env=self._env, check=False, **kwargs + ) + + def ssh(self, command: str, *, tty: bool = False) -> subprocess.CompletedProcess: + result = self._run("ssh", [*(["-t"] if tty else []), self.host, command]) + if result.returncode != 0: + raise DeployError( + f"ssh {self.host} failed with exit code {result.returncode}" + ) + return result + + def status( + self, + command: str, + *, + tty: bool = False, + timeout: int = 0, + stdin: str | None = None, + ) -> int: + """Run a command and return its exit status instead of raising (255 when ssh itself fails).""" + opts = ["-o", f"ConnectTimeout={timeout}"] if timeout else [] + extra: dict = {"capture_output": True} if timeout else {} + if stdin is not None: + extra.update(input=stdin, text=True) + return self._run( + "ssh", [*opts, *(["-t"] if tty else []), self.host, command], **extra + ).returncode + + def output(self, command: str, *, stdin: str | None = None) -> tuple[int, str]: + """Run a command; return (exit status, stdout). Its stderr still goes to this terminal.""" + extra: dict = ( + {"input": stdin, "text": True} if stdin is not None else {"text": True} + ) + result = self._run("ssh", [self.host, command], stdout=subprocess.PIPE, **extra) + return result.returncode, result.stdout + + def sudo(self, command: str) -> None: + """Run a command as root: the password from the environment on stdin, else an interactive prompt.""" + if self.password: + result = self._run( + "ssh", + [self.host, f"sudo -S -p '' {command}"], + input=self.password + "\n", + text=True, + ) + if result.returncode != 0: + raise DeployError( + f"sudo on {self.host} failed with exit code {result.returncode}" + ) + else: + self.ssh(f"sudo {command}", tty=True) + + def validate_connection(self) -> None: + """Fail early, with the reason, when the Orin cannot be reached or lacks what the deployment runs on it.""" + probe = self._run( + "ssh", + [ + "-o", + "ConnectTimeout=10", + self.host, + "command -v python3 >/dev/null && uname -m", + ], + capture_output=True, + text=True, + ) + if probe.returncode != 0: + reason = probe.stderr.strip() or "python3 is not installed on the target" + raise DeployError(f"cannot use {self.host} over ssh: {reason}") + if probe.stdout.strip() != "aarch64": + raise DeployError( + f"{self.host} is {probe.stdout.strip() or 'unknown'}, not an aarch64 Jetson" + ) + + def capture(self, command: str) -> str: + out = self._run("ssh", [self.host, command], capture_output=True, text=True) + if out.returncode != 0: + raise DeployError( + f"ssh {self.host} {command!r} failed: {out.stderr.strip()}" + ) + return out.stdout.strip() + + def readable(self, path: str) -> bool: + return self._run("ssh", [self.host, "test", "-r", path]).returncode == 0 + + @staticmethod + def _scp_quiet() -> list[str]: + """scp shows its own transfer meter on a terminal; stay quiet when output is piped.""" + return [] if sys.stdout.isatty() else ["-q"] + + def put(self, local: Path, remote: str) -> None: + if ( + self._run( + "scp", [*self._scp_quiet(), str(local), f"{self.host}:{remote}"] + ).returncode + != 0 + ): + raise DeployError(f"copying {local.name} to {self.host} failed") + + def close(self) -> None: + self._run("ssh", ["-O", "exit", self.host], capture_output=True) + shutil.rmtree(self._dir, ignore_errors=True) + + +def upload_tools(target: "Target", cache: str) -> None: + """Put the setup and camera-check scripts, with the config they stream with, under the Orin's cache.""" + target.capture(f"mkdir -p {cache}/tools/scripts {cache}/tools/configs") + for local_file, relative in tool_files(): + target.put(local_file, f"{cache}/tools/{relative}") + + +def show_logs(target: "Target") -> None: + """Print the producer service's state and recent journal; nothing on the Orin changes.""" + target.validate_connection() + target.sudo(f"systemctl status {SERVICE} --no-pager -l") + target.sudo(f"journalctl -u {SERVICE} -n 120 --no-pager") + + +def run_tools(args: argparse.Namespace, target: "Target") -> None: + """Install the Orin's prerequisites and/or check its cameras with the uploaded scripts; deploy nothing. + + The output shows live. Given ORIN_PASSWORD the Orin's sudo is answered for you; otherwise it prompts in the + terminal. + """ + if args.logs: + show_logs(target) + return + step(1, f"Connecting to {target.host}") + target.validate_connection() + cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") + step(2, "Upload the setup and camera-check scripts") + upload_tools(target, cache) + check = args.check_camera or args.scan_cameras + flags = [] + if args.install_prereqs and not check: + flags.append("--prereqs-only") + elif not args.install_prereqs: + flags.append("--check-only") + if args.scan_cameras: + flags.append("--scan") + label = " and ".join( + part + for part, wanted in ( + ("Install prerequisites", args.install_prereqs), + ( + "Scan the camera links" if args.scan_cameras else "Check the cameras", + check, + ), + ) + if wanted + ) + step(3, label) + code = run_orin_setup(target, cache, " ".join(flags)) + if code != 0: + raise DeployError(f"{label.lower()} failed on {target.host} (exit {code})") + + +def detect_cameras(target: "Target", cache: str, rescan: bool = False) -> dict: + """Find where cameras are connected, on any of the carrier's 8 inputs, and write the platform config for them. + + The inputs found are remembered on the Orin and tried first next time; all 8 are scanned again only when a + remembered camera does not answer, or with rescan. The Orin's producer service holds the cameras, so it is + stopped first. Returns the JSON orin_setup.sh prints: + the platform config's path on the Orin, the link masks, and how many cameras were found and used. + """ + print("==> Finding the cameras (any of the carrier's 8 inputs)", flush=True) + target.sudo(f"sh -c {shlex.quote(f'systemctl stop {SERVICE} || true')}") + code, text = target.output( + f"bash {cache}/tools/scripts/orin_setup.sh --detect {cache}/cameras" + + (" --rescan" if rescan else "") + ) + if code != 0: + raise DeployError( + "no camera answers on any of the 8 inputs; check the carrier's 12 V supply and each cable, " + "or run --scan-cameras" + ) + try: + found = json.loads(text.strip().splitlines()[-1]) + found["config"], found["masks"], found["used"] + except (ValueError, IndexError, KeyError) as exc: + raise DeployError( + f"camera detection printed no usable result: {text!r}" + ) from exc + for place in found.get("positions", []): + print(f" camera: {place}") + return found + + +def apply_detected(config: dict, found: dict) -> dict: + """The deploy config with the detected cameras: their platform config and masks, and streams for them only. + + Sensors are numbered from 0 in the order of the detected cameras, so a stream for a sensor that was not found + is dropped. All streams dropped is an error. + """ + used = found["used"] + kept, dropped = [], [] + for arg in config["args"]: + if arg.startswith("--add-stream="): + sensors = configured_sensors({"args": [arg]}) + if sensors and sensors[0] >= used: + dropped.append(arg) + continue + kept.append(arg) + if not configured_sensors({"args": kept}): + raise DeployError( + f"the config streams sensors the detected cameras do not have ({used} found)" + ) + if dropped: + print( + f" {used} camera(s) found; dropping {len(dropped)} stream(s) for sensors that are not there", + flush=True, + ) + return { + **config, + "args": [ + *kept, + f"--platform-config={PREFIX}/{PLATFORM_FILE}", + f"--config-name={PLATFORM_CONFIG_NAME}", + f"--link-masks={found['masks']}", + ], + } + + +def find_sipl_package(explicit: str | None) -> Path | None: + """The L4T SIPL API headers archive on this desktop: the named file, else the newest SDK Manager download.""" + if explicit: + path = Path(explicit).expanduser() + if not path.is_file(): + raise DeployError(f"--sipl-api {explicit}: no such file") + return path + found = sorted( + Path(SDKM_DOWNLOADS).expanduser().glob("Jetson_SIPL_API_R*_aarch64.tbz2"), + key=lambda p: p.stat().st_mtime, + ) + return found[-1] if found else None + + +def wait_for_reboot(target: "Target") -> None: + """Block until the Orin has gone down and answered ssh again; fail when it never reboots or never returns.""" + print("==> Waiting for the Orin to reboot", flush=True) + start = time.monotonic() + went_down = False + while time.monotonic() - start < REBOOT_TIMEOUT_S: + up = target.status("true", timeout=5) == 0 + if not up: + went_down = True + elif went_down: + return + elif time.monotonic() - start > REBOOT_START_S: + break + time.sleep(5) + raise DeployError( + "the Orin did not reboot and come back; reboot it yourself and run this command again" + if not went_down + else f"the Orin did not answer ssh within {REBOOT_TIMEOUT_S} s of rebooting" + ) + + +def prerequisites_ready(target: "Target", cache: str, sdk_root: str) -> bool: + """The Orin passes its own preflight and has the drivers, overlay and SIPL headers the cameras need.""" + remote = f"python3 {cache}/deploy_orin.py remote" + if target.status(f"{remote} preflight --sdk-root {shlex.quote(sdk_root)}") != 0: + return False + check = ( + f"bash {cache}/tools/scripts/orin_setup.sh --prereqs-only --no-apt --no-drivers" + ) + return target.status(f"{check} --sdk-root {shlex.quote(sdk_root or '/')}") == 0 + + +def run_orin_setup(target: "Target", cache: str, flags: str) -> int: + """Run orin_setup.sh on the Orin and return its exit status. + + With the password in the environment nothing prompts: it goes to the script on stdin, for its sudo calls. + Without one the script runs in a terminal and sudo prompts there. + """ + command = f"bash {cache}/tools/scripts/orin_setup.sh {flags}".rstrip() + if target.password: + return target.status( + f"{command} --sudo-password-stdin", stdin=target.password + "\n" + ) + return target.status(command, tty=True) + + +def install_prerequisites( + args: argparse.Namespace, target: "Target", cache: str +) -> None: + """Install what the preflight found missing: packages, uv, SIPL headers, the SENSING drivers and overlay. + + The drivers step selects the overlay through Jetson-IO's command line and reboots the Orin; the deploy waits out + the reboot and carries on. With no ORIN_PASSWORD it runs in a terminal and sudo prompts. + """ + print("==> Installing the Orin's missing prerequisites", flush=True) + flags = "--prereqs-only" + if args.sdk_root: + flags += f" --sdk-root {shlex.quote(args.sdk_root)}" + sipl = find_sipl_package(args.sipl_api) + if sipl: + target.put(sipl, f"{cache}/{sipl.name}") + flags += f" --sipl-api {cache}/{sipl.name}" + + def run_setup() -> int: + return run_orin_setup(target, cache, flags) + + code = run_setup() + if code in ( + NEEDS_REBOOT, + 255, + ): # 255: ssh dropped because Jetson-IO rebooted the Orin + wait_for_reboot(target) + target.validate_connection() + code = run_setup() + if code != 0: + raise DeployError( + f"prerequisite setup did not finish (exit {code}); fix what it reported and run this command again" + ) + if not prerequisites_ready(target, cache, args.sdk_root): + raise DeployError("the Orin still fails the preflight after setup") + + +def deploy(args: argparse.Namespace) -> None: + tools_only = ( + args.install_prereqs or args.check_camera or args.scan_cameras or args.logs + ) + if not tools_only and not args.config: + raise DeployError("--config is required to deploy") + config = None if tools_only else load_config(Path(args.config)) + target = Target(target_host(args.target)) + try: + if tools_only: + run_tools(args, target) + else: + run_deploy(args, config, target) + finally: + target.close() + + +def git_toplevel(folder: Path) -> Path: + found = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=folder, + capture_output=True, + text=True, + check=False, + ) + if found.returncode != 0: + raise DeployError(f"{folder} is not inside a git checkout") + return Path(found.stdout.strip()) + + +@contextlib.contextmanager +def open_source(args: argparse.Namespace): + """The source to deploy as (root, files, git state). + + By default the tracked files of the working tree of the checkout holding this script, with local edits. `--source` + names another checkout (a worktree, say) to deploy instead. `--branch` deploys that branch's committed files from + that checkout, ignoring the working tree, exported to a temporary directory. + """ + repo = git_toplevel(Path(args.source)) if args.source else repo_root() + if not args.branch: + yield repo, tracked_files(repo), git_state(repo) + return + found = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{args.branch}^{{commit}}"], + cwd=repo, + capture_output=True, + text=True, + check=False, + ) + if found.returncode != 0: + raise DeployError(f"branch {args.branch!r} not found in {repo}") + revision = found.stdout.strip() + with tempfile.TemporaryDirectory() as tmp: + export = Path(tmp) / "tree" + export.mkdir() + archive = subprocess.Popen( + ["git", "archive", "--format=tar", revision], + cwd=repo, + stdout=subprocess.PIPE, + ) + with tarfile.open(fileobj=archive.stdout, mode="r|") as tar: + tar.extractall(export, filter="data") + if archive.wait() != 0: + raise DeployError(f"git archive of {args.branch!r} failed") + files = sorted( + p.relative_to(export).as_posix() for p in export.rglob("*") if p.is_file() + ) + yield export, files, {"base_revision": revision, "dirty": False} + + +def run_deploy(args: argparse.Namespace, config: dict, target: Target) -> None: + with open_source(args) as (root, files, state): + deploy_source(args, config, target, root, files, state) + + +def deploy_source( + args: argparse.Namespace, + config: dict, + target: Target, + root: Path, + files: list[str], + state: dict, +) -> None: + step(1, f"Connecting to {target.host}") + target.validate_connection() + user = args.service_user or target.capture("id -un") + + step(2, f"Preflight on {target.host}") + cache = target.capture(f"mkdir -p {CACHE} && cd {CACHE} && pwd") + if config.get("token_source"): + # The file is owned by the ssh user with mode 0600, so only a service running as that user can read it. + if args.service_user and args.service_user != target.capture("id -un"): + raise DeployError( + "token_source uploads the token readable by the ssh user only; " + "with a different --service-user, place the token file yourself" + ) + install_token(target, Path(config["token_source"]), config["token_file"]) + if config.get("mode") == "livekit" and not target.readable(config["token_file"]): + raise DeployError( + f"the token file {config['token_file']} is not readable on the Orin" + ) + # The helper always travels alone and fresh, so the Orin-side steps never depend on an older source copy. + helper = f"{cache}/deploy_orin.py" + target.put(Path(__file__).resolve(), helper) + # The setup and camera-check scripts go up before the preflight, so a missing prerequisite can be fixed with them. + upload_tools(target, cache) + remote = f"python3 {helper} remote" + if not prerequisites_ready(target, cache, args.sdk_root): + if args.no_setup: + raise DeployError( + f"the Orin fails the preflight; run {SETUP_HINT} --prereqs-only or drop --no-setup" + ) + install_prerequisites(args, target, cache) + + platform_file = "" + if config.get("cameras") == "auto": + found = detect_cameras(target, cache, args.rescan_cameras) + config = apply_detected(config, found) + platform_file = found["config"] + unit = render_unit(config, PREFIX, user, SERVICE_GROUPS) + + with tempfile.TemporaryDirectory() as tmp: + step(3, "Compare the source with the Orin's and upload what differs") + meta = dict(state) + local = hash_files(root, files) + meta["manifest_sha256"] = hashlib.sha256( + json.dumps(local, sort_keys=True).encode() + ).hexdigest() + release = f"{time.strftime('%Y%m%d-%H%M%S')}-{meta['base_revision'][:9]}{'-dirty' if meta['dirty'] else ''}" + print( + f" base {meta['base_revision'][:12]}, dirty={meta['dirty']}, {len(files)} tracked files" + ) + # One fixed source path, so the persistent build directory's CMake cache keeps matching it; files that are + # unchanged keep their modification times and are not rebuilt. + src = f"{cache}/src" + remote_manifest = read_remote_manifest(target, cache) + send, removed = diff_manifests(local, remote_manifest) + size = sum((root / n).stat().st_size for n in send) + print( + f" {len(local) - len(send)} identical, {len(send)} to send ({size / 1e6:.1f} MB), " + f"{len(removed)} to remove" + ) + if send or removed: + delta = Path(tmp) / "delta.tar.gz" + build_delta(root, send, removed, local, delta) + target.put(delta, f"{cache}/delta.tar.gz") + target.ssh( + f"{remote} apply --archive {cache}/delta.tar.gz --src {src} --manifest {cache}/src.manifest.json" + + ("" if remote_manifest else " --fresh") + ) + else: + print(" the Orin's source is identical; nothing to send") + step(4, "Build and package on the Orin (the running producer is untouched)") + stage, archive = ( + f"{cache}/stage-{release}", + f"{cache}/producer-{release}.tar.gz", + ) + target.ssh( + f"{remote} build --src {src} --stage {stage} --jobs {args.jobs} " + f"--sdk-root {shlex.quote(args.sdk_root)}" + + "".join(f" --cmake-arg={shlex.quote(a)}" for a in args.cmake_arg) + ) + manifest = { + **meta, + "release": release, + "arch": "aarch64", + "config": args.config, + } + manifest_path = Path(tmp) / "MANIFEST.json" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + target.put(manifest_path, f"{stage}/MANIFEST.json") + target.ssh(f"{remote} package --stage {stage} --archive {archive}") + unit_path = Path(tmp) / SERVICE + unit_path.write_text(unit) + target.put(unit_path, f"{cache}/{SERVICE}") + + if args.no_activate: + print(f"==> Built release {release}; not activated (--no-activate)") + return + step(5, "Install and replace the running producer (needs sudo on the Orin)") + activate = ( + f"{remote} activate --archive {archive} --prefix {PREFIX} --release {release} " + f"--unit {cache}/{SERVICE} --owner {user} --ready-timeout {READY_TIMEOUT_S} " + f"--sensors={','.join(str(n) for n in configured_sensors(config))} " + f"--direct-sensors={','.join(str(n) for n, direct in configured_streams(config) if direct)} " + f"--mode={config.get('mode', 'direct')}" + + (f" --platform-file {shlex.quote(platform_file)}" if platform_file else "") + + (" --replace-unmanaged" if args.replace_unmanaged else "") + ) + target.sudo(activate) + print(f"==> Deployed {release}; logs: ssh {target.host} journalctl -u {SERVICE} -f") + + +# --------------------------------------------------------------------------- +# Target side +# --------------------------------------------------------------------------- + + +def tail(cmd: list[str]) -> str: + out = subprocess.run(cmd, capture_output=True, text=True, check=False) + return (out.stdout + out.stderr).strip() + + +def sdk_dirs(sdk_root: str) -> tuple[Path, Path]: + """The Multimedia API and SIPL header trees: under --sdk-root when given, else the system locations. + + Only where the headers live is decided here; whether a given source tree needs them is its own CMake's concern. + """ + base = Path(sdk_root) / "usr/src" if sdk_root else Path("/usr/src") + return base / "jetson_multimedia_api", base / "jetson_sipl_api" + + +def build_path() -> str: + """PATH for the build: an ssh command runs without the login profile, which is where uv and CUDA are added.""" + extra = [str(Path.home() / ".local/bin"), "/usr/local/cuda/bin"] + return ":".join([*extra, os.environ.get("PATH", "")]) + + +def remote_preflight(args: argparse.Namespace) -> None: + if os.uname().machine != "aarch64": + raise DeployError(f"the target is {os.uname().machine}, not aarch64") + release_file = Path("/etc/nv_tegra_release") + if not release_file.exists(): + raise DeployError("no /etc/nv_tegra_release: this is not a Jetson L4T system") + print(f" L4T: {release_file.read_text().splitlines()[0]}") + on_path = {**os.environ, "PATH": build_path()} + for tool in ("cmake", "g++", "systemctl", "uv"): + found = subprocess.run( + ["which", tool], capture_output=True, check=False, env=on_path + ) + if found.returncode != 0: + raise DeployError( + f"{tool} is not installed; run {SETUP_HINT} --prereqs-only" + ) + if ( + not Path("/usr/local/cuda/bin/nvcc").exists() + and subprocess.run( + ["which", "nvcc"], capture_output=True, check=False + ).returncode + ): + raise DeployError( + f"nvcc not found (CUDA toolkit); run {SETUP_HINT} --prereqs-only" + ) + stat = os.statvfs(Path.home()) + if stat.f_bavail * stat.f_frsize < 4 << 30: + raise DeployError("less than 4 GB free in the home directory") + mmapi, _ = sdk_dirs(args.sdk_root) + if not mmapi.is_dir(): + raise DeployError( + f"the Jetson Multimedia API is missing at {mmapi}; run {SETUP_HINT} --prereqs-only " + "or pass --sdk-root" + ) + print(" preflight ok") + + +def cache_value(build_dir: Path, name: str) -> str: + """One entry of the build's CMakeCache.txt, or empty.""" + for line in (build_dir / "CMakeCache.txt").read_text().splitlines(): + if line.startswith(f"{name}:"): + return line.split("=", 1)[1] + return "" + + +def livekit_notice(build_dir: Path, name: str) -> Path: + """The LiveKit SDK's LICENSE or NOTICE: where the build recorded it, else where the fetch or a local SDK put it.""" + sdk = cache_value(build_dir, "LIVEKIT_SDK_DIR") + candidates = [ + cache_value(build_dir, f"SENSING_LIVEKIT_{name}_FILE"), + str(build_dir / "_deps" / f"livekit_sdk_{name.lower()}-src" / name), + f"{sdk}/{name}" if sdk else "", + f"{sdk}/../{name}" if sdk else "", + ] + for candidate in candidates: + if candidate and Path(candidate).is_file(): + return Path(candidate) + raise DeployError(f"the LiveKit SDK's {name} file was not found for the release") + + +def stage_producer(build_dir: Path, stage: Path) -> None: + """Assemble the release from the build tree itself, so it does not depend on the source's install rules. + + The release is the binary, the SIPL platform configs the build puts beside it, the setup and camera-check + scripts, the LiveKit runtime libraries the binary resolves, and the LiveKit licence files. The service sets LD_LIBRARY_PATH to the release directory, + so the copied libraries win over the build tree's RUNPATH. + """ + binaries = [p for p in build_dir.rglob("camera_plugin_sensing") if p.is_file()] + if len(binaries) != 1: + raise DeployError( + f"expected one camera_plugin_sensing in the build, found {len(binaries)}" + ) + built = binaries[0] + out = stage / Path(BIN_REL).parent + out.mkdir(parents=True) + binary = stage / BIN_REL + shutil.copy2(built, binary) + configs = built.parent / "configs" + if not configs.is_dir(): + raise DeployError(f"the build staged no platform configs beside {built}") + shutil.copytree(configs, out / "configs") + # The setup and camera-check scripts resolve the platform config as ../configs from their own directory. + (out / "scripts").mkdir() + for local_file, relative in tool_files(): + if relative.startswith("scripts/"): + shutil.copy2(local_file, out / relative) + # The libraries are found by name among what the binary resolves in the build, not by the SDK's layout. + for line in tail(["ldd", str(built)]).splitlines(): + name, _, rest = line.strip().partition(" => ") + if name.startswith("liblivekit") and rest and not rest.startswith("not found"): + shutil.copy2(rest.split(" (")[0], out / name) + # A producer that sends only over its direct links links no LiveKit; one that does carries its libraries and the + # SDK's licence files. + if list(out.glob("liblivekit*.so*")): + notices = out / "licenses" / "livekit-sdk" + notices.mkdir(parents=True) + for name in ("LICENSE", "NOTICE"): + shutil.copy2(livekit_notice(build_dir, name), notices / name) + # Everything must resolve from the release or the system: a library found only in the build tree would vanish + # with the next clean build. + env = {**os.environ, "LD_LIBRARY_PATH": str(out)} + resolved = subprocess.run( + ["ldd", str(binary)], capture_output=True, text=True, check=False, env=env + ) + listing = (resolved.stdout + resolved.stderr).splitlines() + missing = [ln for ln in listing if "not found" in ln] + if missing: + raise DeployError( + "the producer has unresolved shared libraries on this target:\n" + + "\n".join(missing) + ) + in_build = [ln for ln in listing if str(build_dir) in ln] + if in_build: + raise DeployError( + "the producer still resolves libraries from the build tree:\n" + + "\n".join(in_build) + ) + + +def remote_build(args: argparse.Namespace) -> None: + """Configure and build only the producer, then stage the release from the build tree. + + The build directory persists between deployments and is discarded when the toolchain, L4T release, source + path or CMake arguments change. + """ + cache = Path(os.path.expanduser(CACHE)) + build_dir = cache / "build" + stamp = "|".join( + [ + Path("/etc/nv_tegra_release").read_text().splitlines()[0], + tail(["cmake", "--version"]).splitlines()[0], + tail(["g++", "-dumpfullversion"]), + args.src, + *args.cmake_arg, + ] + ) + stamp_file = build_dir / ".stamp" + if not stamp_file.exists() or stamp_file.read_text() != stamp: + print( + " new toolchain, release, source or CMake arguments: clean build directory" + ) + run(["rm", "-rf", str(build_dir)]) + build_dir.mkdir(parents=True) + stamp_file.write_text(stamp) + mmapi, sipl_api = sdk_dirs(args.sdk_root) + env = {**os.environ, "PATH": build_path()} + sdk_flags = [f"-DJETSON_MMAPI_DIR={mmapi}"] + if sipl_api.is_dir(): + sdk_flags.append(f"-DSIPL_API_DIR={sipl_api}") + # The CUDA driver library is not on the default link path on L4T, and is dropped by --as-needed. + gpu_libcuda = Path("/opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so.1") + link_flags = ( + [f"-DCMAKE_EXE_LINKER_FLAGS=-Wl,--no-as-needed {gpu_libcuda}"] + if gpu_libcuda.exists() + else [] + ) + run( + [ + "cmake", + "-S", + args.src, + "-B", + str(build_dir), + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CUDA_ARCHITECTURES=87", + "-DBUILD_PLUGIN_SENSING=ON", + "-DBUILD_TESTING=OFF", + "-DBUILD_EXAMPLES=OFF", + "-DBUILD_PYTHON_BINDINGS=OFF", + "-DBUILD_VIZ=OFF", + "-DENABLE_CLANG_FORMAT_CHECK=OFF", + "-DCMAKE_CXX_FLAGS=-I/usr/local/cuda/include", + *sdk_flags, + *link_flags, + # Options only some source trees define (a capture backend, say) go last so they can override. + *args.cmake_arg, + ], + env=env, + ) + run( + [ + "cmake", + "--build", + str(build_dir), + "--target", + "camera_plugin_sensing", + "--parallel", + str(args.jobs), + ], + env=env, + ) + run(["rm", "-rf", args.stage]) + stage = Path(args.stage) + stage_producer(build_dir, stage) + print(f" staged {sum(len(files) for _, _, files in os.walk(stage))} files") + + +def remote_apply(args: argparse.Namespace) -> None: + """Apply a source delta: extract the sent files, delete the removed ones, verify, record the new manifest.""" + src = Path(args.src) + if args.fresh: + # No trustworthy record of what is there: start from nothing so no stale file can reach the build. + shutil.rmtree(src, ignore_errors=True) + src.mkdir(parents=True, exist_ok=True) + with tarfile.open(args.archive) as tar: + members = tar.getmembers() + meta = {m.name: m for m in members if m.name.startswith(f"{DEPLOY_DIR}/")} + manifest = json.loads( + tar.extractfile(meta[f"{DEPLOY_DIR}/manifest.json"]).read() + ) + removed = json.loads(tar.extractfile(meta[f"{DEPLOY_DIR}/removed.json"]).read()) + sent = [m for m in members if m.name not in meta] + tar.extractall(src, members=sent, filter="data") + for name in removed: + target = (src / name).resolve() + if src.resolve() not in target.parents: + raise DeployError( + f"refusing to remove {name}: outside the source directory" + ) + target.unlink(missing_ok=True) + for parent in target.parents: + if parent == src.resolve() or any(parent.iterdir()): + break + parent.rmdir() + for member in sent: + expected = manifest[member.name] + if ( + f"{sha256_file(src / member.name)}{'x' if os.access(src / member.name, os.X_OK) else '-'}" + != expected + ): + raise DeployError( + f"{member.name} does not match its recorded hash after extraction" + ) + Path(args.manifest).write_text(json.dumps(manifest)) + print(f" applied {len(sent)} files, removed {len(removed)}") + + +def remote_package(args: argparse.Namespace) -> None: + stage = Path(args.stage) + sums = [] + for path in sorted( + p for p in stage.rglob("*") if p.is_file() and p.name != "SHA256SUMS" + ): + sums.append(f"{sha256_file(path)} ./{path.relative_to(stage)}") + (stage / "SHA256SUMS").write_text("\n".join(sums) + "\n") + with tarfile.open(args.archive, "w:gz") as tar: + tar.add(stage, arcname=".") + print(f" packaged {args.archive}") + + +def unmanaged_producers() -> list[tuple[int, str]]: + """Running camera_plugin_sensing processes the service does not own, as (pid, owner).""" + main_pid = tail(["systemctl", "show", "-p", "MainPID", "--value", SERVICE]) + found = [] + for proc in Path("/proc").iterdir(): + if not proc.name.isdigit() or proc.name == main_pid: + continue + try: + exe = os.readlink(proc / "exe") + owner = proc.stat().st_uid + except OSError: + continue + if Path(exe).name == "camera_plugin_sensing": + found.append((int(proc.name), pwd.getpwuid(owner).pw_name)) + return sorted(found) + + +def stop_unmanaged(procs: list[tuple[int, str]], timeout_s: int = 20) -> None: + """Interrupt exactly these PIDs (the producer stops cleanly on SIGINT) and wait for them to exit.""" + for pid, _ in procs: + os.kill(pid, signal.SIGINT) + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if not any(Path(f"/proc/{pid}").exists() for pid, _ in procs): + return + time.sleep(0.5) + raise DeployError( + f"unmanaged producer PIDs did not exit after SIGINT: {[p for p, _ in procs]}" + ) + + +def remote_activate(args: argparse.Namespace) -> None: + """Install the release, then stop the service, switch the active link and start it. Runs as root.""" + prefix = Path(args.prefix) + unmanaged = unmanaged_producers() + if unmanaged and not args.replace_unmanaged: + listing = ", ".join(f"pid {pid} (user {owner})" for pid, owner in unmanaged) + raise DeployError( + f"a camera_plugin_sensing is already running outside the service: {listing}. " + "Stop it, or pass --replace-unmanaged to stop exactly these processes." + ) + dest = prefix / "releases" / args.release + if dest.exists(): + raise DeployError( + f"release {args.release} already exists under {prefix}/releases" + ) + dest.mkdir(parents=True) + try: + print(" unpacking the release", flush=True) + with tarfile.open(args.archive) as tar: + tar.extractall(dest, filter="data") + sums = (dest / "SHA256SUMS").read_text().splitlines() + bar = Progress("verifying files", len(sums)) + for done, line in enumerate(sums, 1): + digest, name = line.split(" ", 1) + if sha256_file(dest / name) != digest: + raise DeployError( + f"checksum mismatch for {name} in release {args.release}" + ) + bar.update(done) + bar.finish() + except BaseException: + shutil.rmtree(dest, ignore_errors=True) + raise + if args.platform_file: + # The detected platform config outlives the release: the service reads it from a fixed path. + shutil.copy2(args.platform_file, prefix / PLATFORM_FILE) + run(["chown", "-R", f"{args.owner}:", str(prefix)]) + print(f" installed release {args.release}") + + unit_target = Path("/etc/systemd/system") / SERVICE + unit_text = Path(args.unit).read_text() + if not unit_target.exists() or unit_target.read_text() != unit_text: + unit_target.write_text(unit_text) + run(["systemctl", "daemon-reload"]) + run(["systemctl", "enable", SERVICE], capture_output=True) + print(" service unit updated") + print(" stopping the running producer", flush=True) + run(["systemctl", "stop", SERVICE]) + if unmanaged: + print(f" stopping unmanaged producer(s): {[pid for pid, _ in unmanaged]}") + stop_unmanaged(unmanaged) + link, staging_link = prefix / "current", prefix / "current.new" + if staging_link.is_symlink(): + staging_link.unlink() + staging_link.symlink_to(Path("releases") / args.release) + os.replace(staging_link, link) + since = time.strftime("%Y-%m-%d %H:%M:%S") + print(" starting the new producer; waiting for the cameras", flush=True) + run(["systemctl", "start", SERVICE]) + summary = wait_ready( + since, + args.ready_timeout, + [int(n) for n in args.sensors.split(",") if n], + mode=args.mode, + direct_sensors=[int(n) for n in args.direct_sensors.split(",") if n], + ) + for line in summary: + print(f" {line}") + print(f" producer is up (release {args.release})") + + +# Every line the capture driver logs carries "Severity N :"; at any severity they bury the lines that explain a +# failure. DRIVER_FAULT picks the few that do explain one, and BENIGN_DRIVER the ones that look like errors but are +# the driver noting a library or power driver it does not need. +DRIVER_NOISE = re.compile(r"Severity \d+ :") +DRIVER_FAULT = re.compile(r"Failed to|Error processing|HSL I2C error|status: \d+") +BENIGN_DRIVER = re.compile(r"driverCatalog|uddf_discover|No module power driver") + + +# The deserializer's own report that a link did not lock: SIPL names the link by letter, A to D for links 0 to 3. +DEAD_LINK = re.compile(r'Sequence: "enable_link_([A-D])"') + + +def dead_link_lines(lines: list[str]) -> list[str]: + """One plain line per camera link the deserializer could not lock, in the order they failed.""" + found = [] + for line in lines: + match = DEAD_LINK.search(line) + if match and match.group(1) not in found: + found.append(match.group(1)) + return [ + f"link {ord(letter) - ord('A')} (COAX{ord(letter) - ord('A')} on CN1) did not lock: no camera answers there. " + "Check that camera's cable and its harness lead." + for letter in found + ] + + +def failure_report(since: str, sensors: list[int]) -> str: + """What to read when the producer did not become ready. + + The service's state, how much of its log is driver chatter, the first and last real lines of what remains, the + producer's own fault lines, which sensors never published, and where the complete log was saved. + """ + state = tail( + [ + "systemctl", + "show", + SERVICE, + "-p", + "ActiveState", + "-p", + "SubState", + "-p", + "Result", + ] + + ["-p", "ExecMainStatus", "-p", "NRestarts"] + ).replace("\n", ", ") + log = tail( + ["journalctl", "-u", SERVICE, "--since", since, "--no-pager", "-o", "cat"] + ) + saved = Path(os.path.expanduser(CACHE)) / "last-start-failure.log" + saved.parent.mkdir(parents=True, exist_ok=True) + saved.write_text(log + "\n") + lines = log.splitlines() + quiet = [ln for ln in lines if not DRIVER_NOISE.search(ln)] + faults = [ + ln + for ln in quiet + if re.search( + r"camera_|startup_failed|stream_failed|publish_failed|check each camera|driver log:", + ln, + ) + ] + driver_faults = [] + for line in lines: + if ( + DRIVER_NOISE.search(line) + and DRIVER_FAULT.search(line) + and not BENIGN_DRIVER.search(line) + ): + text = line.split(" : ", 1)[-1].strip() + if text not in driver_faults: + driver_faults.append(text) + out = [ + f"service: {state}", + *dead_link_lines(lines), + f"log: {len(lines)} lines since the start, {len(lines) - len(quiet)} of them from the camera driver; " + f"complete log saved on the Orin at {saved}", + ] + if driver_faults: + out.append("camera driver errors:") + out += [f" {ln}" for ln in driver_faults[:4]] + if sensors: + out.append(f"sensors expected to publish: {', '.join(map(str, sensors))}") + if faults: + out.append("producer fault lines:") + out += [f" {ln}" for ln in faults[:8]] + out.append("first lines (driver chatter removed):") + first, rest = quiet[:15], quiet[15:] + out += [f" {ln}" for ln in first] + last = rest[-15:] + if last: + hidden = len(rest) - len(last) + out.append( + f"last lines ({hidden} not shown between):" + if hidden + else "remaining lines:" + ) + out += [f" {ln}" for ln in last] + return "\n".join(out) + + +def current_session(log: str, mode: str) -> str: + """The journal text of the producer's current session: what follows its last start marker. + + A producer that restarts its cameras inside one process (it retries after a fault) logs the marker each time, + so progress from an earlier attempt never counts toward the current one. + """ + marker = "Capturing." if mode == "direct" else "Joined LiveKit room" + return log.rsplit(marker, 1)[1] if marker in log else "" + + +def readiness( + log: str, mode: str, sensors: list[int], direct_sensors: list[int] +) -> tuple[list[str], list[str]]: + """(what is ready, what is not) for every configured sensor, judged on the latest status line of the session. + + direct mode: a sensor is up when its latest line shows frames and, if the config gave it a direct destination, + that link connected; a stream with no destination is reported as capture-only. livekit mode (an older producer + that publishes to LiveKit itself): the latest line shows a positive "livekit N published". + """ + current = current_session(log, mode) + ready, problems = [], [] + if not current: + marker = "capturing" if mode == "direct" else "joined a LiveKit room" + return ready, [f"the producer has not reported {marker} yet"] + for n in sensors: + if mode == "livekit": + found = re.findall(rf"sensor {n}: [^\n]*livekit (\d+) published", current) + if found and int(found[-1]) > 0: + ready.append(f"sensor {n}: publishing to LiveKit") + else: + problems.append(f"sensor {n}: not publishing to LiveKit") + continue + lines = re.findall(rf"sensor {n}: (\d+) frames([^\n]*)", current) + if not lines or int(lines[-1][0]) == 0: + problems.append(f"sensor {n}: no frames captured") + continue + rest = lines[-1][1] + if n not in direct_sensors: + ready.append(f"sensor {n}: capturing (no direct destination configured)") + elif re.search(r"direct: \S+ connected", rest) and "NOT connected" not in rest: + ready.append(f"sensor {n}: capturing and connected to its destination") + else: + reason = re.search(r"direct: (\S+ NOT connected[^\n]*)", rest) + problems.append( + f"sensor {n}: capturing but {reason.group(1) if reason else 'its direct link is not connected'}" + ) + return ready, problems + + +def wait_ready( + since: str, + timeout: int, + sensors: list[int], + mode: str = "direct", + direct_sensors: list[int] | None = None, +) -> list[str]: + """Wait until the current session is up for every configured sensor; return one line per sensor. + + "Up" is the producer's own current output (see readiness()), never a pid and never progress left over from an + earlier attempt. A producer that is not up in time raises, with the sensors that were not. + """ + direct_sensors = direct_sensors or [] + start = time.monotonic() + deadline = start + timeout + bar = Progress("waiting for cameras", timeout) + log = "" + while time.monotonic() < deadline: + if ( + subprocess.run( + ["systemctl", "is-active", "--quiet", SERVICE], check=False + ).returncode + != 0 + ): + break + log = tail( + ["journalctl", "-u", SERVICE, "--since", since, "--no-pager", "-o", "cat"] + ) + ready, problems = readiness(log, mode, sensors, direct_sensors) + if not problems: + bar.finish() + return ready + bar.label = f"waiting for cameras ({len(ready)}/{len(sensors)} up)" + bar.update(int(time.monotonic() - start)) + time.sleep(2) + if sys.stdout.isatty(): + print() + print(failure_report(since, sensors), file=sys.stderr) + # Judged on the log the last check saw, so the error names what that check found missing. + _, problems = readiness(log, mode, sensors, direct_sensors) + detail = f"; {'; '.join(problems)}" if problems else "" + raise DeployError( + f"the producer did not become ready{detail}; logs: journalctl -u {SERVICE}" + ) + + +# --------------------------------------------------------------------------- + + +def parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument( + "target", help=f"host or user@host of the Orin (a bare host uses ${USER_ENV})" + ) + ap.add_argument( + "--config", + help="producer options (JSON); required unless only setting up or checking", + ) + ap.add_argument( + "--install-prereqs", + action="store_true", + help="install the Orin's build prerequisites (asks for its sudo password), then stop without deploying", + ) + ap.add_argument( + "--logs", + action="store_true", + help="show the producer service's status and recent journal, then stop", + ) + ap.add_argument( + "--rescan-cameras", + action="store_true", + help='with "cameras": "auto", scan all 8 inputs instead of trying the cameras found last time first', + ) + ap.add_argument( + "--scan-cameras", + action="store_true", + help="try each camera link alone and report which has a camera (stops the producer service), then stop " + "without deploying", + ) + ap.add_argument( + "--check-camera", + action="store_true", + help="stream both cameras for a few seconds and report, then stop without deploying; " + "with --install-prereqs the prerequisites go first", + ) + ap.add_argument( + "--jobs", type=int, default=2, help="parallel compile jobs on the Orin" + ) + ap.add_argument( + "--service-user", help="account the service runs as (default: the ssh user)" + ) + ap.add_argument( + "--sdk-root", + default="", + help="directory on the Orin holding usr/src/jetson_multimedia_api and jetson_sipl_api (default: /)", + ) + ap.add_argument( + "--replace-unmanaged", + action="store_true", + help="stop a camera_plugin_sensing the service does not own (its exact PIDs are shown) instead of refusing", + ) + ap.add_argument( + "--source", + metavar="DIR", + help="deploy this git checkout (a worktree, say) instead of the one holding this script", + ) + ap.add_argument( + "--branch", + metavar="NAME", + help="deploy this branch's committed files (from --source, or this checkout), ignoring the working tree", + ) + ap.add_argument( + "--cmake-arg", + action="append", + default=[], + metavar="ARG", + help="extra CMake argument for the producer build, repeatable (for options only some source trees have)", + ) + ap.add_argument("--no-activate", action="store_true", help="build and package only") + ap.add_argument( + "--no-setup", + action="store_true", + help="fail when the Orin lacks a prerequisite instead of installing it", + ) + ap.add_argument( + "--sipl-api", + metavar="FILE", + help="Jetson_SIPL_API_R_aarch64.tbz2 to unpack on the Orin when its SIPL headers are missing " + f"(default: the newest in {SDKM_DOWNLOADS})", + ) + return ap + + +def remote_parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser(prog="deploy_orin.py remote") + sub = ap.add_subparsers(dest="step", required=True) + pre = sub.add_parser("preflight") + pre.add_argument("--sdk-root", default="") + build = sub.add_parser("build") + build.add_argument("--src", required=True) + build.add_argument("--cmake-arg", action="append", default=[]) + build.add_argument("--stage", required=True) + build.add_argument("--jobs", type=int, default=2) + build.add_argument("--sdk-root", default="") + apply_ = sub.add_parser("apply") + for name in ("archive", "src", "manifest"): + apply_.add_argument(f"--{name}", required=True) + apply_.add_argument("--fresh", action="store_true") + pack = sub.add_parser("package") + pack.add_argument("--stage", required=True) + pack.add_argument("--archive", required=True) + act = sub.add_parser("activate") + for name in ("archive", "prefix", "release", "unit", "owner"): + act.add_argument(f"--{name}", required=True) + act.add_argument("--ready-timeout", type=int, default=60) + act.add_argument("--sensors", default="") + act.add_argument("--direct-sensors", default="") + act.add_argument("--mode", choices=MODES, default="direct") + act.add_argument("--replace-unmanaged", action="store_true") + act.add_argument("--platform-file", default="") + return ap + + +def main(argv: list[str]) -> int: + try: + if argv and argv[0] == "remote": + args = remote_parser().parse_args(argv[1:]) + { + "preflight": remote_preflight, + "build": remote_build, + "package": remote_package, + "apply": remote_apply, + "activate": remote_activate, + }[args.step](args) + else: + deploy(parser().parse_args(argv)) + except DeployError as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/plugins/sensing/scripts/isaaccapture-producer.service.in b/src/plugins/sensing/scripts/isaaccapture-producer.service.in new file mode 100644 index 0000000000..9ba4113a89 --- /dev/null +++ b/src/plugins/sensing/scripts/isaaccapture-producer.service.in @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Template for the Orin camera producer service; scripts/deploy_orin.py fills the @...@ fields. +[Unit] +Description=Isaac Capture camera producer +After=network-online.target +Wants=network-online.target +# Keeps a producer that cannot start from restarting forever (these are unit settings, not service settings). +StartLimitIntervalSec=120 +StartLimitBurst=5 + +[Service] +Type=simple +User=@SERVICE_USER@ +SupplementaryGroups=@SERVICE_GROUPS@ +WorkingDirectory=@PREFIX@/current/plugins/sensing_camera +# The release carries its own shared libraries; they must win over the build-tree paths the binary was linked with. +Environment=LD_LIBRARY_PATH=@PREFIX@/current/plugins/sensing_camera +@ENVIRONMENT@ +ExecStart=@EXEC_START@ +# A fatal exit (a camera session that will not stop ends the process) restarts the producer; a clean stop does not. +Restart=on-failure +RestartSec=5 +# The producer stops on SIGINT; systemd ends anything left in this cgroup after the timeout. +KillSignal=SIGINT +KillMode=control-group +TimeoutStopSec=20 + +[Install] +WantedBy=multi-user.target diff --git a/src/plugins/sensing/scripts/orin_setup.sh b/src/plugins/sensing/scripts/orin_setup.sh new file mode 100755 index 0000000000..428b032377 --- /dev/null +++ b/src/plugins/sensing/scripts/orin_setup.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Prepare an Orin for the camera producer, then check the cameras work. Run it on the +# Orin. It ships with deploy_orin.py, both ahead of the first build and in each release. +# +# 1. Prerequisites: the build toolchain, the JetPack CUDA and Multimedia API packages and +# uv, then the SENSING drivers and device-tree overlay (setup.sh), then a check of what +# the deploy preflight checks. The SIPL headers come from the L4T SIPL API package. +# 2. Camera check: verify.sh (read-only report), then validate.sh (streams both sensors). When the +# stream fails, or with --scan, all 8 camera inputs are tried one at a time to show where cameras are. +# +# Exit status: 0 if everything requested passed, 2 if a reboot or new login is needed before +# going on, 1 otherwise. +# +# Usage: +# ./orin_setup.sh [--prereqs-only | --check-only] [--no-apt] [--no-drivers] [--sipl-api FILE] [--sdk-root DIR] [--sudo-password-stdin] [--scan | --detect DIR [--rescan]] [-t SECONDS] +# +# Options: +# --prereqs-only install and check prerequisites; skip the camera check +# --check-only run the camera check only +# --no-apt skip apt and uv installation; only check what is present +# --no-drivers skip setup.sh (the SENSING drivers and overlay) +# --sipl-api FILE unpack this Jetson_SIPL_API_R_aarch64.tbz2 if the SIPL headers are missing +# --sdk-root DIR where usr/src/jetson_multimedia_api and jetson_sipl_api live (default /) +# --sudo-password-stdin read the sudo password from stdin, so nothing prompts +# --scan try all 8 camera inputs alone and report where cameras are (stops the producer service) +# --detect DIR the same probe, then write DIR/platform.json for the cameras found and print +# {"config", "masks", "count", "used", "positions"} as JSON on stdout. The inputs found are +# saved in DIR/known-cameras.json and tried first next time; all 8 are scanned only when that +# file is missing or a saved camera does not answer +# --rescan with --detect: ignore the saved cameras and scan all 8 inputs +# -t, --seconds N how long validate.sh streams (default 10) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./common.sh +source "$SCRIPT_DIR/common.sh" + +SERVICE=isaaccapture-producer.service +DO_PREREQS=1 +DO_CHECK=1 +DO_APT=1 +DO_DRIVERS=1 +SIPL_API= +SDK_ROOT= +REBOOT=0 +SUDO_PASSWORD_STDIN=0 +DO_SCAN=0 +DETECT_DIR= +RESCAN=0 +SECONDS_ARG=10 + +usage() { sed -n '4,35p' "${BASH_SOURCE[0]}" | sed 's/^# \?//'; } + +while (( $# )); do + case "$1" in + --prereqs-only) DO_CHECK=0; shift ;; + --check-only) DO_PREREQS=0; shift ;; + --no-apt) DO_APT=0; shift ;; + --no-drivers) DO_DRIVERS=0; shift ;; + --sipl-api) SIPL_API="${2:?--sipl-api needs a file}"; shift 2 ;; + --sdk-root) SDK_ROOT="${2:?--sdk-root needs a directory}"; shift 2 ;; + --sudo-password-stdin) SUDO_PASSWORD_STDIN=1; shift ;; + --scan) DO_SCAN=1; shift ;; + --detect) DETECT_DIR="${2:?--detect needs a directory}"; shift 2 ;; + --rescan) RESCAN=1; shift ;; + -t|--seconds) SECONDS_ARG="${2:?--seconds needs a value}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" "Run $0 --help" ;; + esac +done + +if [[ "$SUDO_PASSWORD_STDIN" -eq 1 ]]; then + # Every sudo below, including the ones in setup.sh and common.sh, answers from a private tmpfs file that is + # removed on exit: the exported function adds -A, and the askpass program prints the file. + umask 077 + SUDO_PWFILE="$(mktemp -p "${XDG_RUNTIME_DIR:-/dev/shm}" orin-sudo.XXXXXX)" + SUDO_ASKPASS="$(mktemp -p "${XDG_RUNTIME_DIR:-/dev/shm}" orin-askpass.XXXXXX)" + trap 'rm -f "$SUDO_PWFILE" "$SUDO_ASKPASS"' EXIT + cat > "$SUDO_PWFILE" + printf '#!/bin/sh\ncat "$SUDO_PWFILE"\n' > "$SUDO_ASKPASS" + chmod 700 "$SUDO_ASKPASS" + export SUDO_PWFILE SUDO_ASKPASS + sudo() { command sudo -A "$@"; } + export -f sudo +fi + +APT_PACKAGES=( + build-essential cmake pkg-config patchelf git curl rsync + python3 python3-dev python3-venv + nvidia-jetpack nvidia-l4t-jetson-multimedia-api +) + +FAIL=0 +check() { + local name="$1"; shift + if "$@" >/dev/null 2>&1; then ok "$name"; else warn "missing: $name"; FAIL=1; fi +} + +prerequisites() { + [[ "$(uname -m)" == aarch64 && -f /etc/nv_tegra_release ]] \ + || die "this is not a Jetson (aarch64 L4T) system." + + if [[ "$DO_APT" -eq 1 ]]; then + step "Installing apt packages" + sudo apt-get update + sudo apt-get install -y "${APT_PACKAGES[@]}" + if ! PATH="$HOME/.local/bin:$PATH" have uv; then + step "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + fi + + if [[ "$DO_DRIVERS" -eq 1 ]]; then + # Idempotent: it reinstalls only what differs. It exits nonzero while a reboot or a new login is still + # needed (the Jetson-IO overlay choice, group membership), so stop there and re-run afterwards. + step "Installing the SENSING drivers (setup.sh)" + # -y: no prompts. Exit 3: the overlay is selected and a reboot is scheduled. Exit 2: only a new login is + # needed for group membership, and every new ssh session has it. + local code=0 + "$SCRIPT_DIR/setup.sh" -y || code=$? + case "$code" in + 0|2) ;; + 3) warn "rebooting to load the device-tree overlay; run this again afterwards." + FAIL=1; REBOOT=1; return ;; + *) warn "setup.sh failed (exit $code)."; FAIL=1; return ;; + esac + fi + + if [[ ! -d "${SDK_ROOT%/}/usr/src/jetson_sipl_api" && -n "$SIPL_API" ]]; then + step "Unpacking the SIPL API headers" + sudo tar -xjf "$SIPL_API" -C "${SDK_ROOT:-/}" + fi + + step "Checking prerequisites" + export PATH="$HOME/.local/bin:/usr/local/cuda/bin:$PATH" + local tool + for tool in cmake g++ systemctl uv nvcc; do check "$tool" have "$tool"; done + check "patchelf" have patchelf + local sdk="${SDK_ROOT%/}/usr/src" + check "Jetson Multimedia API ($sdk/jetson_multimedia_api)" test -d "$sdk/jetson_multimedia_api" + check "SIPL headers ($sdk/jetson_sipl_api)" test -d "$sdk/jetson_sipl_api" + check "SENSING drivers (nvsipl_camera)" have nvsipl_camera + check "ISP tuning ($SIPL_NITO_DIR/$SENSING_NITO)" test -r "$SIPL_NITO_DIR/$SENSING_NITO" + check "device-tree overlay ($SIPL_BOOT_DIR/$SENSING_DTBO)" test -f "$SIPL_BOOT_DIR/$SENSING_DTBO" + check "overlay active: deserializer I2C bus (/dev/i2c-9)" test -e /dev/i2c-9 + check "4 GB free in $HOME" test "$(df --output=avail -B1 "$HOME" | tail -1)" -ge $((4 << 30)) + if [[ "$FAIL" -ne 0 ]]; then + warn "SIPL headers: unpack Jetson_SIPL_API_R_aarch64.tbz2 into / (it holds usr/src/jetson_sipl_api)." + warn "SENSING drivers and overlay: rerun without --no-drivers, then reboot if asked." + fi +} + +# Try every camera input alone: four links on each of the two deserializers. The platform config places its modules +# on links 2 and 3 of deserializer CD (transport 1), so each probe uses a temporary copy whose one module sits on the +# input under test. The vendor file itself is never changed. A link whose camera does not lock fails SIPL +# initialisation (status 10). Fills FOUND (transport:link) and FOUND_LABEL. +FOUND=() +FOUND_LABEL=() + +input_label() { + if [[ "$1" -eq 1 ]]; then + printf 'CN1 COAX%s (deserializer CD, link %s)' "$2" "$2" + else + printf 'CN2 COAX%s (deserializer AB, link %s)' "$((4 + $2))" "$2" + fi +} + +# Does a camera answer on this input (transport, link)? Records it in FOUND on success. +probe_input() { + local transport="$1" link="$2" label config mask slots + label="$(input_label "$transport" "$link")" + config="$PROBE_TMP/shw5g-$transport-$link.json" + python3 - "$SENSING_CONFIGS_DIR/shw5g.json" "$config" "$transport" "$link" <<'PY' +import json, sys +src, out, transport, link = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]) +data = json.load(open(src)) +module = data["cameraConfigs"][0]["cameraModules"][0] +module.update(linkIndex=link, transportIndex=transport) +data["cameraConfigs"][0]["cameraModules"] = [module] +json.dump(data, open(out, "w")) +PY + slots=("0x0000" "0x0000") + slots[transport]="$(printf '0x%04x' $((1 << (4 * link))))" + mask="${slots[0]} ${slots[1]}" + if nvsipl_camera -t "$config" -c "$SENSING_PLATFORM_CONFIG" -m "$mask" \ + --enable-camera-hal -s -Z -r 3 "$PROBE_TMP/log" 2>&1; then + ok "$label: camera streaming" + FOUND+=("$transport:$link") + FOUND_LABEL+=("$label") + return 0 + fi + info "$label: no camera" + return 1 +} + +probe_inputs() { + local transport link + PROBE_TMP="$(mktemp -d)" + step "Scanning all 8 camera inputs (one at a time, about 5 s each)" + for transport in 1 0; do + for link in 0 1 2 3; do + probe_input "$transport" "$link" || true + done + done + rm -rf "$PROBE_TMP" +} + +scan_links() { + probe_inputs + if [[ "${#FOUND[@]}" -eq 0 ]]; then + warn "no camera answers on any input. Check the carrier's 12 V supply and the power switching harness, that each" + warn "camera is clicked into its cable, and that the cable is clicked into its harness lead." + FAIL=1 + return + fi + ok "cameras found on ${#FOUND[@]} input(s); the deploy can use any of them (cameras: auto)" +} + +# Write DIR/platform.json for the cameras found (the SHW5G config with its modules moved onto those inputs, at most +# the two the config names) and print what the producer needs on stdout. Everything else goes to stderr. +detect_cameras() { + local dir="$1" known="$1/known-cameras.json" pair transport link stale=0 saved=() + mkdir -p "$dir" + if [[ "$RESCAN" -eq 0 ]]; then + # The cameras saved last time, as "transport:link". A missing or unreadable file means none. + mapfile -t saved < <(python3 - "$known" <<'PY' +import json, sys +try: + for camera in json.load(open(sys.argv[1]))["cameras"]: + print("%d:%d" % (camera["transport"], camera["link"])) +except (OSError, ValueError, KeyError, TypeError): + pass +PY +) + fi + if [[ "${#saved[@]}" -gt 0 ]]; then + # One short probe per saved camera instead of all 8 inputs. + PROBE_TMP="$(mktemp -d)" + step "Trying the cameras saved last time" >&2 + for pair in "${saved[@]}"; do + transport="${pair%%:*}" + link="${pair##*:}" + probe_input "$transport" "$link" >&2 || stale=1 + done + rm -rf "$PROBE_TMP" + if [[ "$stale" -eq 1 ]]; then + warn "a saved camera did not answer; scanning all inputs." + FOUND=() + FOUND_LABEL=() + fi + fi + [[ "${#FOUND[@]}" -gt 0 ]] || probe_inputs >&2 + [[ "${#FOUND[@]}" -gt 0 ]] || { warn "no camera answers on any input."; return 1; } + python3 - "$known" "${FOUND[@]}" -- "${FOUND_LABEL[@]}" <<'PY' +import json, sys +out, rest = sys.argv[1], sys.argv[2:] +split = rest.index("--") +cameras = [] +for item, label in zip(rest[:split], rest[split + 1:]): + transport, link = map(int, item.split(":")) + cameras.append({"transport": transport, "link": link, "label": label}) +json.dump({"cameras": cameras}, open(out, "w"), indent=1) +PY + python3 - "$SENSING_CONFIGS_DIR/shw5g.json" "$dir/platform.json" "${FOUND[@]}" -- "${FOUND_LABEL[@]}" <<'PY' +import json, sys +src, out = sys.argv[1], sys.argv[2] +rest = sys.argv[3:] +split = rest.index("--") +found = [tuple(map(int, item.split(":"))) for item in rest[:split]] +labels = rest[split + 1:] +data = json.load(open(src)) +config = data["cameraConfigs"][0] +templates = config["cameraModules"] # the two modules the vendor config names +used = found[: len(templates)] +modules, slots = [], [0, 0] +for template, (transport, link) in zip(templates, used): + modules.append(dict(template, linkIndex=link, transportIndex=transport)) + slots[transport] |= 1 << (4 * link) +config["cameraModules"] = modules +json.dump(data, open(out, "w"), indent=1) +print(json.dumps({ + "config": out, + "masks": " ".join("0x%04x" % s for s in slots), + "count": len(found), + "used": len(used), + "positions": labels, +})) +PY +} + +camera_check() { + # SIPL has one client at a time; a running producer would make the checks fail confusingly. + if systemctl is-active --quiet "$SERVICE" 2>/dev/null; then + if [[ "$DO_SCAN" -eq 0 ]]; then + warn "$SERVICE is running and holds the cameras." + warn "Stop it first: sudo systemctl stop $SERVICE" + FAIL=1 + return + fi + step "Stopping $SERVICE so the scan can open the cameras" + sudo systemctl stop "$SERVICE" + fi + if [[ "$DO_SCAN" -eq 1 ]]; then + scan_links + return + fi + step "Camera report (verify.sh)" + "$SCRIPT_DIR/verify.sh" || FAIL=1 + step "Camera stream (validate.sh)" + if ! "$SCRIPT_DIR/validate.sh" -t "$SECONDS_ARG"; then + FAIL=1 + scan_links + fi +} + +if [[ -n "$DETECT_DIR" ]]; then + have nvsipl_camera || die "nvsipl_camera is not on PATH." "Run orin_setup.sh --prereqs-only first." + if systemctl is-active --quiet "$SERVICE" 2>/dev/null; then + warn "$SERVICE is running and holds the cameras; stop it first (sudo systemctl stop $SERVICE)." >&2 + exit 1 + fi + detect_cameras "$DETECT_DIR" + exit $? +fi + +[[ "$DO_PREREQS" -eq 0 ]] || prerequisites +if [[ "$DO_CHECK" -eq 1 ]]; then + if [[ "$FAIL" -ne 0 ]]; then + warn "skipping the camera check until the prerequisites above are fixed" + else + camera_check + fi +fi + +[[ "$FAIL" -eq 0 ]] || exit $(( REBOOT ? 2 : 1 )) +ok "all requested checks passed" diff --git a/src/plugins/sensing/scripts/setup.sh b/src/plugins/sensing/scripts/setup.sh index 3b438ee0b1..8a7eca83d4 100755 --- a/src/plugins/sensing/scripts/setup.sh +++ b/src/plugins/sensing/scripts/setup.sh @@ -16,6 +16,8 @@ # Usage: # ./setup.sh [options] # +# Exit status: 0 done, 2 a new login or manual reboot is needed, 3 with -y: overlay selected and a reboot scheduled. +# # Options: # --pkg DIR vendor driver package dir (default: fetch from SENSING's repo) # --no-fetch do not download the vendor package; fail if it is not on disk @@ -40,7 +42,7 @@ REBOOT_REQUIRED=0 LOGIN_REQUIRED=0 ASSUME_YES=0 -usage() { sed -n '4,27p' "${BASH_SOURCE[0]}" | sed 's/^# \?//'; } +usage() { sed -n '4,29p' "${BASH_SOURCE[0]}" | sed 's/^# \?//'; } while (( $# )); do case "$1" in @@ -212,7 +214,41 @@ else info "nvpmodel not found" fi +# Select the SG8A overlay with Jetson-IO's command line, without its menu. +select_overlay() { + local tool=/opt/nvidia/jetson-io/config-by-hardware.py list header + [[ -x "$tool" ]] || die "$tool not found." "Install the JetPack Jetson-IO tool, or run without -y." + list="$(sudo "$tool" -l)" || die "Jetson-IO could not list its hardware modules." + header="$(awk -v name="$SENSING_OVERLAY_NAME" ' + /^Header [0-9]+/ { h = $2; sub(/[^0-9].*/, "", h) } + index($0, name) { print h; exit }' <<<"$list")" + [[ -n "$header" ]] || die "Jetson-IO lists no '$SENSING_OVERLAY_NAME'." "$list" + sudo "$tool" -n "$header=$SENSING_OVERLAY_NAME" || die "Jetson-IO could not select the overlay." +} + # Complete first-install setup before offering a capture command. +# The deserializer's I2C bus exists only once the overlay is selected and the Orin has rebooted. +OVERLAY_STAMP=/var/lib/sensing/overlay-selected-boot-id +boot_id() { cat /proc/sys/kernel/random/boot_id; } +if [[ "$ASSUME_YES" -eq 1 ]] && { [[ "$REBOOT_REQUIRED" -eq 1 ]] || [[ ! -e /dev/i2c-9 ]]; }; then + step "Activate the device-tree overlay" + previous="$(cat "$OVERLAY_STAMP" 2>/dev/null || true)" + # One selection per boot, and never a second reboot for the same fault: an Orin that has already rebooted + # after selecting and still lacks the bus needs a person. + if [[ "$REBOOT_REQUIRED" -eq 0 && -n "$previous" && "$previous" != "$(boot_id)" ]]; then + die "the overlay was selected and the Orin rebooted, but /dev/i2c-9 is still absent." \ + "Check the Jetson-IO selection, the carrier's 12 V supply and the cables." + fi + if [[ "$REBOOT_REQUIRED" -eq 1 || "$previous" != "$(boot_id)" ]]; then + select_overlay + sudo mkdir -p "$(dirname "$OVERLAY_STAMP")" + boot_id | sudo tee "$OVERLAY_STAMP" >/dev/null + fi + # Reboot after this script has exited, so a caller over ssh still gets the exit status. + sudo systemd-run --quiet --on-active=5 systemctl reboot + warn "overlay selected; rebooting in 5 s to load it." + exit 3 +fi if [[ "$REBOOT_REQUIRED" -eq 1 ]]; then step "Activate the device-tree overlay" hint "Configure Jetson AGX CSI Connector -> Configure for compatible hardware" diff --git a/tests/python/ci/test_deploy_orin.py b/tests/python/ci/test_deploy_orin.py new file mode 100644 index 0000000000..e4f374633e --- /dev/null +++ b/tests/python/ci/test_deploy_orin.py @@ -0,0 +1,2744 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the Orin deploy script; every subprocess call is mocked or runs against a temp git repo.""" + +import argparse +import contextlib +import hashlib +import importlib.util +import io +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from repo_paths import repo_root + +spec = importlib.util.spec_from_file_location( + "deploy_orin", repo_root() / "src/plugins/sensing/scripts/deploy_orin.py" +) +deploy_orin = importlib.util.module_from_spec(spec) +spec.loader.exec_module(deploy_orin) + +DeployError = deploy_orin.DeployError +STREAM = "--add-stream=sensor=0,livekit=left" + + +def completed(returncode=0, stdout="", stderr=""): + return subprocess.CompletedProcess([], returncode, stdout, stderr) + + +class TempDirTestCase(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = Path(tmp.name) + + +class LoadConfigTests(TempDirTestCase): + def load(self, config): + path = self.tmp / "orin.json" + path.write_text(json.dumps(config)) + return deploy_orin.load_config(path) + + def valid(self, **override): + config = { + "mode": "livekit", + "livekit_url": "wss://lk.example.org", + "token_file": "/etc/isaaccapture/join.token", + "args": [STREAM], + } + config.update(override) + return config + + def test_accepts_valid_config(self): + config = self.valid() + self.assertEqual(self.load(config), config) + + def test_rejects_unknown_key(self): + with self.assertRaises(DeployError): + self.load(self.valid(extra=1)) + + def test_livekit_mode_needs_both_url_and_token_file(self): + for key in ("livekit_url", "token_file"): + config = self.valid() + del config[key] + with ( + self.subTest(only_missing=key), + self.assertRaisesRegex(DeployError, "needs livekit_url and token_file"), + ): + self.load(config) + + def test_mode_defaults_to_direct_and_direct_is_accepted(self): + for config in ({"args": [STREAM]}, {"mode": "direct", "args": [STREAM]}): + with self.subTest(config=config): + self.assertEqual(self.load(config), config) + + def test_direct_mode_forbids_livekit_settings(self): + extras = ( + {"livekit_url": "wss://x"}, + {"token_file": "/t"}, + {"livekit_url": "wss://x", "token_file": "/t"}, + ) + for extra in extras: + for mode in ({}, {"mode": "direct"}): + with ( + self.subTest(extra=extra, mode=mode), + self.assertRaisesRegex(DeployError, "belong to"), + ): + self.load({"args": [STREAM], **mode, **extra}) + + def test_livekit_mode_without_any_livekit_settings_is_rejected(self): + with self.assertRaisesRegex(DeployError, "needs livekit_url"): + self.load({"mode": "livekit", "args": [STREAM]}) + + def test_mode_must_be_one_of_the_known_modes(self): + for mode in ("LiveKit", "", "both", None, 3): + with ( + self.subTest(mode=mode), + self.assertRaisesRegex(DeployError, "must be one of direct, livekit"), + ): + self.load({"mode": mode, "args": [STREAM]}) + + def test_mode_is_an_allowed_config_key(self): + self.assertIn("mode", deploy_orin.CONFIG_KEYS) + + def test_env_is_an_allowed_config_key(self): + self.assertIn("env", deploy_orin.CONFIG_KEYS) + + def test_accepts_env(self): + config = self.valid(env={"FOO": "bar baz", "_X1": ""}) + self.assertEqual(self.load(config), config) + + def test_rejects_bad_env(self): + bad = [ + {"1X": "v"}, + {"A-B": "v"}, + {"": "v"}, + {"FOO": 1}, + {"FOO": None}, + {"FOO": "a\nb"}, + {"LD_LIBRARY_PATH": "/x"}, + ["FOO=bar"], + "FOO=bar", + None, + ] + for env in bad: + with self.subTest(env=env), self.assertRaises(DeployError): + self.load(self.valid(env=env)) + + def test_bad_env_message(self): + with self.assertRaisesRegex( + DeployError, "env must map variable names to single-line string values" + ): + self.load(self.valid(env={"A-B": "v"})) + + def test_accepts_config_with_only_args(self): + config = {"args": [STREAM]} + self.assertEqual(self.load(config), config) + + def test_args_are_still_required_in_direct_mode(self): + with self.assertRaisesRegex(DeployError, "add-stream"): + self.load({}) + + def test_rejects_relative_token_file(self): + with self.assertRaisesRegex(DeployError, "absolute"): + self.load(self.valid(token_file="join.token")) + + def test_token_source_is_resolved_against_the_config_directory(self): + (self.tmp / "join.token").write_text("tok\n") + config = self.load(self.valid(token_source="join.token")) + self.assertEqual( + config["token_source"], str((self.tmp / "join.token").resolve()) + ) + + def test_token_source_must_be_an_existing_nonempty_file(self): + (self.tmp / "empty.token").write_text("\n") + for source, message in ( + ("missing.token", "is not a file"), + ("empty.token", "is empty"), + ("", "non-empty string"), + ): + with ( + self.subTest(source=source), + self.assertRaisesRegex(DeployError, message), + ): + self.load(self.valid(token_source=source)) + + def test_token_source_belongs_to_livekit_mode(self): + (self.tmp / "join.token").write_text("tok") + with self.assertRaisesRegex(DeployError, "belong to"): + self.load( + {"mode": "direct", "token_source": "join.token", "args": [STREAM]} + ) + + def test_rejects_missing_add_stream(self): + for args in ([], ["--other=1"]): + with ( + self.subTest(args=args), + self.assertRaisesRegex(DeployError, "add-stream"), + ): + self.load(self.valid(args=args)) + + def test_rejects_url_and_token_flags_in_args(self): + for flag in ("--livekit-url=wss://x", "--livekit-token-file=/t"): + with ( + self.subTest(flag=flag), + self.assertRaisesRegex(DeployError, "not in args"), + ): + self.load(self.valid(args=[STREAM, flag])) + + def test_rejects_non_string_url_and_token_file(self): + for key in ("livekit_url", "token_file"): + for value in (5, None, ["/t"], ""): + with ( + self.subTest(key=key, value=value), + self.assertRaisesRegex(DeployError, "non-empty strings"), + ): + self.load(self.valid(**{key: value})) + + def test_unreadable_or_invalid_json_is_a_deploy_error(self): + with self.assertRaisesRegex(DeployError, "cannot read the config"): + deploy_orin.load_config(self.tmp / "missing.json") + bad = self.tmp / "bad.json" + bad.write_text("{not json") + with self.assertRaisesRegex(DeployError, "cannot read the config"): + deploy_orin.load_config(bad) + + def test_rejects_non_object_and_bad_args(self): + path = self.tmp / "list.json" + path.write_text("[]") + with self.assertRaises(DeployError): + deploy_orin.load_config(path) + with self.assertRaisesRegex(DeployError, "list of strings"): + self.load(self.valid(args=[STREAM, 3])) + + +class UnitRenderTests(unittest.TestCase): + def test_quote_escapes_systemd_specials(self): + self.assertEqual(deploy_orin.systemd_quote("a%b"), "a%%b") + self.assertEqual(deploy_orin.systemd_quote("$HOME"), "$$HOME") + self.assertEqual(deploy_orin.systemd_quote("a\\b"), "a\\\\b") + + def test_quote_wraps_whitespace_and_quotes(self): + self.assertEqual(deploy_orin.systemd_quote("a b"), '"a b"') + self.assertEqual(deploy_orin.systemd_quote('a"b'), '"a\\"b"') + self.assertEqual(deploy_orin.systemd_quote("plain"), "plain") + + def test_quote_wraps_empty_and_single_quote_args(self): + self.assertEqual(deploy_orin.systemd_quote(""), '""') + self.assertEqual(deploy_orin.systemd_quote("a'b"), '"a\'b"') + + def test_render_unit(self): + config = { + "mode": "livekit", + "livekit_url": "wss://lk.example.org", + "token_file": "/etc/t oken", + "args": [STREAM, "--name=100%", "--x=$Y"], + } + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + exec_start = next(ln for ln in text.splitlines() if ln.startswith("ExecStart=")) + self.assertEqual( + exec_start, + "ExecStart=/opt/ic/current/plugins/sensing_camera/camera_plugin_sensing " + "--livekit-url=wss://lk.example.org " + '"--livekit-token-file=/etc/t oken" ' + f"{STREAM} --name=100%% --x=$$Y", + ) + self.assertIn("User=bob", text) + self.assertIn("SupplementaryGroups=video", text) + self.assertIn("WorkingDirectory=/opt/ic/current/plugins/sensing_camera", text) + self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + + @staticmethod + def _unit_sections(text): + sections = {} + current = None + for line in text.splitlines(): + line = line.strip() + if line.startswith("[") and line.endswith("]"): + current = sections.setdefault(line, []) + elif current is not None and line and not line.startswith("#"): + current.append(line) + return sections + + def test_render_unit_restarts_on_failure_with_start_limit(self): + text = deploy_orin.render_unit({"args": [STREAM]}, "/opt/ic", "bob", "video") + sections = self._unit_sections(text) + unit, service = sections["[Unit]"], sections["[Service]"] + for line in ("Restart=on-failure", "RestartSec=5"): + self.assertEqual(service.count(line), 1, line) + # systemd reads the start limit from [Unit] and ignores it in [Service]. + for line in ("StartLimitIntervalSec=120", "StartLimitBurst=5"): + self.assertEqual(unit.count(line), 1, line) + self.assertNotIn(line, service) + self.assertNotIn("Restart=always", unit + service) + self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + + @unittest.skipUnless(shutil.which("systemd-analyze"), "systemd-analyze not found") + def test_render_unit_passes_systemd_analyze_verify(self): + import getpass + + text = deploy_orin.render_unit( + {"args": [STREAM]}, "/opt/ic", getpass.getuser(), "video" + ) + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) / "isaaccapture-producer.service" + unit.write_text(text) + result = subprocess.run( + ["systemd-analyze", "verify", str(unit)], + capture_output=True, + text=True, + check=False, + ) + output = result.stdout + result.stderr + # verify exits 0 even when it ignores a key, so read the diagnostics. + for diagnostic in ("Unknown key name", "ignoring"): + self.assertNotIn(diagnostic, output) + + def test_render_unit_env_lines_follow_ld_library_path_in_order(self): + config = {"args": [STREAM], "env": {"ZED": "1", "ALPHA": "two"}} + lines = deploy_orin.render_unit(config, "/opt/ic", "bob", "video").splitlines() + i = next( + n + for n, ln in enumerate(lines) + if ln.startswith("Environment=LD_LIBRARY_PATH=") + ) + self.assertEqual( + lines[i + 1 : i + 3], ["Environment=ZED=1", "Environment=ALPHA=two"] + ) + self.assertTrue(lines[i + 3].startswith("ExecStart=")) + + def test_render_unit_without_env_adds_no_environment_lines(self): + for config in ({"args": [STREAM]}, {"args": [STREAM], "env": {}}): + with self.subTest(config=config): + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + env_lines = [ + ln for ln in text.splitlines() if ln.startswith("Environment=") + ] + self.assertEqual(len(env_lines), 1) + self.assertTrue(env_lines[0].startswith("Environment=LD_LIBRARY_PATH=")) + self.assertNotIn("@ENVIRONMENT@", text) + self.assertIsNone(re.search(r"@[A-Z_]+@", text)) + + def test_render_unit_env_values_are_quoted(self): + env = {"A": "x y", "B": 'q"q', "C": "100%", "D": "$HOME", "E": ""} + config = {"args": [STREAM], "env": env} + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + lines = text.splitlines() + for name, value in env.items(): + self.assertIn( + f"Environment={deploy_orin.systemd_quote(f'{name}={value}')}", lines + ) + self.assertIn('Environment="A=x y"', lines) + self.assertIn('Environment="B=q\\"q"', lines) + self.assertIn("Environment=C=100%%", lines) + self.assertIn("Environment=D=$$HOME", lines) + + def test_render_unit_direct_mode_has_no_livekit_args(self): + for config in ({"args": [STREAM]}, {"mode": "direct", "args": [STREAM]}): + with self.subTest(config=config): + self.assert_direct_unit(config) + + def assert_direct_unit(self, config): + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + exec_start = next(ln for ln in text.splitlines() if ln.startswith("ExecStart=")) + self.assertEqual( + exec_start, + "ExecStart=/opt/ic/current/plugins/sensing_camera/camera_plugin_sensing " + + STREAM, + ) + self.assertNotIn("--livekit", text) + + +def make_target(test, password=None): + """A Target with its temp dir removed at cleanup; ORIN_PASSWORD is set or cleared explicitly.""" + env = {k: v for k, v in os.environ.items() if k != deploy_orin.PASSWORD_ENV} + if password: + env[deploy_orin.PASSWORD_ENV] = password + with mock.patch.dict(os.environ, env, clear=True): + target = deploy_orin.Target("u@orin") + test.addCleanup(shutil.rmtree, target._dir, True) + return target + + +class ValidateConnectionTests(unittest.TestCase): + def validate(self, result): + target = make_target(self) + with mock.patch.object(deploy_orin.subprocess, "run", return_value=result): + target.validate_connection() + + def test_ssh_failure_surfaces_stderr(self): + with self.assertRaisesRegex(DeployError, "Permission denied"): + self.validate(completed(255, stderr="Permission denied (publickey)\n")) + + def test_missing_python3(self): + with self.assertRaisesRegex(DeployError, "python3 is not installed"): + self.validate(completed(1)) + + def test_non_aarch64(self): + with self.assertRaisesRegex(DeployError, "x86_64"): + self.validate(completed(0, stdout="x86_64\n")) + + def test_success(self): + self.validate(completed(0, stdout="aarch64\n")) + + +class PasswordTests(unittest.TestCase): + PASSWORD = "s3cr3t-pw" + + def test_password_never_in_argv(self): + target = make_target(self, self.PASSWORD) + calls = [] + + def fake_run(cmd, **kwargs): + calls.append((list(cmd), kwargs)) + return completed(0, stdout="aarch64\n") + + with mock.patch.object(deploy_orin.subprocess, "run", fake_run): + target.validate_connection() + target.capture("id -un") + target.readable("/t") + target.ssh("true") + target.sudo("python3 x remote activate --owner bob") + target.put(Path("/tmp/a"), "/r/a") + askpass_text = Path(target._env["SSH_ASKPASS"]).read_text() + target.close() + self.assertGreater(len(calls), 6) + for cmd, _ in calls: + self.assertFalse([w for w in cmd if self.PASSWORD in w], cmd) + _, sudo_kwargs = next( + (c, k) for c, k in calls if "sudo -S -p '' python3" in c[-1] + ) + self.assertEqual(sudo_kwargs["input"], self.PASSWORD + "\n") + self.assertNotIn(self.PASSWORD, askpass_text) + + def test_without_password_sudo_uses_tty_and_no_askpass(self): + target = make_target(self) + with mock.patch.object( + deploy_orin.subprocess, "run", return_value=completed(0) + ) as run: + target.sudo("true") + self.assertIn("-t", run.call_args.args[0]) + self.assertNotIn("SSH_ASKPASS", target._env) + self.assertNotIn("input", run.call_args.kwargs) + + def test_sudo_failure_raises(self): + target = make_target(self, self.PASSWORD) + with mock.patch.object( + deploy_orin.subprocess, "run", return_value=completed(1) + ): + with self.assertRaisesRegex(DeployError, "sudo"): + target.sudo("true") + + +CACHE_DIR = "/home/u/.cache/isaaccapture-deploy" + + +def git_repo(test, files): + """A temp git repo holding the given {path: text} tracked files; returns (root, git runner).""" + root = Path(tempfile.mkdtemp(dir=test.tmp)) + + def git(*args): + return subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@example.org", + "-c", "commit.gpgsign=false", *args], + cwd=root, capture_output=True, text=True, check=True, + ).stdout.strip() # fmt: skip + + git("init", "-q") + for name, text in files.items(): + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_text(text) + git("add", "-A") + git("commit", "-q", "-m", "init") + return root, git + + +class FakeOrin: + """A mock Target that records uploads and keeps the source manifest an `apply` would write.""" + + def __init__(self, manifest=None): + self.manifest = manifest + self.events = [] + self.deltas = [] + self.pending = None + self.target = mock.Mock() + self.target.host = "u@orin" + self.target.readable.return_value = True + self.target.capture.side_effect = self.capture + self.target.put.side_effect = self.put + self.target.ssh.side_effect = self.ssh + self.target.status.side_effect = self.status + self.target.password = None + self.target.output.side_effect = self.output + self.detected = None + self.status_calls = [] + self.preflight_codes = [] + self.rig_codes = [] + self.setup_codes = [] + + def capture(self, cmd): + if "pwd" in cmd: + return CACHE_DIR + if "src.manifest.json" in cmd: + return json.dumps(self.manifest) if self.manifest is not None else "" + return "u" + + def put(self, local, remote): + self.events.append(("put", remote)) + if remote.endswith("delta.tar.gz"): + with tarfile.open(local) as tar: + names = tar.getnames() + self.pending = json.loads( + tar.extractfile(f"{deploy_orin.DEPLOY_DIR}/manifest.json").read() + ) + self.deltas.append(sorted(n for n in names if not n.startswith(".deploy/"))) + + def status(self, cmd, tty=False, timeout=0, stdin=None): + self.events.append(("status", cmd)) + self.status_calls.append((cmd, tty, stdin)) + if " preflight " in cmd: + queue = self.preflight_codes + elif "--no-apt" in cmd: # the read-only rig check + queue = self.rig_codes + elif "orin_setup.sh" in cmd: + queue = self.setup_codes + else: + queue = [] + return queue.pop(0) if queue else 0 + + def output(self, cmd, stdin=None): + self.events.append(("output", cmd)) + if self.detected is None: + return 1, "" + return 0, "noise\n" + json.dumps(self.detected) + "\n" + + def ssh(self, cmd, tty=False): + self.events.append(("ssh", cmd)) + if " apply " in cmd: + self.manifest = self.pending + + def run(self, root, *extra, config=None): + args = deploy_orin.parser().parse_args( + [ + "u@orin", + "--config", + "orin.json", + "--no-activate", + *extra, + ] + ) + config = config or { + "mode": "livekit", + "livekit_url": "wss://x", + "token_file": "/t", + "args": [STREAM], + } + with mock.patch.object(deploy_orin, "repo_root", return_value=root): + deploy_orin.run_deploy(args, config, self.target) + + +class TokenSourceDeployTests(TempDirTestCase): + def deploy(self, **extra): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + token = self.tmp / "join.token" + token.write_text("SECRET-TOKEN-VALUE") + orin = FakeOrin() + config = { + "mode": "livekit", + "livekit_url": "ws://x", + "token_file": "/home/u/lk/join.token", + "token_source": str(token), + "args": [STREAM], + } + return orin, root, config + + def test_token_is_uploaded_private_and_moved_into_place_before_the_readable_check( + self, + ): + orin, root, config = self.deploy() + orin.run(root, config=config) + puts = [v for k, v in orin.events if k == "put" and v.startswith("/home/u/lk/")] + self.assertEqual(puts, ["/home/u/lk/join.token.new"]) + commands = [c.args[0] for c in orin.target.capture.call_args_list] + make = next(c for c in commands if "umask 077" in c) + move = next(c for c in commands if "mv -f" in c) + self.assertIn("mkdir -p /home/u/lk", make) + self.assertIn("chmod 600 /home/u/lk/join.token.new", move) + self.assertIn("/home/u/lk/join.token", move) + self.assertLess(commands.index(make), commands.index(move)) + + def test_the_token_value_never_appears_in_any_command(self): + orin, root, config = self.deploy() + orin.run(root, config=config) + everything = [str(c) for c in orin.target.mock_calls] + [ + str(e) for e in orin.events + ] + self.assertFalse([e for e in everything if "SECRET-TOKEN-VALUE" in e]) + + def test_a_different_service_user_is_refused_before_anything_is_uploaded(self): + orin, root, config = self.deploy() + with self.assertRaisesRegex(DeployError, "service-user"): + orin.run(root, "--service-user", "other", config=config) + self.assertEqual( + [e for e in orin.events if e[0] == "put" and "join.token" in e[1]], [] + ) + + def test_no_token_source_uploads_nothing(self): + orin, root, config = self.deploy() + del config["token_source"] + orin.run(root, config=config) + self.assertEqual( + [e for e in orin.events if e[0] == "put" and "join.token" in e[1]], [] + ) + + +class DeployOrderTests(TempDirTestCase): + def test_failed_validation_blocks_everything_else(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.target.validate_connection.side_effect = DeployError("no route") + with self.assertRaises(DeployError): + orin.run(root) + self.assertEqual(orin.target.mock_calls[0][0], "validate_connection") + self.assertEqual(len(orin.target.mock_calls), 1) + + def test_helper_uploaded_and_preflight_run_before_any_delta(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n", "README.md": "r\n"}) + orin = FakeOrin() + orin.run(root) + kinds = [] + for kind, value in orin.events: + if kind == "put" and value.endswith("/deploy_orin.py"): + kinds.append("helper") + elif kind == "status" and " preflight " in value: + kinds.append("preflight") + elif kind == "put" and value.endswith("delta.tar.gz"): + kinds.append("delta") + self.assertEqual(kinds, ["helper", "preflight", "delta"]) + apply_cmd = next(v for k, v in orin.events if k == "ssh" and " apply " in v) + self.assertIn("--fresh", apply_cmd) + + def test_setup_and_camera_check_scripts_are_uploaded_before_preflight(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.run(root) + wanted = [f"{CACHE_DIR}/tools/{rel}" for _, rel in deploy_orin.tool_files()] + uploads = [ + i for i, (k, v) in enumerate(orin.events) if k == "put" and v in wanted + ] + self.assertEqual(len(uploads), len(wanted)) + preflight = next( + i + for i, (k, v) in enumerate(orin.events) + if k == "status" and " preflight " in v + ) + self.assertTrue(all(i < preflight for i in uploads)) + names = {rel for _, rel in deploy_orin.tool_files()} + self.assertTrue( + { + "scripts/orin_setup.sh", + "scripts/setup.sh", + "scripts/validate.sh", + "scripts/verify.sh", + } + <= names + ) + + def tools_run(self, *flags): + args = deploy_orin.parser().parse_args(["u@orin", *flags]) + orin = FakeOrin() + deploy_orin.run_tools(args, orin.target) + return orin + + def tools_command(self, *flags): + orin = self.tools_run(*flags) + (call,) = [c for c in orin.status_calls if "orin_setup.sh" in c[0]] + return orin, call + + def test_install_prereqs_runs_only_the_prerequisite_half_in_a_terminal(self): + orin, call = self.tools_command("--install-prereqs") + self.assertTrue(call[0].endswith("orin_setup.sh --prereqs-only")) + self.assertTrue(call[1]) + self.assertFalse(any("deploy_orin.py" in v for _, v in orin.events)) + + def test_check_camera_runs_only_the_camera_half(self): + _, call = self.tools_command("--check-camera") + self.assertTrue(call[0].endswith("orin_setup.sh --check-only")) + + def test_both_flags_run_the_whole_script(self): + _, call = self.tools_command("--install-prereqs", "--check-camera") + self.assertTrue(call[0].endswith("orin_setup.sh")) + + def test_scan_cameras_runs_the_camera_half_with_the_scan(self): + _, call = self.tools_command("--scan-cameras") + self.assertTrue(call[0].endswith("orin_setup.sh --check-only --scan")) + + def test_install_and_scan_run_everything_with_the_scan(self): + _, call = self.tools_command("--install-prereqs", "--scan-cameras") + self.assertTrue(call[0].endswith("orin_setup.sh --scan")) + + def test_tools_modes_upload_the_scripts_first_and_need_no_config(self): + events = self.tools_run("--check-camera").events + self.assertEqual(events[-1][0], "status") + self.assertTrue(all(k == "put" for k, _ in events[:-1])) + + def test_a_failing_setup_script_is_a_deploy_error(self): + args = deploy_orin.parser().parse_args(["u@orin", "--scan-cameras"]) + orin = FakeOrin() + orin.setup_codes = [1] + with self.assertRaisesRegex(DeployError, "exit 1"): + deploy_orin.run_tools(args, orin.target) + + def test_tools_run_answers_sudo_from_the_environment_password(self): + args = deploy_orin.parser().parse_args(["u@orin", "--scan-cameras"]) + orin = FakeOrin() + orin.target.password = "s3cret" + deploy_orin.run_tools(args, orin.target) + (call,) = [c for c in orin.status_calls if "orin_setup.sh" in c[0]] + self.assertTrue(call[0].endswith("--sudo-password-stdin")) + self.assertEqual(call[2], "s3cret\n") + self.assertNotIn("s3cret", call[0]) + + def test_logs_only_reads_the_service_and_changes_nothing(self): + orin = self.tools_run("--logs") + commands = [c.args[0] for c in orin.target.sudo.call_args_list] + self.assertEqual(len(commands), 2) + self.assertIn(deploy_orin.SERVICE, commands[0]) + self.assertIn("journalctl", commands[1]) + self.assertEqual(orin.events, []) + + def test_deploying_without_a_config_is_an_error(self): + args = deploy_orin.parser().parse_args(["u@orin"]) + with self.assertRaisesRegex(DeployError, "--config is required"): + deploy_orin.deploy(args) + + def setup_events(self, orin): + return [ + v + for k, v in orin.events + if k == "status" and "orin_setup.sh" in v and "--no-apt" not in v + ] + + def test_passing_preflight_installs_nothing(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.run(root) + self.assertEqual(self.setup_events(orin), []) + + def test_failing_preflight_installs_prerequisites_then_rechecks(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=None): + orin.run(root) + (setup,) = self.setup_events(orin) + self.assertTrue(setup.endswith("orin_setup.sh --prereqs-only")) + kinds = [ + "preflight" if " preflight " in v else "setup" + for k, v in orin.events + if k == "status" + and (" preflight " in v or ("orin_setup.sh" in v and "--no-apt" not in v)) + ] + self.assertEqual(kinds, ["preflight", "setup", "preflight"]) + + def test_missing_drivers_or_overlay_alone_trigger_the_install(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.rig_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=None): + orin.run(root) + self.assertEqual(len(self.setup_events(orin)), 1) + + def test_install_without_a_password_runs_in_a_terminal(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=None): + orin.run(root) + (call,) = [ + c + for c in orin.status_calls + if "orin_setup.sh" in c[0] and "--no-apt" not in c[0] + ] + self.assertTrue(call[1]) + self.assertIsNone(call[2]) + self.assertNotIn("--sudo-password-stdin", call[0]) + + def test_install_with_a_password_sends_it_on_stdin_and_never_in_the_command(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.target.password = "s3cret" + orin.preflight_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=None): + orin.run(root) + (call,) = [ + c + for c in orin.status_calls + if "orin_setup.sh" in c[0] and "--no-apt" not in c[0] + ] + self.assertTrue(call[0].endswith("--sudo-password-stdin")) + self.assertEqual(call[2], "s3cret\n") + self.assertFalse(call[1]) + self.assertTrue(all("s3cret" not in c[0] for c in orin.status_calls)) + + def test_sipl_archive_is_uploaded_and_named_to_the_setup_script(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + archive = self.tmp / "Jetson_SIPL_API_R39.2.1_aarch64.tbz2" + archive.write_bytes(b"x") + orin = FakeOrin() + orin.preflight_codes = [1, 0] + with mock.patch.object(deploy_orin, "find_sipl_package", return_value=archive): + orin.run(root) + self.assertIn(("put", f"{CACHE_DIR}/{archive.name}"), orin.events) + (setup,) = self.setup_events(orin) + self.assertIn(f"--sipl-api {CACHE_DIR}/{archive.name}", setup) + + def test_no_setup_refuses_instead_of_installing(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1] + with self.assertRaisesRegex(DeployError, "fails the preflight"): + orin.run(root, "--no-setup") + self.assertEqual(self.setup_events(orin), []) + + def test_reboot_request_waits_then_reruns_setup(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1, 0] + orin.setup_codes = [deploy_orin.NEEDS_REBOOT, 0] + with ( + mock.patch.object(deploy_orin, "find_sipl_package", return_value=None), + mock.patch.object(deploy_orin, "wait_for_reboot") as wait, + ): + orin.run(root) + wait.assert_called_once_with(orin.target) + self.assertEqual(len(self.setup_events(orin)), 2) + + def test_dropped_ssh_during_setup_is_treated_as_a_reboot(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1, 0] + orin.setup_codes = [255, 0] + with ( + mock.patch.object(deploy_orin, "find_sipl_package", return_value=None), + mock.patch.object(deploy_orin, "wait_for_reboot") as wait, + ): + orin.run(root) + wait.assert_called_once() + + def test_setup_failure_stops_the_deploy(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.preflight_codes = [1] + orin.setup_codes = [1] + with ( + mock.patch.object(deploy_orin, "find_sipl_package", return_value=None), + self.assertRaisesRegex(DeployError, "prerequisite setup did not finish"), + ): + orin.run(root) + self.assertFalse( + any(k == "put" and v.endswith("delta.tar.gz") for k, v in orin.events) + ) + + def test_unchanged_second_run_sends_nothing_and_one_change_sends_only_it(self): + root, _ = git_repo( + self, {"src/a.cpp": "a\n", "src/b.cpp": "b\n", "README.md": "r\n"} + ) + orin = FakeOrin() + orin.run(root) + self.assertEqual(orin.deltas, [["README.md", "src/a.cpp", "src/b.cpp"]]) + + orin.run(root) + self.assertEqual(len(orin.deltas), 1) + + (root / "src/b.cpp").write_text("changed\n") + orin.run(root) + self.assertEqual(orin.deltas[1:], [["src/b.cpp"]]) + applies = [v for k, v in orin.events if k == "ssh" and " apply " in v] + self.assertIn("--fresh", applies[0]) + self.assertNotIn("--fresh", applies[-1]) + + def test_same_src_path_for_different_edits(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.run(root) + (root / "src/a.cpp").write_text("z\n") + orin.run(root) + srcs = { + m.group(1) + for k, v in orin.events + if k == "ssh" and (m := re.search(r"--src (\S+)", v)) + } + self.assertEqual(srcs, {f"{CACHE_DIR}/src"}) + + def test_cmake_args_are_forwarded_to_the_remote_build_in_order(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.run(root, "--cmake-arg=-DA=1", "--cmake-arg=-DB=a b") + build = next(v for k, v in orin.events if k == "ssh" and " build " in v) + self.assertTrue(build.endswith(" --cmake-arg=-DA=1 --cmake-arg='-DB=a b'")) + + def activate_command(self, config): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + args = deploy_orin.parser().parse_args(["u@orin", "--config", "orin.json"]) + with mock.patch.object(deploy_orin, "repo_root", return_value=root): + deploy_orin.run_deploy(args, config, orin.target) + (activate,), _ = orin.target.sudo.call_args + return activate + + def activate_options(self, config): + activate = self.activate_command(config) + return activate[activate.index(" --sensors=") :] + + def test_activate_command_direct_two_sensors(self): + config = { + "args": [ + "--add-stream=sensor=0,direct=10.0.0.1:5000", + "--add-stream=sensor=1,direct=10.0.0.1:5001", + ] + } + self.assertEqual( + self.activate_options(config), + " --sensors=0,1 --direct-sensors=0,1 --mode=direct", + ) + + def test_activate_command_only_streams_with_a_direct_key_are_direct_sensors(self): + config = { + "args": [ + "--add-stream=sensor=0,direct=10.0.0.1:5000", + "--add-stream=sensor=1,livekit=right", + ] + } + self.assertEqual( + self.activate_options(config), + " --sensors=0,1 --direct-sensors=0 --mode=direct", + ) + + def test_activate_command_with_no_direct_stream_has_an_empty_direct_list(self): + self.assertEqual( + self.activate_options({"args": [STREAM]}), + " --sensors=0 --direct-sensors= --mode=direct", + ) + + def test_activate_command_livekit_mode(self): + config = { + "mode": "livekit", + "livekit_url": "wss://x", + "token_file": "/t", + "args": [STREAM], + } + activate = self.activate_command(config) + self.assertIn(" --sensors=0 --direct-sensors= --mode=livekit", activate) + self.assertNotIn("--no-livekit", activate) + + def test_activate_command_line_uses_the_module_constants(self): + activate = self.activate_command({"args": [STREAM]}) + self.assertIn(" activate ", activate) + self.assertIn("--prefix /opt/isaaccapture", activate) + self.assertIn("--ready-timeout 60", activate) + self.assertNotIn("--no-livekit", activate) + self.assertEqual( + ( + deploy_orin.PREFIX, + deploy_orin.SERVICE_GROUPS, + deploy_orin.READY_TIMEOUT_S, + ), + ("/opt/isaaccapture", "video", 60), + ) + + def test_unit_is_rendered_with_prefix_and_service_groups(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + with mock.patch.object( + deploy_orin, "render_unit", return_value="unit\n" + ) as render: + orin.run(root) + _, prefix, _, groups = render.call_args.args + self.assertEqual( + (prefix, groups), (deploy_orin.PREFIX, deploy_orin.SERVICE_GROUPS) + ) + + def test_deploy_closes_target_on_failure(self): + config = self.tmp / "orin.json" + config.write_text( + json.dumps( + { + "mode": "livekit", + "livekit_url": "wss://x", + "token_file": "/t", + "args": [STREAM], + } + ) + ) + args = deploy_orin.parser().parse_args(["u@orin", "--config", str(config)]) + target = mock.Mock() + with ( + mock.patch.object(deploy_orin, "Target", return_value=target), + mock.patch.object(deploy_orin, "run_deploy", side_effect=DeployError("x")), + ): + with self.assertRaises(DeployError): + deploy_orin.deploy(args) + target.close.assert_called_once() + + +class OpenSourceTests(TempDirTestCase): + def args(self, *extra): + return deploy_orin.parser().parse_args(["u@orin", "--config", "c", *extra]) + + def repo_with_branch(self): + root, git = git_repo( + self, {"src/a.cpp": "committed\n", "run.sh": "#!/bin/sh\n"} + ) + (root / "run.sh").chmod(0o755) + git("add", "run.sh") + git("commit", "-q", "-m", "exec") + git("branch", "feature") + (root / "src/a.cpp").write_text("edited later\n") + (root / "untracked.txt").write_text("x") + return root, git + + def test_cli_parses_source_and_branch(self): + args = self.args("--source", "/w", "--branch", "feat") + self.assertEqual((args.source, args.branch), ("/w", "feat")) + args = self.args() + self.assertFalse(args.source) + self.assertFalse(args.branch) + + def test_default_yields_script_checkout_with_local_edits(self): + root, _ = git_repo(self, {"a.txt": "1\n", "b.txt": "2\n"}) + (root / "a.txt").write_text("edited\n") + (root / "untracked.txt").write_text("x") + with mock.patch.object(deploy_orin, "repo_root", return_value=root): + with deploy_orin.open_source(self.args()) as (src, files, state): + self.assertEqual(src, root) + self.assertEqual(files, ["a.txt", "b.txt"]) + self.assertTrue(state["dirty"]) + + def test_source_option_uses_another_checkout(self): + root, git = git_repo(self, {"a.txt": "1\n"}) + sub = root / "sub" + sub.mkdir() + with mock.patch.object( + deploy_orin, "repo_root", side_effect=AssertionError("own checkout used") + ): + with deploy_orin.open_source(self.args("--source", str(sub))) as ( + src, + files, + state, + ): + self.assertEqual(src.resolve(), root.resolve()) + self.assertEqual(files, ["a.txt"]) + self.assertEqual(state["base_revision"], git("rev-parse", "HEAD")) + self.assertFalse(state["dirty"]) + + def test_branch_yields_committed_content_only(self): + root, git = self.repo_with_branch() + with deploy_orin.open_source( + self.args("--source", str(root), "--branch", "feature") + ) as (export, files, state): + self.assertEqual(files, ["run.sh", "src/a.cpp"]) + self.assertEqual((export / "src/a.cpp").read_text(), "committed\n") + self.assertFalse((export / "untracked.txt").exists()) + self.assertTrue(os.access(export / "run.sh", os.X_OK)) + self.assertFalse(os.access(export / "src/a.cpp", os.X_OK)) + self.assertEqual( + state, {"base_revision": git("rev-parse", "feature"), "dirty": False} + ) + kept = export + self.assertFalse(kept.exists()) + + def test_unknown_branch_and_non_git_source_raise(self): + root, _ = git_repo(self, {"a.txt": "1\n"}) + with self.assertRaisesRegex(DeployError, "not found"): + with deploy_orin.open_source( + self.args("--source", str(root), "--branch", "nope") + ): + pass + plain = self.tmp / "plain" + plain.mkdir() + with self.assertRaisesRegex(DeployError, "not inside a git"): + with deploy_orin.open_source(self.args("--source", str(plain))): + pass + + def test_run_deploy_with_branch_uploads_the_branch_files(self): + root, _ = self.repo_with_branch() + orin = FakeOrin() + orin.run(root, "--source", str(root), "--branch", "feature") + self.assertEqual(orin.deltas, [["run.sh", "src/a.cpp"]]) + self.assertEqual( + orin.pending["src/a.cpp"], hashlib.sha256(b"committed\n").hexdigest() + "-" + ) + + +class SyncTests(TempDirTestCase): + def test_tracked_files_include_edits_not_untracked(self): + root, _ = git_repo(self, {"a.txt": "1\n", "sub/b.txt": "2\n"}) + (root / "a.txt").write_text("edited\n") + (root / "untracked.txt").write_text("x") + self.assertEqual(deploy_orin.tracked_files(root), ["a.txt", "sub/b.txt"]) + + def test_tracked_files_skip_deleted(self): + root, _ = git_repo(self, {"a.txt": "1\n", "b.txt": "2\n"}) + (root / "b.txt").unlink() + self.assertEqual(deploy_orin.tracked_files(root), ["a.txt"]) + + def test_hash_files_records_exec_bit(self): + root, _ = git_repo(self, {"a.txt": "1\n", "run.sh": "#!/bin/sh\n"}) + (root / "run.sh").chmod(0o755) + entries = deploy_orin.hash_files(root, ["a.txt", "run.sh"]) + self.assertEqual( + entries["a.txt"], deploy_orin.sha256_file(root / "a.txt") + "-" + ) + self.assertEqual( + entries["run.sh"], deploy_orin.sha256_file(root / "run.sh") + "x" + ) + + def test_diff_identical_sends_nothing(self): + entries = {"src/a": "1-", "README.md": "2-"} + self.assertEqual(deploy_orin.diff_manifests(entries, dict(entries)), ([], [])) + + def test_diff_changed_new_and_removed(self): + local = {"src/a": "1-", "src/new": "n-", "src/same": "s-", "README.md": "r2-", "doc": "d-"} # fmt: skip + remote = {"src/a": "0-", "src/same": "s-", "src/gone": "g-", "README.md": "r1-", "doc": "d-", "old.txt": "o-"} # fmt: skip + send, removed = deploy_orin.diff_manifests(local, remote) + self.assertEqual(send, ["README.md", "src/a", "src/new"]) + self.assertEqual(removed, ["old.txt", "src/gone"]) + + def test_diff_is_per_file_whatever_the_path(self): + local = {"src/a": "1x", "src/b": "2-", "deps/x": "3-", "z": "4-"} + remote = {"src/a": "1-", "src/b": "2-", "deps/x": "9-", "gone/y": "5-"} + self.assertEqual( + deploy_orin.diff_manifests(local, remote), + (["deps/x", "src/a", "z"], ["gone/y"]), + ) + + def test_grouping_helpers_are_gone(self): + for name in ("GROUPS", "VOLATILE_PREFIXES", "file_group", "group_digest"): + self.assertFalse(hasattr(deploy_orin, name), name) + + def test_diff_against_empty_remote_sends_everything(self): + local = {"src/a": "1-", "README.md": "2-"} + self.assertEqual( + deploy_orin.diff_manifests(local, {}), (["README.md", "src/a"], []) + ) + + def test_build_delta_has_only_sent_files_and_deploy_members(self): + root, _ = git_repo(self, {"src/a": "a\n", "src/b": "b\n", "README.md": "r\n"}) + manifest = deploy_orin.hash_files(root, deploy_orin.tracked_files(root)) + dest = self.tmp / "delta.tar.gz" + deploy_orin.build_delta(root, ["src/b"], ["old/x"], manifest, dest) + with tarfile.open(dest) as tar: + self.assertEqual( + sorted(tar.getnames()), + [".deploy/manifest.json", ".deploy/removed.json", "src/b"], + ) + self.assertEqual( + json.loads(tar.extractfile(".deploy/manifest.json").read()), manifest + ) + self.assertEqual( + json.loads(tar.extractfile(".deploy/removed.json").read()), ["old/x"] + ) + + def test_read_remote_manifest_tolerates_missing_or_garbage(self): + def read(text): + target = mock.Mock() + target.capture.return_value = text + return deploy_orin.read_remote_manifest(target, "/c") + + self.assertEqual(read('{"a": "h-"}'), {"a": "h-"}) + for text in ("", "{oops", "[1]"): + self.assertEqual(read(text), {}) + + +class ApplyTests(TempDirTestCase): + def setUp(self): + super().setUp() + self.src = self.tmp / "orin" / "src" + self.manifest_path = self.tmp / "orin" / "src.manifest.json" + self.manifest_path.parent.mkdir() + + def delta(self, files, removed=(), manifest=None): + root = self.tmp / "local" + root.mkdir(exist_ok=True) + for name, text in files.items(): + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_text(text) + if manifest is None: + manifest = deploy_orin.hash_files(root, sorted(files)) + dest = self.tmp / "delta.tar.gz" + deploy_orin.build_delta(root, sorted(files), list(removed), manifest, dest) + return dest + + def apply(self, archive, fresh=False): + deploy_orin.remote_apply( + argparse.Namespace( + archive=str(archive), + src=str(self.src), + manifest=str(self.manifest_path), + fresh=fresh, + ) + ) + + def test_extracts_and_writes_manifest(self): + self.apply(self.delta({"src/a.cpp": "a\n", "README.md": "r\n"})) + self.assertEqual((self.src / "src/a.cpp").read_text(), "a\n") + self.assertFalse((self.src / ".deploy").exists()) + manifest = json.loads(self.manifest_path.read_text()) + self.assertEqual(sorted(manifest), ["README.md", "src/a.cpp"]) + + def test_removes_files_and_prunes_empty_dirs(self): + (self.src / "gone/deep").mkdir(parents=True) + (self.src / "gone/deep/x").write_text("x") + (self.src / "keep").mkdir() + (self.src / "keep/a").write_text("a") + (self.src / "keep/b").write_text("b") + self.apply(self.delta({"new": "n\n"}, removed=["gone/deep/x", "keep/b"])) + self.assertFalse((self.src / "gone").exists()) + self.assertTrue((self.src / "keep/a").exists()) + self.assertFalse((self.src / "keep/b").exists()) + self.assertTrue(self.src.is_dir()) + + def test_hash_mismatch_raises_and_leaves_manifest_unwritten(self): + bad = {"a": "0" * 64 + "-"} + with self.assertRaisesRegex(DeployError, "does not match"): + self.apply(self.delta({"a": "a\n"}, manifest=bad)) + self.assertFalse(self.manifest_path.exists()) + + def test_fresh_wipes_stale_files(self): + self.src.mkdir() + (self.src / "stale").write_text("old") + self.apply(self.delta({"a": "a\n"}), fresh=True) + self.assertFalse((self.src / "stale").exists()) + self.assertTrue((self.src / "a").exists()) + + def test_without_fresh_keeps_untouched_files(self): + self.src.mkdir() + (self.src / "stale").write_text("old") + self.apply(self.delta({"a": "a\n"})) + self.assertTrue((self.src / "stale").exists()) + + def test_refuses_removal_outside_src(self): + victim = self.tmp / "orin" / "victim" + victim.write_text("keep") + for name in ("../victim", "/etc/hostname"): + with ( + self.subTest(name=name), + self.assertRaisesRegex(DeployError, "outside"), + ): + self.apply(self.delta({"a": "a\n"}, removed=[name])) + self.assertTrue(victim.exists()) + + def test_remote_parser_apply_arguments(self): + parsed = deploy_orin.remote_parser().parse_args( + ["apply", "--archive", "a", "--src", "s", "--manifest", "m", "--fresh"] + ) + self.assertTrue(parsed.fresh) + + def test_remote_parser_activate_accepts_sensors(self): + base = ["activate"] + for name in ("archive", "prefix", "release", "unit", "owner"): + base += [f"--{name}", "x"] + self.assertEqual(deploy_orin.remote_parser().parse_args(base).sensors, "") + parsed = deploy_orin.remote_parser().parse_args(base + ["--sensors", "0,1"]) + self.assertEqual(parsed.sensors, "0,1") + + def test_remote_parser_activate_mode_and_direct_sensors(self): + base = ["activate"] + for name in ("archive", "prefix", "release", "unit", "owner"): + base += [f"--{name}", "x"] + parsed = deploy_orin.remote_parser().parse_args(base) + self.assertEqual((parsed.mode, parsed.direct_sensors), ("direct", "")) + parsed = deploy_orin.remote_parser().parse_args( + base + ["--mode=livekit", "--direct-sensors=0,1"] + ) + self.assertEqual((parsed.mode, parsed.direct_sensors), ("livekit", "0,1")) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + deploy_orin.remote_parser().parse_args(base + ["--mode=both"]) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + deploy_orin.remote_parser().parse_args(base + ["--no-livekit"]) + + +class ActivateTests(TempDirTestCase): + """remote_activate with subprocess faked; the unit path is redirected into the temp dir.""" + + def setUp(self): + super().setUp() + self.prefix = self.tmp / "opt" + self.prefix.mkdir() + self.unit = self.tmp / "in.service" + self.unit.write_text("[Unit]\n") + # Path("/etc/systemd/system") / resolves to the absolute path. + self.service = str(self.tmp / "isaaccapture-producer.service") + self.calls = [] + self.events = [] + self.journal = "" + self.unmanaged = [] + for p in ( + mock.patch.object( + deploy_orin, "unmanaged_producers", lambda: list(self.unmanaged) + ), + mock.patch.object(deploy_orin, "SERVICE", self.service), + mock.patch.object(deploy_orin.subprocess, "run", self.fake_run), + mock.patch.object(deploy_orin.time, "sleep"), + ): + p.start() + self.addCleanup(p.stop) + + def link_target(self): + link = self.prefix / "current" + return os.readlink(link) if link.is_symlink() else None + + def fake_run(self, cmd, **kwargs): + self.calls.append(list(cmd)) + if cmd[:2] == ["systemctl", "stop"]: + self.events.append(("stop", self.link_target())) + elif cmd[:2] == ["systemctl", "start"]: + self.events.append(("start", self.link_target())) + elif cmd[0] == "journalctl": + return completed(0, stdout=self.journal) + return completed(0) + + def make_archive(self, good=True): + stage = self.tmp / "stage" + (stage / "bin").mkdir(parents=True) + (stage / "bin/producer").write_text("binary") + digest = deploy_orin.sha256_file(stage / "bin/producer") if good else "0" * 64 + (stage / "SHA256SUMS").write_text(f"{digest} ./bin/producer\n") + archive = self.tmp / "producer.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(stage, arcname=".") + return archive + + def activate( + self, + archive, + timeout=60, + replace_unmanaged=False, + platform_file="", + sensors="", + mode="livekit", + direct_sensors="", + ): + deploy_orin.remote_activate( + argparse.Namespace( + archive=str(archive), + prefix=str(self.prefix), + release="r1", + unit=str(self.unit), + owner="bob", + ready_timeout=timeout, + sensors=sensors, + mode=mode, + direct_sensors=direct_sensors, + replace_unmanaged=replace_unmanaged, + platform_file=platform_file, + ) + ) + + def test_platform_file_is_installed_under_the_prefix(self): + platform = self.tmp / "platform.json" + platform.write_text("{}") + self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" + self.activate(self.make_archive(), platform_file=str(platform)) + self.assertEqual((self.prefix / deploy_orin.PLATFORM_FILE).read_text(), "{}") + + def test_checksum_mismatch_aborts_before_stop(self): + with self.assertRaisesRegex(DeployError, "checksum mismatch"): + self.activate(self.make_archive(good=False)) + self.assertFalse([c for c in self.calls if c[0] == "systemctl"]) + self.assertIsNone(self.link_target()) + + def test_checksum_mismatch_removes_the_new_release_dir(self): + with self.assertRaisesRegex(DeployError, "checksum mismatch"): + self.activate(self.make_archive(good=False)) + self.assertFalse((self.prefix / "releases" / "r1").exists()) + self.assertFalse([c for c in self.calls if c[0] == "chown"]) + + def test_unmanaged_producer_refuses_before_installing_anything(self): + self.unmanaged = [(4242, "alice"), (4243, "bob")] + with mock.patch.object(deploy_orin.os, "kill") as kill: + with self.assertRaises(DeployError) as ctx: + self.activate(self.make_archive()) + message = str(ctx.exception) + self.assertIn("pid 4242 (user alice)", message) + self.assertIn("pid 4243 (user bob)", message) + self.assertIn("--replace-unmanaged", message) + kill.assert_not_called() + self.assertEqual(self.calls, []) + self.assertFalse((self.prefix / "releases").exists()) + self.assertIsNone(self.link_target()) + self.assertFalse(Path(self.service).exists()) + + def test_replace_unmanaged_interrupts_only_listed_pids_between_stop_and_start( + self, + ): + self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" + self.unmanaged = [(4242, "alice")] + order = [] + self.events = order + + def fake_kill(pid, sig): + order.append(("kill", pid, sig)) + + def fake_stop(procs, timeout_s=20): + real_kill = deploy_orin.os.kill + for pid, _ in procs: + real_kill(pid, deploy_orin.signal.SIGINT) + + with mock.patch.object(deploy_orin.os, "kill", fake_kill): + with mock.patch.object( + deploy_orin, "stop_unmanaged", side_effect=fake_stop + ): + self.activate(self.make_archive(), replace_unmanaged=True) + self.assertEqual([e[0] for e in order], ["stop", "kill", "start"], order) + self.assertEqual(order[1], ("kill", 4242, signal.SIGINT)) + self.assertEqual(order[2], ("start", "releases/r1")) + + def test_without_unmanaged_nothing_is_killed(self): + self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" + with mock.patch.object(deploy_orin.os, "kill") as kill: + self.activate(self.make_archive(), replace_unmanaged=True) + kill.assert_not_called() + + def test_direct_activation_prints_ready_lines_and_up(self): + self.journal = ( + "Capturing.\nsensor 0: 150 frames | direct: h:1 connected\n" + "sensor 1: 9 frames | encoded 9\n" + ) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.activate( + self.make_archive(), + sensors="0,1", + mode="direct", + direct_sensors="0", + ) + text = out.getvalue() + self.assertIn("sensor 0: capturing and connected to its destination", text) + self.assertIn("sensor 1: capturing (no direct destination configured)", text) + self.assertIn("producer is up (release r1)", text) + + def test_direct_activation_not_connected_fails(self): + self.journal = ( + "Capturing.\nsensor 0: 5 frames | direct: h:1 NOT connected (x)\n" + ) + with ( + contextlib.redirect_stderr(io.StringIO()), + self.assertRaisesRegex(DeployError, "NOT connected"), + ): + self.activate( + self.make_archive(), + sensors="0", + mode="direct", + direct_sensors="0", + timeout=1, + ) + + def test_livekit_activation_still_needs_the_join(self): + self.journal = "sensor 0: 150 frames | livekit 5 published\n" + with self.assertRaises(DeployError): + self.activate(self.make_archive(), sensors="0", timeout=1) + + def test_success_switches_link_between_stop_and_start(self): + self.journal = "Joined LiveKit room x\nstats livekit 12 published\n" + replaced = [] + real_replace = os.replace + + def spy(src, dst): + replaced.append((Path(src).name, Path(dst).name)) + real_replace(src, dst) + + with mock.patch.object(deploy_orin.os, "replace", spy): + self.activate(self.make_archive()) + self.assertEqual(replaced, [("current.new", "current")]) + self.assertEqual(self.events, [("stop", None), ("start", "releases/r1")]) + self.assertFalse((self.prefix / "current.new").exists()) + + def test_readiness_times_out_without_markers(self): + self.journal = "Joined LiveKit room x\nno stats yet\n" + ticks = iter(range(0, 1000, 10)) + with mock.patch.object(deploy_orin.time, "monotonic", lambda: next(ticks)): + with mock.patch.object(deploy_orin, "print", create=True): + with self.assertRaisesRegex(DeployError, "did not become ready"): + self.activate(self.make_archive(), sensors="0", timeout=25) + self.assertEqual(self.events[-1], ("start", "releases/r1")) + + +class FakeStdout(io.StringIO): + def __init__(self, tty): + super().__init__() + self.tty = tty + + def isatty(self): + return self.tty + + +class ProgressTests(unittest.TestCase): + def drawn(self, tty, total, updates): + out = FakeStdout(tty) + with mock.patch.object(deploy_orin.sys, "stdout", out): + bar = deploy_orin.Progress("pack", total) + for done in updates: + bar.update(done) + bar.finish() + return out.getvalue() + + def test_non_tty_prints_at_most_one_line_per_ten_percent(self): + text = self.drawn(False, 1000, range(0, 1001, 7)) + lines = text.splitlines() + self.assertLessEqual(len(lines), 11) + self.assertNotIn("\r", text) + self.assertIn("100%", lines[-1]) + self.assertEqual(len(lines), len(set(lines))) + + def test_non_tty_finish_reports_100_once(self): + lines = self.drawn(False, 4, [4]).splitlines() + self.assertEqual(lines, [" pack 100%"]) + + def test_tty_draws_a_30_char_bar_redrawn_in_place(self): + text = self.drawn(True, 10, [5]) + self.assertIn("\r pack [" + "#" * 15 + "-" * 15 + "] 50% (5/10)", text) + self.assertIn("[" + "#" * 30 + "] 100% (10/10)", text) + self.assertTrue(text.endswith("\n")) + + def test_scp_is_quiet_only_when_stdout_is_not_a_tty(self): + for tty, expected in ((False, ["-q"]), (True, [])): + with ( + self.subTest(tty=tty), + mock.patch.object(deploy_orin.sys, "stdout", FakeStdout(tty)), + ): + self.assertEqual(deploy_orin.Target._scp_quiet(), expected) + + def test_put_passes_quiet_flag_to_scp(self): + target = make_target(self) + with ( + mock.patch.object(deploy_orin.sys, "stdout", FakeStdout(False)), + mock.patch.object( + deploy_orin.subprocess, "run", return_value=completed(0) + ) as run, + ): + target.put(Path("/tmp/a"), "/r/a") + self.assertEqual(len(run.call_args_list), 1) + for call in run.call_args_list: + self.assertEqual(call.args[0][0], "scp") + self.assertIn("-q", call.args[0]) + + def test_step_heading_format(self): + out = FakeStdout(False) + with mock.patch.object(deploy_orin.sys, "stdout", out): + deploy_orin.step(2, "Preflight") + self.assertEqual(out.getvalue(), f"==> [2/{deploy_orin.STEPS}] Preflight\n") + + +class StopUnmanagedTests(unittest.TestCase): + def test_sigints_exactly_the_listed_pids_and_returns_when_gone(self): + with ( + mock.patch.object(deploy_orin.os, "kill") as kill, + mock.patch.object(Path, "exists", return_value=False), + ): + deploy_orin.stop_unmanaged([(11, "a"), (12, "b")]) + self.assertEqual( + kill.call_args_list, + [mock.call(11, signal.SIGINT), mock.call(12, signal.SIGINT)], + ) + + def test_raises_when_a_pid_survives(self): + ticks = iter(range(0, 1000, 10)) + with ( + mock.patch.object(deploy_orin.os, "kill"), + mock.patch.object(Path, "exists", return_value=True), + mock.patch.object(deploy_orin.time, "sleep"), + mock.patch.object(deploy_orin.time, "monotonic", lambda: next(ticks)), + ): + with self.assertRaisesRegex(DeployError, r"\[11\]"): + deploy_orin.stop_unmanaged([(11, "a")], timeout_s=25) + + +class SdkDirsTests(unittest.TestCase): + def test_default_uses_system_locations(self): + self.assertEqual( + deploy_orin.sdk_dirs(""), + ( + Path("/usr/src/jetson_multimedia_api"), + Path("/usr/src/jetson_sipl_api"), + ), + ) + + def test_sdk_root_prefixes_usr_src(self): + self.assertEqual( + deploy_orin.sdk_dirs("/mnt/sdk"), + ( + Path("/mnt/sdk/usr/src/jetson_multimedia_api"), + Path("/mnt/sdk/usr/src/jetson_sipl_api"), + ), + ) + + def test_cli_parsers_accept_sdk_root_and_replace_unmanaged(self): + args = deploy_orin.parser().parse_args( + ["u@orin", "--config", "c", "--sdk-root", "/s"] + ) + self.assertEqual(args.sdk_root, "/s") + self.assertFalse(args.replace_unmanaged) + self.assertEqual( + deploy_orin.parser().parse_args(["h", "--config", "c"]).sdk_root, + "", + ) + remote = deploy_orin.remote_parser() + for step, extra in ( + ("preflight", []), + ("build", ["--src", "s", "--stage", "t"]), + ): + parsed = remote.parse_args([step, "--sdk-root", "/s", *extra]) + self.assertEqual(parsed.sdk_root, "/s") + + def test_backend_option_is_rejected_everywhere(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + deploy_orin.parser().parse_args( + ["u@orin", "--backend", "sipl", "--config", "c"] + ) + with self.assertRaises(SystemExit): + deploy_orin.remote_parser().parse_args( + ["preflight", "--backend", "sipl"] + ) + + def test_removed_local_options_are_rejected(self): + for flag in ("--prefix", "--service-groups", "--ready-timeout"): + with self.subTest(flag=flag): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + deploy_orin.parser().parse_args( + ["u@orin", "--config", "c", flag, "1"] + ) + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + deploy_orin.parser().parse_args( + ["u@orin", "--config", "c", "--fetch-archive"] + ) + + def test_cmake_arg_is_repeatable_and_defaults_empty(self): + parser = deploy_orin.parser() + self.assertEqual(parser.parse_args(["h", "--config", "c"]).cmake_arg, []) + args = parser.parse_args( + ["h", "--config", "c", "--cmake-arg=-DA=1", "--cmake-arg=-DB=2"] + ) + self.assertEqual(args.cmake_arg, ["-DA=1", "-DB=2"]) + remote = deploy_orin.remote_parser().parse_args( + ["build", "--src", "s", "--stage", "t", "--cmake-arg=-DA=1"] + ) + self.assertEqual(remote.cmake_arg, ["-DA=1"]) + + def test_preflight_names_the_sdk_root_path_when_headers_are_missing(self): + args = argparse.Namespace(sdk_root="/nonexistent-sdk") + with ( + mock.patch.object(deploy_orin.os, "uname") as uname, + mock.patch.object(Path, "exists", return_value=True), + mock.patch.object(Path, "read_text", return_value="# R36\n"), + mock.patch.object(deploy_orin.os, "statvfs") as statvfs, + mock.patch.object(deploy_orin.subprocess, "run", return_value=completed(0)), + ): + uname.return_value.machine = "aarch64" + statvfs.return_value.f_bavail = 1 << 30 + statvfs.return_value.f_frsize = 4096 + with self.assertRaisesRegex(DeployError, "/nonexistent-sdk/usr/src"): + deploy_orin.remote_preflight(args) + + +class BuildPathTests(unittest.TestCase): + def test_build_path_prefixes_uv_then_cuda_before_the_inherited_path(self): + with mock.patch.dict(os.environ, {"PATH": "/usr/bin:/bin"}): + self.assertEqual( + deploy_orin.build_path(), + f"{Path.home()}/.local/bin:/usr/local/cuda/bin:/usr/bin:/bin", + ) + + def preflight(self, missing=()): + seen = [] + + def fake_run(cmd, **kwargs): + seen.append((cmd, kwargs)) + return completed(1 if cmd[0] == "which" and cmd[1] in missing else 0) + + args = argparse.Namespace(sdk_root="/nonexistent-sdk") + with ( + mock.patch.object(deploy_orin.os, "uname") as uname, + mock.patch.object(Path, "exists", return_value=True), + mock.patch.object(Path, "read_text", return_value="# R36\n"), + mock.patch.object(Path, "is_dir", return_value=True), + mock.patch.object(deploy_orin.os, "statvfs") as statvfs, + mock.patch.object(deploy_orin.subprocess, "run", fake_run), + mock.patch.dict(os.environ, {"PATH": "/usr/bin"}), + ): + uname.return_value.machine = "aarch64" + statvfs.return_value.f_bavail = 1 << 30 + statvfs.return_value.f_frsize = 4096 + try: + deploy_orin.remote_preflight(args) + finally: + self.seen = seen + + def test_preflight_requires_uv_using_the_build_path(self): + with self.assertRaisesRegex(DeployError, "uv is not installed.*orin_setup.sh"): + self.preflight(missing=("uv",)) + uv_call = next(k for c, k in self.seen if c == ["which", "uv"]) + self.assertEqual( + uv_call["env"]["PATH"], + f"{Path.home()}/.local/bin:/usr/local/cuda/bin:/usr/bin", + ) + + def test_preflight_succeeds_when_uv_present(self): + self.preflight() + self.assertIn(["which", "uv"], [c for c, _ in self.seen]) + + +class RemoteBuildTests(TempDirTestCase): + LIBCUDA = "/opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so.1" + EXPECTED_PATH = f"{Path.home()}/.local/bin:/usr/local/cuda/bin:/usr/bin" + + def build( + self, + sdk_root="", + libcuda=False, + path="/usr/bin", + src="/src", + cmake_arg=(), + sipl=True, + ): + stage = self.tmp / "stage" + args = argparse.Namespace( + src=src, + cmake_arg=list(cmake_arg), + stage=str(stage), + jobs=3, + sdk_root=sdk_root, + ) + real_exists, real_read_text = Path.exists, Path.read_text + + def fake_exists(p, *a, **k): + return libcuda if str(p) == self.LIBCUDA else real_exists(p, *a, **k) + + def fake_read_text(p, *a, **k): + if str(p) == "/etc/nv_tegra_release": + return "# R36 (release)\n" + return real_read_text(p, *a, **k) + + calls = [] + + def fake_run(cmd, **kwargs): + calls.append((list(cmd), kwargs)) + if cmd[:2] == ["rm", "-rf"] and cmd[2].endswith("/build"): + shutil.rmtree(cmd[2], ignore_errors=True) + return completed(0) + + with ( + mock.patch.object(deploy_orin, "CACHE", str(self.tmp / "cache")), + mock.patch.object(deploy_orin, "run", fake_run), + mock.patch.object(deploy_orin, "tail", return_value="x"), + mock.patch.object(deploy_orin, "stage_producer") as self.stage_producer, + mock.patch.object(Path, "exists", fake_exists), + mock.patch.object(Path, "is_dir", return_value=sipl), + mock.patch.object(Path, "read_text", fake_read_text), + mock.patch.dict(os.environ, {"PATH": path}), + ): + deploy_orin.remote_build(args) + return calls + + def configure(self, calls): + return next(c for c in calls if c[0][:2] == ["cmake", "-S"]) + + def test_configure_flags_and_cuda_path(self): + cmd, kwargs = self.configure(self.build(sdk_root="/mnt/sdk")) + for flag in ( + "-DJETSON_MMAPI_DIR=/mnt/sdk/usr/src/jetson_multimedia_api", + "-DSIPL_API_DIR=/mnt/sdk/usr/src/jetson_sipl_api", + "-DENABLE_CLANG_FORMAT_CHECK=OFF", + "-DCMAKE_CXX_FLAGS=-I/usr/local/cuda/include", + ): + self.assertIn(flag, cmd) + self.assertEqual(kwargs["env"]["PATH"], self.EXPECTED_PATH) + + def cleaned(self, calls): + return [ + c for c, _ in calls if c[:2] == ["rm", "-rf"] and c[2].endswith("/build") + ] + + def test_changed_source_path_cleans_build_dir_but_same_path_keeps_it(self): + self.assertTrue(self.cleaned(self.build(src="/cache/src"))) + self.assertFalse(self.cleaned(self.build(src="/cache/src"))) + self.assertTrue(self.cleaned(self.build(src="/cache/src-abc123"))) + self.assertFalse(self.cleaned(self.build(src="/cache/src-abc123"))) + + def test_release_is_staged_from_the_build_tree_not_cmake_install(self): + calls = self.build() + self.assertEqual( + self.stage_producer.call_args.args, + (self.tmp / "cache/build", self.tmp / "stage"), + ) + self.assertFalse([c for c, _ in calls if "--install" in c]) + self.assertFalse([a for c, _ in calls for a in c if "component" in a]) + + def test_build_step_also_gets_the_cuda_path(self): + calls = self.build() + cmd, kwargs = next(c for c in calls if c[0][:2] == ["cmake", "--build"]) + self.assertEqual(kwargs["env"]["PATH"], self.EXPECTED_PATH) + self.assertIn("camera_plugin_sensing", cmd) + + def test_sipl_dir_omitted_when_absent(self): + cmd, _ = self.configure(self.build(sipl=False)) + self.assertIn("-DJETSON_MMAPI_DIR=/usr/src/jetson_multimedia_api", cmd) + self.assertFalse([a for a in cmd if a.startswith("-DSIPL_API_DIR")]) + + def test_cmake_args_are_appended_last_in_order(self): + extra = ["-DENABLE_CLANG_FORMAT_CHECK=ON", "-DX=1"] + cmd, _ = self.configure(self.build(cmake_arg=extra, libcuda=True)) + self.assertEqual(cmd[-2:], extra) + self.assertGreater( + cmd.index(extra[0]), + max(i for i, a in enumerate(cmd) if a.startswith("-DCMAKE_EXE_LINKER")), + ) + + def test_changed_cmake_args_clean_build_dir(self): + self.assertTrue(self.cleaned(self.build(cmake_arg=["-DX=1"]))) + self.assertFalse(self.cleaned(self.build(cmake_arg=["-DX=1"]))) + self.assertTrue(self.cleaned(self.build(cmake_arg=["-DX=2"]))) + + def test_linker_flag_only_when_gpu_libcuda_exists(self): + absent, _ = self.configure(self.build(libcuda=False)) + self.assertFalse([a for a in absent if "LINKER_FLAGS" in a]) + present, _ = self.configure(self.build(libcuda=True)) + self.assertIn( + f"-DCMAKE_EXE_LINKER_FLAGS=-Wl,--no-as-needed {self.LIBCUDA}", present + ) + + +LIB_LINE = "\tliblivekit_ffi.so => {} (0x0000ffff)" + + +class CacheValueTests(TempDirTestCase): + def test_reads_typed_entries_and_ignores_prefix_lookalikes(self): + (self.tmp / "CMakeCache.txt").write_text( + "// comment\nFOO_BAR:STRING=wrong\nFOO:PATH=/a=b\nEMPTY:STRING=\n" + ) + self.assertEqual(deploy_orin.cache_value(self.tmp, "FOO"), "/a=b") + self.assertEqual(deploy_orin.cache_value(self.tmp, "EMPTY"), "") + self.assertEqual(deploy_orin.cache_value(self.tmp, "ABSENT"), "") + + +class StageProducerTests(TempDirTestCase): + """stage_producer on a fabricated build tree, with both ldd runs faked.""" + + def setUp(self): + super().setUp() + self.build = self.tmp / "build" + self.stage = self.tmp / "stage" + self.plugin = self.build / "src/plugins/sensing/app" + (self.plugin / "configs").mkdir(parents=True) + (self.plugin / "configs/platform.json").write_text("{}") + (self.plugin / "camera_plugin_sensing").write_text("bin") + self.libdir = self.build / "_deps/sdk/lib" + self.libdir.mkdir(parents=True) + (self.libdir / "liblivekit_ffi.so").write_text("lib") + self.cache = "" + self.build_ldd = LIB_LINE.format(self.libdir / "liblivekit_ffi.so") + self.release_ldd = "\tlibc.so.6 => /lib/aarch64-linux-gnu/libc.so.6 (0x1)" + self.licenses = self.tmp / "lic" + self.licenses.mkdir() + (self.licenses / "LICENSE").write_text("license") + (self.licenses / "NOTICE").write_text("notice") + + def stage_it(self, cache=None): + """Run stage_producer; cache defaults to recording both licence files.""" + if cache is None: + cache = ( + f"SENSING_LIVEKIT_LICENSE_FILE:FILEPATH={self.licenses}/LICENSE\n" + f"SENSING_LIVEKIT_NOTICE_FILE:FILEPATH={self.licenses}/NOTICE\n" + ) + (self.build / "CMakeCache.txt").write_text(cache) + runs = [] + + def fake_run(cmd, **kwargs): + runs.append((list(cmd), kwargs)) + return completed(0, stdout=self.release_ldd) + + with ( + mock.patch.object(deploy_orin, "tail", return_value=self.build_ldd), + mock.patch.object(deploy_orin.subprocess, "run", fake_run), + ): + deploy_orin.stage_producer(self.build, self.stage) + return runs + + def test_happy_path_stages_binary_configs_libs_and_licences(self): + runs = self.stage_it() + out = self.stage / "plugins/sensing_camera" + self.assertEqual((self.stage / deploy_orin.BIN_REL).read_text(), "bin") + self.assertTrue((out / "configs/platform.json").is_file()) + self.assertEqual((out / "liblivekit_ffi.so").read_text(), "lib") + self.assertEqual((out / "licenses/livekit-sdk/LICENSE").read_text(), "license") + self.assertEqual((out / "licenses/livekit-sdk/NOTICE").read_text(), "notice") + cmd, kwargs = runs[-1] + self.assertEqual(cmd, ["ldd", str(self.stage / deploy_orin.BIN_REL)]) + self.assertEqual(kwargs["env"]["LD_LIBRARY_PATH"], str(out)) + + def test_release_carries_the_setup_and_camera_check_scripts(self): + self.stage_it() + scripts = self.stage / "plugins/sensing_camera/scripts" + for name in deploy_orin.TOOL_SCRIPTS: + self.assertTrue((scripts / name).is_file(), name) + + def test_ldd_not_found_for_a_livekit_library_is_not_copied(self): + self.build_ldd = "\tliblivekit_ffi.so => not found" + self.stage_it() + out = self.stage / "plugins/sensing_camera" + self.assertEqual(list(out.glob("liblivekit*")), []) + self.assertFalse((out / "licenses").exists()) + + def test_no_binary_is_an_error(self): + (self.plugin / "camera_plugin_sensing").unlink() + with self.assertRaisesRegex(DeployError, "found 0"): + self.stage_it() + + def test_a_directory_with_the_binary_name_is_not_a_binary(self): + (self.plugin / "camera_plugin_sensing").unlink() + (self.build / "camera_plugin_sensing").mkdir() + with self.assertRaisesRegex(DeployError, "found 0"): + self.stage_it() + + def test_two_binaries_is_an_error(self): + other = self.build / "other" + other.mkdir() + (other / "camera_plugin_sensing").write_text("bin2") + with self.assertRaisesRegex(DeployError, "found 2"): + self.stage_it() + + def test_missing_configs_dir_is_an_error(self): + shutil.rmtree(self.plugin / "configs") + with self.assertRaisesRegex(DeployError, "platform configs"): + self.stage_it() + + def test_no_livekit_library_resolved_stages_without_libs_or_licences(self): + self.build_ldd = "\tlibc.so.6 => /lib/libc.so.6 (0x1)" + # No licence cache entries either: nothing should look for them. + self.stage_it(cache="") + out = self.stage / "plugins/sensing_camera" + self.assertEqual((self.stage / deploy_orin.BIN_REL).read_text(), "bin") + self.assertTrue((out / "configs/platform.json").is_file()) + self.assertEqual(list(out.glob("liblivekit*")), []) + self.assertFalse((out / "licenses").exists()) + + def test_livekit_library_resolved_stages_libs_and_licences(self): + self.stage_it() + out = self.stage / "plugins/sensing_camera" + self.assertTrue((out / "liblivekit_ffi.so").is_file()) + self.assertTrue((out / "licenses/livekit-sdk/LICENSE").is_file()) + + def test_missing_licence_is_an_error(self): + for name in ("LICENSE", "NOTICE"): + (self.licenses / name).unlink() + with self.subTest(missing=name), self.assertRaisesRegex(DeployError, name): + self.stage_it() + (self.licenses / name).write_text("x") + shutil.rmtree(self.stage) + + def test_library_still_resolved_from_the_build_tree_is_an_error(self): + self.release_ldd = LIB_LINE.format(self.libdir / "liblivekit_ffi.so") + with self.assertRaisesRegex(DeployError, "from the build tree"): + self.stage_it() + + def test_unresolved_library_is_an_error(self): + self.release_ldd = "\tlibfoo.so.1 => not found" + with self.assertRaisesRegex(DeployError, "unresolved shared libraries"): + self.stage_it() + + def test_licence_found_via_the_cache_entry(self): + self.stage_it() + self.assertEqual( + deploy_orin.livekit_notice(self.build, "LICENSE"), self.licenses / "LICENSE" + ) + + def test_licence_found_in_the_fetchcontent_directory(self): + for name in ("LICENSE", "NOTICE"): + fetched = self.build / "_deps" / f"livekit_sdk_{name.lower()}-src" + fetched.mkdir(parents=True) + (fetched / name).write_text(f"fetched {name}") + self.stage_it(cache="") + notices = self.stage / "plugins/sensing_camera/licenses/livekit-sdk" + self.assertEqual((notices / "LICENSE").read_text(), "fetched LICENSE") + self.assertEqual((notices / "NOTICE").read_text(), "fetched NOTICE") + + def test_licence_found_in_or_beside_a_local_sdk_directory(self): + sdk = self.tmp / "sdk-root/install" + sdk.mkdir(parents=True) + cache = f"LIVEKIT_SDK_DIR:PATH={sdk}\n" + (sdk / "LICENSE").write_text("in sdk") + (sdk.parent / "NOTICE").write_text("beside sdk") + self.stage_it(cache=cache) + notices = self.stage / "plugins/sensing_camera/licenses/livekit-sdk" + self.assertEqual((notices / "LICENSE").read_text(), "in sdk") + self.assertEqual((notices / "NOTICE").read_text(), "beside sdk") + + def test_cache_entry_wins_over_the_other_locations(self): + fetched = self.build / "_deps/livekit_sdk_license-src" + fetched.mkdir(parents=True) + (fetched / "LICENSE").write_text("fetched") + self.stage_it() + self.assertEqual( + ( + self.stage / "plugins/sensing_camera/licenses/livekit-sdk/LICENSE" + ).read_text(), + "license", + ) + + def test_unit_sets_ld_library_path_to_the_release_directory(self): + config = { + "mode": "livekit", + "livekit_url": "wss://x", + "token_file": "/t", + "args": [STREAM], + } + text = deploy_orin.render_unit(config, "/opt/ic", "bob", "video") + self.assertIn( + "Environment=LD_LIBRARY_PATH=/opt/ic/current/plugins/sensing_camera", + text.splitlines(), + ) + + +class ConfiguredSensorsTests(unittest.TestCase): + def sensors(self, *args): + return deploy_orin.configured_sensors({"args": list(args)}) + + def test_single_and_multiple_streams_in_order(self): + self.assertEqual(self.sensors(STREAM), [0]) + self.assertEqual( + self.sensors( + "--add-stream=sensor=0,livekit=left", + "--add-stream=sensor=1,livekit=right", + ), + [0, 1], + ) + + def test_sensor_not_first_field_and_multi_digit(self): + self.assertEqual(self.sensors("--add-stream=livekit=a,sensor=12"), [12]) + + def test_ignores_other_arguments(self): + self.assertEqual(self.sensors("--verbose", "--other=sensor=3", STREAM), [0]) + + def test_no_match(self): + self.assertEqual(self.sensors("--add-stream=livekit=x"), []) + self.assertEqual(deploy_orin.configured_sensors({}), []) + + +class ConfiguredStreamsTests(unittest.TestCase): + def streams(self, *args): + return deploy_orin.configured_streams({"args": list(args)}) + + def test_direct_key_with_a_value_marks_a_direct_stream(self): + self.assertEqual( + self.streams( + "--add-stream=sensor=0,direct=10.0.0.1:5000", + "--add-stream=sensor=1,livekit=right", + "--add-stream=livekit=a,direct=h:1,sensor=2", + ), + [(0, True), (1, False), (2, True)], + ) + + def test_direct_as_the_first_key_is_a_direct_stream(self): + self.assertEqual(self.streams("--add-stream=direct=h:1,sensor=2"), [(2, True)]) + + def test_empty_direct_value_before_another_key_is_not_a_destination(self): + self.assertEqual( + self.streams("--add-stream=sensor=1,direct=,livekit=a"), [(1, False)] + ) + + def test_empty_direct_value_at_the_end_is_not_a_destination(self): + self.assertEqual(self.streams("--add-stream=sensor=0,direct="), [(0, False)]) + + def test_direct_inside_another_key_is_not_a_direct_key(self): + self.assertEqual( + self.streams("--add-stream=sensor=0,indirect=x,livekit=direct=y"), + [(0, False)], + ) + + def test_ignores_other_arguments_and_streams_without_a_sensor(self): + self.assertEqual( + self.streams("--verbose", "--add-stream=direct=h:1", "--other=sensor=3"), + [], + ) + self.assertEqual(deploy_orin.configured_streams({}), []) + + def test_configured_sensors_derives_from_the_streams(self): + config = {"args": ["--add-stream=sensor=4,direct=h:1", STREAM]} + self.assertEqual(deploy_orin.configured_sensors(config), [4, 0]) + + +DIRECT_OK = "direct: 10.0.0.1:5000 connected" + + +class CurrentSessionTests(unittest.TestCase): + def test_text_after_the_last_marker(self): + log = "a Capturing. b Capturing. c" + self.assertEqual(deploy_orin.current_session(log, "direct"), " c") + log = "x Joined LiveKit room r1 y Joined LiveKit room r2 z" + self.assertEqual(deploy_orin.current_session(log, "livekit"), " r2 z") + + def test_absent_marker_is_empty_and_markers_are_per_mode(self): + self.assertEqual( + deploy_orin.current_session("sensor 0: 5 frames", "direct"), "" + ) + self.assertEqual(deploy_orin.current_session("Capturing.\n", "livekit"), "") + self.assertEqual( + deploy_orin.current_session("Joined LiveKit room r\n", "direct"), "" + ) + + +class ReadinessFunctionTests(unittest.TestCase): + def check(self, log, sensors, direct=(), mode="direct"): + return deploy_orin.readiness(log, mode, sensors, list(direct)) + + def test_no_marker_is_one_problem_per_mode(self): + ready, problems = self.check("sensor 0: 5 frames\n", [0]) + self.assertEqual(ready, []) + self.assertEqual(problems, ["the producer has not reported capturing yet"]) + _, problems = self.check("livekit 5 published\n", [0], mode="livekit") + self.assertEqual( + problems, ["the producer has not reported joined a LiveKit room yet"] + ) + + def test_connected_direct_sensor_is_ready(self): + log = f"Capturing.\nsensor 0: 30 frames | {DIRECT_OK}\n" + self.assertEqual( + self.check(log, [0], [0]), + (["sensor 0: capturing and connected to its destination"], []), + ) + + def test_restart_inside_the_unit_discards_the_earlier_attempt(self): + log = ( + f"Capturing.\nsensor 0: 30 frames | {DIRECT_OK}\n" + "Capturing.\nsensor 1: 30 frames | " + DIRECT_OK + "\n" + ) + ready, problems = self.check(log, [0, 1], [0, 1]) + self.assertEqual( + ready, ["sensor 1: capturing and connected to its destination"] + ) + self.assertEqual(problems, ["sensor 0: no frames captured"]) + + def test_not_connected_is_a_problem_even_with_frames(self): + log = ( + "Capturing.\nsensor 0: 30 frames | direct: 10.0.0.1:5000 NOT connected " + "(cannot connect to 10.0.0.1:5000)\n" + ) + ready, problems = self.check(log, [0], [0]) + self.assertEqual(ready, []) + self.assertEqual(len(problems), 1) + self.assertIn("sensor 0", problems[0]) + self.assertIn("NOT connected (cannot connect to 10.0.0.1:5000)", problems[0]) + + def test_configured_direct_sensor_with_no_direct_text_is_a_problem(self): + _, problems = self.check( + "Capturing.\nsensor 0: 30 frames | encoded 30\n", [0], [0] + ) + self.assertEqual( + problems, ["sensor 0: capturing but its direct link is not connected"] + ) + + def test_sensor_without_a_destination_is_capture_only(self): + log = "Capturing.\nsensor 0: 30 frames | encoded 30\n" + self.assertEqual( + self.check(log, [0], []), + (["sensor 0: capturing (no direct destination configured)"], []), + ) + + def test_destinationless_sensor_ignores_a_not_connected_text(self): + log = "Capturing.\nsensor 0: 30 frames | direct: h:1 NOT connected (x)\n" + ready, problems = self.check(log, [0], []) + self.assertEqual(problems, []) + self.assertEqual(len(ready), 1) + + def test_latest_line_wins_connected_then_not_connected(self): + log = ( + f"Capturing.\nsensor 0: 10 frames | {DIRECT_OK}\n" + "sensor 0: 20 frames | direct: 10.0.0.1:5000 NOT connected (link lost)\n" + ) + ready, problems = self.check(log, [0], [0]) + self.assertEqual(ready, []) + self.assertIn("link lost", problems[0]) + + def test_latest_line_wins_not_connected_then_connected(self): + log = ( + "Capturing.\nsensor 0: 10 frames | direct: h:1 NOT connected (x)\n" + f"sensor 0: 20 frames | {DIRECT_OK}\n" + ) + ready, problems = self.check(log, [0], [0]) + self.assertEqual(problems, []) + self.assertEqual(len(ready), 1) + + def test_latest_zero_frames_line_is_a_problem(self): + log = "Capturing.\nsensor 0: 10 frames\nsensor 0: 0 frames\n" + self.assertEqual(self.check(log, [0])[1], ["sensor 0: no frames captured"]) + + def test_zero_frames_with_a_later_digit_on_the_line(self): + log = "Capturing.\nsensor 0: 0 frames | encoded 5, dropped 7\n" + self.assertEqual(self.check(log, [0])[1], ["sensor 0: no frames captured"]) + + def test_sensor_one_is_not_confused_with_sensor_ten(self): + log = f"Capturing.\nsensor 10: 30 frames | {DIRECT_OK}\n" + ready, problems = self.check(log, [1, 10], [1, 10]) + self.assertEqual(problems, ["sensor 1: no frames captured"]) + self.assertEqual(len(ready), 1) + self.assertIn("sensor 10", ready[0]) + + def test_livekit_mode_latest_published_count_decides(self): + log = ( + "Joined LiveKit room r\n" + "sensor 0: 30 fps | livekit 12 published\n" + "sensor 1: 30 fps | livekit 0 published\n" + "sensor 10: 30 fps | livekit 5 published\n" + ) + ready, problems = self.check(log, [0, 1], mode="livekit") + self.assertEqual(ready, ["sensor 0: publishing to LiveKit"]) + self.assertEqual(problems, ["sensor 1: not publishing to LiveKit"]) + + def test_livekit_mode_latest_line_wins_and_restart_resets(self): + log = ( + "Joined LiveKit room r\nsensor 0: x | livekit 5 published\n" + "sensor 0: x | livekit 0 published\n" + ) + self.assertEqual(len(self.check(log, [0], mode="livekit")[1]), 1) + log = ( + "Joined LiveKit room r\nsensor 0: x | livekit 5 published\n" + "Joined LiveKit room r2\n" + ) + self.assertEqual(len(self.check(log, [0], mode="livekit")[1]), 1) + + def test_livekit_mode_ignores_direct_markers(self): + log = "Capturing.\nsensor 0: 5 frames | " + DIRECT_OK + "\n" + self.assertEqual( + self.check(log, [0], [0], mode="livekit")[1], + ["the producer has not reported joined a LiveKit room yet"], + ) + + +class ReadinessTests(TempDirTestCase): + JOINED = "Joined LiveKit room r\n" + CAPTURING = "Capturing.\n" + + def run_wait( + self, journal, sensors, active=True, timeout=10, mode="livekit", direct=None + ): + def fake_run(cmd, **kwargs): + return completed(0 if active else 3) + + self.journal_calls = 0 + + def fake_tail(cmd): + if cmd[0] != "journalctl": + return "ActiveState=active" + self.journal_calls += 1 + return journal + + ticks = iter(range(0, 1000, 5)) + stderr = io.StringIO() + with ( + mock.patch.object(deploy_orin.subprocess, "run", fake_run), + mock.patch.object(deploy_orin, "tail", fake_tail), + mock.patch.object(deploy_orin.time, "sleep"), + mock.patch.object(deploy_orin.time, "monotonic", lambda: next(ticks)), + mock.patch.object(deploy_orin, "CACHE", str(self.tmp)), + contextlib.redirect_stderr(stderr), + ): + try: + return deploy_orin.wait_ready( + "2026-01-01 00:00:00", + timeout, + sensors, + mode=mode, + direct_sensors=direct, + ) + finally: + self.stderr = stderr.getvalue() + + def test_direct_is_the_default_mode(self): + journal = f"Capturing.\nsensor 0: 3 frames | {DIRECT_OK}\n" + self.assertEqual(deploy_orin.wait_ready.__defaults__[0:1], ("direct",)) + with ( + mock.patch.object(deploy_orin, "tail", lambda cmd: journal), + mock.patch.object( + deploy_orin.subprocess, "run", lambda *a, **k: completed(0) + ), + ): + ready = deploy_orin.wait_ready("t", 10, [0], direct_sensors=[0]) + self.assertEqual( + ready, ["sensor 0: capturing and connected to its destination"] + ) + + def test_direct_returns_one_line_per_sensor(self): + journal = ( + self.CAPTURING + + f"sensor 0: 150 frames | {DIRECT_OK}\n" + + "sensor 1: 10 frames | encoded 10\n" + ) + self.assertEqual( + self.run_wait(journal, [0, 1], mode="direct", direct=[0]), + [ + "sensor 0: capturing and connected to its destination", + "sensor 1: capturing (no direct destination configured)", + ], + ) + + def test_direct_restart_inside_the_unit_is_not_ready_for_sensor_zero(self): + journal = ( + f"Capturing.\nsensor 0: 30 frames | {DIRECT_OK}\n" + f"Capturing.\nsensor 1: 30 frames | {DIRECT_OK}\n" + ) + with self.assertRaises(DeployError) as ctx: + self.run_wait(journal, [0, 1], mode="direct", direct=[0, 1]) + message = str(ctx.exception) + self.assertIn("sensor 0: no frames captured", message) + self.assertNotIn("sensor 1:", message) + + def test_direct_not_connected_fails_naming_the_reason(self): + journal = ( + self.CAPTURING + + "sensor 0: 30 frames | direct: h:1 NOT connected (cannot connect to h:1)\n" + ) + with self.assertRaisesRegex( + DeployError, + r"did not become ready; sensor 0: capturing but h:1 NOT connected", + ): + self.run_wait(journal, [0], mode="direct", direct=[0]) + self.assertIn("sensor 0", self.stderr) + + def test_direct_no_marker_names_the_missing_capture_report(self): + with self.assertRaisesRegex(DeployError, "has not reported capturing yet"): + self.run_wait("sensor 0: 5 frames\n", [0], mode="direct") + + def test_direct_sensor_with_zero_frames_is_named(self): + journal = self.CAPTURING + "sensor 0: 150 frames\nsensor 1: 0 frames\n" + with self.assertRaises(DeployError) as ctx: + self.run_wait(journal, [0, 1], mode="direct") + self.assertIn("sensor 1: no frames captured", str(ctx.exception)) + self.assertNotIn("sensor 0:", str(ctx.exception)) + + def test_livekit_both_sensors_publishing_returns_lines(self): + journal = ( + self.JOINED + + "sensor 0: 30 fps | livekit 12 published\n" + + "sensor 1: 30 fps | livekit 3 published\n" + ) + self.assertEqual( + self.run_wait(journal, [0, 1]), + ["sensor 0: publishing to LiveKit", "sensor 1: publishing to LiveKit"], + ) + + def test_livekit_one_of_two_sensors_names_the_other(self): + journal = ( + self.JOINED + + "sensor 0: 30 fps | livekit 12 published\n" + + "sensor 1: 0 fps | livekit 0 published\n" + ) + with self.assertRaises(DeployError) as ctx: + self.run_wait(journal, [0, 1]) + message = str(ctx.exception) + self.assertIn("sensor 1: not publishing to LiveKit", message) + self.assertNotIn("sensor 0:", message) + + def test_livekit_sensor_number_does_not_match_a_longer_number(self): + journal = self.JOINED + "sensor 10: 30 fps | livekit 5 published\n" + with self.assertRaisesRegex(DeployError, "sensor 1: not publishing"): + self.run_wait(journal, [1]) + + def test_livekit_frames_alone_are_not_readiness(self): + with self.assertRaises(DeployError): + self.run_wait(self.JOINED + "sensor 0: 150 frames\n", [0]) + + def test_livekit_requires_joining_the_room(self): + with self.assertRaisesRegex(DeployError, "joined a LiveKit room"): + self.run_wait("sensor 0: x | livekit 5 published\n", [0]) + + def test_inactive_service_stops_waiting_and_reports(self): + ticks_used = [] + with mock.patch.object(deploy_orin.time, "sleep", ticks_used.append): + with self.assertRaises(DeployError): + self.run_wait(self.JOINED + "livekit 5 published\n", [0], active=False) + self.assertEqual(ticks_used, []) + + def test_inactive_service_reports_the_producer_never_reported(self): + with self.assertRaisesRegex(DeployError, "has not reported"): + self.run_wait("sensor 0: x | livekit 5 published\n", [0, 1], active=False) + + def test_inactive_service_never_fetches_for_the_loop(self): + with self.assertRaises(DeployError): + self.run_wait(self.JOINED, [0], active=False) + self.assertEqual(self.journal_calls, 1) # the report's fetch only + + def test_timeout_adds_no_journal_fetch_beyond_loop_and_report(self): + # Clock ticks 0 (deadline = 10), 5 (one loop fetch), 10 (expired); + # the failure report then makes the only other fetch. + with self.assertRaises(DeployError): + self.run_wait( + self.JOINED + "sensor 0: x | livekit 0 published\n", [0], timeout=10 + ) + self.assertEqual(self.journal_calls, 2) + + def test_failure_prints_the_report_to_stderr(self): + with self.assertRaises(DeployError): + self.run_wait("boom\n", [0], active=False) + self.assertIn("service: ActiveState=active", self.stderr) + self.assertIn("boom", self.stderr) + + +class FailureReportTests(TempDirTestCase): + def report(self, log, sensors=(0, 1)): + def fake_tail(cmd): + if cmd[0] == "systemctl": + return "ActiveState=failed\nSubState=failed\nNRestarts=2" + return log + + with ( + mock.patch.object(deploy_orin, "tail", fake_tail), + mock.patch.object(deploy_orin, "CACHE", str(self.tmp / "cache")), + ): + return deploy_orin.failure_report("since", list(sensors)) + + def test_noise_is_filtered_out_and_counted(self): + log = "\n".join( + [ + "Severity 5 : driver info", + "Severity 6 : driver debug", + "real line one", + "real line two", + ] + ) + out = self.report(log) + self.assertNotIn("driver info", out) + self.assertNotIn("driver debug", out) + self.assertIn("4 lines since the start, 2 of them from the camera driver", out) + self.assertIn(" real line one", out) + + def test_a_link_that_did_not_lock_is_named_in_plain_words(self): + log = "\n".join( + [ + 'x [Deser_9] Sequence: "enable_link_C" (operation 4/7)', + 'x [Deser_9] Sequence: "enable_link_C" (operation 4/7)', + ] + ) + out = self.report(log) + self.assertEqual(out.count("link 2 (COAX2 on CN1) did not lock"), 1) + + def test_driver_chatter_at_every_severity_is_hidden(self): + log = "\n".join( + [ + "[TID=1] Module_id 30 Severity 2 : [PIPELINE (0,0)] CNvFusaRawBlock.cpp:2143: DBG GetOutputBuffer", + "[TID=1] Module_id 30 Severity 3 : [CameraHAL] WARNING: No module power driver found for GMSL transport 'Deser_9'", + "producer line", + ] + ) + out = self.report(log) + self.assertNotIn("GetOutputBuffer", out) + self.assertIn(" producer line", out) + + def test_only_real_driver_errors_are_listed_and_deduplicated(self): + log = "\n".join( + [ + "[TID=1] Module_id 30 Severity 2 : [CameraHAL] ERROR: CameraHalPortal.cpp:280: driverCatalog: path='/x'", + "[TID=1] Module_id 30 Severity 3 : dlsym failed for 'uddf_discover_drivers'", + "[TID=1] Module_id 30 Severity 2 : Failed to initialize camera module: 0", + "[TID=2] Module_id 30 Severity 2 : Failed to initialize camera module: 0", + "[TID=1] Module_id 30 Severity 2 : Master initialization failed. status: 10", + ] + ) + out = self.report(log) + errors = out.split("camera driver errors:")[1].split("first lines")[0] + self.assertEqual(errors.count("Failed to initialize camera module"), 1) + self.assertIn("Master initialization failed. status: 10", errors) + self.assertNotIn("driverCatalog", errors) + self.assertNotIn("uddf_discover", errors) + + def test_no_driver_error_section_without_driver_errors(self): + self.assertNotIn("camera driver errors", self.report("plain")) + + def test_no_link_failure_adds_no_link_line(self): + self.assertNotIn("did not lock", self.report("real line")) + + def test_service_state_and_sensors_listed(self): + out = self.report("x") + self.assertIn("service: ActiveState=failed, SubState=failed, NRestarts=2", out) + self.assertIn("sensors expected to publish: 0, 1", out) + self.assertNotIn("sensors expected", self.report("x", sensors=())) + + def test_fault_lines_listed_and_capped_at_eight(self): + log = "\n".join(f"camera_{i} startup_failed" for i in range(12)) + out = self.report(log) + self.assertIn("producer fault lines:", out) + faults = out.split("producer fault lines:")[1].split("first lines")[0] + self.assertEqual(len([ln for ln in faults.splitlines() if ln.strip()]), 8) + + def test_no_fault_section_without_fault_lines(self): + self.assertNotIn("producer fault lines", self.report("plain")) + + @staticmethod + def numbered(n): + return "\n".join(f"line {i:02d}" for i in range(n)) + + @staticmethod + def listed(out): + return [ln.strip() for ln in out.splitlines() if ln.startswith(" line ")] + + def test_first_and_last_lines_with_elision_count(self): + out = self.report(self.numbered(40)) + self.assertIn("last lines (10 not shown between):", out) + head, tail_part = out.split("last lines") + self.assertIn("line 00", head) + self.assertIn("line 14", head) + self.assertNotIn("line 15", head) + self.assertNotIn("line 24", tail_part) + self.assertIn("line 25", tail_part) + self.assertIn("line 39", tail_part) + + def test_16_to_29_quiet_lines_have_no_duplicates(self): + for n in (16, 22, 29, 30): + with self.subTest(n=n): + out = self.report(self.numbered(n)) + shown = self.listed(out) + self.assertEqual(shown, [f"line {i:02d}" for i in range(n)]) + self.assertIn("remaining lines:", out) + self.assertNotIn("not shown", out) + self.assertNotIn("last lines", out) + + def test_31_lines_hides_one(self): + out = self.report(self.numbered(31)) + self.assertIn("last lines (1 not shown between):", out) + self.assertEqual(len(self.listed(out)), 30) + + def test_15_or_fewer_lines_have_no_last_section(self): + for n in (1, 14, 15): + with self.subTest(n=n): + out = self.report(self.numbered(n)) + self.assertNotIn("last lines", out) + self.assertNotIn("remaining lines", out) + self.assertNotIn("not shown", out) + self.assertEqual(len(self.listed(out)), n) + + def test_complete_journal_saved_under_cache(self): + log = "Severity 5 : noise\nreal\n" + out = self.report(log) + saved = self.tmp / "cache" / "last-start-failure.log" + self.assertEqual(saved.read_text(), log + "\n") + self.assertIn(str(saved), out) + + def test_empty_log(self): + out = self.report("") + self.assertIn("log: 0 lines since the start, 0 of them", out) + self.assertTrue((self.tmp / "cache" / "last-start-failure.log").exists()) + self.assertNotIn("last lines", out) + + +if __name__ == "__main__": + unittest.main() + + +class SiplPackageTests(TempDirTestCase): + def test_explicit_file_wins_and_a_missing_one_is_an_error(self): + archive = self.tmp / "a.tbz2" + archive.write_bytes(b"x") + self.assertEqual(deploy_orin.find_sipl_package(str(archive)), archive) + with self.assertRaisesRegex(DeployError, "no such file"): + deploy_orin.find_sipl_package(str(self.tmp / "missing.tbz2")) + + def test_default_is_the_newest_sdk_manager_download(self): + old = self.tmp / "Jetson_SIPL_API_R39.2.0_aarch64.tbz2" + new = self.tmp / "Jetson_SIPL_API_R39.2.1_aarch64.tbz2" + for path, when in ((old, 100), (new, 200)): + path.write_bytes(b"x") + os.utime(path, (when, when)) + with mock.patch.object(deploy_orin, "SDKM_DOWNLOADS", str(self.tmp)): + self.assertEqual(deploy_orin.find_sipl_package(None), new) + + def test_no_download_means_none(self): + with mock.patch.object(deploy_orin, "SDKM_DOWNLOADS", str(self.tmp)): + self.assertIsNone(deploy_orin.find_sipl_package(None)) + + +class WaitForRebootTests(unittest.TestCase): + def wait(self, codes): + target = mock.Mock() + target.status.side_effect = codes + clock = iter(range(0, 100000, 5)) + with ( + mock.patch.object(deploy_orin.time, "sleep"), + mock.patch.object(deploy_orin.time, "monotonic", lambda: next(clock)), + ): + deploy_orin.wait_for_reboot(target) + + def test_returns_once_the_orin_has_gone_down_and_come_back(self): + self.wait([0, 255, 255, 0]) + + def test_an_orin_that_never_goes_down_is_an_error(self): + with self.assertRaisesRegex(DeployError, "did not reboot"): + self.wait([0] * 1000) + + def test_an_orin_that_never_returns_is_an_error(self): + with self.assertRaisesRegex(DeployError, "did not answer ssh"): + self.wait([255] * 1000) + + +FOUND_ONE = { + "config": f"{CACHE_DIR}/cameras/platform.json", + "masks": "0x0100 0x0000", + "count": 1, + "used": 1, + "positions": ["CN2 COAX6 (deserializer AB, link 2)"], +} +TWO_STREAMS = [ + "--add-stream=sensor=0,name=left,direct=10.0.0.1:5600", + "--add-stream=sensor=1,name=right,direct=10.0.0.1:5601", +] + + +class AutoCamerasConfigTests(TempDirTestCase): + def load(self, config): + path = self.tmp / "c.json" + path.write_text(json.dumps(config)) + return deploy_orin.load_config(path) + + def test_auto_is_accepted(self): + self.assertEqual( + self.load({"cameras": "auto", "args": TWO_STREAMS})["cameras"], "auto" + ) + + def test_only_auto_is_a_valid_value(self): + with self.assertRaisesRegex(DeployError, '"cameras" must be "auto"'): + self.load({"cameras": "manual", "args": TWO_STREAMS}) + + def test_auto_forbids_the_arguments_the_deploy_supplies(self): + for arg in ( + "--link-masks=0x0000 0x1100", + "--platform-config=/x", + "--config-name=Y", + ): + with self.subTest(arg), self.assertRaisesRegex(DeployError, "supplies"): + self.load({"cameras": "auto", "args": [*TWO_STREAMS, arg]}) + + +class ApplyDetectedTests(unittest.TestCase): + def test_streams_for_missing_sensors_are_dropped(self): + config = deploy_orin.apply_detected({"args": TWO_STREAMS}, FOUND_ONE) + self.assertEqual(deploy_orin.configured_sensors(config), [0]) + + def test_platform_arguments_are_appended(self): + config = deploy_orin.apply_detected({"args": TWO_STREAMS}, FOUND_ONE) + self.assertIn( + f"--platform-config={deploy_orin.PREFIX}/camera-platform.json", + config["args"], + ) + self.assertIn("--config-name=SHW5G_2", config["args"]) + self.assertIn("--link-masks=0x0100 0x0000", config["args"]) + + def test_other_keys_and_stream_order_are_kept(self): + config = deploy_orin.apply_detected( + {"mode": "direct", "env": {"A": "b"}, "args": ["--fps=30", TWO_STREAMS[0]]}, + {**FOUND_ONE, "used": 2}, + ) + self.assertEqual(config["env"], {"A": "b"}) + self.assertEqual(config["args"][:2], ["--fps=30", TWO_STREAMS[0]]) + + def test_two_cameras_keep_both_streams(self): + found = {**FOUND_ONE, "used": 2, "masks": "0x0000 0x1100"} + config = deploy_orin.apply_detected({"args": TWO_STREAMS}, found) + self.assertEqual(deploy_orin.configured_sensors(config), [0, 1]) + + def test_no_stream_left_is_an_error(self): + with self.assertRaisesRegex(DeployError, "do not have"): + deploy_orin.apply_detected({"args": [TWO_STREAMS[1]]}, FOUND_ONE) + + +class DetectCamerasTests(unittest.TestCase): + def target(self, code, text): + target = mock.Mock() + target.output.return_value = (code, text) + return target + + def test_returns_the_json_printed_last(self): + target = self.target(0, "junk\n" + json.dumps(FOUND_ONE) + "\n") + self.assertEqual(deploy_orin.detect_cameras(target, CACHE_DIR), FOUND_ONE) + + def test_stops_the_service_before_probing(self): + target = self.target(0, json.dumps(FOUND_ONE)) + deploy_orin.detect_cameras(target, CACHE_DIR) + self.assertIn(deploy_orin.SERVICE, target.sudo.call_args.args[0]) + self.assertIn("--detect", target.output.call_args.args[0]) + + def test_remembered_cameras_are_used_unless_a_rescan_is_asked_for(self): + target = self.target(0, json.dumps(FOUND_ONE)) + deploy_orin.detect_cameras(target, CACHE_DIR) + self.assertNotIn("--rescan", target.output.call_args.args[0]) + deploy_orin.detect_cameras(target, CACHE_DIR, rescan=True) + self.assertTrue(target.output.call_args.args[0].endswith("--rescan")) + + def test_no_camera_is_an_error_that_points_at_the_scan(self): + with self.assertRaisesRegex(DeployError, "--scan-cameras"): + deploy_orin.detect_cameras(self.target(1, ""), CACHE_DIR) + + def test_unusable_output_is_an_error(self): + with self.assertRaisesRegex(DeployError, "no usable result"): + deploy_orin.detect_cameras(self.target(0, "not json"), CACHE_DIR) + + +class AutoCamerasDeployTests(TempDirTestCase): + def deploy(self, detected=FOUND_ONE): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + orin.detected = detected + written = {} + original = deploy_orin.render_unit + + def spy(config, *rest): + written["args"] = config["args"] + return original(config, *rest) + + with mock.patch.object(deploy_orin, "render_unit", spy): + orin.run( + root, config={"mode": "direct", "cameras": "auto", "args": TWO_STREAMS} + ) + return orin, written + + def test_the_unit_uses_the_detected_cameras(self): + _, written = self.deploy() + self.assertIn("--link-masks=0x0100 0x0000", written["args"]) + self.assertEqual(deploy_orin.configured_sensors({"args": written["args"]}), [0]) + + def test_detection_runs_after_the_prerequisites_and_before_the_upload(self): + orin, _ = self.deploy() + kinds = [ + "detect" + if k == "output" + else "upload" + if v.endswith("delta.tar.gz") + else None + for k, v in orin.events + if k == "output" or (k == "put" and v.endswith("delta.tar.gz")) + ] + self.assertEqual(kinds, ["detect", "upload"]) + + def test_no_detected_cameras_stops_before_anything_is_uploaded(self): + root, _ = git_repo(self, {"src/a.cpp": "a\n"}) + orin = FakeOrin() + with self.assertRaises(DeployError): + orin.run( + root, config={"mode": "direct", "cameras": "auto", "args": TWO_STREAMS} + ) + self.assertFalse( + any(k == "put" and v.endswith("delta.tar.gz") for k, v in orin.events) + )