From cafae59ba687241ad8dc388c99dd539b043df249 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 09:00:19 -0800 Subject: [PATCH 1/6] sensing: add the capture boundary the sender will select a backend through A vendor-neutral CameraCapture interface, a move-only frame lease and a tagged pixel representation (CUDA RGBA for SIPL, NV12 DMA-BUF for Argus). Header only; nothing uses it yet, so behaviour is unchanged. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/camera_capture.hpp | 131 ++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 src/plugins/sensing/core/camera_capture.hpp diff --git a/src/plugins/sensing/core/camera_capture.hpp b/src/plugins/sensing/core/camera_capture.hpp new file mode 100644 index 0000000000..92d1d475ac --- /dev/null +++ b/src/plugins/sensing/core/camera_capture.hpp @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// The capture boundary of the sender: what a camera backend (SIPL, Argus) hands to the encoder. No vendor SDK +// type appears here, so the sender can select a backend at startup without including either SDK. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +enum class CameraBackend +{ + Auto, + Argus, + Sipl, +}; + +/// One camera as a backend reports it. `id` is the backend's own index (a SIPL pipeline index, an Argus device +/// index); the two are unrelated, so configuration names cameras by logical stream, not by id. +struct CameraInfo +{ + uint32_t id = 0; + std::string name; + uint32_t width = 0; + uint32_t height = 0; + double fps = 0.0; +}; + +/// RGBA8 in GPU memory, as the SIPL backend converts it. +struct CudaRgbaPixels +{ + uintptr_t device_ptr = 0; + size_t pitch_bytes = 0; +}; + +/// NV12 in a DMA-BUF, as an Argus buffer stream produces it. The plane layout is what the encoder must accept. +struct DmaBufNv12Pixels +{ + int fd = -1; + uint32_t pitch_bytes[2] = { 0, 0 }; + uint32_t offset_bytes[2] = { 0, 0 }; +}; + +/// Returns a frame's buffer to its backend when destroyed or reset. Move-only: exactly one holder owns the buffer, +/// and the backend must outlive every lease it issued. +class FrameLease +{ +public: + FrameLease() = default; + explicit FrameLease(std::function release) : m_release(std::move(release)) + { + } + FrameLease(FrameLease&& other) noexcept : m_release(std::exchange(other.m_release, nullptr)) + { + } + FrameLease& operator=(FrameLease&& other) noexcept + { + if (this != &other) + { + reset(); + m_release = std::exchange(other.m_release, nullptr); + } + return *this; + } + FrameLease(const FrameLease&) = delete; + FrameLease& operator=(const FrameLease&) = delete; + ~FrameLease() + { + reset(); + } + + void reset() + { + if (m_release) + { + std::exchange(m_release, nullptr)(); + } + } + +private: + std::function m_release; +}; + +/// One captured frame. The pixels stay valid until `lease` is destroyed or reset; a consumer that needs them +/// longer (an encoder holding an input buffer) keeps the lease. +struct CaptureFrame +{ + uint32_t camera_id = 0; + uint64_t sequence = 0; + uint32_t width = 0; + uint32_t height = 0; + /// CLOCK_MONOTONIC and CLOCK_REALTIME read together when the backend had the frame ready: processing time, + /// not the sensor's capture time. + uint64_t monotonic_ns = 0; + uint64_t unix_ns = 0; + /// The backend's own sensor timestamp, unscaled and in its own clock domain; never convert it to another + /// domain here. Zero when the backend reports none. + uint64_t sensor_time = 0; + std::variant pixels; + FrameLease lease; +}; + +/// A backend instance owns its whole rig: SIPL hands out one process-wide object, so there is one instance per +/// backend, never one per camera. +class CameraCapture +{ +public: + virtual ~CameraCapture() = default; + + virtual const std::vector& cameras() const = 0; + virtual void start() = 0; + /// Stops capture. Frames already handed out stay valid until their leases are released. + virtual void stop() = 0; + /// The newest frame of `camera_id` not yet taken, or nullopt when none has arrived. Never blocks. + virtual std::optional acquire_latest(uint32_t camera_id) = 0; +}; + +} // namespace sensing +} // namespace plugins From 689aa3ff5252a9eff83adb5a96345044f6722a22 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 09:15:57 -0800 Subject: [PATCH 2/6] sensing: run the SIPL capture through the CameraCapture interface SiplCapture adapts SiplCamera without copying: frames stay the borrowed CUDA RGBA views, which the output still copies out before it returns, after the pacing check as before. SensingCamera and LiveKitVideoOutput now take the interface's frame type; other pixel types are rejected until an encoder input exists. Behaviour is unchanged. Syntax-checked here; the Jetson build and run are not. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/CMakeLists.txt | 1 + src/plugins/sensing/core/camera_capture.hpp | 11 ++-- .../sensing/core/livekit_video_output.cpp | 31 ++++++----- .../sensing/core/livekit_video_output.hpp | 7 +-- src/plugins/sensing/core/sensing_camera.cpp | 13 ++--- src/plugins/sensing/core/sensing_camera.hpp | 4 +- src/plugins/sensing/core/sipl_capture.cpp | 51 +++++++++++++++++++ src/plugins/sensing/core/sipl_capture.hpp | 36 +++++++++++++ 8 files changed, 127 insertions(+), 27 deletions(-) create mode 100644 src/plugins/sensing/core/sipl_capture.cpp create mode 100644 src/plugins/sensing/core/sipl_capture.hpp diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt index 22b466c2df..4e091beea2 100644 --- a/src/plugins/sensing/core/CMakeLists.txt +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -5,6 +5,7 @@ add_library(sensing_core STATIC sensing_camera.cpp sipl_camera.cpp + sipl_capture.cpp jetson_encoder.cpp livekit_video_output.cpp yuv_to_rgba.cu diff --git a/src/plugins/sensing/core/camera_capture.hpp b/src/plugins/sensing/core/camera_capture.hpp index 92d1d475ac..586cc420bf 100644 --- a/src/plugins/sensing/core/camera_capture.hpp +++ b/src/plugins/sensing/core/camera_capture.hpp @@ -38,14 +38,16 @@ struct CameraInfo double fps = 0.0; }; -/// RGBA8 in GPU memory, as the SIPL backend converts it. +/// RGBA8 in GPU memory, as the SIPL backend converts it. Borrowed: the backend recycles the buffer, so a consumer +/// copies it out before its next acquire_latest() for the same camera, and the frame's lease is empty. struct CudaRgbaPixels { uintptr_t device_ptr = 0; size_t pitch_bytes = 0; }; -/// NV12 in a DMA-BUF, as an Argus buffer stream produces it. The plane layout is what the encoder must accept. +/// NV12 in a DMA-BUF, as an Argus buffer stream produces it. Leased: valid until the frame's lease is released. +/// The plane layout is what the encoder must accept. struct DmaBufNv12Pixels { int fd = -1; @@ -93,8 +95,9 @@ class FrameLease std::function m_release; }; -/// One captured frame. The pixels stay valid until `lease` is destroyed or reset; a consumer that needs them -/// longer (an encoder holding an input buffer) keeps the lease. +/// One captured frame. Pixels that are leased stay valid until `lease` is destroyed or reset, so a consumer that +/// needs them longer (an encoder holding an input buffer) keeps the lease; borrowed pixels (see CudaRgbaPixels) +/// are valid only until the next acquire_latest() for the same camera. struct CaptureFrame { uint32_t camera_id = 0; diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index bdf5749a9b..9ea0b65224 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace plugins { @@ -146,15 +147,21 @@ struct LiveKitVideoOutput::Impl } // Polling thread. - void submit(const FrameView& view) + void submit(const CaptureFrame& frame) { + const auto* rgba = std::get_if(&frame.pixels); + if (rgba == nullptr) + { + fail("LiveKit output: this pixel type has no encoder input yet"); + return; + } ++offered; if (failed_flag.load()) return; // Pace on the capture clock, so poll jitter does not drop a frame that is due. A tenth of a // period of slack lets 60 fps capture feed a 30 fps track every second capture. - if (have_last && view.timestamp_ns < last_accepted_ns + period_ns * 9 / 10) + if (have_last && frame.monotonic_ns < last_accepted_ns + period_ns * 9 / 10) { ++paced_out; return; @@ -183,16 +190,16 @@ struct LiveKitVideoOutput::Impl // Complete before returning: the camera may rewrite its slot at the second following poll. Slot& slot = slots[target]; - check_cuda(cudaMemcpy2DAsync(slot.ptr, slot.pitch, reinterpret_cast(view.ptr), view.pitch, - static_cast(config.source_width) * 4, config.source_height, - cudaMemcpyDeviceToDevice, snapshot_stream), + check_cuda(cudaMemcpy2DAsync(slot.ptr, slot.pitch, reinterpret_cast(rgba->device_ptr), + rgba->pitch_bytes, static_cast(config.source_width) * 4, + config.source_height, cudaMemcpyDeviceToDevice, snapshot_stream), "snapshot copy"); check_cuda(cudaStreamSynchronize(snapshot_stream), "snapshot sync"); - slot.timestamp_ns = view.timestamp_ns; - slot.unix_ns = view.unix_ns; - slot.capture_tsc = view.capture_tsc; - slot.sequence = view.sequence; + slot.timestamp_ns = frame.monotonic_ns; + slot.unix_ns = frame.unix_ns; + slot.capture_tsc = frame.sensor_time; + slot.sequence = frame.sequence; pending = target; } catch (const std::exception& e) @@ -202,7 +209,7 @@ struct LiveKitVideoOutput::Impl } ++snapshots; have_last = true; - last_accepted_ns = view.timestamp_ns; + last_accepted_ns = frame.monotonic_ns; wake.notify_one(); } @@ -448,9 +455,9 @@ LiveKitVideoOutput::LiveKitVideoOutput(std::unique_ptr sink, con LiveKitVideoOutput::~LiveKitVideoOutput() = default; -void LiveKitVideoOutput::submit(const FrameView& view) +void LiveKitVideoOutput::submit(const CaptureFrame& frame) { - m_impl->submit(view); + m_impl->submit(frame); } LiveKitOutputStats LiveKitVideoOutput::stats() const diff --git a/src/plugins/sensing/core/livekit_video_output.hpp b/src/plugins/sensing/core/livekit_video_output.hpp index a0526c17cc..52a98c7e14 100644 --- a/src/plugins/sensing/core/livekit_video_output.hpp +++ b/src/plugins/sensing/core/livekit_video_output.hpp @@ -3,7 +3,7 @@ #pragma once -#include "sipl_camera.hpp" +#include "camera_capture.hpp" #include @@ -77,8 +77,9 @@ class LiveKitVideoOutput LiveKitVideoOutput(const LiveKitVideoOutput&) = delete; LiveKitVideoOutput& operator=(const LiveKitVideoOutput&) = delete; - /// Never blocks on the network. A view older than the publication rate allows is skipped. - void submit(const FrameView& view); + /// Never blocks on the network. A frame sooner than the publication rate allows is skipped. Only borrowed + /// CUDA RGBA pixels are accepted for now; they are copied out before this returns. + void submit(const CaptureFrame& frame); LiveKitOutputStats stats() const; diff --git a/src/plugins/sensing/core/sensing_camera.cpp b/src/plugins/sensing/core/sensing_camera.cpp index 0e1c0dcc77..103ee2c1ac 100644 --- a/src/plugins/sensing/core/sensing_camera.cpp +++ b/src/plugins/sensing/core/sensing_camera.cpp @@ -4,6 +4,7 @@ #include "sensing_camera.hpp" #include "livekit_video_output.hpp" +#include "sipl_capture.hpp" #include @@ -35,7 +36,7 @@ SiplConfig make_sipl_config(const SensingConfig& config) return sipl; } -LiveKitOutputConfig make_output_config(const SensingConfig& config, const SensorInfo& sensor) +LiveKitOutputConfig make_output_config(const SensingConfig& config, const CameraInfo& sensor) { LiveKitOutputConfig output; output.source_width = sensor.width; @@ -57,8 +58,8 @@ SensingCamera::SensingCamera(const SensingConfig& config, const std::vector(make_sipl_config(config)); - const auto& sensors = m_camera->sensors(); + m_camera = std::make_unique(make_sipl_config(config)); + const auto& sensors = m_camera->cameras(); // Everything that needs no network is checked first, so a bad name or divisor fails before any // connection is made. @@ -68,7 +69,7 @@ SensingCamera::SensingCamera(const SensingConfig& config, const std::vectorlatest(stream.sensor_id); + auto view = m_camera->acquire_latest(stream.sensor_id); if (view.has_value() && (!stream.have_last_sequence || view->sequence != stream.last_sequence)) { if (stream.have_last_sequence && view->sequence > stream.last_sequence + 1) diff --git a/src/plugins/sensing/core/sensing_camera.hpp b/src/plugins/sensing/core/sensing_camera.hpp index 0822da3460..b5271c0d41 100644 --- a/src/plugins/sensing/core/sensing_camera.hpp +++ b/src/plugins/sensing/core/sensing_camera.hpp @@ -3,8 +3,8 @@ #pragma once +#include "camera_capture.hpp" #include "sensing_types.hpp" -#include "sipl_camera.hpp" #include #include @@ -59,7 +59,7 @@ class SensingCamera }; SensingConfig m_config; - std::unique_ptr m_camera; + std::unique_ptr m_camera; /// Declared before the streams so it is destroyed after their tracks have been unpublished. std::unique_ptr m_livekit; std::vector m_streams; diff --git a/src/plugins/sensing/core/sipl_capture.cpp b/src/plugins/sensing/core/sipl_capture.cpp new file mode 100644 index 0000000000..9576b5db44 --- /dev/null +++ b/src/plugins/sensing/core/sipl_capture.cpp @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "sipl_capture.hpp" + +namespace plugins +{ +namespace sensing +{ + +SiplCapture::SiplCapture(const SiplConfig& config) : m_camera(config) +{ + for (const auto& sensor : m_camera.sensors()) + m_cameras.push_back({ sensor.id, sensor.name, sensor.width, sensor.height, sensor.fps }); +} + +const std::vector& SiplCapture::cameras() const +{ + return m_cameras; +} + +void SiplCapture::start() +{ + m_camera.start(); +} + +void SiplCapture::stop() +{ + m_camera.stop(); +} + +std::optional SiplCapture::acquire_latest(uint32_t camera_id) +{ + const auto view = m_camera.latest(camera_id); + if (!view) + return std::nullopt; + + CaptureFrame frame; + frame.camera_id = camera_id; + frame.sequence = view->sequence; + frame.width = view->width; + frame.height = view->height; + frame.monotonic_ns = view->timestamp_ns; + frame.unix_ns = view->unix_ns; + frame.sensor_time = view->capture_tsc; + frame.pixels = CudaRgbaPixels{ view->ptr, view->pitch }; + return frame; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/sipl_capture.hpp b/src/plugins/sensing/core/sipl_capture.hpp new file mode 100644 index 0000000000..dd4f4a86db --- /dev/null +++ b/src/plugins/sensing/core/sipl_capture.hpp @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// The SIPL backend behind the sender's capture interface. A thin adapter over SiplCamera: frames are the +// borrowed CUDA RGBA views SiplCamera already produces, passed on without a copy. + +#pragma once + +#include "camera_capture.hpp" +#include "sipl_camera.hpp" + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +class SiplCapture : public CameraCapture +{ +public: + explicit SiplCapture(const SiplConfig& config); + + const std::vector& cameras() const override; + void start() override; + void stop() override; + std::optional acquire_latest(uint32_t camera_id) override; + +private: + SiplCamera m_camera; + std::vector m_cameras; +}; + +} // namespace sensing +} // namespace plugins From 0d231f6f418ba73081abef3501cf8ecc619101fd Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 09:17:58 -0800 Subject: [PATCH 3/6] sensing: choose the camera backend once at startup, with no silent fallback select_camera_backend takes each backend's probe result: an explicit request must match or fail with its own reason; auto takes the only match and otherwise names the --camera-backend override. main.cpp parses the flag and probes SIPL, the only backend built in. Syntax-checked here; Jetson build not run. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/app/main.cpp | 36 +++++++- src/plugins/sensing/core/CMakeLists.txt | 1 + src/plugins/sensing/core/camera_backend.cpp | 93 +++++++++++++++++++++ src/plugins/sensing/core/camera_backend.hpp | 49 +++++++++++ 4 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 src/plugins/sensing/core/camera_backend.cpp create mode 100644 src/plugins/sensing/core/camera_backend.hpp diff --git a/src/plugins/sensing/app/main.cpp b/src/plugins/sensing/app/main.cpp index 242e2c7660..84cbf44968 100644 --- a/src/plugins/sensing/app/main.cpp +++ b/src/plugins/sensing/app/main.cpp @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#include "core/camera_backend.hpp" #include "core/sensing_camera.hpp" #include "core/sipl_camera.hpp" @@ -124,6 +125,8 @@ static void print_usage(const char* program_name) << "Streams (repeat for each sensor):\n" << " --add-stream=sensor=N,livekit=NAME SIPL pipeline index N, published as track NAME\n" << " --list-sensors print the pipelines, then exit\n" + << " --camera-backend=auto|sipl|argus auto picks the one backend that matches (default);\n" + << " only sipl is built in\n" << "\n" << "Platform:\n" << " --platform-config=PATH platform JSON (default: configs/shw5g.json beside this\n" @@ -164,6 +167,7 @@ try std::map stream_map; std::string livekit_token_file; bool list_sensors = false; + CameraBackend requested_backend = CameraBackend::Auto; for (int i = 1; i < argc; ++i) { @@ -200,6 +204,16 @@ try { list_sensors = true; } + else if (arg.find("--camera-backend=") == 0) + { + const auto parsed = parse_camera_backend(arg.substr(17)); + if (!parsed) + { + std::cerr << "Error: --camera-backend must be auto, argus or sipl." << std::endl; + return 1; + } + requested_backend = *parsed; + } else if (arg.find("--livekit-url=") == 0) { camera_config.livekit.url = arg.substr(14); @@ -269,8 +283,26 @@ try // Query is hardware-free -- it only parses the driver database and the JSON -- so --list-sensors // works on a rig whose cameras are unplugged, and the poll interval below can be derived before // anything is opened. - const auto sensors = SiplCamera::query( - camera_config.platform_config_json, camera_config.platform_config_name, camera_config.link_masks); + // Only SIPL is built in, so it is the only probe. Its query parses the configuration; it does not prove + // that the cameras are present, so a matching rig is a configuration that names sensors. + std::vector sensors; + BackendProbe sipl_probe; + sipl_probe.backend = CameraBackend::Sipl; + sipl_probe.available = true; + try + { + sensors = SiplCamera::query( + camera_config.platform_config_json, camera_config.platform_config_name, camera_config.link_masks); + sipl_probe.matches = !sensors.empty(); + if (!sipl_probe.matches) + sipl_probe.detail = "the platform configuration names no sensors"; + } + catch (const std::exception& e) + { + sipl_probe.detail = e.what(); + } + const CameraBackend backend = select_camera_backend(requested_backend, { sipl_probe }); + std::cout << "Camera backend: " << camera_backend_name(backend) << std::endl; if (list_sensors) { diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt index 4e091beea2..4f6e67157e 100644 --- a/src/plugins/sensing/core/CMakeLists.txt +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -6,6 +6,7 @@ add_library(sensing_core STATIC sensing_camera.cpp sipl_camera.cpp sipl_capture.cpp + camera_backend.cpp jetson_encoder.cpp livekit_video_output.cpp yuv_to_rgba.cu diff --git a/src/plugins/sensing/core/camera_backend.cpp b/src/plugins/sensing/core/camera_backend.cpp new file mode 100644 index 0000000000..1b9028b592 --- /dev/null +++ b/src/plugins/sensing/core/camera_backend.cpp @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "camera_backend.hpp" + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +std::optional parse_camera_backend(const std::string& text) +{ + if (text == "auto") + return CameraBackend::Auto; + if (text == "argus") + return CameraBackend::Argus; + if (text == "sipl") + return CameraBackend::Sipl; + return std::nullopt; +} + +const char* camera_backend_name(CameraBackend backend) +{ + switch (backend) + { + case CameraBackend::Auto: + return "auto"; + case CameraBackend::Argus: + return "argus"; + case CameraBackend::Sipl: + return "sipl"; + } + return "unknown"; +} + +namespace +{ + +std::string describe(const BackendProbe& probe) +{ + const std::string reason = probe.detail.empty() ? "no reason given" : probe.detail; + return std::string(camera_backend_name(probe.backend)) + + (probe.available ? " (found no matching cameras: " : " (unavailable: ") + reason + ")"; +} + +} // namespace + +CameraBackend select_camera_backend(CameraBackend requested, const std::vector& probes) +{ + if (requested != CameraBackend::Auto) + { + const auto probe = + std::find_if(probes.begin(), probes.end(), [&](const BackendProbe& p) { return p.backend == requested; }); + if (probe == probes.end()) + throw std::runtime_error(std::string("--camera-backend=") + camera_backend_name(requested) + + ": this build does not include that backend"); + if (!probe->matches) + throw std::runtime_error(std::string("--camera-backend=") + camera_backend_name(requested) + ": " + + describe(*probe)); + return requested; + } + + std::vector matching; + for (const auto& probe : probes) + { + if (probe.matches) + matching.push_back(probe.backend); + } + if (matching.size() == 1) + return matching.front(); + + std::string message; + if (matching.empty()) + { + message = "no camera backend matches the configured rig:"; + for (const auto& probe : probes) + message += " " + describe(probe); + message += ". Name the backend with --camera-backend=argus or --camera-backend=sipl"; + } + else + { + message = + "both camera backends match the configured rig; choose one with --camera-backend=argus or " + "--camera-backend=sipl"; + } + throw std::runtime_error(message); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/camera_backend.hpp b/src/plugins/sensing/core/camera_backend.hpp new file mode 100644 index 0000000000..263b0bd901 --- /dev/null +++ b/src/plugins/sensing/core/camera_backend.hpp @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Choosing the camera backend at startup. This is the decision only: probing the hardware is each backend's job, +// and its result arrives here as a BackendProbe. No vendor SDK is involved, so the rules are testable alone. + +#pragma once + +#include "camera_capture.hpp" + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/// What one backend found when asked whether it can open the configured rig. +struct BackendProbe +{ + CameraBackend backend = CameraBackend::Sipl; + /// The backend is built in and its SDK loaded; false means it cannot run here at all. + bool available = false; + /// The backend is available and found the configured cameras. + bool matches = false; + /// Why not, for the message when this backend was wanted: a missing library, no device, a bad configuration. + std::string detail; +}; + +/// "auto", "argus" or "sipl"; nullopt for anything else. +std::optional parse_camera_backend(const std::string& text); + +const char* camera_backend_name(CameraBackend backend); + +/** + * @brief The backend to open. + * + * An explicit request must match or the call throws with that backend's own reason: it never falls back to the + * other one. Auto selects the only matching backend; none matching, or both matching, throws and names the + * explicit override, because guessing which camera to open could take the wrong rig. + * + * @throws std::runtime_error with the reason. + */ +CameraBackend select_camera_backend(CameraBackend requested, const std::vector& probes); + +} // namespace sensing +} // namespace plugins From a1e0073676103a7179b0417aa11e0a5bfec21689 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 09:18:57 -0800 Subject: [PATCH 4/6] sensing: state the frame release and DMA-BUF contract, drop the unused encoder stamp Release callbacks must not throw and may run on any thread; a DMA-BUF frame is producer-complete pitch-linear NV12 on return. FrameMeta no longer carries a stamp the FIFO pairing does not use. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CMakeLists.txt | 4 ++++ src/plugins/sensing/core/camera_capture.hpp | 7 +++++-- src/plugins/sensing/core/livekit_video_output.cpp | 6 ++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index da64e4fbdc..826531830f 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -205,6 +205,10 @@ add_subdirectory(livekit) add_subdirectory(core) add_subdirectory(app) +if(BUILD_TESTING) + add_subdirectory(tests) +endif() + install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/plugin.yaml" "${CMAKE_CURRENT_SOURCE_DIR}/README.md" diff --git a/src/plugins/sensing/core/camera_capture.hpp b/src/plugins/sensing/core/camera_capture.hpp index 586cc420bf..d8a65edae9 100644 --- a/src/plugins/sensing/core/camera_capture.hpp +++ b/src/plugins/sensing/core/camera_capture.hpp @@ -47,7 +47,9 @@ struct CudaRgbaPixels }; /// NV12 in a DMA-BUF, as an Argus buffer stream produces it. Leased: valid until the frame's lease is released. -/// The plane layout is what the encoder must accept. +/// Contract: the producer has finished writing when the frame is returned, and the surface is pitch-linear NV12 with +/// the pitches and offsets below. A backend that cannot deliver that (block-linear, or still being written) must +/// convert or wait before returning, not describe it here for the encoder to guess. struct DmaBufNv12Pixels { int fd = -1; @@ -56,7 +58,8 @@ struct DmaBufNv12Pixels }; /// Returns a frame's buffer to its backend when destroyed or reset. Move-only: exactly one holder owns the buffer, -/// and the backend must outlive every lease it issued. +/// and the backend must outlive every lease it issued. The release callback must not throw (it runs from a +/// destructor), and it runs on whichever thread drops the lease, so it must be thread-safe. class FrameLease { public: diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index 9ea0b65224..e8473f8835 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -322,8 +322,7 @@ struct LiveKitVideoOutput::Impl ++encoder_busy; return; } - in_encoder.push_back( - { stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc, slot.unix_ns / 1000 }); + in_encoder.push_back({ static_cast(slot.sequence), slot.capture_tsc, slot.unix_ns / 1000 }); // The encoder holds a bounded number of frames, so a longer queue means it dropped some inside it. // Trimming bounds how stale the oldest metadata can be; it does not resynchronise units with captures, // so frame ids after an encoder-internal drop can be shifted until a supported per-frame correlation exists. @@ -379,10 +378,9 @@ struct LiveKitVideoOutput::Impl struct FrameMeta { - int64_t stamp_us = 0; uint32_t sequence = 0; uint64_t capture_tsc = 0; - /// Unix time of the same instant as stamp_us, for the frame's application timestamp. + /// Unix time when the frame was handed over, for the frame's application timestamp. uint64_t unix_us = 0; }; From f416a86b8ffdd0379223aed9cd8e00c9f373d132 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 09:20:27 -0800 Subject: [PATCH 5/6] sensing: unit tests for backend selection, report a stopped encoder from poll, document the flag Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 5 + src/plugins/sensing/core/jetson_encoder.cpp | 20 +++ src/plugins/sensing/tests/CMakeLists.txt | 16 ++ .../sensing/tests/test_camera_backend.cpp | 160 ++++++++++++++++++ 4 files changed, 201 insertions(+) create mode 100644 src/plugins/sensing/tests/CMakeLists.txt create mode 100644 src/plugins/sensing/tests/test_camera_backend.cpp diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 36316449ea..0ccec71a06 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -70,6 +70,11 @@ build/src/plugins/sensing/app/camera_plugin_sensing \ The `livekit=` names must equal the `track` names in the PC's streams file. Stop with Ctrl-C. `--help` lists every option (`--livekit-fps`, `--livekit-bitrate`, `--livekit-scale-div`, `--platform-config`, `--link-masks`). +`--camera-backend=auto|sipl|argus` selects the capture backend once at startup. Only SIPL is built in today: +`argus` fails with a message saying so, and `auto` picks the only matching backend (and, with several, names the +override instead of guessing). Argus capture is not implemented yet and has never been run: there is no Argus SDK +or Argus camera available to the author. + `SENSING_TSC_PROBE=1` logs the SIPL capture counter beside the host clocks twice a second, as separate uncalibrated samples. The counter's units are unconfirmed, so do not subtract or fit them. diff --git a/src/plugins/sensing/core/jetson_encoder.cpp b/src/plugins/sensing/core/jetson_encoder.cpp index 767079f077..b74082afeb 100644 --- a/src/plugins/sensing/core/jetson_encoder.cpp +++ b/src/plugins/sensing/core/jetson_encoder.cpp @@ -56,6 +56,8 @@ struct JetsonEncoder::Impl std::mutex mutex; std::deque ready; bool eos_reached = false; + /// Why the capture thread stopped early; guarded by mutex. + std::string error; bool eos_sent = false; // driving thread only ~Impl() @@ -67,13 +69,24 @@ struct JetsonEncoder::Impl delete encoder; } + // The capture thread stops after a failure; poll() reports it so the sender does not look healthy. + void fail(const char* what) + { + std::lock_guard lock(mutex); + if (error.empty()) + error = what; + } + // Capture-plane dequeue callback; runs on the encoder's own thread. static bool capture_dq(struct v4l2_buffer* v4l2_buf, NvBuffer* buffer, NvBuffer* /*shared*/, void* arg) { auto* self = static_cast(arg); if (!v4l2_buf) + { + self->fail("the capture plane returned no buffer"); return false; + } if (buffer && buffer->planes[0].bytesused > 0) { @@ -99,7 +112,10 @@ struct JetsonEncoder::Impl } if (self->encoder->capture_plane.qBuffer(*v4l2_buf, nullptr) < 0) + { + self->fail("requeueing a capture buffer failed"); return false; + } return true; } @@ -253,7 +269,11 @@ EncodedUnit JetsonEncoder::poll() { std::lock_guard lock(m_impl->mutex); if (m_impl->ready.empty()) + { + if (!m_impl->error.empty()) + throw std::runtime_error("JetsonEncoder: " + m_impl->error); return {}; + } auto out = std::move(m_impl->ready.front()); m_impl->ready.pop_front(); diff --git a/src/plugins/sensing/tests/CMakeLists.txt b/src/plugins/sensing/tests/CMakeLists.txt new file mode 100644 index 0000000000..966e8e9089 --- /dev/null +++ b/src/plugins/sensing/tests/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Backend selection and the frame lease are pure C++; compile camera_backend.cpp directly so the test links +# neither sensing_core (CUDA, SIPL, the Multimedia API) nor any vendor SDK. +add_executable(sensing_backend_tests + test_camera_backend.cpp + ../core/camera_backend.cpp +) + +# Reaches ../core headers as "core/..." without adding the plugin's own core/ include path. +target_include_directories(sensing_backend_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/..) + +target_link_libraries(sensing_backend_tests PRIVATE Catch2::Catch2WithMain) + +catch_discover_tests(sensing_backend_tests) diff --git a/src/plugins/sensing/tests/test_camera_backend.cpp b/src/plugins/sensing/tests/test_camera_backend.cpp new file mode 100644 index 0000000000..9b701100a5 --- /dev/null +++ b/src/plugins/sensing/tests/test_camera_backend.cpp @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "core/camera_backend.hpp" + +#include +#include + +#include +#include +#include +#include + +using Catch::Matchers::ContainsSubstring; +using plugins::sensing::BackendProbe; +using plugins::sensing::CameraBackend; +using plugins::sensing::FrameLease; +using plugins::sensing::parse_camera_backend; +using plugins::sensing::select_camera_backend; + +namespace +{ + +BackendProbe probe(CameraBackend backend, bool available, bool matches, std::string detail = "") +{ + BackendProbe p; + p.backend = backend; + p.available = available; + p.matches = matches; + p.detail = std::move(detail); + return p; +} + +std::string error_of(CameraBackend requested, const std::vector& probes) +{ + try + { + select_camera_backend(requested, probes); + } + catch (const std::runtime_error& e) + { + return e.what(); + } + FAIL("select_camera_backend did not throw"); + return {}; +} + +} // namespace + +TEST_CASE("parse_camera_backend accepts the three names and nothing else", "[sensing][backend]") +{ + CHECK(parse_camera_backend("auto") == CameraBackend::Auto); + CHECK(parse_camera_backend("argus") == CameraBackend::Argus); + CHECK(parse_camera_backend("sipl") == CameraBackend::Sipl); + CHECK_FALSE(parse_camera_backend("garbage")); + CHECK_FALSE(parse_camera_backend("")); + CHECK_FALSE(parse_camera_backend("SIPL")); +} + +TEST_CASE("Auto picks the only matching backend", "[sensing][backend]") +{ + CHECK(select_camera_backend(CameraBackend::Auto, { probe(CameraBackend::Argus, true, true), + probe(CameraBackend::Sipl, true, false, "no pipelines") }) == + CameraBackend::Argus); + CHECK(select_camera_backend(CameraBackend::Auto, { probe(CameraBackend::Argus, false, false, "no libargus"), + probe(CameraBackend::Sipl, true, true) }) == CameraBackend::Sipl); +} + +TEST_CASE("Auto with no match throws and names the override", "[sensing][backend]") +{ + const auto message = error_of(CameraBackend::Auto, { probe(CameraBackend::Argus, false, false, "no libargus"), + probe(CameraBackend::Sipl, true, false, "no pipelines") }); + CHECK_THAT(message, ContainsSubstring("--camera-backend")); + CHECK_THAT(message, ContainsSubstring("no libargus")); + CHECK_THAT(message, ContainsSubstring("no pipelines")); +} + +TEST_CASE("Auto with both matching throws and names the override", "[sensing][backend]") +{ + const auto message = error_of( + CameraBackend::Auto, { probe(CameraBackend::Argus, true, true), probe(CameraBackend::Sipl, true, true) }); + CHECK_THAT(message, ContainsSubstring("both")); + CHECK_THAT(message, ContainsSubstring("--camera-backend")); +} + +TEST_CASE("An explicit request that matches is returned", "[sensing][backend]") +{ + const std::vector both = { probe(CameraBackend::Argus, true, true), + probe(CameraBackend::Sipl, true, true) }; + CHECK(select_camera_backend(CameraBackend::Argus, both) == CameraBackend::Argus); + CHECK(select_camera_backend(CameraBackend::Sipl, both) == CameraBackend::Sipl); +} + +TEST_CASE("An explicit request that does not match throws with its own detail and never falls back", "[sensing][backend]") +{ + const auto message = error_of(CameraBackend::Sipl, { probe(CameraBackend::Argus, true, true), + probe(CameraBackend::Sipl, true, false, "sipl found nothing") }); + CHECK_THAT(message, ContainsSubstring("sipl found nothing")); + CHECK_THAT(message, ContainsSubstring("--camera-backend=sipl")); +} + +TEST_CASE("An explicit request for a backend missing from the probes throws", "[sensing][backend]") +{ + const auto message = error_of(CameraBackend::Argus, { probe(CameraBackend::Sipl, true, true) }); + CHECK_THAT(message, ContainsSubstring("--camera-backend=argus")); +} + +TEST_CASE("FrameLease releases once on destruction", "[sensing][lease]") +{ + int released = 0; + { + FrameLease lease([&] { ++released; }); + CHECK(released == 0); + } + CHECK(released == 1); +} + +TEST_CASE("A moved-from FrameLease does not release", "[sensing][lease]") +{ + int released = 0; + { + FrameLease a([&] { ++released; }); + { + FrameLease b(std::move(a)); + } + CHECK(released == 1); + } + CHECK(released == 1); +} + +TEST_CASE("FrameLease reset releases once, then not again", "[sensing][lease]") +{ + int released = 0; + { + FrameLease lease([&] { ++released; }); + lease.reset(); + CHECK(released == 1); + lease.reset(); + CHECK(released == 1); + } + CHECK(released == 1); +} + +TEST_CASE("FrameLease move-assign releases the previous target first", "[sensing][lease]") +{ + std::vector order; + { + FrameLease a([&] { order.push_back(1); }); + FrameLease b([&] { order.push_back(2); }); + a = std::move(b); + REQUIRE(order == std::vector{ 1 }); + } + CHECK(order == std::vector{ 1, 2 }); +} + +TEST_CASE("A default FrameLease releases nothing", "[sensing][lease]") +{ + FrameLease lease; + lease.reset(); +} From 75ba3392d4877a12c6b03a12d587f9676dc37a27 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 09:21:53 -0800 Subject: [PATCH 6/6] sensing: report a failed encoder input dequeue instead of counting it as a dropped frame Only EAGAIN and EINTR are the ordinary no-buffer result; any other errno is recorded for poll() to surface. Relies on the Multimedia API plane leaving errno set by the failed ioctl, unverified here (no Jetson); the first Orin run shows whether a healthy encoder trips it. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/jetson_encoder.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/plugins/sensing/core/jetson_encoder.cpp b/src/plugins/sensing/core/jetson_encoder.cpp index b74082afeb..1a687332d5 100644 --- a/src/plugins/sensing/core/jetson_encoder.cpp +++ b/src/plugins/sensing/core/jetson_encoder.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -233,7 +234,12 @@ bool JetsonEncoder::submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_byte } else if (enc->output_plane.dqBuffer(v4l2_buf, &buffer, nullptr, 0) < 0) { - return false; // encoder still busy; drop this frame rather than block + // The plane logs every failure but EAGAIN, which is the non-blocking "no input buffer free yet": drop this + // frame rather than block. Any other errno is the device failing, which poll() reports. + const int err = errno; + if (err != EAGAIN && err != EINTR) + m_impl->fail((std::string("dequeueing an input buffer failed: ") + std::strerror(err)).c_str()); + return false; } // An input buffer is ours now, so convert: a frame that would be dropped costs no conversion. A