sensing: capture boundary and backend selection ahead of Argus (draft, Argus untested)

src/plugins/sensing/CMakeLists.txt

@@ -205,6 +205,10 @@ add_subdirectory(livekit)
 add_subdirectory(core)
 add_subdirectory(app)
 
+if(BUILD_TESTING)
+    add_subdirectory(tests)
+endif()
+
 install(FILES
     "${CMAKE_CURRENT_SOURCE_DIR}/plugin.yaml"
     "${CMAKE_CURRENT_SOURCE_DIR}/README.md"

src/plugins/sensing/README.md

@@ -70,6 +70,11 @@ build/src/plugins/sensing/app/camera_plugin_sensing \
 
 The `livekit=` names must equal the `track` names in the PC's streams file. Stop with Ctrl-C. `--help` lists every
 option (`--livekit-fps`, `--livekit-bitrate`, `--livekit-scale-div`, `--platform-config`, `--link-masks`).
+`--camera-backend=auto|sipl|argus` selects the capture backend once at startup. Only SIPL is built in today:
+`argus` fails with a message saying so, and `auto` picks the only matching backend (and, with several, names the
+override instead of guessing). Argus capture is not implemented yet and has never been run: there is no Argus SDK
+or Argus camera available to the author.
+
 `SENSING_TSC_PROBE=1` logs the SIPL capture counter beside the host clocks twice a second, as separate uncalibrated
 samples. The counter's units are unconfirmed, so do not subtract or fit them.
 

src/plugins/sensing/app/main.cpp

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+#include "core/camera_backend.hpp"
 #include "core/sensing_camera.hpp"
 #include "core/sipl_camera.hpp"
 
@@ -124,6 +125,8 @@ static void print_usage(const char* program_name)
               << "Streams (repeat for each sensor):\n"
               << "  --add-stream=sensor=N,livekit=NAME   SIPL pipeline index N, published as track NAME\n"
               << "  --list-sensors                       print the pipelines, then exit\n"
+              << "  --camera-backend=auto|sipl|argus     auto picks the one backend that matches (default);\n"
+              << "                                       only sipl is built in\n"
               << "\n"
               << "Platform:\n"
               << "  --platform-config=PATH  platform JSON (default: configs/shw5g.json beside this\n"
@@ -164,6 +167,7 @@ try
     std::map<uint32_t, StreamConfig> stream_map;
     std::string livekit_token_file;
     bool list_sensors = false;
+    CameraBackend requested_backend = CameraBackend::Auto;
 
     for (int i = 1; i < argc; ++i)
     {
@@ -200,6 +204,16 @@ try
         {
             list_sensors = true;
         }
+        else if (arg.find("--camera-backend=") == 0)
+        {
+            const auto parsed = parse_camera_backend(arg.substr(17));
+            if (!parsed)
+            {
+                std::cerr << "Error: --camera-backend must be auto, argus or sipl." << std::endl;
+                return 1;
+            }
+            requested_backend = *parsed;
+        }
         else if (arg.find("--livekit-url=") == 0)
         {
             camera_config.livekit.url = arg.substr(14);
@@ -269,8 +283,26 @@ try
     // Query is hardware-free -- it only parses the driver database and the JSON -- so --list-sensors
     // works on a rig whose cameras are unplugged, and the poll interval below can be derived before
     // anything is opened.
-    const auto sensors = SiplCamera::query(
-        camera_config.platform_config_json, camera_config.platform_config_name, camera_config.link_masks);
+    // Only SIPL is built in, so it is the only probe. Its query parses the configuration; it does not prove
+    // that the cameras are present, so a matching rig is a configuration that names sensors.
+    std::vector<SensorInfo> sensors;
+    BackendProbe sipl_probe;
+    sipl_probe.backend = CameraBackend::Sipl;
+    sipl_probe.available = true;
+    try
+    {
+        sensors = SiplCamera::query(
+            camera_config.platform_config_json, camera_config.platform_config_name, camera_config.link_masks);
+        sipl_probe.matches = !sensors.empty();
+        if (!sipl_probe.matches)
+            sipl_probe.detail = "the platform configuration names no sensors";
+    }
+    catch (const std::exception& e)
+    {
+        sipl_probe.detail = e.what();
+    }
+    const CameraBackend backend = select_camera_backend(requested_backend, { sipl_probe });
+    std::cout << "Camera backend: " << camera_backend_name(backend) << std::endl;
 
     if (list_sensors)
     {

src/plugins/sensing/core/CMakeLists.txt

@@ -5,6 +5,8 @@
 add_library(sensing_core STATIC
     sensing_camera.cpp
     sipl_camera.cpp
+    sipl_capture.cpp
+    camera_backend.cpp
     jetson_encoder.cpp
     livekit_video_output.cpp
     yuv_to_rgba.cu

src/plugins/sensing/core/camera_backend.cpp

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "camera_backend.hpp"
+
+#include <algorithm>
+#include <stdexcept>
+
+namespace plugins
+{
+namespace sensing
+{
+
+std::optional<CameraBackend> parse_camera_backend(const std::string& text)
+{
+    if (text == "auto")
+        return CameraBackend::Auto;
+    if (text == "argus")
+        return CameraBackend::Argus;
+    if (text == "sipl")
+        return CameraBackend::Sipl;
+    return std::nullopt;
+}
+
+const char* camera_backend_name(CameraBackend backend)
+{
+    switch (backend)
+    {
+    case CameraBackend::Auto:
+        return "auto";
+    case CameraBackend::Argus:
+        return "argus";
+    case CameraBackend::Sipl:
+        return "sipl";
+    }
+    return "unknown";
+}
+
+namespace
+{
+
+std::string describe(const BackendProbe& probe)
+{
+    const std::string reason = probe.detail.empty() ? "no reason given" : probe.detail;
+    return std::string(camera_backend_name(probe.backend)) +
+           (probe.available ? " (found no matching cameras: " : " (unavailable: ") + reason + ")";
+}
+
+} // namespace
+
+CameraBackend select_camera_backend(CameraBackend requested, const std::vector<BackendProbe>& probes)
+{
+    if (requested != CameraBackend::Auto)
+    {
+        const auto probe =
+            std::find_if(probes.begin(), probes.end(), [&](const BackendProbe& p) { return p.backend == requested; });
+        if (probe == probes.end())
+            throw std::runtime_error(std::string("--camera-backend=") + camera_backend_name(requested) +
+                                     ": this build does not include that backend");
+        if (!probe->matches)
+            throw std::runtime_error(std::string("--camera-backend=") + camera_backend_name(requested) + ": " +
+                                     describe(*probe));
+        return requested;
+    }
+
+    std::vector<CameraBackend> matching;
+    for (const auto& probe : probes)
+    {
+        if (probe.matches)
+            matching.push_back(probe.backend);
+    }
+    if (matching.size() == 1)
+        return matching.front();
+
+    std::string message;
+    if (matching.empty())
+    {
+        message = "no camera backend matches the configured rig:";
+        for (const auto& probe : probes)
+            message += " " + describe(probe);
+        message += ". Name the backend with --camera-backend=argus or --camera-backend=sipl";
+    }
+    else
+    {
+        message =
+            "both camera backends match the configured rig; choose one with --camera-backend=argus or "
+            "--camera-backend=sipl";
+    }
+    throw std::runtime_error(message);
+}
+
+} // namespace sensing
+} // namespace plugins

src/plugins/sensing/core/camera_backend.hpp

@@ -0,0 +1,49 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Choosing the camera backend at startup. This is the decision only: probing the hardware is each backend's job,
+// and its result arrives here as a BackendProbe. No vendor SDK is involved, so the rules are testable alone.
+
+#pragma once
+
+#include "camera_capture.hpp"
+
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace plugins
+{
+namespace sensing
+{
+
+/// What one backend found when asked whether it can open the configured rig.
+struct BackendProbe
+{
+    CameraBackend backend = CameraBackend::Sipl;
+    /// The backend is built in and its SDK loaded; false means it cannot run here at all.
+    bool available = false;
+    /// The backend is available and found the configured cameras.
+    bool matches = false;
+    /// Why not, for the message when this backend was wanted: a missing library, no device, a bad configuration.
+    std::string detail;
+};
+
+/// "auto", "argus" or "sipl"; nullopt for anything else.
+std::optional<CameraBackend> parse_camera_backend(const std::string& text);
+
+const char* camera_backend_name(CameraBackend backend);
+
+/**
+ * @brief The backend to open.
+ *
+ * An explicit request must match or the call throws with that backend's own reason: it never falls back to the
+ * other one. Auto selects the only matching backend; none matching, or both matching, throws and names the
+ * explicit override, because guessing which camera to open could take the wrong rig.
+ *
+ * @throws std::runtime_error with the reason.
+ */
+CameraBackend select_camera_backend(CameraBackend requested, const std::vector<BackendProbe>& probes);
+
+} // namespace sensing
+} // namespace plugins

src/plugins/sensing/core/camera_capture.hpp

@@ -0,0 +1,137 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// The capture boundary of the sender: what a camera backend (SIPL, Argus) hands to the encoder. No vendor SDK
+// type appears here, so the sender can select a backend at startup without including either SDK.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <optional>
+#include <string>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace plugins
+{
+namespace sensing
+{
+
+enum class CameraBackend
+{
+    Auto,
+    Argus,
+    Sipl,
+};
+
+/// One camera as a backend reports it. `id` is the backend's own index (a SIPL pipeline index, an Argus device
+/// index); the two are unrelated, so configuration names cameras by logical stream, not by id.
+struct CameraInfo
+{
+    uint32_t id = 0;
+    std::string name;
+    uint32_t width = 0;
+    uint32_t height = 0;
+    double fps = 0.0;
+};
+
+/// RGBA8 in GPU memory, as the SIPL backend converts it. Borrowed: the backend recycles the buffer, so a consumer
+/// copies it out before its next acquire_latest() for the same camera, and the frame's lease is empty.
+struct CudaRgbaPixels
+{
+    uintptr_t device_ptr = 0;
+    size_t pitch_bytes = 0;
+};
+
+/// NV12 in a DMA-BUF, as an Argus buffer stream produces it. Leased: valid until the frame's lease is released.
+/// Contract: the producer has finished writing when the frame is returned, and the surface is pitch-linear NV12 with
+/// the pitches and offsets below. A backend that cannot deliver that (block-linear, or still being written) must
+/// convert or wait before returning, not describe it here for the encoder to guess.
+struct DmaBufNv12Pixels
+{
+    int fd = -1;
+    uint32_t pitch_bytes[2] = { 0, 0 };
+    uint32_t offset_bytes[2] = { 0, 0 };
+};
+
+/// Returns a frame's buffer to its backend when destroyed or reset. Move-only: exactly one holder owns the buffer,
+/// and the backend must outlive every lease it issued. The release callback must not throw (it runs from a
+/// destructor), and it runs on whichever thread drops the lease, so it must be thread-safe.
+class FrameLease
+{
+public:
+    FrameLease() = default;
+    explicit FrameLease(std::function<void()> release) : m_release(std::move(release))
+    {
+    }
+    FrameLease(FrameLease&& other) noexcept : m_release(std::exchange(other.m_release, nullptr))
+    {
+    }
+    FrameLease& operator=(FrameLease&& other) noexcept
+    {
+        if (this != &other)
+        {
+            reset();
+            m_release = std::exchange(other.m_release, nullptr);
+        }
+        return *this;
+    }
+    FrameLease(const FrameLease&) = delete;
+    FrameLease& operator=(const FrameLease&) = delete;
+    ~FrameLease()
+    {
+        reset();
+    }
+
+    void reset()
+    {
+        if (m_release)
+        {
+            std::exchange(m_release, nullptr)();
+        }
+    }
+
+private:
+    std::function<void()> m_release;
+};
+
+/// One captured frame. Pixels that are leased stay valid until `lease` is destroyed or reset, so a consumer that
+/// needs them longer (an encoder holding an input buffer) keeps the lease; borrowed pixels (see CudaRgbaPixels)
+/// are valid only until the next acquire_latest() for the same camera.
+struct CaptureFrame
+{
+    uint32_t camera_id = 0;
+    uint64_t sequence = 0;
+    uint32_t width = 0;
+    uint32_t height = 0;
+    /// CLOCK_MONOTONIC and CLOCK_REALTIME read together when the backend had the frame ready: processing time,
+    /// not the sensor's capture time.
+    uint64_t monotonic_ns = 0;
+    uint64_t unix_ns = 0;
+    /// The backend's own sensor timestamp, unscaled and in its own clock domain; never convert it to another
+    /// domain here. Zero when the backend reports none.
+    uint64_t sensor_time = 0;
+    std::variant<CudaRgbaPixels, DmaBufNv12Pixels> pixels;
+    FrameLease lease;
+};
+
+/// A backend instance owns its whole rig: SIPL hands out one process-wide object, so there is one instance per
+/// backend, never one per camera.
+class CameraCapture
+{
+public:
+    virtual ~CameraCapture() = default;
+
+    virtual const std::vector<CameraInfo>& cameras() const = 0;
+    virtual void start() = 0;
+    /// Stops capture. Frames already handed out stay valid until their leases are released.
+    virtual void stop() = 0;
+    /// The newest frame of `camera_id` not yet taken, or nullopt when none has arrived. Never blocks.
+    virtual std::optional<CaptureFrame> acquire_latest(uint32_t camera_id) = 0;
+};
+
+} // namespace sensing
+} // namespace plugins