sensing: report camera faults and offer the management channel from the producer

CMakeLists.txt

@@ -73,6 +73,7 @@ option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C
 option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF)
 option(BUILD_PLUGIN_SENSING "Build SENSING GMSL camera plugin (Jetson only; requires SIPL and NVENC)" OFF)
 option(BUILD_VIDEO_STAGE "Build the video stage: the Orin-to-PC video transport and the PC relay" OFF)
+option(BUILD_SENSING_CONTROL "Build the camera producer management channel (no camera SDK needed; builds and tests on a PC)" OFF)
 option(BUILD_EXAMPLES "Build examples" ON)
 option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF)
 option(BUILD_TESTING "Build unit tests" ON)

src/CMakeLists.txt

@@ -36,6 +36,11 @@ elseif(BUILD_PLUGIN_SENSING)
     add_subdirectory(video_stage/sender)
 endif()
 
+# The camera producer's management channel: no video stage, camera SDK or GPU dependency.
+if(BUILD_PLUGIN_SENSING OR BUILD_SENSING_CONTROL)
+    add_subdirectory(sensing_control)
+endif()
+
 if(BUILD_PLUGINS)
     add_subdirectory(plugins)
 endif()

src/plugins/sensing/README.md

@@ -54,3 +54,25 @@ guessing). SIPL is built by default (`-DSENSING_WITH_SIPL=OFF` for an Argus-only
 DMA-BUFs that the hardware encoder reads without a copy. **The Argus code has never been built or run: there is no
 Argus SDK or Argus camera available to the author.** Argus DMA-BUF input cannot be combined with
 `--scale-div` above 1. An Argus-only build (`-DSENSING_WITH_SIPL=OFF -DSENSING_WITH_ARGUS=ON`) also builds on JetPack 6.2 / L4T R36.4.3.
+
+## Management channel and camera faults
+
+A camera that cannot start (no power cable, no GMSL link) does not end the producer. It reports the problem in plain
+words, retries every few seconds (`--retry-seconds`), and starts streaming once the cable is fixed. The same process
+can offer a WebSocket management channel, off unless `--control-token-file=PATH` names a file holding a shared secret:
+
+```bash
+camera_plugin_sensing ... --control-token-file=/etc/isaaccapture/control.token   # listens on 0.0.0.0:5610
+```
+
+Clients send one JSON object per message and get one reply; the producer also pushes `state_changed` and `error`
+events. `get_status` reports the state (`starting`, `streaming`, `camera_fault`, `error`, `stopping`), the fault with
+a code and what to check, each stream's frame counts, and where its direct link goes. `set_direct_target` sends a
+stream's encoded video to a new PC address and port from now on (an empty host turns the direct link off); the choice
+is remembered across camera restarts but not across a producer restart, where the command line applies. Every bad
+request gets an error reply with a code and a sentence. The token goes in `Authorization: Bearer ...` (or `?token=`).
+
+```bash
+src/sensing_control/tools/orin_control.py ORIN_HOST --token-file control.token status
+src/sensing_control/tools/orin_control.py ORIN_HOST --token-file control.token set-target sensing-left 192.168.1.205 5600
+```

src/plugins/sensing/app/CMakeLists.txt

@@ -5,7 +5,7 @@ add_executable(camera_plugin_sensing main.cpp)
 
 set_target_properties(camera_plugin_sensing PROPERTIES INSTALL_RPATH "$ORIGIN")
 
-target_link_libraries(camera_plugin_sensing PRIVATE sensing::core)
+target_link_libraries(camera_plugin_sensing PRIVATE sensing::core sensing::control)
 
 # The plugin resolves configs/shw5g.json relative to its own executable, so the
 # config has to sit beside the binary in the build tree as well as the install
@@ -17,14 +17,19 @@ add_custom_command(TARGET camera_plugin_sensing POST_BUILD
     COMMENT "Staging SIPL platform configs beside camera_plugin_sensing"
 )
 
+# Everything below is the `producer` install component: `cmake --install <build> --component producer` stages
+# exactly what scripts/deploy_orin.py packages for an Orin.
 install(TARGETS camera_plugin_sensing
     RUNTIME DESTINATION plugins/sensing_camera
+    COMPONENT producer
 )
 install(DIRECTORY "${SENSING_CONFIGS_DIR}/"
     DESTINATION plugins/sensing_camera/configs
+    COMPONENT producer
 )
 
 # The vendored encoder classes are compiled into the binary, so their license text ships with it.
 install(FILES "${SENSING_MMAPI_VENDORED_DIR}/LICENSE"
     DESTINATION plugins/sensing_camera/licenses/jetson-multimedia-api
+    COMPONENT producer
 )

src/plugins/sensing/app/main.cpp

@@ -1,6 +1,11 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+#include <sensing_control/camera_fault.hpp>
+#include <sensing_control/control_handler.hpp>
+#include <sensing_control/control_server.hpp>
+#include <sensing_control/driver_log_capture.hpp>
+#include <sensing_control/producer_supervisor.hpp>
 #include <sensing_core/camera_backend.hpp>
 #include <sensing_core/capture_factory.hpp>
 #include <sensing_core/sensing_camera.hpp>
@@ -21,6 +26,7 @@
 #include <fstream>
 #include <iostream>
 #include <map>
+#include <memory>
 #include <sstream>
 #include <string>
 #include <thread>
@@ -143,7 +149,8 @@ static void print_usage(const char* program_name)
         << "Streams (repeat for each sensor):\n"
         << "  --add-stream=sensor=N,name=NAME[,direct=HOST:PORT]\n"
         << "                                       camera N (SIPL or Argus device), called NAME;\n"
-        << "                                       direct= is where the PC video stage listens for it\n"
+        << "                                       direct= is where the PC video stage listens for it (it can also be\n"
+        << "                                       set later over the management channel)\n"
         << "  --list-sensors                       print the cameras, then exit\n"
         << "  --camera-backend=auto|sipl|argus     auto picks the one backend that matches (default);\n"
         << "                                       (a backend must also be built in)\n"
@@ -170,6 +177,12 @@ static void print_usage(const char* program_name)
         << "  --bitrate=N               target bitrate of each stream's encoder in bps (default: 8000000)\n"
         << "  --gop=N                   IDR period in frames (default: fps*5)\n"
         << "\n"
+        << "Management channel (a WebSocket server; off unless a token file is given):\n"
+        << "  --control-token-file=PATH file holding the shared secret clients must present\n"
+        << "  --control-port=N          port to listen on (default: 5610)\n"
+        << "  --control-bind=ADDR       IPv4 address to listen on (default: 0.0.0.0)\n"
+        << "  --retry-seconds=N         wait between attempts to start the cameras (default: 5)\n"
+        << "\n"
         << "Other:\n"
         << "  --gpu-id=N          CUDA device index (default: 0)\n"
         << "  --isp0-buffers=N    ISP0 buffers per sensor (default: 6)\n"
@@ -181,6 +194,85 @@ static void print_usage(const char* program_name)
         << "      --add-stream=sensor=1,name=sensing-right,direct=192.168.1.205:5601\n";
 }
 
+// =============================================================================
+// Producer session and release
+// =============================================================================
+
+/// The release this binary was installed from (".../releases/<id>/..."), or a placeholder when run by hand.
+static std::string release_id()
+{
+    char buffer[PATH_MAX];
+    const ssize_t length = ::readlink("/proc/self/exe", buffer, sizeof(buffer) - 1);
+    if (length <= 0)
+        return "unknown";
+    const std::string path(buffer, static_cast<size_t>(length));
+    const std::string marker = "/releases/";
+    const auto at = path.find(marker);
+    if (at == std::string::npos)
+        return "unmanaged";
+    const auto start = at + marker.size();
+    return path.substr(start, path.find('/', start) - start);
+}
+
+/// One capture-and-publish pipeline as the supervisor sees it.
+class ProducerSession : public CameraSession
+{
+public:
+    ProducerSession(const SensingConfig& config,
+                    std::unique_ptr<CameraCapture> capture,
+                    const std::vector<StreamConfig>& streams)
+        : m_camera(config, std::move(capture), streams), m_last_stats(std::chrono::steady_clock::now())
+    {
+        std::cout << "------------------------------------------------------------" << std::endl;
+        std::cout << "Capturing." << std::endl;
+    }
+
+    void update() override
+    {
+        m_camera.update();
+        const auto now = std::chrono::steady_clock::now();
+        if (now - m_last_stats >= std::chrono::seconds(5))
+        {
+            m_camera.print_stats();
+            m_last_stats = now;
+        }
+    }
+
+    void flush() override
+    {
+        m_camera.flush();
+        m_camera.print_stats();
+    }
+
+    std::vector<StreamStatus> streams() const override
+    {
+        std::vector<StreamStatus> out;
+        for (const auto& figures : m_camera.figures())
+        {
+            StreamStatus entry;
+            entry.track = figures.track;
+            entry.sensor_id = figures.sensor_id;
+            entry.frames = figures.frames;
+            entry.published = figures.published;
+            entry.direct_host = figures.direct.host;
+            entry.direct_port = figures.direct.port;
+            entry.direct_connected = figures.direct.connected;
+            entry.direct_error = figures.direct.error;
+            out.push_back(std::move(entry));
+        }
+        return out;
+    }
+
+    void set_direct_target(const std::string& track, const std::string& host, uint16_t port) override
+    {
+        m_camera.set_direct_target(track, host, port);
+    }
+
+private:
+    SensingCamera m_camera;
+    std::chrono::steady_clock::time_point m_last_stats;
+};
+
 // =============================================================================
 // Main
 // =============================================================================
@@ -190,6 +282,9 @@ try
 {
     SensingConfig camera_config;
     std::map<uint32_t, StreamConfig> stream_map;
+    ControlServerConfig control_config;
+    std::string control_token_file;
+    uint32_t retry_seconds = 5;
     bool list_sensors = false;
     CameraBackend requested_backend = CameraBackend::Auto;
 
@@ -242,6 +337,25 @@ try
             }
             requested_backend = *parsed;
         }
+        else if (arg.find("--control-token-file=") == 0)
+        {
+            control_token_file = arg.substr(21);
+        }
+        else if (arg.find("--control-port=") == 0)
+        {
+            const unsigned long port = std::stoul(arg.substr(15));
+            if (port < 1 || port > 65535)
+                throw std::runtime_error("--control-port must be from 1 to 65535");
+            control_config.port = static_cast<uint16_t>(port);
+        }
+        else if (arg.find("--control-bind=") == 0)
+        {
+            control_config.bind_address = arg.substr(15);
+        }
+        else if (arg.find("--retry-seconds=") == 0)
+        {
+            retry_seconds = static_cast<uint32_t>(std::stoul(arg.substr(16)));
+        }
         else if (arg.find("--fps=") == 0)
         {
             camera_config.encode.fps = static_cast<uint32_t>(std::stoul(arg.substr(6)));
@@ -426,9 +540,10 @@ try
     {
         const bool any_direct = std::any_of(
             stream_configs.begin(), stream_configs.end(), [](const StreamConfig& s) { return !s.direct_host.empty(); });
-        if (!any_direct)
-            std::cerr << "Warning: no stream has a direct=HOST:PORT destination: nothing will receive the video. "
-                         "Give --add-stream a direct=HOST:PORT."
+        if (!any_direct && control_token_file.empty())
+            std::cerr << "Warning: no stream has a direct=HOST:PORT destination and there is no management channel: "
+                         "nothing will receive the video. Give --add-stream a direct=HOST:PORT or set "
+                         "--control-token-file."
                       << std::endl;
     }
 
@@ -439,42 +554,104 @@ try
     std::cout << "SENSING Camera Plugin Starting (" << camera_backend_name(backend) << ")" << std::endl;
     std::cout << "============================================================" << std::endl;
 
-    SensingCamera camera(camera_config, make_capture(backend, camera_config, camera_ids), stream_configs);
-
-    std::cout << "------------------------------------------------------------" << std::endl;
-    std::cout << "Running capture loop. Press Ctrl+C to stop." << std::endl;
-
-    constexpr auto stats_interval = std::chrono::seconds(5);
-    auto last_stats_time = std::chrono::steady_clock::now();
+    // A stream naming a sensor that does not exist is a configuration error, not a fault to retry.
+    for (const auto& stream : stream_configs)
+    {
+        const bool known =
+            std::any_of(cameras.begin(), cameras.end(), [&](const CameraInfo& c) { return c.id == stream.sensor_id; });
+        if (!known)
+            throw std::runtime_error("sensor=" + std::to_string(stream.sensor_id) + " is not a camera of '" +
+                                     camera_config.platform_config_name + "'; run --list-sensors");
+    }
 
     // acquire_latest() is non-blocking, so poll at roughly twice the fastest sensor's frame rate rather than
-    // spinning a core flat out. SIPL captures on its own threads and is unaffected; with Argus this loop also
+    // spinning a core flat out. SIPL captures on its own threads and is unaffected; with Argus the polling loop also
     // returns buffers to the camera, so a stalled loop stalls capture.
     double fastest_fps = 30.0;
     for (const auto& c : cameras)
-    {
         fastest_fps = std::max(fastest_fps, c.fps);
-    }
-    const auto poll_interval = std::chrono::microseconds(static_cast<int64_t>(500'000.0 / fastest_fps));
 
-    while (!g_stop_requested.load(std::memory_order_relaxed))
+    ProducerStatus status(release_id());
+    std::vector<StreamStatus> configured;
+    for (const auto& stream : stream_configs)
     {
-        camera.update();
+        StreamStatus entry;
+        entry.track = stream.name;
+        entry.sensor_id = stream.sensor_id;
+        entry.direct_host = stream.direct_host;
+        entry.direct_port = stream.direct_port;
+        configured.push_back(entry);
+    }
 
-        auto now = std::chrono::steady_clock::now();
-        if (now - last_stats_time >= stats_interval)
+    // The cameras and the room are opened by the factory, again after any failure, so an unplugged camera is a state
+    // to report and retry, not the end of the process.
+    const SessionFactory factory = [&](const DirectTargets& targets) -> std::unique_ptr<CameraSession>
+    {
+        std::vector<StreamConfig> streams = stream_configs;
+        for (auto& stream : streams)
         {
-            camera.print_stats();
-            last_stats_time = now;
+            const auto found = targets.find(stream.name);
+            if (found != targets.end())
+            {
+                stream.direct_host = found->second.host;
+                stream.direct_port = found->second.port;
+            }
+        }
+        // The camera driver explains a failure only in its own log lines, so they are collected while the cameras
+        // start and attached to the failure; the fault table reads them (an open circuit, a lock timeout, an I2C NACK).
+        DriverLogCapture driver_log;
+        try
+        {
+            auto capture = make_capture(backend, camera_config, camera_ids);
+            auto session = std::make_unique<ProducerSession>(camera_config, std::move(capture), streams);
+            driver_log.finish();
+            return session;
+        }
+        catch (const std::exception& e)
+        {
+            const std::string lines = driver_log.finish();
+            throw std::runtime_error(lines.empty() ? std::string(e.what()) :
+                                                     std::string(e.what()) + " | driver log: " + lines);
         }
+    };
 
-        std::this_thread::sleep_for(poll_interval);
+    // Destruction runs in reverse declaration order, so the server (declared last) is stopped and joined while the
+    // handler, supervisor and status its thread calls into still exist, on return and on exception alike.
+    // `channel` is what the supervisor publishes through; it is set once the server exists.
+    ControlServer* channel = nullptr;
+    const auto publish = [&channel](const std::string& text)
+    {
+        if (channel != nullptr)
+            channel->broadcast(text);
+    };
+    SupervisorConfig supervisor_config;
+    supervisor_config.retry_interval = std::chrono::seconds(std::max<uint32_t>(retry_seconds, 1));
+    supervisor_config.poll_interval = std::chrono::milliseconds(static_cast<int64_t>(500.0 / fastest_fps));
+    ProducerSupervisor supervisor(status, configured, factory, publish, supervisor_config);
+    ControlHandler handler(status, supervisor);
+    std::unique_ptr<ControlServer> server;
+
+    if (control_token_file.empty())
+    {
+        std::cout << "Management channel: off (give --control-token-file to turn it on)" << std::endl;
+    }
+    else
+    {
+        control_config.token = read_secret_file(control_token_file);
+        server = std::make_unique<ControlServer>(
+            control_config, [&handler](const std::string& text) { return handler.handle(text); });
+        channel = server.get();
+        std::cout << "Management channel: listening on " << control_config.bind_address << ":" << server->port()
+                  << std::endl;
     }
 
     std::cout << "------------------------------------------------------------" << std::endl;
-    std::cout << "Shutting down SENSING Camera Plugin..." << std::endl;
-    camera.flush();
-    camera.print_stats();
+    std::cout << "Running. Press Ctrl+C to stop." << std::endl;
+
+    supervisor.run([] { return g_stop_requested.load(std::memory_order_relaxed); });
+    channel = nullptr; // nothing publishes once the run has ended
+
+    std::cout << "------------------------------------------------------------" << std::endl;
     std::cout << "Plugin stopped" << std::endl;
     std::cout << "============================================================" << std::endl;