From cb2f9cae49b1cf193e67e04b614c4cf4190cdb14 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 14:08:43 -0500 Subject: [PATCH 1/2] sensing_control: add the camera producer's management channel library An authenticated WebSocket server, a JSON request handler, the status a producer reports, a supervisor that retries a camera that cannot start, a plain-language fault table and a capture of the camera driver's own log lines. It has no video, camera-SDK or GPU dependency, so it builds and its unit tests run on a PC with BUILD_SENSING_CONTROL=ON. A standard-library client, tools/orin_control.py, drives the channel from the command line. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- CMakeLists.txt | 1 + src/CMakeLists.txt | 5 + src/sensing_control/CMakeLists.txt | 52 ++ src/sensing_control/camera_fault.cpp | 182 ++++ src/sensing_control/control_handler.cpp | 153 ++++ src/sensing_control/control_server.cpp | 377 +++++++++ src/sensing_control/driver_log_capture.cpp | 122 +++ .../inc/sensing_control/camera_fault.hpp | 38 + .../inc/sensing_control/control_handler.hpp | 70 ++ .../inc/sensing_control/control_server.hpp | 88 ++ .../sensing_control/driver_log_capture.hpp | 46 ++ .../inc/sensing_control/producer_status.hpp | 90 ++ .../sensing_control/producer_supervisor.hpp | 143 ++++ .../inc/sensing_control/websocket.hpp | 94 +++ src/sensing_control/producer_status.cpp | 79 ++ src/sensing_control/producer_supervisor.cpp | 236 ++++++ src/sensing_control/tests/CMakeLists.txt | 21 + .../tests/test_camera_fault.cpp | 181 ++++ .../tests/test_control_handler.cpp | 280 +++++++ .../tests/test_control_server.cpp | 764 +++++++++++++++++ .../tests/test_driver_log_capture.cpp | 233 ++++++ .../tests/test_producer_status.cpp | 84 ++ .../tests/test_producer_supervisor.cpp | 780 ++++++++++++++++++ src/sensing_control/tests/test_websocket.cpp | 317 +++++++ src/sensing_control/tools/orin_control.py | 205 +++++ src/sensing_control/websocket.cpp | 369 +++++++++ 26 files changed, 5010 insertions(+) create mode 100644 src/sensing_control/CMakeLists.txt create mode 100644 src/sensing_control/camera_fault.cpp create mode 100644 src/sensing_control/control_handler.cpp create mode 100644 src/sensing_control/control_server.cpp create mode 100644 src/sensing_control/driver_log_capture.cpp create mode 100644 src/sensing_control/inc/sensing_control/camera_fault.hpp create mode 100644 src/sensing_control/inc/sensing_control/control_handler.hpp create mode 100644 src/sensing_control/inc/sensing_control/control_server.hpp create mode 100644 src/sensing_control/inc/sensing_control/driver_log_capture.hpp create mode 100644 src/sensing_control/inc/sensing_control/producer_status.hpp create mode 100644 src/sensing_control/inc/sensing_control/producer_supervisor.hpp create mode 100644 src/sensing_control/inc/sensing_control/websocket.hpp create mode 100644 src/sensing_control/producer_status.cpp create mode 100644 src/sensing_control/producer_supervisor.cpp create mode 100644 src/sensing_control/tests/CMakeLists.txt create mode 100644 src/sensing_control/tests/test_camera_fault.cpp create mode 100644 src/sensing_control/tests/test_control_handler.cpp create mode 100644 src/sensing_control/tests/test_control_server.cpp create mode 100644 src/sensing_control/tests/test_driver_log_capture.cpp create mode 100644 src/sensing_control/tests/test_producer_status.cpp create mode 100644 src/sensing_control/tests/test_producer_supervisor.cpp create mode 100644 src/sensing_control/tests/test_websocket.cpp create mode 100755 src/sensing_control/tools/orin_control.py create mode 100644 src/sensing_control/websocket.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b02232bd7..08800bda6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetc option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF) option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF) option(BUILD_PLUGIN_SENSING "Build SENSING GMSL camera plugin (Jetson only; requires SIPL, NVENC and the LiveKit SDK)" OFF) +option(BUILD_SENSING_CONTROL "Build the camera producer management channel (no camera SDK needed; builds and tests on a PC)" OFF) option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) option(BUILD_TESTING "Build unit tests" ON) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cdef8fe380..a95b15c86d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,6 +25,11 @@ if(BUILD_VIZ) add_subdirectory(viz) endif() +# The camera producer's management channel: no video stage, camera SDK or GPU dependency. +if(BUILD_SENSING_CONTROL) + add_subdirectory(sensing_control) +endif() + if(BUILD_PLUGINS) add_subdirectory(plugins) endif() diff --git a/src/sensing_control/CMakeLists.txt b/src/sensing_control/CMakeLists.txt new file mode 100644 index 0000000000..4432a45d25 --- /dev/null +++ b/src/sensing_control/CMakeLists.txt @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The camera producer's management channel: an authenticated WebSocket server, the request protocol, the status the +# producer reports, and plain-language camera fault messages. No CUDA or SIPL, so it builds and is tested on a PC. +include(FetchContent) +FetchContent_Declare( + nlohmann_json + GIT_REPOSITORY https://github.com/nlohmann/json.git + GIT_TAG v3.11.3 + GIT_SHALLOW TRUE +) +set(JSON_BuildTests OFF CACHE INTERNAL "") +isaac_teleop_third_party_scope_begin() +FetchContent_MakeAvailable(nlohmann_json) +isaac_teleop_third_party_scope_end() + +add_library(sensing_control STATIC + websocket.cpp + producer_status.cpp + camera_fault.cpp + control_handler.cpp + control_server.cpp + producer_supervisor.cpp + driver_log_capture.cpp + inc/sensing_control/websocket.hpp + inc/sensing_control/producer_status.hpp + inc/sensing_control/camera_fault.hpp + inc/sensing_control/control_handler.hpp + inc/sensing_control/control_server.hpp + inc/sensing_control/producer_supervisor.hpp + inc/sensing_control/driver_log_capture.hpp +) +add_library(sensing::control ALIAS sensing_control) + +set_target_properties(sensing_control PROPERTIES POSITION_INDEPENDENT_CODE ON) + +target_include_directories(sensing_control + INTERFACE + $ +) + +target_link_libraries(sensing_control + PUBLIC + log_bridge::log_bridge_core + PRIVATE + nlohmann_json::nlohmann_json +) + +if(BUILD_TESTING) + add_subdirectory(tests) +endif() diff --git a/src/sensing_control/camera_fault.cpp b/src/sensing_control/camera_fault.cpp new file mode 100644 index 0000000000..26226c9b6a --- /dev/null +++ b/src/sensing_control/camera_fault.cpp @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_control/camera_fault.hpp" + +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +struct Signature +{ + /// A fragment of the text the component was seen to produce for that situation. + std::string_view needle; + std::string_view code; + ProducerState state; + std::string_view message; +}; + +constexpr ProducerState kCamera = ProducerState::CameraFault; +constexpr ProducerState kOther = ProducerState::Error; + +constexpr std::string_view kPowerAndCables = + "Check that each camera's power cable is connected and that its GMSL cable is seated at both ends."; + +// Most specific first; the first needle found wins. Extend this table, not the logic, when a new failure is seen. +// Messages say what to check; the original text is appended by diagnose_failure(). +constexpr Signature kSignatures[] = { + // --- What the capture driver itself says about an absent camera (its log lines are attached to the failure) -- + { "OPEN_CIRCUIT", "camera_power_missing", kCamera, + "A camera's power switch reports an open circuit: the camera is not drawing power. Connect the camera's power " + "cable." }, + { "CIRCUIT_OPEN", "camera_power_missing", kCamera, + "A camera's power switch reports an open circuit: the camera is not drawing power. Connect the camera's power " + "cable." }, + { "lock timeout", "camera_link_not_locked", kCamera, + "A camera's GMSL link did not lock. The camera is unpowered or its GMSL cable is not connected. Check its power " + "cable and that its GMSL cable is seated at both ends." }, + { "DES_LOCK_ERR", "camera_link_not_locked", kCamera, + "A camera's GMSL link lost lock. The camera lost power or its GMSL cable came loose. Check its power cable and " + "that its GMSL cable is seated at both ends." }, + { "CAMERA_DETECT", "camera_not_detected", kCamera, + "A camera was not detected. It is unpowered or its GMSL cable is not connected. Check its power cable and that " + "its GMSL cable is seated at both ends." }, + { "Parse failed after", "camera_not_responding", kCamera, + "A camera stopped answering while it was being set up. It is unpowered or its GMSL cable is not connected. Check " + "its power cable and that its GMSL cable is seated at both ends." }, + { "SIPLStatus 8", "camera_partly_started", kCamera, + "Some cameras started and at least one did not. Check the power cable and GMSL cable of the camera that is " + "missing." }, + // --- A camera is gone or never came up --------------------------------------------------------------------- + { "sensor control write failure", "camera_control_bus_failed", kCamera, + "A camera's control bus is not answering (settings cannot be written to the sensor). The camera lost power or " + "its GMSL cable came loose. Check its power cable and GMSL cable." }, + { "input stream failure", "camera_stream_failed", kCamera, + "A camera's video stream is corrupt or missing. Check its GMSL cable and connectors for damage or a loose " + "fit, and its power." }, + { "image processing failure", "camera_isp_failed", kCamera, + "The camera image pipeline failed. Check the camera's cables; if it persists, restart the producer." }, + { "interrupt localization failure", "camera_hardware_fault", kCamera, + "The camera hardware reported a fault that could not be traced. Power-cycle the cameras." }, + { "deserializer failure", "camera_link_failed", kCamera, + "A GMSL deserializer reported a failure. A camera lost power or its GMSL cable came loose. Check each camera's " + "power cable and GMSL cable." }, + { "serializer failure", "camera_link_failed", kCamera, + "A camera's serializer stopped responding. The camera lost power or its GMSL cable came loose. Check its power " + "cable and GMSL cable." }, + { "sensor failure", "camera_sensor_failed", kCamera, + "A camera's image sensor reported a failure. Check the camera's power and cables; if it persists, power-cycle " + "the camera." }, + { "link lock", "camera_link_not_locked", kCamera, + "A camera's GMSL link did not lock. Check each camera's power cable and that its GMSL cable is seated at both " + "ends." }, + { "Failed to detect", "camera_not_detected", kCamera, + "A camera was not detected. Check each camera's power cable and that its GMSL cable is seated at both ends." }, + { "no frame for", "camera_no_frames", kCamera, + "A camera stopped delivering frames. Check its power and GMSL cable; the producer restarts it automatically." }, + { "INvSIPLCamera::Init failed", "camera_start_failed", kCamera, + "The camera system could not start. The usual cause is a camera with no power or no GMSL link. Check each " + "camera's power cable and GMSL cable." }, + { "INvSIPLCamera::SetPlatformCfg failed", "camera_config_rejected", kCamera, + "The camera system rejected the platform configuration. Check --platform-config, --config-name and " + "--link-masks against the installed camera drivers." }, + { "INvSIPLCamera::Start failed", "camera_start_failed", kCamera, + "The cameras initialised but would not start streaming. Check each camera's power and GMSL cable and try " + "again; if it persists, power-cycle the cameras." }, + { "SIPLStatus 10", "camera_start_failed", kCamera, + "The camera system reported an error. The usual cause is a camera with no power or no GMSL link. Check each " + "camera's power cable and GMSL cable." }, + { "pipeline error", "camera_pipeline_error", kCamera, + "A camera pipeline reported an error while streaming. Check each camera's power cable and GMSL cable." }, + { "capture error", "camera_capture_error", kCamera, + "Capturing from a camera failed. Check each camera's power cable and GMSL cable." }, + { "no modules left in", "camera_link_mask_mismatch", kCamera, + "No camera matches the configured link masks. Check --link-masks and --config-name against how the cameras " + "are wired." }, + { "declares no sensors", "camera_config_empty", kCamera, + "The platform configuration names no cameras. Check --platform-config and --config-name." }, + { "no SIPL pipeline with index", "camera_not_in_config", kCamera, + "A requested sensor is not in the platform configuration. Run --list-sensors for the valid numbers." }, + { "link masks are required", "camera_config_missing", kCamera, + "The camera configuration needs --link-masks (one mask per deserializer)." }, + { "cannot open NITO file", "camera_tuning_missing", kCamera, + "The camera's image-processing tuning file is missing. Install the camera tuning files (see --nito-dir)." }, + { "short read on NITO file", "camera_tuning_invalid", kCamera, + "The camera's image-processing tuning file is truncated or corrupt. Reinstall the camera tuning files." }, + { "Argus: no camera provider", "camera_service_down", kCamera, + "The Argus camera service is not running. Start it with: sudo systemctl start nvargus-daemon." }, + { "Argus: no camera device", "camera_not_detected", kCamera, + "The requested Argus camera was not found. Check the camera's cable and power and the --add-stream sensor " + "number." }, + { "Argus: no camera", "camera_not_detected", kCamera, + "The requested Argus camera was not found. Check the camera's cable and power and the --add-stream sensor " + "number." }, + { "Argus:", "camera_error", kCamera, + "The Argus camera stack reported an error. Check the camera's cable and power; the Argus service " + "(nvargus-daemon) may need a restart." }, + + // --- Frames could not reach the GPU or the encoder ---------------------------------------------------------- + { "NvBufSurfaceMapEglImage", "gpu_buffer_error", kCamera, + "Camera frames could not be mapped for the GPU. This is usually the graphics environment: the producer must " + "not run with DISPLAY set to a remote X server, and the Tegra EGL libraries must be installed." }, + { "NvmmNvSciBufToNvBufSurface", "gpu_buffer_error", kCamera, + "Camera frames could not be converted for the GPU. Check that the Jetson multimedia libraries match the " + "installed L4T release." }, + { "unsupported CUDA EGL frame type", "gpu_buffer_error", kCamera, + "The camera delivered a frame format the GPU conversion does not support. Check the camera mode and L4T " + "release." }, + { "CUDA", "gpu_error", kOther, + "A CUDA call failed. Check that no other process is using the GPU and that the driver matches the installed " + "L4T release." }, + { "createVideoEncoder failed", "encoder_unavailable", kOther, + "The hardware video encoder is not available (/dev/v4l2-nvenc is missing or the driver is not loaded). Check " + "the L4T installation." }, + { "JetsonEncoder", "encoder_error", kOther, + "The hardware video encoder reported an error. Another process may be holding the encoder; if it persists, " + "reduce --bitrate or the stream count." }, + + { "bad_alloc", "out_of_memory", kOther, + "The producer ran out of memory. Close other programs on the Orin or reduce the number of streams." }, + { "Permission denied", "permission_denied", kOther, + "The producer is not allowed to use a device or file. Check that its service user is in the video group." }, + { "No such file or directory", "file_missing", kOther, + "A file or device the producer needs is missing. Check the paths in its configuration." }, + { "Device or resource busy", "device_busy", kOther, + "A camera or encoder device is in use by another process. Stop any other camera program (for example a " + "second copy of this producer) and try again." }, +}; + +} // namespace + +Diagnosis diagnose_failure(std::string_view what) +{ + for (const auto& signature : kSignatures) + { + if (what.find(signature.needle) != std::string_view::npos) + { + return { { std::string(signature.code), std::string(signature.message) + + " The producer retries automatically. (" + std::string(what) + + ")" }, + signature.state }; + } + } + // Unknown text: still a code and the original words, and a camera fault because a start-up failure that nothing + // else explains is most often the cameras. + return { { "startup_failed", + "The producer could not start. The producer retries automatically. (" + std::string(what) + ")" }, + ProducerState::CameraFault }; +} + +Fault diagnose_camera_failure(std::string_view what) +{ + return diagnose_failure(what).fault; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/control_handler.cpp b/src/sensing_control/control_handler.cpp new file mode 100644 index 0000000000..8b939607bd --- /dev/null +++ b/src/sensing_control/control_handler.cpp @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_control/control_handler.hpp" + +#include +#include + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +using Json = nlohmann::json; + +Json error_object(const std::string& code, const std::string& message) +{ + return Json{ { "code", code }, { "message", message } }; +} + +std::string failure(const Json& id, const std::string& code, const std::string& message) +{ + return Json{ { "id", id }, { "ok", false }, { "error", error_object(code, message) } }.dump(); +} + +/// An IP literal or a plain host name; anything else (spaces, slashes, a URL) is refused with a reason. +bool valid_host(const std::string& host) +{ + if (host.empty() || host.size() > 253) + return false; + unsigned char buffer[sizeof(in6_addr)]; + if (::inet_pton(AF_INET, host.c_str(), buffer) == 1 || ::inet_pton(AF_INET6, host.c_str(), buffer) == 1) + return true; + return std::all_of(host.begin(), host.end(), [](unsigned char c) { return std::isalnum(c) || c == '.' || c == '-'; }) && + host.front() != '-' && host.front() != '.'; +} + +} // namespace + +std::string status_json(const StatusSnapshot& snapshot) +{ + Json streams = Json::array(); + for (const auto& stream : snapshot.streams) + { + streams.push_back(Json{ + { "stream", stream.track }, + { "sensor", stream.sensor_id }, + { "direct", Json{ { "host", stream.direct_host }, + { "port", stream.direct_port }, + { "connected", stream.direct_connected }, + { "error", stream.direct_error } } }, + { "frames", stream.frames }, + { "published", stream.published }, + }); + } + Json out{ + { "state", to_string(snapshot.state) }, + { "release", snapshot.release }, + { "uptime_seconds", snapshot.uptime_seconds }, + { "streams", streams }, + }; + if (snapshot.fault) + out["fault"] = error_object(snapshot.fault->code, snapshot.fault->message); + return out.dump(); +} + +std::string state_event(const StatusSnapshot& snapshot) +{ + return Json{ + { "type", "event" }, { "event", "state_changed" }, { "status", Json::parse(status_json(snapshot)) } + }.dump(); +} + +std::string error_event(const Fault& fault) +{ + return Json{ { "type", "event" }, { "event", "error" }, { "error", error_object(fault.code, fault.message) } }.dump(); +} + +ControlHandler::ControlHandler(const ProducerStatus& status, ControlActions& actions) + : m_status(status), m_actions(actions) +{ +} + +std::string ControlHandler::handle(const std::string& text) const +{ + Json request = Json::parse(text, nullptr, /*allow_exceptions=*/false); + if (request.is_discarded() || !request.is_object()) + return failure(nullptr, "bad_json", "the message must be one JSON object"); + + const Json id = request.contains("id") ? request["id"] : Json(nullptr); + if (!request.contains("type") || !request["type"].is_string()) + return failure(id, "missing_type", "the request needs a string \"type\""); + const std::string type = request["type"]; + + if (type == "ping") + return Json{ { "id", id }, { "ok", true }, { "type", "pong" } }.dump(); + + if (type == "get_status") + { + return Json{ + { "id", id }, { "ok", true }, { "type", "status" }, { "status", Json::parse(status_json(m_status.snapshot())) } + }.dump(); + } + + if (type == "set_direct_target") + { + if (!request.contains("stream") || !request["stream"].is_string()) + return failure(id, "missing_stream", "set_direct_target needs a string \"stream\" (a track name)"); + if (!request.contains("host") || !request["host"].is_string()) + return failure(id, "missing_host", "set_direct_target needs a string \"host\" (\"\" turns the link off)"); + const std::string stream = request["stream"]; + const std::string host = request["host"]; + + std::uint16_t port = 0; + if (!host.empty()) + { + if (!valid_host(host)) + return failure(id, "invalid_host", "\"host\" must be an IP address or a host name, not \"" + host + "\""); + if (!request.contains("port") || !request["port"].is_number_integer() || request["port"] < 1 || + request["port"] > 65535) + return failure(id, "invalid_port", "\"port\" must be an integer from 1 to 65535"); + port = request["port"].get(); + } + + const auto known = m_status.snapshot().streams; + if (std::none_of(known.begin(), known.end(), [&](const StreamStatus& s) { return s.track == stream; })) + { + std::string names; + for (const auto& s : known) + names += (names.empty() ? "" : ", ") + s.track; + return failure(id, "unknown_stream", "no stream named \"" + stream + "\"; the streams are: " + names); + } + + const ActionResult result = m_actions.set_direct_target(stream, host, port); + if (!result.ok) + return failure(id, result.code, result.message); + return Json{ + { "id", id }, { "ok", true }, { "type", "status" }, { "status", Json::parse(status_json(m_status.snapshot())) } + }.dump(); + } + + return failure( + id, "unknown_type", "unknown request type \"" + type + "\"; supported: ping, get_status, set_direct_target"); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/control_server.cpp b/src/sensing_control/control_server.cpp new file mode 100644 index 0000000000..36b95b37c9 --- /dev/null +++ b/src/sensing_control/control_server.cpp @@ -0,0 +1,377 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_control/control_server.hpp" + +#include "inc/sensing_control/websocket.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +std::shared_ptr logger() +{ + return isaaccapture::Logger::get("isaaccapture.plugins.sensing.ControlServer"); +} + +constexpr std::size_t kMaxHandshakeBytes = 8 * 1024; +/// Room beyond one maximum message for the frame header, the mask and a following control frame. +constexpr std::size_t kFrameHeadroom = 4096; +constexpr std::size_t kMaxOutputBytes = 1 << 20; +/// Frames handled per client per poll round, so one chatty peer cannot hold the server thread. +constexpr int kMaxFramesPerRound = 16; +constexpr auto kHandshakeTimeout = std::chrono::seconds(5); + +void set_non_blocking(int fd) +{ + ::fcntl(fd, F_SETFL, ::fcntl(fd, F_GETFL, 0) | O_NONBLOCK); +} + +} // namespace + +std::string read_secret_file(const std::string& path) +{ + std::ifstream file(path); + if (!file) + throw std::runtime_error("cannot read " + path); + std::string secret; + std::getline(file, secret); + const auto last = secret.find_last_not_of(" \t\r\n"); + secret.erase(last == std::string::npos ? 0 : last + 1); + const auto first = secret.find_first_not_of(" \t"); + secret.erase(0, first == std::string::npos ? secret.size() : first); + if (secret.empty()) + throw std::runtime_error(path + " is empty"); + return secret; +} + +struct ControlServer::Client +{ + int fd = -1; + bool open = false; ///< the handshake completed + bool closing = false; ///< flush the output, then drop the connection + std::string input; + std::string output; + std::string message; ///< a fragmented text message being assembled + bool fragmenting = false; ///< a text message was started without its final fragment + std::string peer; + std::chrono::steady_clock::time_point accepted = std::chrono::steady_clock::now(); +}; + +ControlServer::ControlServer(const ControlServerConfig& config, MessageHandler on_message) + : m_config(config), m_on_message(std::move(on_message)) +{ + if (m_config.token.empty()) + throw std::runtime_error("the management channel needs a token: it can redirect camera video"); + + m_listen_fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (m_listen_fd < 0) + throw std::runtime_error(std::string("management channel: cannot create a socket: ") + std::strerror(errno)); + const int one = 1; + ::setsockopt(m_listen_fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_port = htons(m_config.port); + if (::inet_pton(AF_INET, m_config.bind_address.c_str(), &address.sin_addr) != 1) + { + ::close(m_listen_fd); + throw std::runtime_error("management channel: \"" + m_config.bind_address + "\" is not an IPv4 address"); + } + if (::bind(m_listen_fd, reinterpret_cast(&address), sizeof(address)) != 0 || ::listen(m_listen_fd, 16) != 0) + { + const std::string why = std::strerror(errno); + ::close(m_listen_fd); + throw std::runtime_error("management channel: cannot listen on " + m_config.bind_address + ":" + + std::to_string(m_config.port) + ": " + why); + } + socklen_t length = sizeof(address); + ::getsockname(m_listen_fd, reinterpret_cast(&address), &length); + m_port = ntohs(address.sin_port); + set_non_blocking(m_listen_fd); + m_wake_fd = ::eventfd(0, EFD_NONBLOCK); + + logger()->info("management channel listening on {}:{}", m_config.bind_address, m_port); + m_thread = std::thread(&ControlServer::run, this); +} + +ControlServer::~ControlServer() +{ + m_stop = true; + wake(); + if (m_thread.joinable()) + m_thread.join(); + ::close(m_listen_fd); + ::close(m_wake_fd); +} + +void ControlServer::wake() +{ + const std::uint64_t one = 1; + [[maybe_unused]] const ssize_t written = ::write(m_wake_fd, &one, sizeof(one)); +} + +void ControlServer::broadcast(const std::string& text) +{ + { + std::lock_guard lock(m_mutex); + // Notifications are advisory: if nobody drains them, keep only the newest. + if (m_broadcasts.size() >= 64) + m_broadcasts.erase(m_broadcasts.begin()); + m_broadcasts.push_back(text); + } + wake(); +} + +void ControlServer::run() +{ + std::vector> clients; + + const auto drop = [&](std::size_t index, const char* why) + { + logger()->info("management client {} disconnected: {}", clients[index]->peer, why); + ::close(clients[index]->fd); + clients.erase(clients.begin() + static_cast(index)); + }; + const auto refuse = [&](Client& client, const websocket::Handshake& handshake) + { + client.output += websocket::handshake_response(handshake); + client.closing = true; + }; + + while (!m_stop) + { + std::vector fds; + fds.push_back({ m_listen_fd, POLLIN, 0 }); + fds.push_back({ m_wake_fd, POLLIN, 0 }); + for (const auto& client : clients) + fds.push_back({ client->fd, static_cast(POLLIN | (client->output.empty() ? 0 : POLLOUT)), 0 }); + + const int ready = ::poll(fds.data(), fds.size(), 500); + if (ready < 0 && errno != EINTR) + { + logger()->error("management channel poll failed: {}", std::strerror(errno)); + break; + } + + if (fds[1].revents & POLLIN) + { + std::uint64_t counter = 0; + [[maybe_unused]] const ssize_t got = ::read(m_wake_fd, &counter, sizeof(counter)); + std::vector pending; + { + std::lock_guard lock(m_mutex); + pending.swap(m_broadcasts); + } + for (const auto& text : pending) + for (auto& client : clients) + if (client->open && !client->closing) + client->output += websocket::encode_frame(websocket::kText, text); + } + + if (fds[0].revents & POLLIN) + { + while (true) + { + sockaddr_in peer{}; + socklen_t length = sizeof(peer); + const int fd = ::accept(m_listen_fd, reinterpret_cast(&peer), &length); + if (fd < 0) + break; + if (clients.size() >= m_config.max_clients) + { + ::close(fd); + continue; + } + set_non_blocking(fd); + const int one = 1; + ::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + auto client = std::make_unique(); + client->fd = fd; + char text[INET_ADDRSTRLEN] = {}; + ::inet_ntop(AF_INET, &peer.sin_addr, text, sizeof(text)); + client->peer = std::string(text) + ":" + std::to_string(ntohs(peer.sin_port)); + clients.push_back(std::move(client)); + } + } + + // The clients that were polled are the first ones in the list; those accepted just now wait for the next turn. + const std::size_t polled = fds.size() - 2; + for (std::size_t i = polled; i-- > 0;) + { + Client& client = *clients[i]; + const short events = fds[i + 2].revents; + bool dead = (events & (POLLERR | POLLNVAL)) != 0; + + if (!dead && (events & (POLLIN | POLLHUP))) + { + // Read only what the protocol could still accept: the rest waits in the kernel (TCP backpressure) until + // the frames already buffered are handled. A peer that streams without end therefore cannot grow this + // buffer or keep the thread here. + const std::size_t input_cap = + client.open ? m_config.max_message_bytes + kFrameHeadroom : kMaxHandshakeBytes + 1; + char buffer[4096]; + while (client.input.size() < input_cap) + { + const ssize_t n = ::recv(client.fd, buffer, sizeof(buffer), 0); + if (n > 0) + { + client.input.append(buffer, static_cast(n)); + continue; + } + if (n == 0 || (errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) + dead = true; + break; + } + } + + // Handshake: one HTTP request, then WebSocket frames. + if (!dead && !client.open && !client.closing) + { + const std::size_t end = client.input.find("\r\n\r\n"); + if (end != std::string::npos) + { + websocket::Handshake handshake = websocket::parse_handshake(client.input.substr(0, end + 4)); + client.input.erase(0, end + 4); + if (handshake.ok && !websocket::constant_time_equal(handshake.token, m_config.token)) + { + handshake.ok = false; + handshake.status = 401; + handshake.reason = "a valid token is required"; + logger()->warn("management client {} refused: missing or wrong token", client.peer); + } + if (handshake.ok) + { + client.open = true; + m_clients.fetch_add(1); + client.output += websocket::handshake_response(handshake); + logger()->info("management client {} connected", client.peer); + } + else + { + refuse(client, handshake); + } + } + else if (client.input.size() > kMaxHandshakeBytes) + { + websocket::Handshake bad; + bad.reason = "request too large"; + refuse(client, bad); + } + else if (std::chrono::steady_clock::now() - client.accepted > kHandshakeTimeout) + { + dead = true; + } + } + + for (int handled = 0; !dead && client.open && !client.closing && handled < kMaxFramesPerRound; ++handled) + { + if (client.output.size() > kMaxOutputBytes) + { + dead = true; // a client that does not read its replies is cut off rather than buffered for + break; + } + const auto parsed = websocket::parse_frame(client.input, m_config.max_message_bytes); + if (parsed.status == websocket::ParseStatus::NeedMore) + break; + if (parsed.status == websocket::ParseStatus::Error) + { + client.output += websocket::encode_close(parsed.close_code, parsed.error); + client.closing = true; + break; + } + client.input.erase(0, parsed.consumed); + const auto& frame = parsed.frame; + if (frame.opcode == websocket::kPing) + { + client.output += websocket::encode_frame(websocket::kPong, frame.payload); + } + else if (frame.opcode == websocket::kClose) + { + client.output += websocket::encode_close(1000, ""); + client.closing = true; + } + else if (frame.opcode == websocket::kText || frame.opcode == websocket::kContinuation) + { + const bool continuation = frame.opcode == websocket::kContinuation; + // A message is one text frame, or a text frame without FIN followed by continuations ending + // with FIN. Anything else (an orphan continuation, a new message inside an unfinished one) is a + // protocol error. + if (continuation != client.fragmenting) + { + client.output += websocket::encode_close(1002, "bad fragmentation"); + client.closing = true; + continue; + } + client.message += frame.payload; + client.fragmenting = !frame.fin; + if (client.message.size() > m_config.max_message_bytes) + { + client.output += websocket::encode_close(1009, "message too big"); + client.closing = true; + } + else if (frame.fin) + { + const std::string request = std::move(client.message); + client.message.clear(); + client.output += websocket::encode_frame(websocket::kText, m_on_message(request)); + } + } + else if (frame.opcode == websocket::kBinary) + { + client.output += websocket::encode_close(1003, "text messages only"); + client.closing = true; + } + } + + if (!dead && !client.output.empty()) + { + const ssize_t n = ::send(client.fd, client.output.data(), client.output.size(), MSG_NOSIGNAL); + if (n > 0) + client.output.erase(0, static_cast(n)); + else if (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) + dead = true; + if (client.output.size() > kMaxOutputBytes) + dead = true; // a client that does not read is cut off rather than buffered for + } + + if (dead || (client.closing && client.output.empty())) + { + if (client.open) + m_clients.fetch_sub(1); + drop(i, dead ? "connection ended" : "closed"); + } + } + } + + for (const auto& client : clients) + { + if (client->open) + m_clients.fetch_sub(1); + ::close(client->fd); + } +} + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/driver_log_capture.cpp b/src/sensing_control/driver_log_capture.cpp new file mode 100644 index 0000000000..c220c0a148 --- /dev/null +++ b/src/sensing_control/driver_log_capture.cpp @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_control/driver_log_capture.hpp" + +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +bool looks_like_failure(const std::string& line) +{ + static const char* const kWords[] = { "ERROR", "error", "fail", "Fail", "timeout", + "OPEN_CIRCUIT", "CIRCUIT_OPEN", "DES_LOCK_ERR", "CAMERA_DETECT" }; + return std::any_of( + std::begin(kWords), std::end(kWords), [&](const char* word) { return line.find(word) != std::string::npos; }); +} + +/// Drop the driver's own prefix noise ("[TID=1234] Module_id 30 Severity 2 : ") so the useful words lead. +std::string trim_prefix(const std::string& line) +{ + const auto at = line.find(" : "); + return (line.find("Severity") != std::string::npos && at != std::string::npos) ? line.substr(at + 3) : line; +} + +} // namespace + +DriverLogCapture::DriverLogCapture() +{ + std::fflush(stdout); + std::fflush(stderr); + char path[] = "/tmp/sensing-driver-log-XXXXXX"; + m_file = ::mkstemp(path); + if (m_file < 0) + return; // no capture: start-up still works, only the driver's words are lost + ::unlink(path); // the file lives only as long as the descriptor + m_saved_out = ::dup(STDOUT_FILENO); + m_saved_err = ::dup(STDERR_FILENO); + if (m_saved_out < 0 || m_saved_err < 0 || ::dup2(m_file, STDOUT_FILENO) < 0 || ::dup2(m_file, STDERR_FILENO) < 0) + { + if (m_saved_out >= 0) + ::dup2(m_saved_out, STDOUT_FILENO); + if (m_saved_err >= 0) + ::dup2(m_saved_err, STDERR_FILENO); + return; + } + m_active = true; +} + +DriverLogCapture::~DriverLogCapture() +{ + finish(); + if (m_saved_out >= 0) + ::close(m_saved_out); + if (m_saved_err >= 0) + ::close(m_saved_err); + if (m_file >= 0) + ::close(m_file); +} + +std::string DriverLogCapture::finish() +{ + if (!m_active) + return {}; + m_active = false; + std::fflush(stdout); + std::fflush(stderr); + ::dup2(m_saved_out, STDOUT_FILENO); + ::dup2(m_saved_err, STDERR_FILENO); + + std::string text; + ::lseek(m_file, 0, SEEK_SET); + char buffer[4096]; + ssize_t n = 0; + while ((n = ::read(m_file, buffer, sizeof(buffer))) > 0) + text.append(buffer, static_cast(n)); + // The journal keeps the driver's lines exactly as if nothing had intervened. + for (std::size_t done = 0; done < text.size();) + { + const ssize_t w = ::write(STDERR_FILENO, text.data() + done, text.size() - done); + if (w <= 0) + break; + done += static_cast(w); + } + return interesting_lines(text); +} + +std::string DriverLogCapture::interesting_lines(const std::string& text) +{ + std::vector lines; + std::istringstream stream(text); + std::string line; + while (std::getline(stream, line)) + { + if (looks_like_failure(line)) + { + std::string cleaned = trim_prefix(line); + if (lines.empty() || lines.back() != cleaned) + lines.push_back(std::move(cleaned)); + } + } + // Keep the first lines: the cause comes first and the rest is the teardown that follows it. + std::string joined; + for (std::size_t i = 0; i < lines.size() && i < 6; ++i) + joined += (joined.empty() ? "" : "; ") + lines[i]; + if (joined.size() > 600) + joined.resize(600); + return joined; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/inc/sensing_control/camera_fault.hpp b/src/sensing_control/inc/sensing_control/camera_fault.hpp new file mode 100644 index 0000000000..48a1ae1fd3 --- /dev/null +++ b/src/sensing_control/inc/sensing_control/camera_fault.hpp @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "producer_status.hpp" + +#include + +namespace plugins +{ +namespace sensing +{ + +/// A failure explained for an operator, and the state the producer is in because of it. +struct Diagnosis +{ + Fault fault; + ProducerState state = ProducerState::Error; +}; + +/** + * @brief Turn the text of a failure into what an operator should check. + * + * The producer's own components (the SIPL and Argus capture backends, the Jetson encoder) + * report failures as exception text, and SIPL as bare status codes ("Init failed with SIPLStatus 10"). A camera that + * is unpowered or unplugged produces the same failure as several other conditions, so the message names the physical + * checks first and keeps the original text for anyone who needs it. Camera failures put the producer in + * CameraFault; encoder and other failures in Error. Text nothing matches still gets a code and the + * original text, never silence. + */ +Diagnosis diagnose_failure(std::string_view what); + +/// The fault part of diagnose_failure(). +Fault diagnose_camera_failure(std::string_view what); + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/inc/sensing_control/control_handler.hpp b/src/sensing_control/inc/sensing_control/control_handler.hpp new file mode 100644 index 0000000000..bcb51a6240 --- /dev/null +++ b/src/sensing_control/inc/sensing_control/control_handler.hpp @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "producer_status.hpp" + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/// The outcome of an action the channel asked the producer to take. +struct ActionResult +{ + bool ok = true; + std::string code; + std::string message; +}; + +/// What a management client can change. The producer implements it; the handler only validates and forwards. +class ControlActions +{ +public: + virtual ~ControlActions() = default; + + /// Send stream `track` to host:port from now on, or stop the direct link when `host` is empty. + virtual ActionResult set_direct_target(const std::string& track, const std::string& host, std::uint16_t port) = 0; +}; + +/** + * @brief The management protocol: one JSON object per WebSocket text message. + * + * request {"id": 7, "type": "get_status"} + * {"id": 8, "type": "set_direct_target", "stream": "sensing-left", "host": "192.168.1.205", "port": 5600} + * {"id": 9, "type": "ping"} + * reply {"id": 7, "ok": true, "type": "status", "status": {...}} + * {"id": 8, "ok": false, "error": {"code": "invalid_host", "message": "..."}} + * events {"type": "event", "event": "state_changed", "status": {...}} + * {"type": "event", "event": "error", "error": {"code": "...", "message": "..."}} + * + * Every failure is answered with a code and a sentence; nothing is dropped silently. + */ +class ControlHandler +{ +public: + ControlHandler(const ProducerStatus& status, ControlActions& actions); + + /// The reply to one received text message. + std::string handle(const std::string& text) const; + +private: + const ProducerStatus& m_status; + ControlActions& m_actions; +}; + +/// The status as a JSON object text. +std::string status_json(const StatusSnapshot& snapshot); + +/// A pushed notification that the state changed. +std::string state_event(const StatusSnapshot& snapshot); + +/// A pushed notification of a failure. +std::string error_event(const Fault& fault); + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/inc/sensing_control/control_server.hpp b/src/sensing_control/inc/sensing_control/control_server.hpp new file mode 100644 index 0000000000..4b791809a4 --- /dev/null +++ b/src/sensing_control/inc/sensing_control/control_server.hpp @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct ControlServerConfig +{ + std::string bind_address = "0.0.0.0"; + /// 0 picks a free port (tests); port() reports it. + std::uint16_t port = 5610; + /// Shared secret every client must present (`Authorization: Bearer` or `?token=`). Required: a channel that can + /// redirect camera video is never offered without one. + std::string token; + std::size_t max_clients = 8; + std::size_t max_message_bytes = 64 * 1024; +}; + +/// The first line of a secret file with surrounding whitespace removed; throws naming the file if it is unreadable or +/// empty. The secret is read from a file so it never appears in a process listing. +std::string read_secret_file(const std::string& path); + +/** + * @brief An authenticated WebSocket server on its own thread. + * + * Each received text message goes to `on_message` and its return value is sent back to that client; the callback + * runs on the server thread and must not block. broadcast() pushes a text message to every connected client from + * any thread. A client that sends a bad handshake, a wrong token, an oversized or malformed message, or that stops + * reading is disconnected with a reason where the protocol allows one. + */ +class ControlServer +{ +public: + using MessageHandler = std::function; + + /// Starts listening. Throws std::runtime_error with the reason if the token is empty or the port cannot be bound. + ControlServer(const ControlServerConfig& config, MessageHandler on_message); + ~ControlServer(); + + ControlServer(const ControlServer&) = delete; + ControlServer& operator=(const ControlServer&) = delete; + + std::uint16_t port() const + { + return m_port; + } + + /// Queue a text message for every connected client. + void broadcast(const std::string& text); + + /// Clients that have completed the handshake. + std::size_t client_count() const + { + return m_clients.load(); + } + +private: + struct Client; + void run(); + void wake(); + + ControlServerConfig m_config; + MessageHandler m_on_message; + int m_listen_fd = -1; + int m_wake_fd = -1; + std::uint16_t m_port = 0; + std::atomic m_stop{ false }; + std::atomic m_clients{ 0 }; + std::mutex m_mutex; + std::vector m_broadcasts; ///< guarded by m_mutex + std::thread m_thread; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/inc/sensing_control/driver_log_capture.hpp b/src/sensing_control/inc/sensing_control/driver_log_capture.hpp new file mode 100644 index 0000000000..55d8f3af03 --- /dev/null +++ b/src/sensing_control/inc/sensing_control/driver_log_capture.hpp @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace plugins +{ +namespace sensing +{ + +/** + * @brief Collects what native code prints to stdout and stderr while it runs, for the length of one scope. + * + * The camera driver explains why a camera failed only in its own log lines ("lock timeout", "OPEN_CIRCUIT", an I2C + * NACK); the status code it returns says nothing. This redirects descriptors 1 and 2 into a temporary file for the + * scope, and on finish() restores them, writes what was captured back to them so the service journal still has it, + * and returns the lines that matter for a failure. The descriptors are restored on every path, including + * destruction without finish(). Use it only around a start-up call, never for the process lifetime. + */ +class DriverLogCapture +{ +public: + DriverLogCapture(); + ~DriverLogCapture(); + + DriverLogCapture(const DriverLogCapture&) = delete; + DriverLogCapture& operator=(const DriverLogCapture&) = delete; + + /// Restore the descriptors, replay the captured text, and return the captured lines that look like failures. + std::string finish(); + + /// The failure-looking lines of `text` (an error, failure, timeout, NACK, open circuit), newest last, joined with + /// "; " and limited in length. + static std::string interesting_lines(const std::string& text); + +private: + int m_saved_out = -1; + int m_saved_err = -1; + int m_file = -1; + bool m_active = false; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/inc/sensing_control/producer_status.hpp b/src/sensing_control/inc/sensing_control/producer_status.hpp new file mode 100644 index 0000000000..d02b17cbb6 --- /dev/null +++ b/src/sensing_control/inc/sensing_control/producer_status.hpp @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +enum class ProducerState +{ + /// Opening the cameras and joining the room. + Starting, + /// Cameras are capturing and tracks are published. + Streaming, + /// The cameras could not be started; the producer retries and says why. + CameraFault, + /// Another failure (for example the encoder or a stalled stream); the producer retries. + Error, + Stopping, +}; + +const char* to_string(ProducerState state); + +/// A failure as an operator should read it: a stable code and a sentence that says what to check. +struct Fault +{ + std::string code; + std::string message; +}; + +struct StreamStatus +{ + std::string track; + std::uint32_t sensor_id = 0; + /// Where this stream's encoded video is also sent (the PC video stage); empty host means no direct link. + std::string direct_host; + std::uint16_t direct_port = 0; + bool direct_connected = false; + /// Why the direct link is not up (for example "cannot connect to 192.168.1.5:5600: Connection refused"). + std::string direct_error; + std::uint64_t frames = 0; + std::uint64_t published = 0; +}; + +struct StatusSnapshot +{ + ProducerState state = ProducerState::Starting; + std::optional fault; + std::vector streams; + std::string release; + std::uint64_t uptime_seconds = 0; +}; + +/// What the management channel reports. Written by the producer's main loop, read by the channel's thread. +class ProducerStatus +{ +public: + explicit ProducerStatus(std::string release); + + /// Set the state; a fault is kept only for the states that have one. + void set_state(ProducerState state, std::optional fault = std::nullopt); + + /// Replace the per-stream figures (the configured streams stay listed while the cameras are down). + void set_streams(std::vector streams); + + /// Change one stream's direct-link target; false if no stream has that track name. + bool set_direct_target(const std::string& track, const std::string& host, std::uint16_t port); + + StatusSnapshot snapshot() const; + +private: + mutable std::mutex m_mutex; + std::string m_release; + ProducerState m_state = ProducerState::Starting; + std::optional m_fault; + std::vector m_streams; + std::chrono::steady_clock::time_point m_started = std::chrono::steady_clock::now(); +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/inc/sensing_control/producer_supervisor.hpp b/src/sensing_control/inc/sensing_control/producer_supervisor.hpp new file mode 100644 index 0000000000..832b56487e --- /dev/null +++ b/src/sensing_control/inc/sensing_control/producer_supervisor.hpp @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "control_handler.hpp" +#include "producer_status.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/// Where a stream's encoded video goes (the PC video stage); chosen at startup or at runtime over the management +/// channel. +struct DirectTarget +{ + std::string host; + std::uint16_t port = 0; +}; + +using DirectTargets = std::map; ///< by track name + +/// A start-up failure the factory has already explained: the state to report and the fault to show. +class StartupFailure : public std::runtime_error +{ +public: + StartupFailure(Fault fault, ProducerState state) + : std::runtime_error(fault.message), m_fault(std::move(fault)), m_state(state) + { + } + const Fault& fault() const + { + return m_fault; + } + ProducerState state() const + { + return m_state; + } + +private: + Fault m_fault; + ProducerState m_state; +}; + +/// One running capture-and-publish pipeline. A producer provides one. +class CameraSession +{ +public: + virtual ~CameraSession() = default; + + /// Poll for frames and hand them on. Throws once the session can no longer work (room lost, camera gone). + virtual void update() = 0; + virtual void flush() = 0; + /// Current per-stream figures, including each direct link's target and whether it is connected. + virtual std::vector streams() const = 0; + virtual void set_direct_target(const std::string& track, const std::string& host, std::uint16_t port) = 0; +}; + +/// Builds a session, applying `targets` over the configured direct links. Throws with the failure's text. +using SessionFactory = std::function(const DirectTargets& targets)>; + +struct SupervisorConfig +{ + /// How long to wait before trying to start the cameras again after a failure. + std::chrono::milliseconds retry_interval{ 5000 }; + std::chrono::milliseconds poll_interval{ 15 }; + /// How often the figures in the status are refreshed from the running session. + std::chrono::milliseconds status_interval{ 1000 }; + /// A running session whose stream delivers no new frame for this long is failed and restarted. Zero disables it. + std::chrono::milliseconds stall_timeout{ 5000 }; +}; + +/** + * @brief Keeps the producer running and reporting. + * + * A camera that cannot be started (no power, no link) is a state to report and retry, not a reason to exit: the + * management channel stays up, says why in plain words, and the stream comes back when the cable is fixed. + * + * One thread, the one in run(), owns the session: it creates, updates, retargets, flushes and destroys it. The + * channel's thread only records what was asked (set_direct_target) and run() applies it between updates, so a slow + * camera start never blocks a command and a command never races a session that is being built or torn down. Direct + * targets are remembered, so a restarted session starts with the latest. + */ +class ProducerSupervisor : public ControlActions +{ +public: + /// `publish` receives each pushed notification as JSON text (the channel's broadcast); it may be empty. + ProducerSupervisor(ProducerStatus& status, + std::vector configured, + SessionFactory factory, + std::function publish, + const SupervisorConfig& config = {}); + + /// Run until `stop()` returns true. + void run(const std::function& stop); + + ActionResult set_direct_target(const std::string& track, const std::string& host, std::uint16_t port) override; + +private: + void sleep_unless_stopped(std::chrono::milliseconds duration, const std::function& stop) const; + void announce(ProducerState state, std::optional fault); + /// Compare the session's figures with the last ones: report new direct-link errors, and return a description of a + /// stalled stream (empty when every stream is delivering). + std::string watch(const std::vector& streams, std::chrono::steady_clock::time_point now); + + ProducerStatus& m_status; + std::vector m_configured; + SessionFactory m_factory; + std::function m_publish; + SupervisorConfig m_config; + + /// Per stream: the last frame count seen, when it last changed, and the direct-link error last reported. + struct Watch + { + std::uint64_t frames = 0; + std::chrono::steady_clock::time_point changed; + std::string direct_error; + }; + std::map m_watch; ///< main thread only + + /// Applies the targets that changed since the session was built, on the owning thread. + void apply_pending(CameraSession& session); + + std::mutex m_targets_mutex; ///< guards m_targets and m_pending + DirectTargets m_targets; ///< what the operator last chose, by track + std::set m_pending; ///< tracks whose target changed since the running session last heard of it +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/inc/sensing_control/websocket.hpp b/src/sensing_control/inc/sensing_control/websocket.hpp new file mode 100644 index 0000000000..faa85f278f --- /dev/null +++ b/src/sensing_control/inc/sensing_control/websocket.hpp @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ +namespace websocket +{ + +/// RFC 6455 opcodes this server understands. +enum Opcode : std::uint8_t +{ + kContinuation = 0x0, + kText = 0x1, + kBinary = 0x2, + kClose = 0x8, + kPing = 0x9, + kPong = 0xA, +}; + +/// SHA-1 of `data` as 20 raw bytes; used only for the opening handshake, never for security. +std::string sha1(std::string_view data); + +std::string base64_encode(std::string_view data); + +/// The Sec-WebSocket-Accept value for a client's Sec-WebSocket-Key. +std::string accept_key(std::string_view client_key); + +/// A parsed client upgrade request. +struct Handshake +{ + bool ok = false; + /// HTTP status to answer with when !ok (400, 401, 404 or 426), and a plain-text reason. + int status = 400; + std::string reason; + std::string key; + /// The bearer token from `Authorization: Bearer ...` or the `token=` query parameter; empty when absent. + std::string token; + std::string path; +}; + +/// Parse the request head (everything up to and including the blank line). +Handshake parse_handshake(std::string_view head); + +/// The server's answer to a successful or failed handshake. +std::string handshake_response(const Handshake& handshake); + +/// Compare two secrets without leaking where they differ. +bool constant_time_equal(std::string_view a, std::string_view b); + +struct Frame +{ + bool fin = true; + std::uint8_t opcode = kText; + std::string payload; +}; + +enum class ParseStatus +{ + NeedMore, + Complete, + Error, +}; + +struct ParseResult +{ + ParseStatus status = ParseStatus::NeedMore; + std::size_t consumed = 0; + Frame frame; + /// WebSocket close code to send when status is Error (1002 protocol error, 1009 too big). + std::uint16_t close_code = 1002; + std::string error; +}; + +/// Parse one client frame from the front of `buffer`. Client frames must be masked. +ParseResult parse_frame(std::string_view buffer, std::size_t max_payload); + +/// Encode one unmasked server frame. +std::string encode_frame(std::uint8_t opcode, std::string_view payload, bool fin = true); + +/// A close frame carrying `code` and a short reason. +std::string encode_close(std::uint16_t code, std::string_view reason); + +} // namespace websocket +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/producer_status.cpp b/src/sensing_control/producer_status.cpp new file mode 100644 index 0000000000..bea614320f --- /dev/null +++ b/src/sensing_control/producer_status.cpp @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_control/producer_status.hpp" + +#include + +namespace plugins +{ +namespace sensing +{ + +const char* to_string(ProducerState state) +{ + switch (state) + { + case ProducerState::Starting: + return "starting"; + case ProducerState::Streaming: + return "streaming"; + case ProducerState::CameraFault: + return "camera_fault"; + case ProducerState::Error: + return "error"; + case ProducerState::Stopping: + return "stopping"; + } + return "unknown"; +} + +ProducerStatus::ProducerStatus(std::string release) : m_release(std::move(release)) +{ +} + +void ProducerStatus::set_state(ProducerState state, std::optional fault) +{ + std::lock_guard lock(m_mutex); + m_state = state; + // Only the failing states carry a fault; leaving one clears it. + m_fault = (state == ProducerState::CameraFault || state == ProducerState::Error) ? std::move(fault) : std::nullopt; +} + +void ProducerStatus::set_streams(std::vector streams) +{ + std::lock_guard lock(m_mutex); + m_streams = std::move(streams); +} + +bool ProducerStatus::set_direct_target(const std::string& track, const std::string& host, std::uint16_t port) +{ + std::lock_guard lock(m_mutex); + for (auto& stream : m_streams) + { + if (stream.track == track) + { + stream.direct_host = host; + stream.direct_port = host.empty() ? 0 : port; + stream.direct_connected = false; + return true; + } + } + return false; +} + +StatusSnapshot ProducerStatus::snapshot() const +{ + std::lock_guard lock(m_mutex); + StatusSnapshot out; + out.state = m_state; + out.fault = m_fault; + out.streams = m_streams; + out.release = m_release; + out.uptime_seconds = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - m_started).count()); + return out; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/producer_supervisor.cpp b/src/sensing_control/producer_supervisor.cpp new file mode 100644 index 0000000000..38087cabc5 --- /dev/null +++ b/src/sensing_control/producer_supervisor.cpp @@ -0,0 +1,236 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_control/producer_supervisor.hpp" + +#include "inc/sensing_control/camera_fault.hpp" + +#include + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +std::shared_ptr logger() +{ + return isaaccapture::Logger::get("isaaccapture.plugins.sensing.ProducerSupervisor"); +} + +} // namespace + +ProducerSupervisor::ProducerSupervisor(ProducerStatus& status, + std::vector configured, + SessionFactory factory, + std::function publish, + const SupervisorConfig& config) + : m_status(status), + m_configured(std::move(configured)), + m_factory(std::move(factory)), + m_publish(std::move(publish)), + m_config(config) +{ + // The configured direct targets are the starting point; a runtime change replaces them. + for (const auto& stream : m_configured) + m_targets[stream.track] = { stream.direct_host, stream.direct_port }; + m_status.set_streams(m_configured); +} + +void ProducerSupervisor::announce(ProducerState state, std::optional fault) +{ + m_status.set_state(state, fault); + if (fault) + logger()->error("{}: {}", fault->code, fault->message); + if (m_publish) + { + const auto snapshot = m_status.snapshot(); + if (fault) + m_publish(error_event(*fault)); + m_publish(state_event(snapshot)); + } +} + +std::string ProducerSupervisor::watch(const std::vector& streams, std::chrono::steady_clock::time_point now) +{ + std::string stalled; + for (const auto& stream : streams) + { + auto& entry = m_watch[stream.track]; + if (stream.frames != entry.frames) + { + entry.frames = stream.frames; + entry.changed = now; + } + else if (m_config.stall_timeout.count() > 0 && now - entry.changed >= m_config.stall_timeout) + { + const auto seconds = std::chrono::duration_cast(now - entry.changed).count(); + stalled += (stalled.empty() ? "" : "; ") + std::string("no frame for ") + std::to_string(seconds) + + " s from sensor " + std::to_string(stream.sensor_id) + " (stream " + stream.track + ")"; + } + + // A direct link that cannot reach the PC is reported once per distinct reason, not every second. + if (stream.direct_error != entry.direct_error) + { + entry.direct_error = stream.direct_error; + if (!stream.direct_error.empty() && m_publish) + { + m_publish(error_event( + { "direct_link_failed", "Stream " + stream.track + ": " + stream.direct_error + + ". Check that the PC worker is running and listening on that port, " + "and that the address is the PC's (set it with set_direct_target)." })); + } + } + } + return stalled; +} + +void ProducerSupervisor::sleep_unless_stopped(std::chrono::milliseconds duration, const std::function& stop) const +{ + const auto until = std::chrono::steady_clock::now() + duration; + while (!stop() && std::chrono::steady_clock::now() < until) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); +} + +ActionResult ProducerSupervisor::set_direct_target(const std::string& track, const std::string& host, std::uint16_t port) +{ + // Only records the request: the thread that owns the session applies it (see apply_pending). + if (!m_status.set_direct_target(track, host, port)) + return { false, "unknown_stream", "no stream named \"" + track + "\"" }; + { + std::lock_guard lock(m_targets_mutex); + m_targets[track] = { host, host.empty() ? std::uint16_t(0) : port }; + m_pending.insert(track); + } + logger()->info( + "stream {} will send to {}", track, host.empty() ? std::string("nowhere") : host + ":" + std::to_string(port)); + if (m_publish) + m_publish(state_event(m_status.snapshot())); + return {}; +} + +void ProducerSupervisor::apply_pending(CameraSession& session) +{ + DirectTargets changed; + { + std::lock_guard lock(m_targets_mutex); + for (const auto& track : m_pending) + changed[track] = m_targets[track]; + m_pending.clear(); + } + for (const auto& [track, target] : changed) + session.set_direct_target(track, target.host, target.port); +} + +void ProducerSupervisor::run(const std::function& stop) +{ + // A fault that persists across retries is announced once, not on every attempt. + std::optional last_fault; + bool first = true; + while (!stop()) + { + if (first || !last_fault) + announce(ProducerState::Starting, std::nullopt); + first = false; + + std::unique_ptr session; + DirectTargets targets; + { + // The session is built with everything chosen so far; only later changes are pending. + std::lock_guard lock(m_targets_mutex); + targets = m_targets; + m_pending.clear(); + } + try + { + session = m_factory(targets); + } + catch (const std::exception& e) + { + // The factory explains the failures it understands; anything else is read as a camera failure. + const auto* explained = dynamic_cast(&e); + const Diagnosis diagnosis = + explained ? Diagnosis{ explained->fault(), explained->state() } : diagnose_failure(e.what()); + Fault fault = diagnosis.fault; + const ProducerState state = diagnosis.state; + if (!last_fault || last_fault->message != fault.message) + announce(state, fault); + last_fault = std::move(fault); + sleep_unless_stopped(m_config.retry_interval, stop); + continue; + } + + last_fault.reset(); + announce(ProducerState::Streaming, std::nullopt); + + std::optional failure; + ProducerState failure_state = ProducerState::Error; + m_watch.clear(); + for (const auto& stream : m_configured) + { + m_watch[stream.track].changed = std::chrono::steady_clock::now(); + } + auto next_status = std::chrono::steady_clock::now(); + while (!stop() && !failure) + { + try + { + // A change that arrived while the cameras were starting is applied before the first update. + apply_pending(*session); + session->update(); + if (std::chrono::steady_clock::now() >= next_status) + { + next_status += m_config.status_interval; + // The session's own figures, including where each stream really sends. + auto streams = session->streams(); + const std::string stalled = watch(streams, std::chrono::steady_clock::now()); + m_status.set_streams(std::move(streams)); + if (!stalled.empty()) + throw std::runtime_error(stalled); + } + } + catch (const std::exception& e) + { + failure_state = ProducerState::Error; + Diagnosis diagnosis = diagnose_failure(e.what()); + if (diagnosis.fault.code == "startup_failed") + { + // A session that was running and failed for an unrecognised reason did not "fail to start". + diagnosis = { { "stream_failed", std::string("The stream stopped. The producer restarts it " + "automatically. (") + + e.what() + ")" }, + ProducerState::Error }; + } + failure = diagnosis.fault; + failure_state = diagnosis.state; + } + if (!failure) + std::this_thread::sleep_for(m_config.poll_interval); + } + + try + { + session->flush(); + } + catch (const std::exception& e) + { + logger()->warn("flush on shutdown failed: {}", e.what()); + } + session.reset(); + if (failure) + { + announce(failure_state, failure); + last_fault.reset(); + sleep_unless_stopped(m_config.retry_interval, stop); + } + } + announce(ProducerState::Stopping, std::nullopt); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/sensing_control/tests/CMakeLists.txt b/src/sensing_control/tests/CMakeLists.txt new file mode 100644 index 0000000000..0581b44eb7 --- /dev/null +++ b/src/sensing_control/tests/CMakeLists.txt @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(sensing_control_tests + test_websocket.cpp + test_camera_fault.cpp + test_driver_log_capture.cpp + test_producer_status.cpp + test_control_handler.cpp + test_producer_supervisor.cpp + test_control_server.cpp +) + +target_link_libraries(sensing_control_tests + PRIVATE + sensing::control + nlohmann_json::nlohmann_json + Catch2::Catch2WithMain +) + +catch_discover_tests(sensing_control_tests) diff --git a/src/sensing_control/tests/test_camera_fault.cpp b/src/sensing_control/tests/test_camera_fault.cpp new file mode 100644 index 0000000000..df04c26b15 --- /dev/null +++ b/src/sensing_control/tests/test_camera_fault.cpp @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include + +using plugins::sensing::diagnose_camera_failure; +using plugins::sensing::diagnose_failure; +using plugins::sensing::ProducerState; + +namespace +{ + +bool contains(const std::string& text, const std::string& part) +{ + return text.find(part) != std::string::npos; +} + +bool ends_with(const std::string& text, const std::string& suffix) +{ + return text.size() >= suffix.size() && text.compare(text.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +struct Expectation +{ + std::string text; + std::string code; + ProducerState state; +}; + +constexpr ProducerState kCamera = ProducerState::CameraFault; +constexpr ProducerState kError = ProducerState::Error; + +// One representative text per signature in camera_fault.cpp, in table order. +const std::vector& expectations() +{ + static const std::vector table = { + { "Severity 2 : OPEN_CIRCUIT on GPIO 4", "camera_power_missing", kCamera }, + { "fault: CIRCUIT_OPEN_DISCONNECTION", "camera_power_missing", kCamera }, + { "deserializer lock timeout on link 1", "camera_link_not_locked", kCamera }, + { "DES_LOCK_ERR reported", "camera_link_not_locked", kCamera }, + { "CAMERA_DETECT failed for module 2", "camera_not_detected", kCamera }, + { "Parse failed after 3 retries", "camera_not_responding", kCamera }, + { "Init returned SIPLStatus 8", "camera_partly_started", kCamera }, + { "sensor control write failure on sensor 3", "camera_control_bus_failed", kCamera }, + { "input stream failure on pipeline 0", "camera_stream_failed", kCamera }, + { "image processing failure on pipeline 1", "camera_isp_failed", kCamera }, + { "interrupt localization failure", "camera_hardware_fault", kCamera }, + { "deserializer failure on link 0", "camera_link_failed", kCamera }, + { "serializer failure on link 0", "camera_link_failed", kCamera }, + { "sensor failure on sensor 2", "camera_sensor_failed", kCamera }, + { "link lock lost on deserializer 1", "camera_link_not_locked", kCamera }, + { "Failed to detect camera on link 2", "camera_not_detected", kCamera }, + { "no frame for 5 s from sensor 2 (stream sensing-left)", "camera_no_frames", kCamera }, + { "INvSIPLCamera::Init failed", "camera_start_failed", kCamera }, + { "INvSIPLCamera::SetPlatformCfg failed", "camera_config_rejected", kCamera }, + { "INvSIPLCamera::Start failed", "camera_start_failed", kCamera }, + { "Init failed with SIPLStatus 10", "camera_start_failed", kCamera }, + { "pipeline error on sensor 1", "camera_pipeline_error", kCamera }, + { "capture error on sensor 1", "camera_capture_error", kCamera }, + { "no modules left in the link masks", "camera_link_mask_mismatch", kCamera }, + { "configuration declares no sensors", "camera_config_empty", kCamera }, + { "no SIPL pipeline with index 9", "camera_not_in_config", kCamera }, + { "link masks are required", "camera_config_missing", kCamera }, + { "cannot open NITO file /var/nito/x.nito", "camera_tuning_missing", kCamera }, + { "short read on NITO file", "camera_tuning_invalid", kCamera }, + { "Argus: no camera provider", "camera_service_down", kCamera }, + { "Argus: no camera device 2", "camera_not_detected", kCamera }, + { "Argus: no camera at index 3", "camera_not_detected", kCamera }, + { "Argus: session failed", "camera_error", kCamera }, + { "NvBufSurfaceMapEglImage failed", "gpu_buffer_error", kCamera }, + { "NvmmNvSciBufToNvBufSurface failed", "gpu_buffer_error", kCamera }, + { "unsupported CUDA EGL frame type 7", "gpu_buffer_error", kCamera }, + { "CUDA error 700 in cudaMemcpy", "gpu_error", kError }, + { "createVideoEncoder failed", "encoder_unavailable", kError }, + { "JetsonEncoder: dequeue timed out", "encoder_error", kError }, + { "std::bad_alloc", "out_of_memory", kError }, + { "open /dev/video0: Permission denied", "permission_denied", kError }, + { "open /dev/foo: No such file or directory", "file_missing", kError }, + { "open /dev/video1: Device or resource busy", "device_busy", kError }, + }; + return table; +} + +} // namespace + +TEST_CASE("every signature maps its text to the intended code and state", "[camera_fault]") +{ + for (const auto& e : expectations()) + { + INFO("text: " << e.text); + const auto d = diagnose_failure(e.text); + CHECK(d.fault.code == e.code); + CHECK(d.state == e.state); + CHECK_FALSE(d.fault.message.empty()); + CHECK(ends_with(d.fault.message, "(" + e.text + ")")); + CHECK(diagnose_camera_failure(e.text).code == e.code); + } +} + +TEST_CASE("known capture failures are explained and keep the original text", "[camera_fault]") +{ + const std::string sipl = "INvSIPLCamera::Init failed with SIPLStatus 10"; + const auto a = diagnose_failure(sipl); + CHECK(a.fault.code == "camera_start_failed"); + CHECK(a.state == ProducerState::CameraFault); + CHECK(contains(a.fault.message, "power cable")); + CHECK(contains(a.fault.message, "retries automatically")); + CHECK(contains(a.fault.message, sipl)); +} + +TEST_CASE("the first matching signature wins", "[camera_fault]") +{ + // Both the generic SIPL status and the driver's own explanation: the explanation is listed first. + CHECK(diagnose_failure("Init failed with SIPLStatus 10; OPEN_CIRCUIT").fault.code == "camera_power_missing"); + CHECK(diagnose_failure("SIPLStatus 10 CIRCUIT_OPEN_DISCONNECTION").fault.code == "camera_power_missing"); + CHECK(diagnose_failure("link lock failed, SIPLStatus 10").fault.code == "camera_link_not_locked"); + // A driver NACK line attached to an otherwise unknown failure does not change the mapping. + CHECK(diagnose_failure("something unknown; I2C: NACK ... errno=121").fault.code == "startup_failed"); + // The camera signature still wins when it is mentioned beside a publishing word. + CHECK(diagnose_failure("Init failed with SIPLStatus 10; LiveKit: cannot join the room").fault.code == + "camera_start_failed"); + // "Argus:" alone is the last Argus entry, not the first. + CHECK(diagnose_failure("Argus: no camera provider").fault.code == "camera_service_down"); + // Camera signatures are listed before the generic CUDA one. + CHECK(diagnose_failure("unsupported CUDA EGL frame type 3").state == ProducerState::CameraFault); +} + +TEST_CASE("an I2C NACK alone is not a signature", "[camera_fault]") +{ + // The driver prints NACK lines on healthy starts too. + for (const char* text : { "I2C: NACK", "CAMI2C_ERROR_BUS_NACK on 0x1a", "I2C: NACK from address 0x36 errno=121" }) + { + INFO(text); + CHECK(diagnose_failure(text).fault.code == "startup_failed"); + } +} + +TEST_CASE("no signature mentions LiveKit any more", "[camera_fault]") +{ + // The Orin streams over its direct links only, so LiveKit texts are not diagnosed specially. + for (const char* text : { "LiveKit: cannot join the room", "LiveKit: cannot publish track sensing-left", + "LiveKit: no local participant", "LiveKit: disconnected" }) + { + INFO(text); + const auto d = diagnose_failure(text); + CHECK(d.fault.code == "startup_failed"); + CHECK(d.state == ProducerState::CameraFault); + CHECK_FALSE(contains(d.fault.message, "livekit-url")); + CHECK_FALSE(contains(d.fault.message, "livekit-token")); + CHECK_FALSE(contains(d.fault.message, "--livekit")); + CHECK(ends_with(d.fault.message, std::string("(") + text + ")")); + } +} + +TEST_CASE("the signature advice names no retired --livekit flag", "[camera_fault]") +{ + for (const auto& e : expectations()) + { + INFO(e.text); + CHECK_FALSE(contains(diagnose_failure(e.text).fault.message, "--livekit")); + } + CHECK(contains(diagnose_failure("JetsonEncoder: dequeue timed out").fault.message, "--bitrate")); +} + +TEST_CASE("an unknown failure is a startup_failed camera fault with the original text", "[camera_fault]") +{ + const std::string text = "something entirely new (errno 5)"; + const auto d = diagnose_failure(text); + CHECK(d.fault.code == "startup_failed"); + CHECK(d.state == ProducerState::CameraFault); + CHECK(contains(d.fault.message, "retries automatically")); + CHECK(ends_with(d.fault.message, "(" + text + ")")); + + const auto empty = diagnose_failure(""); + CHECK(empty.fault.code == "startup_failed"); + CHECK(ends_with(empty.fault.message, "()")); +} diff --git a/src/sensing_control/tests/test_control_handler.cpp b/src/sensing_control/tests/test_control_handler.cpp new file mode 100644 index 0000000000..b9fc884f48 --- /dev/null +++ b/src/sensing_control/tests/test_control_handler.cpp @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +using namespace plugins::sensing; +using Json = nlohmann::json; + +namespace +{ + +class FakeActions : public ControlActions +{ +public: + ActionResult set_direct_target(const std::string& track, const std::string& host, std::uint16_t port) override + { + ++calls; + last_track = track; + last_host = host; + last_port = port; + return result; + } + + int calls = 0; + std::string last_track; + std::string last_host; + std::uint16_t last_port = 0; + ActionResult result; +}; + +struct Fixture +{ + Fixture() : status("test-release"), handler(status, actions) + { + StreamStatus left; + left.track = "sensing-left"; + left.sensor_id = 3; + StreamStatus right; + right.track = "sensing-right"; + status.set_streams({ left, right }); + } + + Json call(const Json& request) + { + return Json::parse(handler.handle(request.dump())); + } + + ProducerStatus status; + FakeActions actions; + ControlHandler handler; +}; + +} // namespace + +TEST_CASE("ping answers pong and echoes the id", "[control_handler]") +{ + Fixture f; + const auto reply = f.call({ { "id", 7 }, { "type", "ping" } }); + CHECK(reply["ok"] == true); + CHECK(reply["type"] == "pong"); + CHECK(reply["id"] == 7); +} + +TEST_CASE("a string id is echoed and a missing id is null", "[control_handler]") +{ + Fixture f; + CHECK(f.call({ { "id", "abc" }, { "type", "ping" } })["id"] == "abc"); + CHECK(f.call({ { "type", "ping" } })["id"].is_null()); +} + +TEST_CASE("get_status reports the state, release and streams", "[control_handler]") +{ + Fixture f; + f.status.set_state(ProducerState::CameraFault, Fault{ "camera_error", "check the power" }); + const auto reply = f.call({ { "id", 1 }, { "type", "get_status" } }); + REQUIRE(reply["ok"] == true); + CHECK(reply["type"] == "status"); + const auto& status = reply["status"]; + CHECK(status["state"] == "camera_fault"); + CHECK(status["release"] == "test-release"); + CHECK(status["fault"]["code"] == "camera_error"); + CHECK(status["fault"]["message"] == "check the power"); + REQUIRE(status["streams"].size() == 2); + CHECK(status["streams"][0]["stream"] == "sensing-left"); + CHECK(status["streams"][0]["sensor"] == 3); + CHECK(status["streams"][0]["direct"]["connected"] == false); +} + +TEST_CASE("get_status reports why a direct link is down", "[control_handler]") +{ + Fixture f; + StreamStatus left; + left.track = "sensing-left"; + left.direct_host = "10.0.0.5"; + left.direct_port = 5600; + left.direct_error = "cannot connect to 10.0.0.5:5600: Connection refused"; + StreamStatus right; + right.track = "sensing-right"; + f.status.set_streams({ left, right }); + const auto status = f.call({ { "type", "get_status" } })["status"]; + CHECK(status["streams"][0]["direct"]["error"] == "cannot connect to 10.0.0.5:5600: Connection refused"); + CHECK(status["streams"][0]["direct"]["connected"] == false); + // A link with no error still carries the key, empty. + REQUIRE(status["streams"][1]["direct"].contains("error")); + CHECK(status["streams"][1]["direct"]["error"] == ""); +} + +TEST_CASE("the status has no fault key when healthy", "[control_handler]") +{ + Fixture f; + f.status.set_state(ProducerState::Streaming); + CHECK_FALSE(f.call({ { "type", "get_status" } })["status"].contains("fault")); +} + +TEST_CASE("malformed requests are answered with a code, never dropped", "[control_handler]") +{ + Fixture f; + + SECTION("bad_json") + { + for (const char* text : { "not json", "", "[1,2]", "42", "\"str\"" }) + { + const auto reply = Json::parse(f.handler.handle(text)); + CHECK(reply["ok"] == false); + CHECK(reply["error"]["code"] == "bad_json"); + CHECK_FALSE(reply["error"]["message"].get().empty()); + } + } + SECTION("missing_type") + { + CHECK(f.call({ { "id", 4 } })["error"]["code"] == "missing_type"); + const auto reply = f.call({ { "id", 5 }, { "type", 3 } }); + CHECK(reply["error"]["code"] == "missing_type"); + CHECK(reply["id"] == 5); + } + SECTION("unknown_type") + { + const auto reply = f.call({ { "id", 6 }, { "type", "reboot" } }); + CHECK(reply["ok"] == false); + CHECK(reply["error"]["code"] == "unknown_type"); + CHECK(reply["error"]["message"].get().find("reboot") != std::string::npos); + CHECK(reply["id"] == 6); + } + CHECK(f.actions.calls == 0); +} + +TEST_CASE("set_direct_target validation", "[control_handler]") +{ + Fixture f; + const Json good = { { "id", 9 }, + { "type", "set_direct_target" }, + { "stream", "sensing-left" }, + { "host", "192.168.1.205" }, + { "port", 5600 } }; + + const auto code_of = [&](Json request) { return f.call(request)["error"]["code"]; }; + + SECTION("missing_stream") + { + Json r = good; + r.erase("stream"); + CHECK(code_of(r) == "missing_stream"); + r["stream"] = 5; + CHECK(code_of(r) == "missing_stream"); + } + SECTION("missing_host") + { + Json r = good; + r.erase("host"); + CHECK(code_of(r) == "missing_host"); + r["host"] = 5; + CHECK(code_of(r) == "missing_host"); + } + SECTION("invalid_host") + { + for (const char* host : { "http://x", "a b", "a/b", "-bad", ".bad", "host:5600", "a_b" }) + { + Json r = good; + r["host"] = host; + CHECK(code_of(r) == "invalid_host"); + } + Json r = good; + r["host"] = std::string(254, 'a'); + CHECK(code_of(r) == "invalid_host"); + } + SECTION("invalid_port") + { + Json r = good; + r.erase("port"); + CHECK(code_of(r) == "invalid_port"); + for (const Json& port : { Json(0), Json(-1), Json(65536), Json("5600"), Json(5600.5), Json(nullptr) }) + { + r["port"] = port; + CHECK(code_of(r) == "invalid_port"); + } + } + SECTION("unknown_stream lists the real ones") + { + Json r = good; + r["stream"] = "sensing-up"; + const auto reply = f.call(r); + CHECK(reply["error"]["code"] == "unknown_stream"); + const std::string message = reply["error"]["message"]; + CHECK(message.find("sensing-up") != std::string::npos); + CHECK(message.find("sensing-left") != std::string::npos); + CHECK(message.find("sensing-right") != std::string::npos); + } + CHECK(f.actions.calls == 0); +} + +TEST_CASE("set_direct_target forwards to the action", "[control_handler]") +{ + Fixture f; + + SECTION("success returns the status and echoes the id") + { + const auto reply = f.call({ { "id", 11 }, + { "type", "set_direct_target" }, + { "stream", "sensing-right" }, + { "host", "pc.example" }, + { "port", 65535 } }); + CHECK(reply["ok"] == true); + CHECK(reply["id"] == 11); + CHECK(reply["type"] == "status"); + CHECK(f.actions.calls == 1); + CHECK(f.actions.last_track == "sensing-right"); + CHECK(f.actions.last_host == "pc.example"); + CHECK(f.actions.last_port == 65535); + } + SECTION("IPv4 and IPv6 literals are accepted") + { + for (const char* host : { "127.0.0.1", "::1", "fe80::1" }) + { + const auto reply = f.call( + { { "type", "set_direct_target" }, { "stream", "sensing-left" }, { "host", host }, { "port", 1 } }); + CHECK(reply["ok"] == true); + CHECK(f.actions.last_host == host); + } + } + SECTION("an empty host disables and needs no port") + { + const auto reply = + f.call({ { "id", 12 }, { "type", "set_direct_target" }, { "stream", "sensing-left" }, { "host", "" } }); + CHECK(reply["ok"] == true); + CHECK(f.actions.calls == 1); + CHECK(f.actions.last_host.empty()); + CHECK(f.actions.last_port == 0); + } + SECTION("an action failure is passed through with its own code") + { + f.actions.result = { false, "busy", "the sender is restarting" }; + const auto reply = f.call({ { "id", 13 }, + { "type", "set_direct_target" }, + { "stream", "sensing-left" }, + { "host", "10.1.1.1" }, + { "port", 5600 } }); + CHECK(reply["ok"] == false); + CHECK(reply["id"] == 13); + CHECK(reply["error"]["code"] == "busy"); + CHECK(reply["error"]["message"] == "the sender is restarting"); + } +} + +TEST_CASE("events", "[control_handler]") +{ + ProducerStatus status("r"); + status.set_state(ProducerState::Error, Fault{ "stream_failed", "gone" }); + const auto state = Json::parse(state_event(status.snapshot())); + CHECK(state["type"] == "event"); + CHECK(state["event"] == "state_changed"); + CHECK(state["status"]["state"] == "error"); + CHECK(state["status"]["fault"]["code"] == "stream_failed"); + + const auto error = Json::parse(error_event(Fault{ "c", "m" })); + CHECK(error["event"] == "error"); + CHECK(error["error"]["code"] == "c"); + CHECK(error["error"]["message"] == "m"); +} diff --git a/src/sensing_control/tests/test_control_server.cpp b/src/sensing_control/tests/test_control_server.cpp new file mode 100644 index 0000000000..e8ca1c2e7a --- /dev/null +++ b/src/sensing_control/tests/test_control_server.cpp @@ -0,0 +1,764 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::sensing; +using namespace std::chrono_literals; +namespace ws = plugins::sensing::websocket; + +namespace +{ + +constexpr const char* kToken = "test-token-123"; + +/// A received server frame (server frames are unmasked). +struct ServerFrame +{ + std::uint8_t opcode = 0; + std::string payload; +}; + +/// A bare TCP client that speaks just enough WebSocket to drive the server. +class RawClient +{ +public: + explicit RawClient(std::uint16_t port) + { + m_fd = ::socket(AF_INET, SOCK_STREAM, 0); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + REQUIRE(::connect(m_fd, reinterpret_cast(&addr), sizeof(addr)) == 0); + } + ~RawClient() + { + if (m_fd >= 0) + ::close(m_fd); + } + RawClient(const RawClient&) = delete; + RawClient& operator=(const RawClient&) = delete; + + void send_raw(const std::string& bytes) + { + std::size_t done = 0; + while (done < bytes.size()) + { + const ssize_t n = ::send(m_fd, bytes.data() + done, bytes.size() - done, MSG_NOSIGNAL); + REQUIRE(n > 0); + done += static_cast(n); + } + } + + static std::string upgrade(const std::string& target, const std::string& auth_header) + { + return "GET " + target + + " HTTP/1.1\r\nHost: x\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n" + + auth_header + "\r\n"; + } + + /// Handshake with the right token; returns the response head. + std::string handshake(const std::string& token = kToken) + { + send_raw(upgrade("/", "Authorization: Bearer " + token + "\r\n")); + return read_head(); + } + + /// Reads up to the end of an HTTP head; for a refusal also reads the body that follows. Empty on EOF/timeout. + std::string read_head() + { + while (m_buffer.find("\r\n\r\n") == std::string::npos) + if (!fill(3000)) + break; + const auto end = m_buffer.find("\r\n\r\n"); + if (end == std::string::npos) + return {}; + std::string head = m_buffer.substr(0, end + 4); + m_buffer.erase(0, end + 4); + return head; + } + + /// Reads whatever remains until the peer closes (or the timeout), for HTTP bodies. + std::string read_until_eof(int timeout_ms = 3000) + { + while (fill(timeout_ms)) + { + } + std::string out = std::move(m_buffer); + m_buffer.clear(); + return out; + } + + /// Sends as much as the peer accepts; false once the peer has closed or stopped reading for `timeout_ms`. + bool try_send_raw(const std::string& bytes, int timeout_ms = 3000) + { + std::size_t done = 0; + while (done < bytes.size()) + { + pollfd pfd{ m_fd, POLLOUT, 0 }; + if (::poll(&pfd, 1, timeout_ms) <= 0) + return false; + const ssize_t n = ::send(m_fd, bytes.data() + done, bytes.size() - done, MSG_NOSIGNAL | MSG_DONTWAIT); + if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) + continue; + if (n <= 0) + return false; + done += static_cast(n); + } + return true; + } + + void send_frame(std::uint8_t opcode, const std::string& payload, bool fin = true, bool masked = true) + { + send_raw(build_frame(opcode, payload, fin, masked)); + } + + static std::string build_frame(std::uint8_t opcode, const std::string& payload, bool fin = true, bool masked = true) + { + const unsigned char mask[4] = { 1, 2, 3, 4 }; + std::string out; + out.push_back(static_cast((fin ? 0x80 : 0) | opcode)); + const std::uint8_t mask_bit = masked ? 0x80 : 0; + if (payload.size() < 126) + { + out.push_back(static_cast(mask_bit | payload.size())); + } + else if (payload.size() <= 0xFFFF) + { + out.push_back(static_cast(mask_bit | 126)); + out.push_back(static_cast(payload.size() >> 8)); + out.push_back(static_cast(payload.size() & 0xFF)); + } + else + { + out.push_back(static_cast(mask_bit | 127)); + for (int shift = 56; shift >= 0; shift -= 8) + out.push_back(static_cast((static_cast(payload.size()) >> shift) & 0xFF)); + } + if (masked) + out.append(reinterpret_cast(mask), 4); + for (std::size_t i = 0; i < payload.size(); ++i) + out.push_back(static_cast(masked ? (static_cast(payload[i]) ^ mask[i % 4]) : payload[i])); + return out; + } + + /// The next server frame, or nullopt on EOF/timeout. + std::optional read_frame(int timeout_ms = 3000) + { + while (true) + { + if (m_buffer.size() >= 2) + { + const auto b = [&](std::size_t i) { return static_cast(m_buffer[i]); }; + std::uint64_t length = b(1) & 0x7F; + std::size_t header = 2; + if (length == 126 && m_buffer.size() >= 4) + { + length = (std::uint64_t(b(2)) << 8) | b(3); + header = 4; + } + else if (length == 127 && m_buffer.size() >= 10) + { + length = 0; + for (int i = 0; i < 8; ++i) + length = (length << 8) | b(2 + i); + header = 10; + } + const bool header_known = + !((b(1) & 0x7F) == 126 && m_buffer.size() < 4) && !((b(1) & 0x7F) == 127 && m_buffer.size() < 10); + if (header_known && m_buffer.size() >= header + length) + { + ServerFrame frame; + frame.opcode = b(0) & 0x0F; + frame.payload = m_buffer.substr(header, static_cast(length)); + m_buffer.erase(0, header + static_cast(length)); + return frame; + } + } + if (!fill(timeout_ms)) + return std::nullopt; + } + } + + /// The close code of the next frame, which must be a close frame; 0 otherwise. + std::uint16_t read_close_code(int timeout_ms = 3000) + { + const auto frame = read_frame(timeout_ms); + if (!frame || frame->opcode != ws::kClose || frame->payload.size() < 2) + return 0; + return static_cast((static_cast(frame->payload[0]) << 8) | + static_cast(frame->payload[1])); + } + + /// True if the peer closes the TCP connection (EOF or reset) within the timeout. + bool peer_closes(int timeout_ms = 3000) + { + const auto until = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + while (std::chrono::steady_clock::now() < until) + { + pollfd pfd{ m_fd, POLLIN, 0 }; + if (::poll(&pfd, 1, 100) > 0) + { + char buffer[4096]; + const ssize_t n = ::recv(m_fd, buffer, sizeof(buffer), 0); + if (n <= 0) + return true; + } + } + return false; + } + +private: + bool fill(int timeout_ms) + { + pollfd pfd{ m_fd, POLLIN, 0 }; + if (::poll(&pfd, 1, timeout_ms) <= 0) + return false; + char buffer[4096]; + const ssize_t n = ::recv(m_fd, buffer, sizeof(buffer), 0); + if (n <= 0) + return false; + m_buffer.append(buffer, static_cast(n)); + return true; + } + + int m_fd = -1; + std::string m_buffer; +}; + +ControlServerConfig config() +{ + ControlServerConfig c; + c.bind_address = "127.0.0.1"; + c.port = 0; + c.token = kToken; + return c; +} + +std::unique_ptr make_server(ControlServerConfig c = config()) +{ + return std::make_unique(c, [](const std::string& text) { return "echo:" + text; }); +} + +bool wait_clients(const ControlServer& server, std::size_t count) +{ + const auto until = std::chrono::steady_clock::now() + 3s; + while (std::chrono::steady_clock::now() < until) + { + if (server.client_count() == count) + return true; + std::this_thread::sleep_for(5ms); + } + return server.client_count() == count; +} + +} // namespace + +TEST_CASE("an empty token is refused at construction", "[control_server]") +{ + ControlServerConfig c = config(); + c.token.clear(); + CHECK_THROWS_AS(make_server(c), std::runtime_error); +} + +TEST_CASE("a bind address that is not IPv4 is refused", "[control_server]") +{ + ControlServerConfig c = config(); + c.bind_address = "not-an-address"; + CHECK_THROWS_AS(make_server(c), std::runtime_error); +} + +TEST_CASE("a port already in use is refused with the reason", "[control_server]") +{ + auto first = make_server(); + ControlServerConfig c = config(); + c.port = first->port(); + // SO_REUSEADDR does not allow two listeners on the same port. + try + { + make_server(c); + FAIL("expected the second bind to fail"); + } + catch (const std::runtime_error& e) + { + CHECK(std::string(e.what()).find("cannot listen") != std::string::npos); + } +} + +TEST_CASE("read_secret_file takes the first line, trimmed", "[control_server]") +{ + const auto dir = std::filesystem::temp_directory_path(); + const auto path = dir / ("sensing_control_secret_" + std::to_string(::getpid()) + ".txt"); + { + std::ofstream(path) << " hunter2 \t\r\nsecond line\n"; + } + CHECK(read_secret_file(path.string()) == "hunter2"); + { + std::ofstream(path) << " \n"; + } + CHECK_THROWS_AS(read_secret_file(path.string()), std::runtime_error); + std::filesystem::remove(path); + CHECK_THROWS_AS(read_secret_file(path.string()), std::runtime_error); +} + +TEST_CASE("a good handshake is accepted with the RFC accept key", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + const std::string head = client.handshake(); + CHECK(head.rfind("HTTP/1.1 101", 0) == 0); + CHECK(head.find("s3pPLMBiTxaQ9kYGzzhZRbK+xOo=") != std::string::npos); + CHECK(wait_clients(*server, 1)); +} + +TEST_CASE("the token may be sent as a query parameter", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + client.send_raw(RawClient::upgrade(std::string("/?token=") + kToken, "")); + CHECK(client.read_head().rfind("HTTP/1.1 101", 0) == 0); +} + +TEST_CASE("a wrong or absent token is 401 and the connection ends", "[control_server]") +{ + auto server = make_server(); + + SECTION("wrong token") + { + RawClient client(server->port()); + const std::string head = client.handshake("wrong-token"); + CHECK(head.rfind("HTTP/1.1 401", 0) == 0); + CHECK(client.peer_closes()); + } + SECTION("same length, one byte off") + { + RawClient client(server->port()); + std::string almost = kToken; + almost.back() ^= 1; + CHECK(client.handshake(almost).rfind("HTTP/1.1 401", 0) == 0); + } + SECTION("no token") + { + RawClient client(server->port()); + client.send_raw(RawClient::upgrade("/", "")); + CHECK(client.read_head().rfind("HTTP/1.1 401", 0) == 0); + } + CHECK(server->client_count() == 0); +} + +TEST_CASE("a plain HTTP GET is 426", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + client.send_raw("GET / HTTP/1.1\r\nHost: x\r\n\r\n"); + const std::string head = client.read_head(); + CHECK(head.rfind("HTTP/1.1 426", 0) == 0); + CHECK(head.find("Upgrade: websocket") != std::string::npos); + CHECK(client.read_until_eof().find("WebSocket endpoint") != std::string::npos); +} + +TEST_CASE("a malformed request line is 400", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + client.send_raw("POST / HTTP/1.1\r\n\r\n"); + CHECK(client.read_head().rfind("HTTP/1.1 400", 0) == 0); +} + +TEST_CASE("an oversized handshake is refused", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + client.send_raw("GET / HTTP/1.1\r\nX-Pad: " + std::string(16 * 1024, 'a')); + CHECK(client.read_head().rfind("HTTP/1.1 400", 0) == 0); +} + +TEST_CASE("ping frames are answered with pong carrying the payload", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kPing, "beat"); + const auto frame = client.read_frame(); + REQUIRE(frame); + CHECK(frame->opcode == ws::kPong); + CHECK(frame->payload == "beat"); +} + +TEST_CASE("a text message goes through the handler and the reply comes back", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + + SECTION("a single frame") + { + client.send_frame(ws::kText, "hello"); + const auto frame = client.read_frame(); + REQUIRE(frame); + CHECK(frame->opcode == ws::kText); + CHECK(frame->payload == "echo:hello"); + } + SECTION("fragments are reassembled") + { + client.send_frame(ws::kText, "he", /*fin=*/false); + client.send_frame(ws::kContinuation, "ll", /*fin=*/false); + client.send_frame(ws::kContinuation, "o", /*fin=*/true); + const auto frame = client.read_frame(); + REQUIRE(frame); + CHECK(frame->payload == "echo:hello"); + } + SECTION("a message that needs a 16-bit length is served") + { + const std::string big(5000, 'z'); + client.send_frame(ws::kText, big); + const auto frame = client.read_frame(); + REQUIRE(frame); + CHECK(frame->payload == "echo:" + big); + } + SECTION("several messages in order") + { + for (int i = 0; i < 5; ++i) + client.send_frame(ws::kText, std::to_string(i)); + for (int i = 0; i < 5; ++i) + { + const auto frame = client.read_frame(); + REQUIRE(frame); + CHECK(frame->payload == "echo:" + std::to_string(i)); + } + } + SECTION("a client close frame is answered with close 1000 and the connection ends") + { + client.send_frame(ws::kClose, std::string("\x03\xe8", 2)); + CHECK(client.read_close_code() == 1000); + CHECK(client.peer_closes()); + CHECK(wait_clients(*server, 0)); + } +} + +TEST_CASE("broadcast reaches every connected client", "[control_server]") +{ + auto server = make_server(); + RawClient a(server->port()); + RawClient b(server->port()); + REQUIRE(a.handshake().rfind("HTTP/1.1 101", 0) == 0); + REQUIRE(b.handshake().rfind("HTTP/1.1 101", 0) == 0); + REQUIRE(wait_clients(*server, 2)); + + server->broadcast("{\"event\":1}"); + server->broadcast("{\"event\":2}"); + for (RawClient* client : { &a, &b }) + { + const auto first = client->read_frame(); + const auto second = client->read_frame(); + REQUIRE(first); + REQUIRE(second); + CHECK(first->opcode == ws::kText); + CHECK(first->payload == "{\"event\":1}"); + CHECK(second->payload == "{\"event\":2}"); + } +} + +TEST_CASE("a client that has not handshaken does not receive broadcasts", "[control_server]") +{ + auto server = make_server(); + RawClient pending(server->port()); + RawClient open(server->port()); + REQUIRE(open.handshake().rfind("HTTP/1.1 101", 0) == 0); + server->broadcast("x"); + REQUIRE(open.read_frame()); + CHECK_FALSE(pending.read_frame(300)); +} + +TEST_CASE("an oversized message closes the connection with 1009", "[control_server]") +{ + ControlServerConfig c = config(); + c.max_message_bytes = 256; + auto server = make_server(c); + + SECTION("one frame") + { + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kText, std::string(1000, 'a')); + CHECK(client.read_close_code() == 1009); + CHECK(client.peer_closes()); + } + SECTION("fragments that add up to too much") + { + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kText, std::string(200, 'a'), /*fin=*/false); + client.send_frame(ws::kContinuation, std::string(200, 'a'), /*fin=*/false); + CHECK(client.read_close_code() == 1009); + } + SECTION("a message exactly at the limit is served") + { + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kText, std::string(256, 'a')); + const auto frame = client.read_frame(); + REQUIRE(frame); + CHECK(frame->opcode == ws::kText); + } +} + +TEST_CASE("a binary frame closes the connection with 1003", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kBinary, "\x01\x02"); + CHECK(client.read_close_code() == 1003); + CHECK(client.peer_closes()); +} + +TEST_CASE("an unmasked client frame closes the connection with 1002", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kText, "hi", true, /*masked=*/false); + CHECK(client.read_close_code() == 1002); + CHECK(client.peer_closes()); + CHECK(wait_clients(*server, 0)); +} + +TEST_CASE("max_clients is enforced and a slot is reusable", "[control_server]") +{ + ControlServerConfig c = config(); + c.max_clients = 2; + auto server = make_server(c); + + auto first = std::make_unique(server->port()); + RawClient second(server->port()); + REQUIRE(first->handshake().rfind("HTTP/1.1 101", 0) == 0); + REQUIRE(second.handshake().rfind("HTTP/1.1 101", 0) == 0); + REQUIRE(wait_clients(*server, 2)); + + { + RawClient third(server->port()); + third.send_raw(RawClient::upgrade("/", std::string("Authorization: Bearer ") + kToken + "\r\n")); + CHECK(third.peer_closes()); + } + CHECK(server->client_count() == 2); + + first.reset(); // a departing client frees a slot + REQUIRE(wait_clients(*server, 1)); + RawClient fourth(server->port()); + CHECK(fourth.handshake().rfind("HTTP/1.1 101", 0) == 0); +} + +TEST_CASE("destroying the server is bounded with idle and half-open clients", "[control_server]") +{ + auto server = make_server(); + RawClient idle(server->port()); + REQUIRE(idle.handshake().rfind("HTTP/1.1 101", 0) == 0); + RawClient half_open(server->port()); // connected, never sends the request + RawClient partial(server->port()); + partial.send_raw("GET / HTTP/1.1\r\nHost: x\r\n"); // an unfinished head + REQUIRE(wait_clients(*server, 1)); + std::this_thread::sleep_for(100ms); + + const auto start = std::chrono::steady_clock::now(); + server.reset(); + CHECK(std::chrono::steady_clock::now() - start < 2s); +} + +TEST_CASE("destroying an idle server is prompt", "[control_server]") +{ + auto server = make_server(); + const auto start = std::chrono::steady_clock::now(); + server.reset(); + CHECK(std::chrono::steady_clock::now() - start < 1s); +} + +TEST_CASE("a client that never reads is cut off instead of buffered for", "[control_server]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + REQUIRE(wait_clients(*server, 1)); + + // Far more than the 1 MiB output cap plus the socket buffers; the client reads none of it. + const std::string chunk(32 * 1024, 'x'); + for (int i = 0; i < 600; ++i) + { + server->broadcast(chunk); + if (i % 16 == 0) + std::this_thread::sleep_for(10ms); + } + CHECK(wait_clients(*server, 0)); +} + +TEST_CASE("an orphan continuation frame closes the connection with 1002", "[control_server][fragmentation]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kContinuation, "tail", /*fin=*/true); + CHECK(client.read_close_code() == 1002); + CHECK(client.peer_closes()); + CHECK(wait_clients(*server, 0)); +} + +TEST_CASE("a new text frame inside an unfinished message closes the connection with 1002", + "[control_server][fragmentation]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kText, "he", /*fin=*/false); + client.send_frame(ws::kText, "other", /*fin=*/true); + CHECK(client.read_close_code() == 1002); + CHECK(client.peer_closes()); +} + +TEST_CASE("a continuation after a finished message is an orphan", "[control_server][fragmentation]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + client.send_frame(ws::kText, "one"); + const auto reply = client.read_frame(); + REQUIRE(reply); + CHECK(reply->payload == "echo:one"); + client.send_frame(ws::kContinuation, "x"); + CHECK(client.read_close_code() == 1002); +} + +TEST_CASE("fragmented messages work back to back, with a ping between fragments", "[control_server][fragmentation]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + for (int round = 0; round < 2; ++round) + { + client.send_frame(ws::kText, "ab", /*fin=*/false); + client.send_frame(ws::kPing, "p"); + client.send_frame(ws::kContinuation, "cd", /*fin=*/true); + const auto pong = client.read_frame(); + REQUIRE(pong); + CHECK(pong->opcode == ws::kPong); + const auto reply = client.read_frame(); + REQUIRE(reply); + CHECK(reply->payload == "echo:abcd"); + } +} + +TEST_CASE("garbage beyond the message limit is cut off and the server stays responsive", "[control_server][limits]") +{ + ControlServerConfig c = config(); + c.max_message_bytes = 1024; + auto server = make_server(c); + + RawClient flooder(server->port()); + REQUIRE(flooder.handshake().rfind("HTTP/1.1 101", 0) == 0); + RawClient other(server->port()); + REQUIRE(other.handshake().rfind("HTTP/1.1 101", 0) == 0); + + // A masked text frame header declaring 1000 bytes, then far more than limit plus headroom of filler without a + // complete valid frame boundary. + std::string flood = std::string("\x81\xFE\x03\xE8", 4) + std::string(4, '\0') + std::string(256 * 1024, 'a'); + const bool all_sent = flooder.try_send_raw(flood); + INFO("all_sent=" << all_sent); + // Either the connection is closed (with or without a close frame) or the sender is held back; never served. + // The leading valid frame may be answered; the filler after it ends in a close, not in service. + std::optional frame; + bool closed_by_frame = false; + while ((frame = flooder.read_frame(3000))) + { + if (frame->opcode == ws::kClose) + { + closed_by_frame = true; + break; + } + CHECK(frame->opcode == ws::kText); + } + INFO("closed_by_frame=" << closed_by_frame); + CHECK(flooder.peer_closes(3000)); + + other.send_frame(ws::kText, "still here"); + const auto reply = other.read_frame(); + REQUIRE(reply); + CHECK(reply->payload == "echo:still here"); + CHECK(wait_clients(*server, 1)); +} + +TEST_CASE("handshake garbage without a terminator is limited and others are served", "[control_server][limits]") +{ + auto server = make_server(); + RawClient flooder(server->port()); + flooder.try_send_raw(std::string(256 * 1024, 'g')); + CHECK(flooder.read_head().rfind("HTTP/1.1 400", 0) == 0); + + RawClient other(server->port()); + REQUIRE(other.handshake().rfind("HTTP/1.1 101", 0) == 0); +} + +TEST_CASE("pipelined requests in one write are all answered", "[control_server][limits]") +{ + auto server = make_server(); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + + // More than the 16 frames a client may have handled per poll round: the rest follow on later rounds. + constexpr int kRequests = 40; + std::string burst; + for (int i = 0; i < kRequests; ++i) + burst += RawClient::build_frame(ws::kText, std::to_string(i)); + client.send_raw(burst); + + const auto begin = std::chrono::steady_clock::now(); + for (int i = 0; i < kRequests; ++i) + { + const auto frame = client.read_frame(2000); + REQUIRE(frame); + CHECK(frame->payload == "echo:" + std::to_string(i)); + } + CHECK(std::chrono::steady_clock::now() - begin < 2s); +} + +TEST_CASE("a client that pipelines requests with large replies and never reads is cut off", "[control_server][limits]") +{ + ControlServerConfig c = config(); + auto server = make_server(c); + RawClient client(server->port()); + REQUIRE(client.handshake().rfind("HTTP/1.1 101", 0) == 0); + RawClient other(server->port()); + REQUIRE(other.handshake().rfind("HTTP/1.1 101", 0) == 0); + REQUIRE(wait_clients(*server, 2)); + + // Each echo is about 60 KiB; the client never reads, so the replies pile up past the output cap. + const std::string request = RawClient::build_frame(ws::kText, std::string(60 * 1024, 'r')); + for (int i = 0; i < 200; ++i) + { + if (!client.try_send_raw(request, 500)) + break; + } + CHECK(wait_clients(*server, 1)); + + other.send_frame(ws::kText, "alive"); + const auto reply = other.read_frame(); + REQUIRE(reply); + CHECK(reply->payload == "echo:alive"); +} diff --git a/src/sensing_control/tests/test_driver_log_capture.cpp b/src/sensing_control/tests/test_driver_log_capture.cpp new file mode 100644 index 0000000000..7bf90a8ff6 --- /dev/null +++ b/src/sensing_control/tests/test_driver_log_capture.cpp @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include +#include +#include + +using plugins::sensing::DriverLogCapture; + +namespace +{ + +bool contains(const std::string& text, const std::string& part) +{ + return text.find(part) != std::string::npos; +} + +std::size_t count_of(const std::string& text, const std::string& part) +{ + std::size_t n = 0; + for (auto at = text.find(part); at != std::string::npos; at = text.find(part, at + 1)) + ++n; + return n; +} + +/// Points fd 2 at a scratch file for the lifetime of the object, so a test can see what the code under test replayed. +class StderrSink +{ +public: + StderrSink() + { + std::fflush(stderr); + char path[] = "/tmp/driver-log-sink-XXXXXX"; + m_file = ::mkstemp(path); + ::unlink(path); + m_saved = ::dup(STDERR_FILENO); + ::dup2(m_file, STDERR_FILENO); + } + + ~StderrSink() + { + restore(); + ::close(m_saved); + ::close(m_file); + } + + std::string restore_and_read() + { + restore(); + std::string text; + ::lseek(m_file, 0, SEEK_SET); + char buffer[4096]; + ssize_t n = 0; + while ((n = ::read(m_file, buffer, sizeof(buffer))) > 0) + text.append(buffer, static_cast(n)); + return text; + } + +private: + void restore() + { + if (!m_restored) + { + std::fflush(stderr); + ::dup2(m_saved, STDERR_FILENO); + m_restored = true; + } + } + + int m_file = -1; + int m_saved = -1; + bool m_restored = false; +}; + +/// What fd `fd` points at (the /proc link text), so two descriptors can be compared. +std::string target_of(int fd) +{ + char target[256] = {}; + const ssize_t n = ::readlink(("/proc/self/fd/" + std::to_string(fd)).c_str(), target, sizeof(target) - 1); + return n < 0 ? std::string() : std::string(target, static_cast(n)); +} + +bool same_file(int fd, int reference) +{ + const auto a = target_of(fd); + return !a.empty() && a == target_of(reference); +} + +void raw_write(int fd, const std::string& text) +{ + REQUIRE(::write(fd, text.data(), text.size()) == static_cast(text.size())); +} + +} // namespace + +TEST_CASE("output on fd 1 and fd 2 inside the scope is captured and replayed to stderr", "[driver_log_capture]") +{ + StderrSink sink; + std::string interesting; + { + DriverLogCapture capture; + raw_write(STDOUT_FILENO, "out: CAMERA_DETECT failed\n"); + raw_write(STDERR_FILENO, "err: lock timeout on link 1\n"); + raw_write(STDERR_FILENO, "harmless chatter\n"); + interesting = capture.finish(); + } + const std::string replayed = sink.restore_and_read(); + + CHECK(contains(interesting, "CAMERA_DETECT failed")); + CHECK(contains(interesting, "lock timeout on link 1")); + CHECK_FALSE(contains(interesting, "harmless chatter")); + // Everything, interesting or not, reaches the journal. + CHECK(contains(replayed, "out: CAMERA_DETECT failed")); + CHECK(contains(replayed, "err: lock timeout on link 1")); + CHECK(contains(replayed, "harmless chatter")); +} + +TEST_CASE("the descriptors are restored after finish and a second finish is empty", "[driver_log_capture]") +{ + StderrSink sink; + const int probe_out = ::dup(STDOUT_FILENO); + const int probe_err = ::dup(STDERR_FILENO); + { + DriverLogCapture capture; + CHECK_FALSE(same_file(STDOUT_FILENO, probe_out)); + capture.finish(); + CHECK(same_file(STDOUT_FILENO, probe_out)); + CHECK(same_file(STDERR_FILENO, probe_err)); + // Text written after finish() goes to the restored stderr, not into a capture. + raw_write(STDERR_FILENO, "after finish\n"); + CHECK(capture.finish().empty()); + } + const std::string replayed = sink.restore_and_read(); + CHECK(count_of(replayed, "after finish") == 1); + ::close(probe_out); + ::close(probe_err); +} + +TEST_CASE("the descriptors are restored when the scope ends without finish", "[driver_log_capture]") +{ + StderrSink sink; + const int probe_out = ::dup(STDOUT_FILENO); + const int probe_err = ::dup(STDERR_FILENO); + { + DriverLogCapture capture; + raw_write(STDERR_FILENO, "lost without finish: error\n"); + CHECK_FALSE(same_file(STDERR_FILENO, probe_err)); + } + CHECK(same_file(STDOUT_FILENO, probe_out)); + CHECK(same_file(STDERR_FILENO, probe_err)); + raw_write(STDERR_FILENO, "after destruction\n"); + const std::string replayed = sink.restore_and_read(); + // The destructor replays what was captured, and later output is not swallowed. + CHECK(contains(replayed, "lost without finish: error")); + CHECK(count_of(replayed, "after destruction") == 1); + ::close(probe_out); + ::close(probe_err); +} + +TEST_CASE("interesting_lines picks failure lines", "[driver_log_capture]") +{ + // Fewer matching lines than the six-line limit per call, so each word is seen. + const std::vector> groups = { + { "an ERROR happened", "operation failed", "Fail to open", "request timeout" }, + { "OPEN_CIRCUIT", "CIRCUIT_OPEN_DISCONNECTION", "DES_LOCK_ERR", "CAMERA_DETECT" } + }; + for (const auto& group : groups) + { + std::string text = "starting up\n"; + for (const auto& line : group) + text += line + "\n"; + text += "all good here\n"; + const auto out = DriverLogCapture::interesting_lines(text); + for (const auto& part : group) + { + INFO(part); + CHECK(contains(out, part)); + } + CHECK_FALSE(contains(out, "starting up")); + CHECK_FALSE(contains(out, "all good here")); + } + CHECK(DriverLogCapture::interesting_lines("").empty()); +} + +TEST_CASE("interesting_lines strips the Severity prefix and joins with semicolons", "[driver_log_capture]") +{ + const std::string text = + "[TID=1234] Module_id 30 Severity 2 : lock timeout on link 1\n" + "[TID=1234] Module_id 30 Severity 2 : open failed\n"; + CHECK(DriverLogCapture::interesting_lines(text) == "lock timeout on link 1; open failed"); + + // A line with " : " but no Severity keeps its text. + CHECK(DriverLogCapture::interesting_lines("camera : error 5\n") == "camera : error 5"); +} + +TEST_CASE("interesting_lines ignores NACK lines", "[driver_log_capture]") +{ + // The driver prints these on healthy starts. + CHECK(DriverLogCapture::interesting_lines("I2C: NACK ... errno=121\nbus NACK at 0x1a\n").empty()); +} + +TEST_CASE("interesting_lines collapses consecutive duplicates", "[driver_log_capture]") +{ + CHECK(DriverLogCapture::interesting_lines("error one\nerror one\nerror one\nerror two\n") == "error one; error two"); +} + +TEST_CASE("interesting_lines keeps the first six lines and at most 600 characters", "[driver_log_capture]") +{ + std::string text; + for (int i = 1; i <= 9; ++i) + text += "error line " + std::to_string(i) + "\n"; + const auto six = DriverLogCapture::interesting_lines(text); + CHECK(contains(six, "error line 1")); + CHECK(contains(six, "error line 6")); + CHECK_FALSE(contains(six, "error line 7")); + CHECK(count_of(six, "; ") == 5); + + const std::string long_line = "error " + std::string(1000, 'x') + "\n"; + CHECK(DriverLogCapture::interesting_lines(long_line).size() == 600); +} + +TEST_CASE("interesting_lines does not match block or clock", "[driver_log_capture]") +{ + CHECK(DriverLogCapture::interesting_lines("block 3 ready\nclock running\nblocked on mutex\n").empty()); + // The substring rule is about "lock", not "block": "lock timeout" style words need the failure word. + CHECK(DriverLogCapture::interesting_lines("pll lock acquired\n").empty()); +} diff --git a/src/sensing_control/tests/test_producer_status.cpp b/src/sensing_control/tests/test_producer_status.cpp new file mode 100644 index 0000000000..d7655c956b --- /dev/null +++ b/src/sensing_control/tests/test_producer_status.cpp @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +using namespace plugins::sensing; + +namespace +{ +StreamStatus stream(const std::string& track) +{ + StreamStatus s; + s.track = track; + s.direct_host = "10.0.0.1"; + s.direct_port = 1111; + s.direct_connected = true; + return s; +} +} // namespace + +TEST_CASE("a new status is Starting with the release and no fault", "[producer_status]") +{ + ProducerStatus status("rel-1"); + const auto s = status.snapshot(); + CHECK(s.state == ProducerState::Starting); + CHECK_FALSE(s.fault.has_value()); + CHECK(s.release == "rel-1"); + CHECK(s.streams.empty()); +} + +TEST_CASE("state names", "[producer_status]") +{ + CHECK(std::string(to_string(ProducerState::Starting)) == "starting"); + CHECK(std::string(to_string(ProducerState::Streaming)) == "streaming"); + CHECK(std::string(to_string(ProducerState::CameraFault)) == "camera_fault"); + CHECK(std::string(to_string(ProducerState::Error)) == "error"); + CHECK(std::string(to_string(ProducerState::Stopping)) == "stopping"); +} + +TEST_CASE("a fault is kept only for the failing states and cleared on leaving them", "[producer_status]") +{ + ProducerStatus status("r"); + const Fault fault{ "code", "message" }; + + status.set_state(ProducerState::CameraFault, fault); + REQUIRE(status.snapshot().fault.has_value()); + CHECK(status.snapshot().fault->code == "code"); + + status.set_state(ProducerState::Error, fault); + CHECK(status.snapshot().fault.has_value()); + + status.set_state(ProducerState::Streaming, fault); // a fault passed with a healthy state is ignored + CHECK(status.snapshot().state == ProducerState::Streaming); + CHECK_FALSE(status.snapshot().fault.has_value()); + + status.set_state(ProducerState::CameraFault, fault); + status.set_state(ProducerState::Stopping); + CHECK_FALSE(status.snapshot().fault.has_value()); + + status.set_state(ProducerState::CameraFault); // a failing state with no fault stays empty + CHECK_FALSE(status.snapshot().fault.has_value()); +} + +TEST_CASE("set_direct_target changes one stream and reports unknown names", "[producer_status]") +{ + ProducerStatus status("r"); + status.set_streams({ stream("left"), stream("right") }); + + CHECK(status.set_direct_target("left", "192.168.1.2", 5600)); + auto s = status.snapshot(); + CHECK(s.streams[0].direct_host == "192.168.1.2"); + CHECK(s.streams[0].direct_port == 5600); + CHECK_FALSE(s.streams[0].direct_connected); + CHECK(s.streams[1].direct_host == "10.0.0.1"); + CHECK(s.streams[1].direct_connected); + + CHECK(status.set_direct_target("left", "", 5600)); // an empty host clears the port too + s = status.snapshot(); + CHECK(s.streams[0].direct_host.empty()); + CHECK(s.streams[0].direct_port == 0); + + CHECK_FALSE(status.set_direct_target("nope", "h", 1)); +} diff --git a/src/sensing_control/tests/test_producer_supervisor.cpp b/src/sensing_control/tests/test_producer_supervisor.cpp new file mode 100644 index 0000000000..cb71b14aa1 --- /dev/null +++ b/src/sensing_control/tests/test_producer_supervisor.cpp @@ -0,0 +1,780 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::sensing; +using namespace std::chrono_literals; +using Json = nlohmann::json; + +namespace +{ + +/// Everything the fake factory and its sessions record, shared with the test thread. +struct World +{ + std::mutex mutex; + int factory_calls = 0; + int fail_factory_until_call = 0; ///< calls up to and including this number throw + std::string factory_failure = "INvSIPLCamera::Init failed with SIPLStatus 10"; + std::function on_factory; ///< may throw; runs under no lock + std::vector targets_seen; + int sessions_created = 0; + int update_calls = 0; + int throw_update_after = -1; ///< a session's update() throws on its N+1th call; -1 never + int flushes = 0; + std::vector> retargets; + std::vector retarget_threads; ///< the thread each retarget was called on + std::vector sequence; ///< "update" and "retarget:::" in call order + std::vector published; + std::string update_failure = "room lost"; ///< the text a throwing update() carries + std::uint64_t frames = 5; ///< what every stream reports + bool advance_frames = false; ///< every streams() call adds a frame + std::map direct_errors; ///< per track +}; + +class FakeSession : public CameraSession +{ +public: + FakeSession(World& world, DirectTargets targets) : m_world(world), m_targets(std::move(targets)) + { + } + void update() override + { + std::lock_guard lock(m_world.mutex); + ++m_world.update_calls; + m_world.sequence.emplace_back("update"); + if (m_world.throw_update_after >= 0 && ++m_updates > m_world.throw_update_after) + throw std::runtime_error(m_world.update_failure); + } + void flush() override + { + std::lock_guard lock(m_world.mutex); + ++m_world.flushes; + } + std::vector streams() const override + { + std::lock_guard lock(m_world.mutex); + std::vector out; + for (const auto& target : m_targets) + { + StreamStatus s; + s.track = target.first; + if (m_world.advance_frames) + ++m_world.frames; + s.frames = m_world.frames; + const auto error = m_world.direct_errors.find(target.first); + if (error != m_world.direct_errors.end()) + s.direct_error = error->second; + // The session's own view of where it sends, as the real producer reports it. + s.direct_host = target.second.host; + s.direct_port = target.second.port; + out.push_back(s); + } + return out; + } + void set_direct_target(const std::string& track, const std::string& host, std::uint16_t port) override + { + std::lock_guard lock(m_world.mutex); + m_world.retargets.emplace_back(track, host, port); + m_world.retarget_threads.push_back(std::this_thread::get_id()); + m_world.sequence.push_back("retarget:" + track + ":" + host + ":" + std::to_string(port)); + m_targets[track] = { host, port }; + } + +private: + World& m_world; + DirectTargets m_targets; + int m_updates = 0; +}; + +struct Harness +{ + explicit Harness(std::vector configured = default_streams(), + bool with_publish = true, + std::chrono::milliseconds stall_timeout = 0ms) + : status("rel") + { + SupervisorConfig config; + config.stall_timeout = stall_timeout; + config.retry_interval = 100ms; + config.poll_interval = 2ms; + config.status_interval = 20ms; + supervisor = std::make_unique( + status, std::move(configured), + [this](const DirectTargets& targets) -> std::unique_ptr + { + std::function hook; + { + std::lock_guard lock(world.mutex); + ++world.factory_calls; + world.targets_seen.push_back(targets); + hook = world.on_factory; + if (world.factory_calls <= world.fail_factory_until_call) + throw std::runtime_error(world.factory_failure); + ++world.sessions_created; + } + if (hook) + hook(world); + return std::make_unique(world, targets); + }, + with_publish ? std::function( + [this](const std::string& text) + { + std::lock_guard lock(world.mutex); + world.published.push_back(text); + }) : + std::function(), + config); + } + + ~Harness() + { + stop_and_join(); + } + + static std::vector default_streams() + { + StreamStatus a; + a.track = "a"; + StreamStatus b; + b.track = "b"; + b.direct_host = "cfg-host"; + b.direct_port = 4000; + return { a, b }; + } + + void start() + { + thread = std::thread( + [this] + { + run_thread_id = std::this_thread::get_id(); + supervisor->run([this] { return stop.load(); }); + }); + } + + void stop_and_join() + { + stop = true; + if (thread.joinable()) + thread.join(); + } + + template + bool wait_for(Pred pred, std::chrono::milliseconds limit = 5s) + { + const auto until = std::chrono::steady_clock::now() + limit; + while (std::chrono::steady_clock::now() < until) + { + if (pred()) + return true; + std::this_thread::sleep_for(5ms); + } + return pred(); + } + + int sessions() + { + std::lock_guard lock(world.mutex); + return world.sessions_created; + } + + int updates() + { + std::lock_guard lock(world.mutex); + return world.update_calls; + } + + bool state_is(ProducerState state) + { + return status.snapshot().state == state; + } + + int count_events(const std::string& event, const std::string& code = "") + { + std::lock_guard lock(world.mutex); + int n = 0; + for (const auto& text : world.published) + { + const auto j = Json::parse(text); + if (j["event"] == event && (code.empty() || j["error"]["code"] == code)) + ++n; + } + return n; + } + + World world; + ProducerStatus status; + std::unique_ptr supervisor; + std::atomic stop{ false }; + std::thread thread; + std::atomic run_thread_id{}; +}; + +} // namespace + +TEST_CASE("a camera that will not start is a reported state, announced once, then recovers", "[supervisor]") +{ + Harness h; + h.world.fail_factory_until_call = 4; + h.start(); + + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::CameraFault); })); + { + const auto s = h.status.snapshot(); + REQUIRE(s.fault.has_value()); + CHECK(s.fault->code == "camera_start_failed"); + CHECK(s.fault->message.find("power cable") != std::string::npos); + CHECK(s.fault->message.find("SIPLStatus 10") != std::string::npos); + CHECK(s.streams.size() == 2); // the configured streams stay listed while the cameras are down + } + + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + CHECK_FALSE(h.status.snapshot().fault.has_value()); + CHECK(h.world.factory_calls == 5); // four failures, then one success + + // Four identical failures produced exactly one announcement. + CHECK(h.count_events("error", "camera_start_failed") == 1); + CHECK(h.count_events("error") == 1); + h.stop_and_join(); +} + +TEST_CASE("a different failure message is announced again", "[supervisor]") +{ + Harness h; + h.world.fail_factory_until_call = 1000; + h.start(); + REQUIRE(h.wait_for([&] { return h.count_events("error") == 1; })); + { + std::lock_guard lock(h.world.mutex); + h.world.factory_failure = "brand new problem"; + } + REQUIRE(h.wait_for([&] { return h.count_events("error") == 2; })); + CHECK(h.status.snapshot().fault->code == "startup_failed"); + h.stop_and_join(); +} + +TEST_CASE("a StartupFailure supplies its own fault and state", "[supervisor]") +{ + // A factory that has already explained its failure. + SupervisorConfig config; + config.retry_interval = 100ms; + config.poll_interval = 2ms; + std::atomic calls{ 0 }; + ProducerStatus status("r"); + ProducerSupervisor supervisor( + status, Harness::default_streams(), + [&](const DirectTargets&) -> std::unique_ptr + { + ++calls; + throw StartupFailure(Fault{ "room_unreachable", "Cannot reach the PC video stage." }, ProducerState::Error); + }, + nullptr, config); + std::atomic stop{ false }; + std::thread thread([&] { supervisor.run([&] { return stop.load(); }); }); + const auto until = std::chrono::steady_clock::now() + 5s; + while (status.snapshot().state != ProducerState::Error && std::chrono::steady_clock::now() < until) + std::this_thread::sleep_for(5ms); + const auto s = status.snapshot(); + stop = true; + thread.join(); + CHECK(s.state == ProducerState::Error); + REQUIRE(s.fault.has_value()); + CHECK(s.fault->code == "room_unreachable"); + CHECK(s.fault->message == "Cannot reach the PC video stage."); + CHECK(calls.load() >= 1); +} + +TEST_CASE("update() throwing reports Error and the session is restarted", "[supervisor]") +{ + Harness h; + h.world.throw_update_after = 3; + h.world.update_failure = "room lost"; + h.start(); + + REQUIRE(h.wait_for([&] { return h.count_events("error", "stream_failed") >= 1; })); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Error); })); + const auto s = h.status.snapshot(); + REQUIRE(s.fault.has_value()); + CHECK(s.fault->code == "stream_failed"); + CHECK(s.fault->message.find("room lost") != std::string::npos); + + // After the retry interval a fresh session starts and streams again. + REQUIRE(h.wait_for([&] { return h.sessions() >= 2; })); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming) || h.state_is(ProducerState::Error); })); + h.stop_and_join(); + CHECK(h.world.flushes >= 1); +} + +TEST_CASE("an unrecognised update() failure is reported as stream_failed with its text", "[supervisor]") +{ + Harness h; + h.world.throw_update_after = 3; + h.world.update_failure = "something odd"; + h.start(); + + REQUIRE(h.wait_for([&] { return h.count_events("error", "stream_failed") >= 1; })); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Error); })); + const auto s = h.status.snapshot(); + REQUIRE(s.fault.has_value()); + CHECK(s.fault->code == "stream_failed"); + CHECK(s.fault->message.find("something odd") != std::string::npos); + CHECK(h.count_events("error", "startup_failed") == 0); + h.stop_and_join(); +} + +TEST_CASE("a restarted session keeps targets set at runtime", "[supervisor]") +{ + Harness h; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + + SECTION("set while streaming reaches the live session and the next factory call") + { + const auto result = h.supervisor->set_direct_target("a", "10.9.8.7", 5730); + CHECK(result.ok); + // The running thread applies it between updates, so it arrives shortly after the call returns. + REQUIRE(h.wait_for( + [&] + { + std::lock_guard lock(h.world.mutex); + return !h.world.retargets.empty(); + })); + { + std::lock_guard lock(h.world.mutex); + REQUIRE(h.world.retargets.size() == 1); + CHECK(std::get<0>(h.world.retargets[0]) == "a"); + CHECK(std::get<1>(h.world.retargets[0]) == "10.9.8.7"); + CHECK(std::get<2>(h.world.retargets[0]) == 5730); + } + // The status shows what the session reports after it applied the change. + REQUIRE(h.wait_for( + [&] + { + const auto s = h.status.snapshot(); + return !s.streams.empty() && s.streams[0].direct_host == "10.9.8.7" && + s.streams[0].direct_port == 5730 && s.streams[0].frames == 5; + })); + // A broadcast notified the change. + CHECK(h.count_events("state_changed") >= 1); + + // Make the session fail; the next factory call must see the remembered target. + { + std::lock_guard lock(h.world.mutex); + h.world.throw_update_after = 0; + } + REQUIRE(h.wait_for([&] { return h.sessions() >= 2; }, 10s)); + std::lock_guard lock(h.world.mutex); + const auto& last = h.world.targets_seen.back(); + REQUIRE(last.count("a") == 1); + CHECK(last.at("a").host == "10.9.8.7"); + CHECK(last.at("a").port == 5730); + CHECK(last.at("b").host == "cfg-host"); // untouched streams keep their configured target + CHECK(last.at("b").port == 4000); + } + SECTION("an empty host disables the link and is remembered as such") + { + CHECK(h.supervisor->set_direct_target("b", "", 1234).ok); + REQUIRE(h.wait_for( + [&] + { + std::lock_guard lock(h.world.mutex); + return !h.world.retargets.empty(); + })); + { + std::lock_guard lock(h.world.mutex); + h.world.throw_update_after = 0; + } + REQUIRE(h.wait_for([&] { return h.sessions() >= 2; }, 10s)); + std::lock_guard lock(h.world.mutex); + CHECK(h.world.targets_seen.back().at("b").host.empty()); + CHECK(h.world.targets_seen.back().at("b").port == 0); + } + SECTION("an unknown stream is rejected and changes nothing") + { + const auto result = h.supervisor->set_direct_target("nope", "h", 1); + CHECK_FALSE(result.ok); + CHECK(result.code == "unknown_stream"); + std::lock_guard lock(h.world.mutex); + CHECK(h.world.retargets.empty()); + } + h.stop_and_join(); +} + +TEST_CASE("a target set while the cameras are down is used when they come up", "[supervisor]") +{ + Harness h; + h.world.fail_factory_until_call = 1000; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::CameraFault); })); + + CHECK(h.supervisor->set_direct_target("a", "10.0.0.5", 5731).ok); + { + std::lock_guard lock(h.world.mutex); + CHECK(h.world.retargets.empty()); // there was no session to retarget + h.world.fail_factory_until_call = 0; + } + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + std::lock_guard lock(h.world.mutex); + CHECK(h.world.targets_seen.back().at("a").host == "10.0.0.5"); + CHECK(h.world.targets_seen.back().at("a").port == 5731); +} + +TEST_CASE("the stop flag ends run() with Stopping, even during a retry wait", "[supervisor]") +{ + SECTION("while streaming") + { + Harness h; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + h.stop_and_join(); + CHECK(h.state_is(ProducerState::Stopping)); + CHECK(h.world.flushes == 1); + } + SECTION("while the cameras are down") + { + Harness h; + h.world.fail_factory_until_call = 1000; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::CameraFault); })); + const auto begin = std::chrono::steady_clock::now(); + h.stop_and_join(); + CHECK(std::chrono::steady_clock::now() - begin < 2s); + CHECK(h.state_is(ProducerState::Stopping)); + CHECK_FALSE(h.status.snapshot().fault.has_value()); + } +} + +TEST_CASE("an empty publish callback is allowed", "[supervisor]") +{ + Harness h(Harness::default_streams(), /*with_publish=*/false); + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + CHECK(h.supervisor->set_direct_target("a", "h", 1).ok); + h.stop_and_join(); +} + +TEST_CASE("a factory failure is diagnosed and sets the state from the diagnosis", "[supervisor]") +{ + SECTION("an unrecognised startup failure is a generic CameraFault") + { + Harness h; + h.world.fail_factory_until_call = 1000; + h.world.factory_failure = "LiveKit: cannot join the room"; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::CameraFault); })); + const auto s = h.status.snapshot(); + REQUIRE(s.fault.has_value()); + CHECK(s.fault->code == "startup_failed"); + CHECK(s.fault->message.find("LiveKit: cannot join the room") != std::string::npos); + CHECK(h.count_events("error", "startup_failed") == 1); + h.stop_and_join(); + } + SECTION("a driver line about an open circuit is a CameraFault") + { + Harness h; + h.world.fail_factory_until_call = 1000; + h.world.factory_failure = "INvSIPLCamera::Init failed with SIPLStatus 10 [Severity 2 : OPEN_CIRCUIT]"; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::CameraFault); })); + const auto s = h.status.snapshot(); + REQUIRE(s.fault.has_value()); + CHECK(s.fault->code == "camera_power_missing"); + h.stop_and_join(); + } +} + +TEST_CASE("a stream whose frames stop is failed and the session restarted", "[supervisor][stall]") +{ + Harness h(Harness::default_streams(), true, 300ms); + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + // The fake session's frame count never moves, so the watchdog fires. + REQUIRE(h.wait_for([&] { return h.count_events("error", "camera_no_frames") >= 1; })); + { + const auto s = h.status.snapshot(); + // The text names the stall; camera_fault.cpp files "no frame for" under CameraFault. + if (s.fault.has_value() && s.fault->code == "camera_no_frames") + { + CHECK(s.fault->message.find("no frame for") != std::string::npos); + CHECK(s.fault->message.find("(stream ") != std::string::npos); + CHECK(s.fault->message.find("from sensor") != std::string::npos); + } + } + REQUIRE(h.wait_for([&] { return h.sessions() >= 2; })); + h.stop_and_join(); + CHECK(h.world.flushes >= 1); +} + +TEST_CASE("the stall state and text are those of the diagnosis", "[supervisor][stall]") +{ + Harness h(Harness::default_streams(), true, 300ms); + h.start(); + REQUIRE(h.wait_for([&] { return h.count_events("error", "camera_no_frames") >= 1; })); + const auto event = [&] + { + std::lock_guard lock(h.world.mutex); + for (const auto& text : h.world.published) + { + const auto j = Json::parse(text); + if (j["event"] == "error" && j["error"]["code"] == "camera_no_frames") + return j; + } + return Json(); + }(); + CHECK(event["error"]["message"].get().find("no frame for") != std::string::npos); + h.stop_and_join(); +} + +TEST_CASE("frames that keep advancing are not a stall", "[supervisor][stall]") +{ + Harness h(Harness::default_streams(), true, 300ms); + h.world.advance_frames = true; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + std::this_thread::sleep_for(1200ms); + CHECK(h.state_is(ProducerState::Streaming)); + CHECK(h.count_events("error") == 0); + h.stop_and_join(); + CHECK(h.world.sessions_created == 1); +} + +TEST_CASE("a stall timeout of zero disables the watchdog", "[supervisor][stall]") +{ + Harness h(Harness::default_streams(), true, 0ms); + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + std::this_thread::sleep_for(1200ms); // frames never move + CHECK(h.state_is(ProducerState::Streaming)); + CHECK(h.count_events("error") == 0); + h.stop_and_join(); + CHECK(h.world.sessions_created == 1); +} + +TEST_CASE("a direct-link error is published once per distinct reason", "[supervisor][direct_error]") +{ + Harness h; + h.world.advance_frames = true; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + CHECK(h.count_events("error", "direct_link_failed") == 0); + + { + std::lock_guard lock(h.world.mutex); + h.world.direct_errors["a"] = "cannot connect to 10.0.0.5:5600: Connection refused"; + } + REQUIRE(h.wait_for([&] { return h.count_events("error", "direct_link_failed") == 1; })); + // Many status refreshes later, still one. + std::this_thread::sleep_for(300ms); + CHECK(h.count_events("error", "direct_link_failed") == 1); + + { + std::lock_guard lock(h.world.mutex); + h.world.direct_errors["a"] = "10.0.0.5:5600 closed the control connection"; + } + REQUIRE(h.wait_for([&] { return h.count_events("error", "direct_link_failed") == 2; })); + + // Clearing publishes nothing. + { + std::lock_guard lock(h.world.mutex); + h.world.direct_errors["a"] = ""; + } + std::this_thread::sleep_for(300ms); + CHECK(h.count_events("error", "direct_link_failed") == 2); + + // The same reason coming back after it cleared is a new occurrence. + { + std::lock_guard lock(h.world.mutex); + h.world.direct_errors["a"] = "10.0.0.5:5600 closed the control connection"; + } + REQUIRE(h.wait_for([&] { return h.count_events("error", "direct_link_failed") == 3; })); + + // It names the stream and the reason, and does not change the producer state. + { + std::lock_guard lock(h.world.mutex); + const auto j = Json::parse(h.world.published.back()); + const auto message = j["error"]["message"].get(); + CHECK(message.find("Stream a: ") != std::string::npos); + CHECK(message.find("closed the control connection") != std::string::npos); + } + CHECK(h.state_is(ProducerState::Streaming)); + h.stop_and_join(); +} + +TEST_CASE("direct-link errors of different streams are independent", "[supervisor][direct_error]") +{ + Harness h; + h.world.advance_frames = true; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + { + std::lock_guard lock(h.world.mutex); + h.world.direct_errors["a"] = "same reason"; + h.world.direct_errors["b"] = "same reason"; + } + REQUIRE(h.wait_for([&] { return h.count_events("error", "direct_link_failed") == 2; })); + h.stop_and_join(); +} + +namespace +{ + +/// A factory hook that parks the first call until released, so the test can act while a session is being built. +struct FactoryGate +{ + explicit FactoryGate(World& world) : released(release.get_future().share()) + { + world.on_factory = [this](World&) + { + if (!entered_once.exchange(true)) + entered.set_value(); + released.wait(); + }; + } + std::promise entered_promise; + std::promise release; + std::shared_future released; + std::atomic entered_once{ false }; + std::promise& entered = entered_promise; +}; + +} // namespace + +TEST_CASE("a change made while the factory runs is applied before the session's first update", "[supervisor][threading]") +{ + Harness h; + FactoryGate gate(h.world); + auto entered = gate.entered_promise.get_future(); + h.start(); + REQUIRE(entered.wait_for(5s) == std::future_status::ready); + + // The factory is blocked: the session does not exist yet. The call must return at once. + const auto begin = std::chrono::steady_clock::now(); + CHECK(h.supervisor->set_direct_target("a", "10.1.1.1", 5741).ok); + CHECK(std::chrono::steady_clock::now() - begin < 1s); + { + std::lock_guard lock(h.world.mutex); + CHECK(h.world.retargets.empty()); + REQUIRE(h.world.targets_seen.size() == 1); + CHECK(h.world.targets_seen[0].at("a").host.empty()); // chosen after the factory took its snapshot + } + gate.release.set_value(); + + REQUIRE(h.wait_for([&] { return h.updates() >= 3; })); + std::lock_guard lock(h.world.mutex); + REQUIRE(h.world.retargets.size() == 1); + CHECK(std::get<0>(h.world.retargets[0]) == "a"); + CHECK(std::get<1>(h.world.retargets[0]) == "10.1.1.1"); + CHECK(std::get<2>(h.world.retargets[0]) == 5741); + REQUIRE(h.world.sequence.size() >= 2); + CHECK(h.world.sequence[0] == "retarget:a:10.1.1.1:5741"); // before the first update + CHECK(h.world.sequence[1] == "update"); +} + +TEST_CASE("a target chosen before the factory runs is passed to it and not applied again", "[supervisor][threading]") +{ + Harness h; + CHECK(h.supervisor->set_direct_target("a", "10.2.2.2", 5742).ok); + h.start(); + REQUIRE(h.wait_for([&] { return h.updates() >= 5; })); + h.stop_and_join(); + std::lock_guard lock(h.world.mutex); + REQUIRE(h.world.targets_seen.size() >= 1); + CHECK(h.world.targets_seen[0].at("a").host == "10.2.2.2"); + CHECK(h.world.targets_seen[0].at("a").port == 5742); + CHECK(h.world.retargets.empty()); +} + +TEST_CASE("set_direct_target never calls into the session from the caller's thread", "[supervisor][threading]") +{ + Harness h; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + for (int i = 0; i < 5; ++i) + CHECK(h.supervisor->set_direct_target("a", "10.3.3." + std::to_string(i), 5743).ok); + REQUIRE(h.wait_for( + [&] + { + std::lock_guard lock(h.world.mutex); + return !h.world.retargets.empty() && std::get<1>(h.world.retargets.back()) == "10.3.3.4"; + })); + h.stop_and_join(); + std::lock_guard lock(h.world.mutex); + REQUIRE_FALSE(h.world.retarget_threads.empty()); + for (const auto& id : h.world.retarget_threads) + { + CHECK(id == h.run_thread_id.load()); + CHECK(id != std::this_thread::get_id()); + } +} + +TEST_CASE("changes to the same track coalesce to the last", "[supervisor][threading]") +{ + Harness h; + FactoryGate gate(h.world); + auto entered = gate.entered_promise.get_future(); + h.start(); + REQUIRE(entered.wait_for(5s) == std::future_status::ready); + + CHECK(h.supervisor->set_direct_target("a", "10.4.4.1", 1).ok); + CHECK(h.supervisor->set_direct_target("a", "10.4.4.2", 2).ok); + CHECK(h.supervisor->set_direct_target("a", "10.4.4.3", 3).ok); + CHECK(h.supervisor->set_direct_target("b", "10.4.4.9", 9).ok); + gate.release.set_value(); + + REQUIRE(h.wait_for([&] { return h.updates() >= 3; })); + h.stop_and_join(); + std::lock_guard lock(h.world.mutex); + REQUIRE(h.world.retargets.size() == 2); // one per track, not one per call + int a_calls = 0; + for (const auto& r : h.world.retargets) + { + if (std::get<0>(r) == "a") + { + ++a_calls; + CHECK(std::get<1>(r) == "10.4.4.3"); + CHECK(std::get<2>(r) == 3); + } + } + CHECK(a_calls == 1); +} + +TEST_CASE("the status shows the session's reported host and port", "[supervisor][threading]") +{ + Harness h; + h.start(); + REQUIRE(h.wait_for([&] { return h.state_is(ProducerState::Streaming); })); + CHECK(h.supervisor->set_direct_target("a", "10.5.5.5", 5745).ok); + REQUIRE(h.wait_for( + [&] + { + const auto s = h.status.snapshot(); + for (const auto& stream : s.streams) + if (stream.track == "a") + return stream.direct_host == "10.5.5.5" && stream.direct_port == 5745; + return false; + })); + const auto s = h.status.snapshot(); + for (const auto& stream : s.streams) + { + if (stream.track == "b") + { + CHECK(stream.direct_host == "cfg-host"); + CHECK(stream.direct_port == 4000); + } + } +} diff --git a/src/sensing_control/tests/test_websocket.cpp b/src/sensing_control/tests/test_websocket.cpp new file mode 100644 index 0000000000..e06c864887 --- /dev/null +++ b/src/sensing_control/tests/test_websocket.cpp @@ -0,0 +1,317 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include + +using namespace plugins::sensing::websocket; + +namespace +{ + +std::string hex(const std::string& bytes) +{ + static constexpr char kDigits[] = "0123456789abcdef"; + std::string out; + for (const unsigned char c : bytes) + { + out.push_back(kDigits[c >> 4]); + out.push_back(kDigits[c & 0x0F]); + } + return out; +} + +/// A client frame: masked, as RFC 6455 requires. +std::string client_frame(std::uint8_t opcode, const std::string& payload, bool fin = true, bool masked = true) +{ + const unsigned char mask[4] = { 0x37, 0xfa, 0x21, 0x3d }; + std::string out; + out.push_back(static_cast((fin ? 0x80 : 0) | opcode)); + const std::uint8_t mask_bit = masked ? 0x80 : 0; + if (payload.size() < 126) + { + out.push_back(static_cast(mask_bit | payload.size())); + } + else if (payload.size() <= 0xFFFF) + { + out.push_back(static_cast(mask_bit | 126)); + out.push_back(static_cast(payload.size() >> 8)); + out.push_back(static_cast(payload.size() & 0xFF)); + } + else + { + out.push_back(static_cast(mask_bit | 127)); + for (int shift = 56; shift >= 0; shift -= 8) + out.push_back(static_cast((static_cast(payload.size()) >> shift) & 0xFF)); + } + if (masked) + out.append(reinterpret_cast(mask), 4); + for (std::size_t i = 0; i < payload.size(); ++i) + out.push_back(static_cast(masked ? (static_cast(payload[i]) ^ mask[i % 4]) : payload[i])); + return out; +} + +std::string upgrade_request(const std::string& target = "/", + const std::string& extra_headers = "", + const std::string& version = "13") +{ + return "GET " + target + + " HTTP/1.1\r\nHost: x\r\nUpgrade: websocket\r\nConnection: keep-alive, Upgrade\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: " + + version + "\r\n" + extra_headers + "\r\n"; +} + +} // namespace + +TEST_CASE("sha1 matches the standard test vectors", "[websocket]") +{ + CHECK(hex(sha1("abc")) == "a9993e364706816aba3e25717850c26c9cd0d89d"); + CHECK(hex(sha1("")) == "da39a3ee5e6b4b0d3255bfef95601890afd80709"); + // Two blocks after padding. + CHECK(hex(sha1("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq")) == + "84983e441c3bd26ebaae4aa1f95129e5e54670f1"); +} + +TEST_CASE("base64 and the RFC 6455 accept key", "[websocket]") +{ + CHECK(base64_encode("") == ""); + CHECK(base64_encode("f") == "Zg=="); + CHECK(base64_encode("fo") == "Zm8="); + CHECK(base64_encode("foo") == "Zm9v"); + CHECK(accept_key("dGhlIHNhbXBsZSBub25jZQ==") == "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="); +} + +TEST_CASE("constant_time_equal", "[websocket]") +{ + CHECK(constant_time_equal("secret", "secret")); + CHECK(constant_time_equal("", "")); + CHECK_FALSE(constant_time_equal("secret", "secreT")); + CHECK_FALSE(constant_time_equal("secret", "secre")); + CHECK_FALSE(constant_time_equal("", "x")); +} + +TEST_CASE("parse_frame decodes the three length forms", "[websocket]") +{ + SECTION("short") + { + const auto r = parse_frame(client_frame(kText, "hello"), 1 << 20); + REQUIRE(r.status == ParseStatus::Complete); + CHECK(r.frame.fin); + CHECK(r.frame.opcode == kText); + CHECK(r.frame.payload == "hello"); + CHECK(r.consumed == 2 + 4 + 5); + } + SECTION("125 is the last short form, 126 uses 16 bits") + { + const std::string p125(125, 'a'); + const std::string p126(126, 'b'); + const auto a = parse_frame(client_frame(kText, p125), 1 << 20); + REQUIRE(a.status == ParseStatus::Complete); + CHECK(a.consumed == 2 + 4 + 125); + const auto b = parse_frame(client_frame(kText, p126), 1 << 20); + REQUIRE(b.status == ParseStatus::Complete); + CHECK(b.frame.payload == p126); + CHECK(b.consumed == 4 + 4 + 126); + } + SECTION("65535 is the 16-bit maximum, 65536 uses 64 bits") + { + const std::string p16(65535, 'c'); + const std::string p64(65536, 'd'); + const auto a = parse_frame(client_frame(kBinary, p16), 1 << 20); + REQUIRE(a.status == ParseStatus::Complete); + CHECK(a.consumed == 4 + 4 + 65535); + const auto b = parse_frame(client_frame(kBinary, p64), 1 << 20); + REQUIRE(b.status == ParseStatus::Complete); + CHECK(b.frame.payload == p64); + CHECK(b.consumed == 10 + 4 + 65536); + } +} + +TEST_CASE("parse_frame waits for more bytes and leaves trailing bytes", "[websocket]") +{ + const std::string frame = client_frame(kText, std::string(300, 'x')); + for (const std::size_t cut : { std::size_t(0), std::size_t(1), std::size_t(3), std::size_t(5), frame.size() - 1 }) + CHECK(parse_frame(std::string_view(frame).substr(0, cut), 1 << 20).status == ParseStatus::NeedMore); + + const std::string two = client_frame(kText, "one") + client_frame(kText, "two"); + const auto first = parse_frame(two, 1 << 20); + REQUIRE(first.status == ParseStatus::Complete); + CHECK(first.frame.payload == "one"); + const auto second = parse_frame(std::string_view(two).substr(first.consumed), 1 << 20); + REQUIRE(second.status == ParseStatus::Complete); + CHECK(second.frame.payload == "two"); +} + +TEST_CASE("parse_frame reports fragments through fin and continuation", "[websocket]") +{ + const auto first = parse_frame(client_frame(kText, "hel", /*fin=*/false), 1 << 20); + REQUIRE(first.status == ParseStatus::Complete); + CHECK_FALSE(first.frame.fin); + CHECK(first.frame.opcode == kText); + const auto last = parse_frame(client_frame(kContinuation, "lo", /*fin=*/true), 1 << 20); + REQUIRE(last.status == ParseStatus::Complete); + CHECK(last.frame.fin); + CHECK(last.frame.opcode == kContinuation); +} + +TEST_CASE("parse_frame errors", "[websocket]") +{ + SECTION("an unmasked client frame is a protocol error") + { + const auto r = parse_frame(client_frame(kText, "hi", true, /*masked=*/false), 1 << 20); + CHECK(r.status == ParseStatus::Error); + CHECK(r.close_code == 1002); + } + SECTION("reserved bits") + { + std::string frame = client_frame(kText, "hi"); + frame[0] = static_cast(frame[0] | 0x40); + const auto r = parse_frame(frame, 1 << 20); + CHECK(r.status == ParseStatus::Error); + CHECK(r.close_code == 1002); + } + SECTION("an oversized payload is 1009, decided from the header alone") + { + const std::string frame = client_frame(kText, std::string(200, 'a')); + const auto r = parse_frame(std::string_view(frame).substr(0, 4), 100); + CHECK(r.status == ParseStatus::Error); + CHECK(r.close_code == 1009); + const auto exact = parse_frame(client_frame(kText, std::string(100, 'a')), 100); + CHECK(exact.status == ParseStatus::Complete); + } + SECTION("a 64-bit length beyond the limit is 1009 without waiting for the payload") + { + std::string header; + header.push_back(static_cast(0x81)); + header.push_back(static_cast(0x80 | 127)); + header.append(std::string("\x00\x00\x00\x01\x00\x00\x00\x00", 8)); + const auto r = parse_frame(header, 1 << 20); + CHECK(r.status == ParseStatus::Error); + CHECK(r.close_code == 1009); + } + SECTION("control frames may not be fragmented or longer than 125 bytes") + { + CHECK(parse_frame(client_frame(kPing, "x", /*fin=*/false), 1 << 20).status == ParseStatus::Error); + CHECK(parse_frame(client_frame(kClose, std::string(126, 'x')), 1 << 20).status == ParseStatus::Error); + CHECK(parse_frame(client_frame(kPing, std::string(125, 'x')), 1 << 20).status == ParseStatus::Complete); + } +} + +TEST_CASE("encode_frame produces unmasked server frames", "[websocket]") +{ + CHECK(encode_frame(kText, "Hello") == std::string("\x81\x05Hello")); + CHECK(encode_frame(kText, "a", /*fin=*/false) == std::string("\x01\x01" + "a")); + + const std::string mid = encode_frame(kText, std::string(126, 'a')); + CHECK(mid.size() == 4 + 126); + CHECK(static_cast(mid[1]) == 126); + CHECK(static_cast(mid[2]) == 0); + CHECK(static_cast(mid[3]) == 126); + + const std::string big = encode_frame(kBinary, std::string(70000, 'a')); + CHECK(big.size() == 10 + 70000); + CHECK(static_cast(big[1]) == 127); + CHECK(static_cast(big[7]) == 1); + CHECK(static_cast(big[8]) == 0x11); + CHECK(static_cast(big[9]) == 0x70); + + const std::string close = encode_close(1009, "big"); + CHECK(close == std::string("\x88\x05\x03\xf1" + "big", + 7)); + CHECK(encode_close(1000, std::string(500, 'r')).size() == 2 + 2 + 120); +} + +TEST_CASE("parse_handshake accepts a valid upgrade", "[websocket]") +{ + const auto h = parse_handshake(upgrade_request("/chan", "Authorization: Bearer s3cret\r\n")); + REQUIRE(h.ok); + CHECK(h.key == "dGhlIHNhbXBsZSBub25jZQ=="); + CHECK(h.token == "s3cret"); + CHECK(h.path == "/chan"); + + const std::string response = handshake_response(h); + CHECK(response.rfind("HTTP/1.1 101 Switching Protocols\r\n", 0) == 0); + CHECK(response.find("Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n") != std::string::npos); +} + +TEST_CASE("parse_handshake finds the token in the header or the query", "[websocket]") +{ + SECTION("header names and the Bearer scheme are case-insensitive") + { + const auto h = parse_handshake(upgrade_request("/", "AUTHORIZATION: bearer abc123\r\n")); + REQUIRE(h.ok); + CHECK(h.token == "abc123"); + } + SECTION("query parameter") + { + const auto h = parse_handshake(upgrade_request("/?x=1&token=qtok&y=2")); + REQUIRE(h.ok); + CHECK(h.token == "qtok"); + CHECK(h.path == "/"); + } + SECTION("the header wins over the query") + { + const auto h = parse_handshake(upgrade_request("/?token=q", "Authorization: Bearer h\r\n")); + REQUIRE(h.ok); + CHECK(h.token == "h"); + } + SECTION("no token leaves it empty and the handshake still parses") + { + const auto h = parse_handshake(upgrade_request()); + REQUIRE(h.ok); + CHECK(h.token.empty()); + } +} + +TEST_CASE("parse_handshake refuses what is not a WebSocket upgrade", "[websocket]") +{ + SECTION("a plain GET is 426 with an Upgrade header in the answer") + { + const auto h = parse_handshake("GET / HTTP/1.1\r\nHost: x\r\n\r\n"); + CHECK_FALSE(h.ok); + CHECK(h.status == 426); + const std::string response = handshake_response(h); + CHECK(response.rfind("HTTP/1.1 426 Upgrade Required\r\n", 0) == 0); + CHECK(response.find("Upgrade: websocket\r\n") != std::string::npos); + CHECK(response.find("Content-Length: " + std::to_string(h.reason.size())) != std::string::npos); + } + SECTION("a missing key is 426") + { + const auto h = parse_handshake( + "GET / HTTP/1.1\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n"); + CHECK(h.status == 426); + CHECK_FALSE(h.ok); + } + SECTION("a wrong version is 426 and says so") + { + const auto h = parse_handshake(upgrade_request("/", "", "8")); + CHECK_FALSE(h.ok); + CHECK(h.status == 426); + CHECK(h.reason.find("13") != std::string::npos); + } + SECTION("bad request lines are 400") + { + CHECK(parse_handshake("POST / HTTP/1.1\r\n\r\n").status == 400); + CHECK(parse_handshake("GET / HTTP/1.0\r\n\r\n").status == 400); + CHECK(parse_handshake("GET\r\n\r\n").status == 400); + CHECK(parse_handshake("garbage").status == 400); + CHECK_FALSE(parse_handshake("GET / HTTP/1.0\r\n\r\n").ok); + } + SECTION("status phrases") + { + Handshake h; + h.status = 401; + h.reason = "no"; + CHECK(handshake_response(h).rfind("HTTP/1.1 401 Unauthorized\r\n", 0) == 0); + h.status = 404; + CHECK(handshake_response(h).rfind("HTTP/1.1 404 Not Found\r\n", 0) == 0); + h.status = 400; + CHECK(handshake_response(h).rfind("HTTP/1.1 400 Bad Request\r\n", 0) == 0); + } +} diff --git a/src/sensing_control/tools/orin_control.py b/src/sensing_control/tools/orin_control.py new file mode 100755 index 0000000000..7183edd212 --- /dev/null +++ b/src/sensing_control/tools/orin_control.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Talk to a camera producer's management channel (for example camera_plugin_sensing). + + orin_control.py HOST[:PORT] --token-file token.txt status + orin_control.py HOST[:PORT] --token-file token.txt set-target sensing-left 192.168.1.205 5600 + orin_control.py HOST[:PORT] --token-file token.txt set-target sensing-left "" # stop the direct link + orin_control.py HOST[:PORT] --token-file token.txt watch # print pushed events + +Standard library only. The token goes in the Authorization header, never on the command line or in the URL. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import socket +import struct +import sys +from pathlib import Path + +DEFAULT_PORT = 5610 +GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + +class ChannelError(Exception): + """The channel could not be used; the message says why.""" + + +def recv_exact(sock: socket.socket, count: int) -> bytes: + data = b"" + while len(data) < count: + chunk = sock.recv(count - len(data)) + if not chunk: + raise ChannelError("the producer closed the connection") + data += chunk + return data + + +class Connection: + def __init__(self, host: str, port: int, token: str, timeout: float = 5.0) -> None: + try: + self._sock = socket.create_connection((host, port), timeout=timeout) + except OSError as exc: + raise ChannelError(f"cannot connect to {host}:{port}: {exc}") from exc + key = base64.b64encode(os.urandom(16)).decode() + request = ( + f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\nAuthorization: Bearer {token}\r\n\r\n" + ) + self._sock.sendall(request.encode()) + head = b"" + while b"\r\n\r\n" not in head: + chunk = self._sock.recv(4096) + if not chunk: + raise ChannelError( + "the producer closed the connection during the handshake" + ) + head += chunk + status_line = head.split(b"\r\n", 1)[0].decode(errors="replace") + if " 101 " not in status_line: + body = head.split(b"\r\n\r\n", 1)[1].decode(errors="replace").strip() + raise ChannelError( + f"the producer refused the connection ({status_line}): {body}" + ) + expected = base64.b64encode( + hashlib.sha1((key + GUID).encode()).digest() + ).decode() + if ( + f"sec-websocket-accept: {expected}".lower() + not in head.decode(errors="replace").lower() + ): + raise ChannelError( + "the server's handshake answer is wrong; this is not a management channel" + ) + self._pending = head.split(b"\r\n\r\n", 1)[1] + + def send(self, message: dict) -> None: + payload = json.dumps(message).encode() + mask = os.urandom(4) + header = bytes([0x81]) + if len(payload) < 126: + header += bytes([0x80 | len(payload)]) + elif len(payload) < 65536: + header += bytes([0x80 | 126]) + struct.pack(">H", len(payload)) + else: + header += bytes([0x80 | 127]) + struct.pack(">Q", len(payload)) + masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + self._sock.sendall(header + mask + masked) + + def _read(self, count: int) -> bytes: + take = self._pending[:count] + self._pending = self._pending[count:] + return take + ( + recv_exact(self._sock, count - len(take)) if len(take) < count else b"" + ) + + def receive(self, timeout: float | None = None) -> dict: + self._sock.settimeout(timeout) + while True: + first, second = self._read(2) + opcode, length = first & 0x0F, second & 0x7F + if length == 126: + (length,) = struct.unpack(">H", self._read(2)) + elif length == 127: + (length,) = struct.unpack(">Q", self._read(8)) + payload = self._read(length) + if opcode == 0x8: + raise ChannelError( + "the producer closed the channel: " + + payload[2:].decode(errors="replace") + ) + if opcode == 0x1: + return json.loads(payload) + # Pings and other frames are ignored; the server sends none that need an answer. + + def request(self, message: dict) -> dict: + message = {"id": 1, **message} + self.send(message) + while True: + reply = self.receive(timeout=10) + if reply.get("id") == message["id"]: + return reply + + +def describe(status: dict) -> str: + lines = [ + f"state: {status['state']} release: {status['release']} up {status['uptime_seconds']} s" + ] + fault = status.get("fault") + if fault: + lines.append(f"PROBLEM [{fault['code']}]: {fault['message']}") + for stream in status["streams"]: + direct = stream["direct"] + target = f"{direct['host']}:{direct['port']}" if direct["host"] else "off" + link = "connected" if direct["connected"] else "not connected" + lines.append( + f" {stream['stream']}: sensor {stream['sensor']}, {stream['frames']} frames, " + f"direct link {target} ({link if direct['host'] else '-'})" + ) + return "\n".join(lines) + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument( + "target", help=f"producer HOST or HOST:PORT (default port {DEFAULT_PORT})" + ) + ap.add_argument( + "--token-file", required=True, type=Path, help="file holding the shared secret" + ) + sub = ap.add_subparsers(dest="command", required=True) + sub.add_parser("status", help="print the producer's state") + sub.add_parser("watch", help="print pushed events until interrupted") + setter = sub.add_parser( + "set-target", + help="send a stream's video to HOST PORT (empty HOST turns it off)", + ) + setter.add_argument("stream") + setter.add_argument("host") + setter.add_argument("port", nargs="?", type=int, default=0) + args = ap.parse_args(argv) + + host, _, port = args.target.partition(":") + try: + token = args.token_file.read_text().strip() + if not token: + raise ChannelError(f"{args.token_file} is empty") + conn = Connection(host, int(port) if port else DEFAULT_PORT, token) + if args.command == "status": + reply = conn.request({"type": "get_status"}) + elif args.command == "set-target": + reply = conn.request( + { + "type": "set_direct_target", + "stream": args.stream, + "host": args.host, + "port": args.port, + } + ) + else: + print(describe(conn.request({"type": "get_status"})["status"])) + while True: + print(json.dumps(conn.receive())) + if not reply["ok"]: + print( + f"Error [{reply['error']['code']}]: {reply['error']['message']}", + file=sys.stderr, + ) + return 1 + print(describe(reply["status"])) + return 0 + except (ChannelError, OSError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/sensing_control/websocket.cpp b/src/sensing_control/websocket.cpp new file mode 100644 index 0000000000..efb77d09c4 --- /dev/null +++ b/src/sensing_control/websocket.cpp @@ -0,0 +1,369 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_control/websocket.hpp" + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ +namespace websocket +{ + +namespace +{ + +constexpr std::string_view kGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + +std::uint32_t rotl(std::uint32_t value, int bits) +{ + return (value << bits) | (value >> (32 - bits)); +} + +std::string lower(std::string_view text) +{ + std::string out(text); + std::transform(out.begin(), out.end(), out.begin(), [](unsigned char c) { return std::tolower(c); }); + return out; +} + +std::string trim(std::string_view text) +{ + std::size_t begin = 0; + std::size_t end = text.size(); + while (begin < end && (text[begin] == ' ' || text[begin] == '\t')) + ++begin; + while (end > begin && (text[end - 1] == ' ' || text[end - 1] == '\t' || text[end - 1] == '\r')) + --end; + return std::string(text.substr(begin, end - begin)); +} + +/// The value of one `name=value` pair of a query string, percent-decoding nothing: tokens are URL-safe. +std::string query_value(std::string_view query, std::string_view name) +{ + std::size_t pos = 0; + while (pos <= query.size()) + { + const std::size_t amp = query.find('&', pos); + const std::string_view pair = query.substr(pos, amp == std::string_view::npos ? amp : amp - pos); + const std::size_t eq = pair.find('='); + if (eq != std::string_view::npos && pair.substr(0, eq) == name) + return std::string(pair.substr(eq + 1)); + if (amp == std::string_view::npos) + break; + pos = amp + 1; + } + return {}; +} + +} // namespace + +std::string sha1(std::string_view data) +{ + std::array h = { 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0 }; + + std::string message(data); + const std::uint64_t bit_length = static_cast(data.size()) * 8; + message.push_back(static_cast(0x80)); + while (message.size() % 64 != 56) + message.push_back('\0'); + for (int shift = 56; shift >= 0; shift -= 8) + message.push_back(static_cast((bit_length >> shift) & 0xFF)); + + for (std::size_t offset = 0; offset < message.size(); offset += 64) + { + std::array w{}; + for (int i = 0; i < 16; ++i) + { + const auto* p = reinterpret_cast(message.data() + offset + 4 * i); + w[i] = (std::uint32_t(p[0]) << 24) | (std::uint32_t(p[1]) << 16) | (std::uint32_t(p[2]) << 8) | p[3]; + } + for (int i = 16; i < 80; ++i) + w[i] = rotl(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1); + + std::uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4]; + for (int i = 0; i < 80; ++i) + { + std::uint32_t f = 0; + std::uint32_t k = 0; + if (i < 20) + { + f = (b & c) | (~b & d); + k = 0x5A827999; + } + else if (i < 40) + { + f = b ^ c ^ d; + k = 0x6ED9EBA1; + } + else if (i < 60) + { + f = (b & c) | (b & d) | (c & d); + k = 0x8F1BBCDC; + } + else + { + f = b ^ c ^ d; + k = 0xCA62C1D6; + } + const std::uint32_t temp = rotl(a, 5) + f + e + k + w[i]; + e = d; + d = c; + c = rotl(b, 30); + b = a; + a = temp; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + } + + std::string digest; + for (const std::uint32_t word : h) + for (int shift = 24; shift >= 0; shift -= 8) + digest.push_back(static_cast((word >> shift) & 0xFF)); + return digest; +} + +std::string base64_encode(std::string_view data) +{ + static constexpr char kAlphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + out.reserve((data.size() + 2) / 3 * 4); + for (std::size_t i = 0; i < data.size(); i += 3) + { + const std::uint32_t b0 = static_cast(data[i]); + const std::uint32_t b1 = i + 1 < data.size() ? static_cast(data[i + 1]) : 0; + const std::uint32_t b2 = i + 2 < data.size() ? static_cast(data[i + 2]) : 0; + const std::uint32_t triple = (b0 << 16) | (b1 << 8) | b2; + out.push_back(kAlphabet[(triple >> 18) & 0x3F]); + out.push_back(kAlphabet[(triple >> 12) & 0x3F]); + out.push_back(i + 1 < data.size() ? kAlphabet[(triple >> 6) & 0x3F] : '='); + out.push_back(i + 2 < data.size() ? kAlphabet[triple & 0x3F] : '='); + } + return out; +} + +std::string accept_key(std::string_view client_key) +{ + std::string joined(client_key); + joined.append(kGuid); + return base64_encode(sha1(joined)); +} + +bool constant_time_equal(std::string_view a, std::string_view b) +{ + // Length is not secret here: it is compared first, and only the content is hidden. + if (a.size() != b.size()) + return false; + unsigned char difference = 0; + for (std::size_t i = 0; i < a.size(); ++i) + difference |= static_cast(a[i]) ^ static_cast(b[i]); + return difference == 0; +} + +Handshake parse_handshake(std::string_view head) +{ + Handshake result; + const std::size_t line_end = head.find("\r\n"); + if (line_end == std::string_view::npos) + { + result.reason = "malformed request"; + return result; + } + const std::string_view request_line = head.substr(0, line_end); + // "GET HTTP/1.1" + const std::size_t first_space = request_line.find(' '); + const std::size_t second_space = request_line.rfind(' '); + if (first_space == std::string_view::npos || second_space == first_space || + request_line.substr(0, first_space) != "GET" || request_line.substr(second_space + 1) != "HTTP/1.1") + { + result.reason = "expected a GET request over HTTP/1.1"; + return result; + } + const std::string_view target = request_line.substr(first_space + 1, second_space - first_space - 1); + const std::size_t question = target.find('?'); + result.path = std::string(target.substr(0, question)); + const std::string_view query = question == std::string_view::npos ? std::string_view() : target.substr(question + 1); + + bool upgrade = false; + bool connection_upgrade = false; + std::string version; + std::string bearer; + std::size_t pos = line_end + 2; + while (pos < head.size()) + { + const std::size_t end = head.find("\r\n", pos); + const std::string_view line = head.substr(pos, end == std::string_view::npos ? end : end - pos); + if (line.empty()) + break; + const std::size_t colon = line.find(':'); + if (colon != std::string_view::npos) + { + const std::string name = lower(trim(line.substr(0, colon))); + const std::string value = trim(line.substr(colon + 1)); + if (name == "upgrade") + upgrade = lower(value) == "websocket"; + else if (name == "connection") + connection_upgrade = lower(value).find("upgrade") != std::string::npos; + else if (name == "sec-websocket-key") + result.key = value; + else if (name == "sec-websocket-version") + version = value; + else if (name == "authorization" && lower(value).rfind("bearer ", 0) == 0) + bearer = trim(std::string_view(value).substr(7)); + } + if (end == std::string_view::npos) + break; + pos = end + 2; + } + + if (!upgrade || !connection_upgrade || result.key.empty()) + { + result.status = 426; + result.reason = "this is a WebSocket endpoint"; + return result; + } + if (version != "13") + { + result.status = 426; + result.reason = "only WebSocket version 13 is supported"; + return result; + } + result.token = bearer.empty() ? query_value(query, "token") : bearer; + result.ok = true; + return result; +} + +std::string handshake_response(const Handshake& handshake) +{ + if (handshake.ok) + { + return "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + "Sec-WebSocket-Accept: " + + accept_key(handshake.key) + "\r\n\r\n"; + } + const char* phrase = handshake.status == 401 ? "Unauthorized" : + handshake.status == 404 ? "Not Found" : + handshake.status == 426 ? "Upgrade Required" : + "Bad Request"; + std::string response = "HTTP/1.1 " + std::to_string(handshake.status) + " " + phrase + "\r\n"; + if (handshake.status == 426) + response += "Upgrade: websocket\r\n"; + response += "Content-Type: text/plain\r\nContent-Length: " + std::to_string(handshake.reason.size()) + + "\r\nConnection: close\r\n\r\n" + handshake.reason; + return response; +} + +ParseResult parse_frame(std::string_view buffer, std::size_t max_payload) +{ + ParseResult result; + if (buffer.size() < 2) + return result; + const auto byte = [&](std::size_t i) { return static_cast(buffer[i]); }; + + const unsigned char b0 = byte(0); + const unsigned char b1 = byte(1); + if ((b0 & 0x70) != 0) + { + result.status = ParseStatus::Error; + result.error = "reserved bits set"; + return result; + } + if ((b1 & 0x80) == 0) + { + result.status = ParseStatus::Error; + result.error = "client frames must be masked"; + return result; + } + const std::uint8_t opcode = b0 & 0x0F; + const bool control = (opcode & 0x08) != 0; + + std::uint64_t length = b1 & 0x7F; + std::size_t header = 2; + if (length == 126) + { + if (buffer.size() < 4) + return result; + length = (std::uint64_t(byte(2)) << 8) | byte(3); + header = 4; + } + else if (length == 127) + { + if (buffer.size() < 10) + return result; + length = 0; + for (int i = 0; i < 8; ++i) + length = (length << 8) | byte(2 + i); + header = 10; + } + if (control && (length > 125 || (b0 & 0x80) == 0)) + { + result.status = ParseStatus::Error; + result.error = "invalid control frame"; + return result; + } + if (length > max_payload) + { + result.status = ParseStatus::Error; + result.close_code = 1009; + result.error = "message too big"; + return result; + } + const std::size_t total = header + 4 + static_cast(length); + if (buffer.size() < total) + return result; + + const unsigned char mask[4] = { byte(header), byte(header + 1), byte(header + 2), byte(header + 3) }; + result.frame.fin = (b0 & 0x80) != 0; + result.frame.opcode = opcode; + result.frame.payload.resize(static_cast(length)); + for (std::size_t i = 0; i < length; ++i) + result.frame.payload[i] = static_cast(byte(header + 4 + i) ^ mask[i % 4]); + result.consumed = total; + result.status = ParseStatus::Complete; + return result; +} + +std::string encode_frame(std::uint8_t opcode, std::string_view payload, bool fin) +{ + std::string out; + out.push_back(static_cast((fin ? 0x80 : 0x00) | (opcode & 0x0F))); + if (payload.size() < 126) + { + out.push_back(static_cast(payload.size())); + } + else if (payload.size() <= 0xFFFF) + { + out.push_back(126); + out.push_back(static_cast(payload.size() >> 8)); + out.push_back(static_cast(payload.size() & 0xFF)); + } + else + { + out.push_back(127); + for (int shift = 56; shift >= 0; shift -= 8) + out.push_back(static_cast((static_cast(payload.size()) >> shift) & 0xFF)); + } + out.append(payload); + return out; +} + +std::string encode_close(std::uint16_t code, std::string_view reason) +{ + std::string payload; + payload.push_back(static_cast(code >> 8)); + payload.push_back(static_cast(code & 0xFF)); + payload.append(reason.substr(0, 120)); + return encode_frame(kClose, payload); +} + +} // namespace websocket +} // namespace sensing +} // namespace plugins From 5aa575c369597ec3182d4d8c1adc568a6128a4af Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 17:43:42 -0500 Subject: [PATCH 2/2] sensing: run the producer under the management channel's supervisor A camera that cannot start no longer ends camera_plugin_sensing: the supervisor reports the fault in plain words, retries, and resumes streaming once it is fixed. --control-token-file turns on the WebSocket management channel, which reports state, faults and per-stream frame counts. SIPL failures name their status and the kind of pipeline notification, which the fault table reads. set_direct_target is refused with not_supported, since this producer publishes to LiveKit. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/CMakeLists.txt | 4 +- src/plugins/sensing/README.md | 20 ++ src/plugins/sensing/app/CMakeLists.txt | 2 +- src/plugins/sensing/app/main.cpp | 213 ++++++++++++++++-- .../core/inc/sensing_core/sensing_camera.hpp | 14 ++ src/plugins/sensing/core/sensing_camera.cpp | 10 + src/plugins/sensing/core/sipl_camera.cpp | 133 ++++++++++- 7 files changed, 367 insertions(+), 29 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a95b15c86d..4816091221 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,8 +25,8 @@ if(BUILD_VIZ) add_subdirectory(viz) endif() -# The camera producer's management channel: no video stage, camera SDK or GPU dependency. -if(BUILD_SENSING_CONTROL) +# The camera producer's management channel: no camera SDK or GPU dependency. The sensing plugin links it. +if(BUILD_PLUGIN_SENSING OR BUILD_SENSING_CONTROL) add_subdirectory(sensing_control) endif() diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 147e65c950..70ef96d11d 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -46,3 +46,23 @@ SIPL drivers. hardware encoder and publishes one LiveKit video track per camera. The [SENSING user guide](https://nvidia.github.io/IsaacCapture/main/device/camera/sensing.html) has the build and run steps. + +## Management channel and camera faults + +A camera that cannot start (no power cable, no GMSL link) does not end the producer. It reports the problem in plain +words, retries every few seconds (`--retry-seconds`), and starts streaming once the cable is fixed. The same process +can offer a WebSocket management channel, off unless `--control-token-file=PATH` names a file holding a shared secret: + +```bash +camera_plugin_sensing ... --control-token-file=/etc/isaaccapture/control.token # listens on 0.0.0.0:5610 +``` + +Clients send one JSON object per message and get one reply; the producer also pushes `state_changed` and `error` +events. `get_status` reports the state (`starting`, `streaming`, `camera_fault`, `error`, `stopping`), the fault with a +code and what to check, and each stream's frame counts. This producer publishes to LiveKit, so `set_direct_target` is +refused with `not_supported`. Every bad request gets an error reply with a code and a sentence. The token goes in +`Authorization: Bearer ...` (or `?token=`). + +```bash +src/sensing_control/tools/orin_control.py ORIN_HOST --token-file control.token status +``` diff --git a/src/plugins/sensing/app/CMakeLists.txt b/src/plugins/sensing/app/CMakeLists.txt index 613d396602..2c9009bc38 100644 --- a/src/plugins/sensing/app/CMakeLists.txt +++ b/src/plugins/sensing/app/CMakeLists.txt @@ -5,7 +5,7 @@ add_executable(camera_plugin_sensing main.cpp) set_target_properties(camera_plugin_sensing PROPERTIES INSTALL_RPATH "$ORIGIN") -target_link_libraries(camera_plugin_sensing PRIVATE sensing::core) +target_link_libraries(camera_plugin_sensing PRIVATE sensing::core sensing::control) # The plugin resolves configs/shw5g.json relative to its own executable, so the # config has to sit beside the binary in the build tree as well as the install diff --git a/src/plugins/sensing/app/main.cpp b/src/plugins/sensing/app/main.cpp index 425197b38f..387a6faebd 100644 --- a/src/plugins/sensing/app/main.cpp +++ b/src/plugins/sensing/app/main.cpp @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#include +#include +#include +#include #include #include @@ -13,9 +17,9 @@ #include #include #include +#include #include #include -#include #include using namespace plugins::sensing; @@ -110,6 +114,90 @@ static std::string default_platform_config() return ::access(candidate.c_str(), R_OK) == 0 ? candidate : std::string{}; } +// ============================================================================= +// Producer session and release +// ============================================================================= + +/// The release this binary was installed from (".../releases//..."), or a placeholder when run by hand. +static std::string release_id() +{ + char buffer[PATH_MAX]; + const ssize_t length = ::readlink("/proc/self/exe", buffer, sizeof(buffer) - 1); + if (length <= 0) + return "unknown"; + const std::string path(buffer, static_cast(length)); + const std::string marker = "/releases/"; + const auto at = path.find(marker); + if (at == std::string::npos) + return "unmanaged"; + const auto start = at + marker.size(); + return path.substr(start, path.find('/', start) - start); +} + +/// One capture-and-publish pipeline as the supervisor sees it. +class ProducerSession : public CameraSession +{ +public: + ProducerSession(const SensingConfig& config, const std::vector& streams) + : m_camera(config, streams), m_last_stats(std::chrono::steady_clock::now()) + { + std::cout << "------------------------------------------------------------" << std::endl; + std::cout << "Capturing." << std::endl; + } + + void update() override + { + m_camera.update(); + const auto now = std::chrono::steady_clock::now(); + if (now - m_last_stats >= std::chrono::seconds(5)) + { + m_camera.print_stats(); + m_last_stats = now; + } + } + + void flush() override + { + m_camera.flush(); + m_camera.print_stats(); + } + + std::vector streams() const override + { + std::vector out; + for (const auto& figures : m_camera.figures()) + { + StreamStatus entry; + entry.track = figures.track; + entry.sensor_id = figures.sensor_id; + entry.frames = figures.frames; + entry.published = figures.published; + out.push_back(std::move(entry)); + } + return out; + } + + /// Never called: ProducerActions refuses direct targets before the supervisor records one. + void set_direct_target(const std::string&, const std::string&, uint16_t) override + { + } + +private: + SensingCamera m_camera; + std::chrono::steady_clock::time_point m_last_stats; +}; + +/// This producer publishes to LiveKit and has no direct link to a PC, so the one action the channel can name is refused +/// with a reason instead of being accepted and ignored. +class ProducerActions : public ControlActions +{ +public: + ActionResult set_direct_target(const std::string&, const std::string&, uint16_t) override + { + return { false, "not_supported", "this producer publishes to LiveKit and has no direct link to set" }; + } +}; + // ============================================================================= // Usage // ============================================================================= @@ -141,6 +229,12 @@ static void print_usage(const char* program_name) << " --livekit-bitrate=N target bitrate of each track's encoder in bps (default: 8000000)\n" << " --gop=N IDR period in frames (default: fps*5)\n" << "\n" + << "Management channel (a WebSocket server; off unless a token file is given):\n" + << " --control-token-file=PATH file holding the shared secret clients must present\n" + << " --control-port=N port to listen on (default: 5610)\n" + << " --control-bind=ADDR IPv4 address to listen on (default: 0.0.0.0)\n" + << " --retry-seconds=N wait between attempts to start the cameras (default: 5)\n" + << "\n" << "Other:\n" << " --gpu-id=N CUDA device index (default: 0)\n" << " --isp0-buffers=N ISP0 buffers per sensor (default: 6)\n" @@ -163,6 +257,9 @@ try SensingConfig camera_config; std::map stream_map; std::string livekit_token_file; + ControlServerConfig control_config; + std::string control_token_file; + uint32_t retry_seconds = 5; bool list_sensors = false; for (int i = 1; i < argc; ++i) @@ -200,6 +297,25 @@ try { list_sensors = true; } + else if (arg.find("--control-token-file=") == 0) + { + control_token_file = arg.substr(21); + } + else if (arg.find("--control-port=") == 0) + { + const unsigned long port = std::stoul(arg.substr(15)); + if (port < 1 || port > 65535) + throw std::runtime_error("--control-port must be from 1 to 65535"); + control_config.port = static_cast(port); + } + else if (arg.find("--control-bind=") == 0) + { + control_config.bind_address = arg.substr(15); + } + else if (arg.find("--retry-seconds=") == 0) + { + retry_seconds = static_cast(std::stoul(arg.substr(16))); + } else if (arg.find("--livekit-url=") == 0) { camera_config.livekit.url = arg.substr(14); @@ -330,42 +446,91 @@ try std::cout << "SENSING Camera Plugin Starting (SIPL)" << std::endl; std::cout << "============================================================" << std::endl; - SensingCamera camera(camera_config, stream_configs); - - std::cout << "------------------------------------------------------------" << std::endl; - std::cout << "Running capture loop. Press Ctrl+C to stop." << std::endl; - - constexpr auto stats_interval = std::chrono::seconds(5); - auto last_stats_time = std::chrono::steady_clock::now(); + // A stream naming a sensor that does not exist is a configuration error, not a fault to retry. + for (const auto& stream : stream_configs) + { + const bool known = + std::any_of(sensors.begin(), sensors.end(), [&](const SensorInfo& s) { return s.id == stream.sensor_id; }); + if (!known) + throw std::runtime_error("sensor=" + std::to_string(stream.sensor_id) + " is not a pipeline of '" + + camera_config.platform_config_name + "'; run --list-sensors"); + } - // SiplCamera::latest() is a non-blocking mailbox read, so poll at roughly - // twice the fastest sensor's frame rate rather than spinning a core flat - // out. Capture itself runs on SIPL's own threads and is unaffected. + // SiplCamera::latest() is a non-blocking mailbox read, so poll at roughly twice the fastest sensor's frame rate + // rather than spinning a core flat out. Capture itself runs on SIPL's own threads and is unaffected. double fastest_fps = 30.0; for (const auto& s : sensors) - { fastest_fps = std::max(fastest_fps, s.fps); - } - const auto poll_interval = std::chrono::microseconds(static_cast(500'000.0 / fastest_fps)); - while (!g_stop_requested.load(std::memory_order_relaxed)) + ProducerStatus status(release_id()); + std::vector configured; + for (const auto& stream : stream_configs) { - camera.update(); + StreamStatus entry; + entry.track = stream.livekit_track; + entry.sensor_id = stream.sensor_id; + configured.push_back(entry); + } - auto now = std::chrono::steady_clock::now(); - if (now - last_stats_time >= stats_interval) + // The cameras and the room are opened by the factory, again after any failure, so an unplugged camera is a state + // to report and retry, not the end of the process. + const SessionFactory factory = [&](const DirectTargets&) -> std::unique_ptr + { + // The camera driver explains a failure only in its own log lines, so they are collected while the cameras + // start and attached to the failure; the fault table reads them (an open circuit, a lock timeout, an I2C NACK). + DriverLogCapture driver_log; + try { - camera.print_stats(); - last_stats_time = now; + auto session = std::make_unique(camera_config, stream_configs); + driver_log.finish(); + return session; } + catch (const std::exception& e) + { + const std::string lines = driver_log.finish(); + throw std::runtime_error(lines.empty() ? std::string(e.what()) : + std::string(e.what()) + " | driver log: " + lines); + } + }; - std::this_thread::sleep_for(poll_interval); + // Destruction runs in reverse declaration order, so the server (declared last) is stopped and joined while the + // handler, supervisor and status its thread calls into still exist, on return and on exception alike. + // `channel` is what the supervisor publishes through; it is set once the server exists. + ControlServer* channel = nullptr; + const auto publish = [&channel](const std::string& text) + { + if (channel != nullptr) + channel->broadcast(text); + }; + SupervisorConfig supervisor_config; + supervisor_config.retry_interval = std::chrono::seconds(std::max(retry_seconds, 1)); + supervisor_config.poll_interval = std::chrono::milliseconds(static_cast(500.0 / fastest_fps)); + ProducerSupervisor supervisor(status, configured, factory, publish, supervisor_config); + ProducerActions actions; + ControlHandler handler(status, actions); + std::unique_ptr server; + + if (control_token_file.empty()) + { + std::cout << "Management channel: off (give --control-token-file to turn it on)" << std::endl; + } + else + { + control_config.token = read_secret_file(control_token_file); + server = std::make_unique( + control_config, [&handler](const std::string& text) { return handler.handle(text); }); + channel = server.get(); + std::cout << "Management channel: listening on " << control_config.bind_address << ":" << server->port() + << std::endl; } std::cout << "------------------------------------------------------------" << std::endl; - std::cout << "Shutting down SENSING Camera Plugin..." << std::endl; - camera.flush(); - camera.print_stats(); + std::cout << "Running. Press Ctrl+C to stop." << std::endl; + + supervisor.run([] { return g_stop_requested.load(std::memory_order_relaxed); }); + channel = nullptr; // nothing publishes once the run has ended + + std::cout << "------------------------------------------------------------" << std::endl; std::cout << "Plugin stopped" << std::endl; std::cout << "============================================================" << std::endl; diff --git a/src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp b/src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp index 0822da3460..9146c146b1 100644 --- a/src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp +++ b/src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp @@ -8,6 +8,7 @@ #include #include +#include #include namespace plugins @@ -18,6 +19,15 @@ namespace sensing class LiveKitSession; class LiveKitVideoOutput; +/// Live figures for one stream, for the management channel. +struct StreamFigures +{ + std::string track; + uint32_t sensor_id = 0; + uint64_t frames = 0; + uint64_t published = 0; +}; + /** * @brief Multi-sensor SENSING camera publisher. * @@ -42,6 +52,9 @@ class SensingCamera /** @brief Flush each track's queued frames. */ void flush(); + /** @brief Per-stream figures. Call from the thread that calls update(). */ + std::vector figures() const; + /** @brief Print per-sensor frame counts to stdout. */ void print_stats() const; @@ -49,6 +62,7 @@ class SensingCamera struct Stream { uint32_t sensor_id = 0; + std::string track; std::unique_ptr livekit; /// SIPL publish counter of the frame already handled; skips re-reads. uint64_t last_sequence = 0; diff --git a/src/plugins/sensing/core/sensing_camera.cpp b/src/plugins/sensing/core/sensing_camera.cpp index b6ed078044..fa60e841dc 100644 --- a/src/plugins/sensing/core/sensing_camera.cpp +++ b/src/plugins/sensing/core/sensing_camera.cpp @@ -99,6 +99,7 @@ SensingCamera::SensingCamera(const SensingConfig& config, const std::vectorid; + stream.track = spec.name; stream.livekit = std::make_unique(m_livekit->publish_video(spec), output); m_streams.push_back(std::move(stream)); @@ -147,6 +148,15 @@ void SensingCamera::flush() stream.livekit->flush(); } +std::vector SensingCamera::figures() const +{ + std::vector out; + out.reserve(m_streams.size()); + for (const auto& stream : m_streams) + out.push_back({ stream.track, stream.sensor_id, stream.frame_count, stream.livekit->stats().published }); + return out; +} + void SensingCamera::print_stats() const { for (const auto& stream : m_streams) diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index 6a362ac25b..9abf6f7a1e 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -95,12 +95,46 @@ void init_sipl_trace() } } +/// The enumerator name for the statuses whose meaning is known; every other value is reported by number only. +const char* status_name(SIPLStatus status) +{ + // The order of NvSIPLCommon.hpp's SIPLStatus. + switch (static_cast(status)) + { + case 1: + return "bad argument"; + case 2: + return "not supported"; + case 3: + return "out of memory"; + case 4: + return "resource error"; + case 5: + return "timed out"; + case 6: + return "invalid state"; + case 7: + return "end of stream"; + case 8: + // Init() reports this when some cameras started and at least one did not. + return "not initialized: some cameras failed to start"; + case 9: + return "fault state"; + case 10: + return "error: no cause given"; + default: + return nullptr; + } +} + void check_sipl(SIPLStatus status, const char* what) { if (status != NVSIPL_STATUS_OK) { std::ostringstream oss; oss << what << " failed with SIPLStatus " << static_cast(status); + if (const char* name = status_name(status)) + oss << " (" << name << ")"; throw std::runtime_error(oss.str()); } } @@ -385,6 +419,9 @@ struct SiplCamera::Impl struct Sensor { SensorInfo info; + /// Event thread only: notifications in a row that mean the camera is not delivering. + uint32_t consecutive_capture_timeouts = 0; + uint32_t consecutive_control_failures = 0; nvsipl::NvSIPLPipelineConfiguration pipeline_config{}; nvsipl::NvSIPLPipelineQueues queues{}; @@ -934,6 +971,52 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) } } +namespace +{ + +/// What a pipeline or device-block notification means for streaming. Values are those of +/// NvSIPLPipelineNotifier::NotificationType in the SIPL reference; the four device-block errors that were already +/// matched by name keep their enumerators, the rest are matched by value so an older header still compiles. +struct NotificationKind +{ + bool fatal = false; + const char* text = nullptr; ///< words the producer's fault table recognises; null for notifications to ignore +}; + +NotificationKind describe_notification(int type) +{ + switch (type) + { + case 200: + return { true, "camera input stream failure (bad input stream)" }; + case 201: + return { true, "camera input stream failure (capture failure)" }; + case 203: + return { true, "image processing failure" }; + case 204: + return { true, "auto control processing failure" }; + case 220: + case 221: + return { true, "interrupt localization failure" }; + case 202: + return { false, "embedded data parse failure" }; + case 205: + return { false, "sensor control write failure" }; + case 210: + return { false, "recoverable image processing failure" }; + case 100: + return { false, "frame dropped" }; + case 101: + return { false, "frame sequence discontinuity" }; + case 102: + return { false, "capture timeout" }; + default: + return {}; + } +} + +} // namespace + void SiplCamera::Impl::event_loop(Sensor& sensor) { // A GMSL link drop arrives here and nowhere else. Without this drain the @@ -954,17 +1037,63 @@ void SiplCamera::Impl::event_loop(Sensor& sensor) case Notif::NOTIF_ERROR_SENSOR_FAILURE: case Notif::NOTIF_ERROR_INTERNAL_FAILURE: { + // The kind is named in words the producer's fault table recognises: a lost link and a failing sensor + // call for different checks. + const char* kind = data.eNotifType == Notif::NOTIF_ERROR_DESERIALIZER_FAILURE ? "deserializer failure" : + data.eNotifType == Notif::NOTIF_ERROR_SERIALIZER_FAILURE ? "serializer failure" : + data.eNotifType == Notif::NOTIF_ERROR_SENSOR_FAILURE ? "sensor failure" : + "internal failure"; std::ostringstream oss; - oss << "SIPL sensor " << sensor.info.id << " pipeline error, notification " - << static_cast(data.eNotifType); + oss << "SIPL sensor " << sensor.info.id << " pipeline error: " << kind << " (notification " + << static_cast(data.eNotifType) << ")"; std::cerr << "[sipl] " << oss.str() << std::endl; set_failure(oss.str()); running.store(false); break; } default: + { + const int type = static_cast(data.eNotifType); + if (type == 0) // ICP processing done: the camera is delivering + { + sensor.consecutive_capture_timeouts = 0; + sensor.consecutive_control_failures = 0; + break; + } + const NotificationKind kind = describe_notification(type); + if (kind.text == nullptr) + break; + std::ostringstream oss; + oss << "SIPL sensor " << sensor.info.id << ": " << kind.text << " (notification " << type << ")"; + if (kind.fatal) + { + std::cerr << "[sipl] " << oss.str() << std::endl; + set_failure(oss.str()); + running.store(false); + break; + } + // A warning once is routine; the same one repeating means the camera has stopped delivering. + if (type == 102 && ++sensor.consecutive_capture_timeouts >= 3) + { + std::ostringstream failure; + failure << "SIPL sensor " << sensor.info.id + << ": no frame for 3 capture timeouts in a row (the camera stopped sending)"; + std::cerr << "[sipl] " << failure.str() << std::endl; + set_failure(failure.str()); + running.store(false); + } + else if (type == 205 && ++sensor.consecutive_control_failures >= 10) + { + std::ostringstream failure; + failure << "SIPL sensor " << sensor.info.id + << ": sensor control write failure repeated 10 times (its control bus is not answering)"; + std::cerr << "[sipl] " << failure.str() << std::endl; + set_failure(failure.str()); + running.store(false); + } break; } + } } }