video_stage: PC worker that forwards Orin cameras to LiveKit over a direct link

CMakeLists.txt

@@ -71,6 +71,8 @@ option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi
 option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF)
 option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF)
 option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF)
+option(BUILD_PLUGIN_SENSING "Build SENSING GMSL camera plugin (Jetson only; requires SIPL, NVENC and the LiveKit SDK)" OFF)
+option(BUILD_VIDEO_STAGE "Build the PC GPU video stage (needs a CUDA toolkit; tests need a CUDA GPU)" OFF)
 option(BUILD_EXAMPLES "Build examples" ON)
 option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF)
 option(BUILD_TESTING "Build unit tests" ON)

src/CMakeLists.txt

@@ -25,6 +25,25 @@ if(BUILD_VIZ)
     add_subdirectory(viz)
 endif()
 
+# The LiveKit SDK and publisher library: the PC video stage publishes to LiveKit. The camera producer on the Orin does
+# not use LiveKit; it sends over its direct links.
+if(BUILD_VIDEO_STAGE)
+    add_subdirectory(livekit_publisher)
+endif()
+
+if(BUILD_VIDEO_STAGE)
+    add_subdirectory(video_stage)
+elseif(BUILD_PLUGIN_SENSING)
+    # The SENSING plugin sends AVS1 records to the video stage and needs only its record codec and sender.
+    add_subdirectory(video_stage/wire)
+    add_subdirectory(video_stage/sender)
+endif()
+
+# The camera producer's management channel, also used by the mock producer.
+if(BUILD_PLUGIN_SENSING OR BUILD_VIDEO_STAGE)
+    add_subdirectory(sensing_control)
+endif()
+
 if(BUILD_PLUGINS)
     add_subdirectory(plugins)
 endif()

src/livekit_publisher/CMakeLists.txt

@@ -0,0 +1,95 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# ==============================================================================
+# LiveKit
+# ==============================================================================
+# Each selected sensor is published as a LiveKit camera track. The SDK is a prebuilt package (headers and shared libraries), pinned to a
+# release and checked against its hash; point LIVEKIT_SDK_DIR at an unpacked copy to use another.
+set(_livekit_local ON)
+if(NOT LIVEKIT_SDK_DIR)
+    set(_livekit_local OFF)
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
+        set(_livekit_arch "arm64")
+        set(_livekit_hash "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad")
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
+        set(_livekit_arch "x64")
+        set(_livekit_hash "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b")
+    else()
+        message(FATAL_ERROR "No prebuilt LiveKit SDK for ${CMAKE_SYSTEM_PROCESSOR}; set LIVEKIT_SDK_DIR.")
+    endif()
+    include(FetchContent)
+    FetchContent_Declare(
+        livekit_sdk
+        URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-24.04-${_livekit_arch}-1.12.4.tar.gz
+        URL_HASH SHA256=${_livekit_hash}
+    )
+    FetchContent_MakeAvailable(livekit_sdk)
+    set(LIVEKIT_SDK_DIR "${livekit_sdk_SOURCE_DIR}")
+endif()
+# The package ships no licence files. The pinned download supplies the SDK's own; a local SDK uses
+# LICENSE/NOTICE from its directory, or LIVEKIT_SDK_LICENSE_FILE / LIVEKIT_SDK_NOTICE_FILE, and needs no network.
+foreach(_notice LICENSE NOTICE)
+    set(_override "${LIVEKIT_SDK_${_notice}_FILE}")
+    if(_override)
+        set(_found "${_override}")
+    elseif(_livekit_local)
+        set(_found "")
+        foreach(_candidate "${LIVEKIT_SDK_DIR}/${_notice}" "${LIVEKIT_SDK_DIR}/../${_notice}")
+            if(NOT _found AND EXISTS "${_candidate}")
+                set(_found "${_candidate}")
+            endif()
+        endforeach()
+        if(NOT _found)
+            message(FATAL_ERROR
+                "LIVEKIT_SDK_DIR is set but ${_notice} was not found in it. Put the SDK's ${_notice} file there "
+                "or pass -DLIVEKIT_SDK_${_notice}_FILE=/path/to/${_notice}.")
+        endif()
+    else()
+        if(_notice STREQUAL "LICENSE")
+            set(_notice_hash "09e8a9bcec8067104652c168685ab0931e7868f9c8284b66f5ae6edae5f1130b")
+        else()
+            set(_notice_hash "6d0f7c5e0deb284362296c5da110dd353831da2e67ca77cb5abc051b951b0fac")
+        endif()
+        FetchContent_Declare(
+            livekit_sdk_${_notice}
+            URL https://raw.githubusercontent.com/livekit/client-sdk-cpp/v1.12.4/${_notice}
+            URL_HASH SHA256=${_notice_hash}
+            DOWNLOAD_NO_EXTRACT TRUE
+        )
+        FetchContent_MakeAvailable(livekit_sdk_${_notice})
+        # FetchContent publishes <lowercased name>_SOURCE_DIR.
+        string(TOLOWER "${_notice}" _notice_lower)
+        set(_found "${livekit_sdk_${_notice_lower}_SOURCE_DIR}/${_notice}")
+    endif()
+    set(SENSING_LIVEKIT_${_notice}_FILE "${_found}" CACHE INTERNAL "")
+endforeach()
+find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH)
+message(STATUS "LiveKit SDK=${LIVEKIT_SDK_DIR}")
+# Cache, so tests/ (a sibling directory) can find the same SDK.
+set(SENSING_LIVEKIT_SDK_DIR "${LIVEKIT_SDK_DIR}" CACHE INTERNAL "")
+
+
+
+# The LiveKit SDK behind a small interface: join a room, publish a video track, submit H.264 access units.
+# No LiveKit type appears in an installed header, and nothing here touches CUDA.
+add_library(sensing_livekit STATIC
+    livekit_session.cpp
+    inc/sensing_livekit/livekit_session.hpp
+    inc/sensing_livekit/video_track_sink.hpp
+)
+
+target_include_directories(sensing_livekit
+    INTERFACE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/inc>
+)
+
+target_link_libraries(sensing_livekit
+    PUBLIC
+        log_bridge::log_bridge_core
+        video_stage::wire
+    PRIVATE
+        LiveKit::livekit
+)
+
+add_library(sensing::livekit ALIAS sensing_livekit)

src/livekit_publisher/inc/sensing_livekit/livekit_session.hpp

@@ -0,0 +1,78 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "video_track_sink.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+namespace plugins
+{
+namespace sensing
+{
+
+struct LiveKitSessionImpl;
+
+/**
+ * @brief Keeps the LiveKit SDK initialised while it is held.
+ *
+ * The SDK is process-wide and must be shut down only when nothing in the process uses it, so every
+ * user holds one of these and the last release shuts it down.
+ */
+class LiveKitSdkReference
+{
+public:
+    LiveKitSdkReference();
+    ~LiveKitSdkReference();
+    LiveKitSdkReference(const LiveKitSdkReference&) = delete;
+    LiveKitSdkReference& operator=(const LiveKitSdkReference&) = delete;
+};
+
+struct PublishSpec
+{
+    /// Name subscribers see for the track.
+    std::string name;
+    uint32_t width = 0;
+    uint32_t height = 0;
+    /// Upper bound the upstream encoder may use; WebRTC adapts below it.
+    uint64_t max_bitrate_bps = 0;
+    double max_fps = 30.0;
+};
+
+/**
+ * @brief One participant in one LiveKit room.
+ *
+ * The room and the participant's identity come from the join token. A terminal loss of the room
+ * (the server went away, the participant was removed) sets failed(); the SDK's own bounded
+ * reconnection runs first and is not wrapped in another supervisor.
+ */
+class LiveKitSession
+{
+public:
+    /// @throws std::runtime_error if the room cannot be joined. The token is never logged.
+    static std::unique_ptr<LiveKitSession> connect(const std::string& url, const std::string& token);
+
+    /// Unpublish every track (destroy them first), then leave the room.
+    ~LiveKitSession();
+
+    LiveKitSession(const LiveKitSession&) = delete;
+    LiveKitSession& operator=(const LiveKitSession&) = delete;
+
+    /// Publishes an H.264 camera track that carries pre-encoded access units; the SDK does not encode.
+    /// @throws std::runtime_error if publication fails.
+    std::unique_ptr<VideoTrackSink> publish_video(const PublishSpec& spec);
+
+    /// True once the room has been lost for good.
+    bool failed() const;
+    std::string failure() const;
+
+private:
+    explicit LiveKitSession(std::shared_ptr<LiveKitSessionImpl> impl);
+    std::shared_ptr<LiveKitSessionImpl> m_impl;
+};
+
+} // namespace sensing
+} // namespace plugins

src/livekit_publisher/inc/sensing_livekit/video_track_sink.hpp

@@ -0,0 +1,57 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <video_stage/encoded_video_frame.hpp>
+
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <string>
+
+namespace plugins
+{
+namespace sensing
+{
+
+using video_stage::EncodedVideoFrame;
+
+/// What the receiving side asked of the encoder since the last call.
+struct EncoderFeedback
+{
+    /// The next frame must be an IDR with its parameter sets.
+    bool keyframe_requested = false;
+    /// Latest bitrate target from congestion control, if one was requested.
+    std::optional<uint32_t> target_bitrate_bps;
+};
+
+/**
+ * @brief One published video track that carries frames already encoded by hardware.
+ *
+ * The SDK forwards each access unit without encoding it again. A single thread must drive one sink.
+ */
+class VideoTrackSink
+{
+public:
+    virtual ~VideoTrackSink() = default;
+
+    /// Size of the encoded pictures this track was published for.
+    virtual uint32_t width() const = 0;
+    virtual uint32_t height() const = 0;
+
+    /// Publishes one access unit; the bytes are copied before this returns.
+    /// @return false when the SDK did not accept the frame, which can break the reference chain:
+    ///         the caller must follow with a keyframe.
+    /// @throws std::runtime_error when the frame cannot be published at all.
+    virtual bool submit(const EncodedVideoFrame& frame) = 0;
+
+    /// Keyframe and rate requests accumulated since the previous call.
+    virtual EncoderFeedback take_feedback() = 0;
+
+    /// One line naming the transport's view of the stream, or empty if unavailable.
+    virtual std::string describe() = 0;
+};
+
+} // namespace sensing
+} // namespace plugins