video_stage: relay the Orin's video to LiveKit on the PC

src/CMakeLists.txt

@@ -25,8 +25,17 @@ if(BUILD_VIZ)
     add_subdirectory(viz)
 endif()
 
+# The LiveKit SDK and publisher library: the PC relay publishes to LiveKit; the camera producer still does until it sends
+# direct to the relay.
+if(BUILD_VIDEO_STAGE OR BUILD_PLUGIN_SENSING)
+    add_subdirectory(livekit_publisher)
+endif()
+
 if(BUILD_VIDEO_STAGE)
     add_subdirectory(video_stage)
+elseif(BUILD_PLUGIN_SENSING)
+    # The LiveKit library uses the record codec and clock-exchange helpers.
+    add_subdirectory(video_stage/wire)
 endif()
 
 if(BUILD_PLUGINS)

src/livekit_publisher/CMakeLists.txt

@@ -0,0 +1,110 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# ==============================================================================
+# LiveKit
+# ==============================================================================
+# Each selected sensor is published as a LiveKit camera track. The SDK is a prebuilt package (headers and shared libraries), pinned to a
+# release and checked against its hash; point LIVEKIT_SDK_DIR at an unpacked copy to use another.
+set(_livekit_local ON)
+if(NOT LIVEKIT_SDK_DIR)
+    set(_livekit_local OFF)
+    # The prebuilt links against the glibc of the Ubuntu release it was built on: the 24.04 build does not load on
+    # JetPack 6 (Ubuntu 22.04). Pick the release matching the host; LIVEKIT_SDK_UBUNTU overrides the detection.
+    if(NOT LIVEKIT_SDK_UBUNTU)
+        set(LIVEKIT_SDK_UBUNTU "24.04")
+        if(EXISTS /etc/os-release)
+            file(STRINGS /etc/os-release _os_version_id REGEX "^VERSION_ID=")
+            if(_os_version_id MATCHES "\"?22\\.04\"?$")
+                set(LIVEKIT_SDK_UBUNTU "22.04")
+            endif()
+        endif()
+    endif()
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
+        set(_livekit_arch "arm64")
+        set(_livekit_hash_24.04 "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad")
+        set(_livekit_hash_22.04 "7a698bbd8f61f019744fd99b3a5d2b6c6118787fd1f81140c9a6fc8732fc87e0")
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
+        set(_livekit_arch "x64")
+        set(_livekit_hash_24.04 "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b")
+        set(_livekit_hash_22.04 "9e568ee6b2bb3c7b0d07faba694930767096a14d0e09422ba01f34077ffb03cb")
+    else()
+        message(FATAL_ERROR "No prebuilt LiveKit SDK for ${CMAKE_SYSTEM_PROCESSOR}; set LIVEKIT_SDK_DIR.")
+    endif()
+    if(NOT DEFINED _livekit_hash_${LIVEKIT_SDK_UBUNTU})
+        message(FATAL_ERROR "No prebuilt LiveKit SDK for Ubuntu ${LIVEKIT_SDK_UBUNTU}; set LIVEKIT_SDK_DIR.")
+    endif()
+    set(_livekit_hash "${_livekit_hash_${LIVEKIT_SDK_UBUNTU}}")
+    include(FetchContent)
+    FetchContent_Declare(
+        livekit_sdk
+        URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-${LIVEKIT_SDK_UBUNTU}-${_livekit_arch}-1.12.4.tar.gz
+        URL_HASH SHA256=${_livekit_hash}
+    )
+    FetchContent_MakeAvailable(livekit_sdk)
+    set(LIVEKIT_SDK_DIR "${livekit_sdk_SOURCE_DIR}")
+endif()
+# The package ships no licence files. The pinned download supplies the SDK's own; a local SDK uses
+# LICENSE/NOTICE from its directory, or LIVEKIT_SDK_LICENSE_FILE / LIVEKIT_SDK_NOTICE_FILE, and needs no network.
+foreach(_notice LICENSE NOTICE)
+    set(_override "${LIVEKIT_SDK_${_notice}_FILE}")
+    if(_override)
+        set(_found "${_override}")
+    elseif(_livekit_local)
+        set(_found "")
+        foreach(_candidate "${LIVEKIT_SDK_DIR}/${_notice}" "${LIVEKIT_SDK_DIR}/../${_notice}")
+            if(NOT _found AND EXISTS "${_candidate}")
+                set(_found "${_candidate}")
+            endif()
+        endforeach()
+        if(NOT _found)
+            message(FATAL_ERROR
+                "LIVEKIT_SDK_DIR is set but ${_notice} was not found in it. Put the SDK's ${_notice} file there "
+                "or pass -DLIVEKIT_SDK_${_notice}_FILE=/path/to/${_notice}.")
+        endif()
+    else()
+        if(_notice STREQUAL "LICENSE")
+            set(_notice_hash "09e8a9bcec8067104652c168685ab0931e7868f9c8284b66f5ae6edae5f1130b")
+        else()
+            set(_notice_hash "6d0f7c5e0deb284362296c5da110dd353831da2e67ca77cb5abc051b951b0fac")
+        endif()
+        FetchContent_Declare(
+            livekit_sdk_${_notice}
+            URL https://raw.githubusercontent.com/livekit/client-sdk-cpp/v1.12.4/${_notice}
+            URL_HASH SHA256=${_notice_hash}
+            DOWNLOAD_NO_EXTRACT TRUE
+        )
+        FetchContent_MakeAvailable(livekit_sdk_${_notice})
+        # FetchContent publishes <lowercased name>_SOURCE_DIR.
+        string(TOLOWER "${_notice}" _notice_lower)
+        set(_found "${livekit_sdk_${_notice_lower}_SOURCE_DIR}/${_notice}")
+    endif()
+    set(SENSING_LIVEKIT_${_notice}_FILE "${_found}" CACHE INTERNAL "")
+endforeach()
+find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH)
+message(STATUS "LiveKit SDK=${LIVEKIT_SDK_DIR}")
+# Cache, so tests/ (a sibling directory) can find the same SDK.
+set(SENSING_LIVEKIT_SDK_DIR "${LIVEKIT_SDK_DIR}" CACHE INTERNAL "")
+
+# The LiveKit SDK behind a small interface: join a room, publish a video track, submit H.264 access units.
+# No LiveKit type appears in an installed header, and nothing here touches CUDA.
+add_library(sensing_livekit STATIC
+    livekit_session.cpp
+    inc/sensing_livekit/livekit_session.hpp
+    inc/sensing_livekit/video_track_sink.hpp
+)
+
+target_include_directories(sensing_livekit
+    INTERFACE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/inc>
+)
+
+target_link_libraries(sensing_livekit
+    PUBLIC
+        log_bridge::log_bridge_core
+        video_stage::wire
+    PRIVATE
+        LiveKit::livekit
+)
+
+add_library(sensing::livekit ALIAS sensing_livekit)

src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp → src/livekit_publisher/inc/sensing_livekit/video_track_sink.hpp

@@ -3,6 +3,8 @@
 
 #pragma once
 
+#include <video_stage/encoded_video_frame.hpp>
+
 #include <cstddef>
 #include <cstdint>
 #include <optional>
@@ -13,26 +15,7 @@ namespace plugins
 namespace sensing
 {
 
-/// One complete encoded H.264 access unit, with what accompanies it.
-struct EncodedVideoFrame
-{
-    /// Annex-B bytes: an IDR carries its SPS/PPS in front of it.
-    const uint8_t* data = nullptr;
-    size_t size = 0;
-    /// The unit is an IDR, decodable on its own.
-    bool keyframe = false;
-    /// CLOCK_MONOTONIC microseconds at capture: the host timeline, not sensor time.
-    uint64_t timestamp_us = 0;
-    /// Unix time in microseconds on the publishing host when the frame finished conversion: processing
-    /// time, not a capture time. Carried as the frame's application timestamp; the transport timestamp
-    /// above stays monotonic.
-    uint64_t user_timestamp_us = 0;
-    /// The camera's own frame sequence.
-    uint32_t frame_id = 0;
-    /// The sensor's capture time on its own clock, carried as user data so a consumer can pair
-    /// frames from different sensors.
-    uint64_t capture_tsc = 0;
-};
+using video_stage::EncodedVideoFrame;
 
 /// What the receiving side asked of the encoder since the last call.
 struct EncoderFeedback

src/plugins/sensing/livekit/livekit_session.cpp → src/livekit_publisher/livekit_session.cpp

@@ -3,11 +3,10 @@
 
 #include "inc/sensing_livekit/livekit_session.hpp"
 
-#include "inc/sensing_livekit/clock_sync.hpp"
-
 #include <livekit/encoded_video_source.h>
 #include <livekit/livekit.h>
 #include <log_bridge/logger.hpp>
+#include <video_stage/clock_sync.hpp>
 
 #include <atomic>
 #include <chrono>
@@ -268,7 +267,10 @@ class LiveKitVideoTrack : public VideoTrackSink
         out.timestamp_us = static_cast<int64_t>(frame.timestamp_us);
         out.data.assign(frame.data, frame.data + frame.size);
         livekit::VideoFrameMetadata metadata;
-        metadata.user_timestamp_us = frame.user_timestamp_us;
+        // Zero means the publisher cannot vouch for a capture time: the frame is sent without one, and a consumer
+        // that needs a timestamp (a recording) says so instead of using a guessed one.
+        if (frame.user_timestamp_us != 0)
+            metadata.user_timestamp_us = frame.user_timestamp_us;
         metadata.frame_id = frame.frame_id;
         std::vector<uint8_t> tsc(8);
         for (int i = 0; i < 8; ++i)

src/plugins/sensing/CMakeLists.txt

@@ -4,8 +4,7 @@
 # ==============================================================================
 # SENSING Camera Plugin: Jetson SIPL capture, hardware H.264 encode, LiveKit publication
 # ==============================================================================
-# Two leaf targets besides the LiveKit wrapper, one per directory (cmake/cmake-structure.md rule 1):
-#   livekit/  sensing_livekit       -- the LiveKit SDK behind a small interface
+# Two leaf targets, one per directory (cmake/cmake-structure.md rule 1); the LiveKit wrapper is src/livekit_publisher:
 #   core/     sensing_core          -- SIPL capture, encoder, conversion, track output
 #   app/      camera_plugin_sensing
 #
@@ -184,90 +183,6 @@ message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}")
 # dependency at all (readelf -d lists only EGL, cuda, dl, pthread, rt, c), and
 # the tegra lib dir ships no libnvjpeg to link against.
 
-# ==============================================================================
-# LiveKit
-# ==============================================================================
-# Each selected sensor is published as a LiveKit camera track. The SDK is a prebuilt package (headers and shared libraries), pinned to a
-# release and checked against its hash; point LIVEKIT_SDK_DIR at an unpacked copy to use another.
-set(_livekit_local ON)
-if(NOT LIVEKIT_SDK_DIR)
-    set(_livekit_local OFF)
-    # The prebuilt links against the glibc of the Ubuntu release it was built on: the 24.04 build does not load on
-    # JetPack 6 (Ubuntu 22.04). Pick the release matching the host; LIVEKIT_SDK_UBUNTU overrides the detection.
-    if(NOT LIVEKIT_SDK_UBUNTU)
-        set(LIVEKIT_SDK_UBUNTU "24.04")
-        if(EXISTS /etc/os-release)
-            file(STRINGS /etc/os-release _os_version_id REGEX "^VERSION_ID=")
-            if(_os_version_id MATCHES "\"?22\\.04\"?$")
-                set(LIVEKIT_SDK_UBUNTU "22.04")
-            endif()
-        endif()
-    endif()
-    if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
-        set(_livekit_arch "arm64")
-        set(_livekit_hash_24.04 "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad")
-        set(_livekit_hash_22.04 "7a698bbd8f61f019744fd99b3a5d2b6c6118787fd1f81140c9a6fc8732fc87e0")
-    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
-        set(_livekit_arch "x64")
-        set(_livekit_hash_24.04 "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b")
-        set(_livekit_hash_22.04 "9e568ee6b2bb3c7b0d07faba694930767096a14d0e09422ba01f34077ffb03cb")
-    else()
-        message(FATAL_ERROR "No prebuilt LiveKit SDK for ${CMAKE_SYSTEM_PROCESSOR}; set LIVEKIT_SDK_DIR.")
-    endif()
-    if(NOT DEFINED _livekit_hash_${LIVEKIT_SDK_UBUNTU})
-        message(FATAL_ERROR "No prebuilt LiveKit SDK for Ubuntu ${LIVEKIT_SDK_UBUNTU}; set LIVEKIT_SDK_DIR.")
-    endif()
-    set(_livekit_hash "${_livekit_hash_${LIVEKIT_SDK_UBUNTU}}")
-    include(FetchContent)
-    FetchContent_Declare(
-        livekit_sdk
-        URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-${LIVEKIT_SDK_UBUNTU}-${_livekit_arch}-1.12.4.tar.gz
-        URL_HASH SHA256=${_livekit_hash}
-    )
-    FetchContent_MakeAvailable(livekit_sdk)
-    set(LIVEKIT_SDK_DIR "${livekit_sdk_SOURCE_DIR}")
-endif()
-# The package ships no licence files. The pinned download supplies the SDK's own; a local SDK uses
-# LICENSE/NOTICE from its directory, or LIVEKIT_SDK_LICENSE_FILE / LIVEKIT_SDK_NOTICE_FILE, and needs no network.
-foreach(_notice LICENSE NOTICE)
-    set(_override "${LIVEKIT_SDK_${_notice}_FILE}")
-    if(_override)
-        set(_found "${_override}")
-    elseif(_livekit_local)
-        set(_found "")
-        foreach(_candidate "${LIVEKIT_SDK_DIR}/${_notice}" "${LIVEKIT_SDK_DIR}/../${_notice}")
-            if(NOT _found AND EXISTS "${_candidate}")
-                set(_found "${_candidate}")
-            endif()
-        endforeach()
-        if(NOT _found)
-            message(FATAL_ERROR
-                "LIVEKIT_SDK_DIR is set but ${_notice} was not found in it. Put the SDK's ${_notice} file there "
-                "or pass -DLIVEKIT_SDK_${_notice}_FILE=/path/to/${_notice}.")
-        endif()
-    else()
-        if(_notice STREQUAL "LICENSE")
-            set(_notice_hash "09e8a9bcec8067104652c168685ab0931e7868f9c8284b66f5ae6edae5f1130b")
-        else()
-            set(_notice_hash "6d0f7c5e0deb284362296c5da110dd353831da2e67ca77cb5abc051b951b0fac")
-        endif()
-        FetchContent_Declare(
-            livekit_sdk_${_notice}
-            URL https://raw.githubusercontent.com/livekit/client-sdk-cpp/v1.12.4/${_notice}
-            URL_HASH SHA256=${_notice_hash}
-            DOWNLOAD_NO_EXTRACT TRUE
-        )
-        FetchContent_MakeAvailable(livekit_sdk_${_notice})
-        set(_found "${livekit_sdk_${_notice}_SOURCE_DIR}/${_notice}")
-    endif()
-    set(SENSING_LIVEKIT_${_notice}_FILE "${_found}" CACHE INTERNAL "")
-endforeach()
-find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH)
-message(STATUS "SENSING plugin: LiveKit SDK=${LIVEKIT_SDK_DIR}")
-# Cache, so tests/ (a sibling directory) can find the same SDK.
-set(SENSING_LIVEKIT_SDK_DIR "${LIVEKIT_SDK_DIR}" CACHE INTERNAL "")
-
-
 # ==============================================================================
 # Targets
 # ==============================================================================
@@ -276,7 +191,6 @@ message(STATUS "Building SENSING camera plugin")
 # The platform configs live beside this file; app/ stages and installs them from here.
 set(SENSING_CONFIGS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/configs")
 
-add_subdirectory(livekit)
 add_subdirectory(core)
 add_subdirectory(app)