From 297ad0ccfee853cba56c113a97ecaddf480617c7 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Wed, 4 Nov 2026 23:17:43 -0800 Subject: [PATCH 01/29] sensing: publish SENSING sensors to LiveKit from the Jetson hardware encoder Add the SENSING SG8A camera plugin as a LiveKit publisher: SIPL capture, colour conversion, one hardware H.264 encode per track, and pre-encoded publication through the LiveKit C++ SDK. The sources are taken from the sensing-livekit branch and trimmed to this one path: no file recording, CUDA IPC, MCAP metadata or schema. Jetson only; built with -DBUILD_PLUGIN_SENSING=ON. Not built or run here. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- CMakeLists.txt | 1 + src/plugins/CMakeLists.txt | 4 + src/plugins/sensing/CMakeLists.txt | 212 +++ src/plugins/sensing/app/CMakeLists.txt | 36 + src/plugins/sensing/app/main.cpp | 383 ++++++ src/plugins/sensing/core/CMakeLists.txt | 50 + src/plugins/sensing/core/jetson_encoder.cpp | 325 +++++ src/plugins/sensing/core/jetson_encoder.hpp | 109 ++ .../sensing/core/livekit_video_output.cpp | 486 +++++++ .../sensing/core/livekit_video_output.hpp | 110 ++ .../sensing/core/rgba_box_downscale.cu | 57 + .../sensing/core/rgba_box_downscale.cuh | 33 + src/plugins/sensing/core/rgba_to_nv12.cu | 103 ++ src/plugins/sensing/core/rgba_to_nv12.cuh | 36 + src/plugins/sensing/core/sensing_camera.cpp | 174 +++ src/plugins/sensing/core/sensing_camera.hpp | 69 + src/plugins/sensing/core/sensing_types.hpp | 65 + src/plugins/sensing/core/sipl_camera.cpp | 1205 +++++++++++++++++ src/plugins/sensing/core/sipl_camera.hpp | 115 ++ src/plugins/sensing/core/yuv_to_rgba.cu | 253 ++++ src/plugins/sensing/core/yuv_to_rgba.cuh | 53 + src/plugins/sensing/livekit/CMakeLists.txt | 24 + .../inc/sensing_livekit/livekit_session.hpp | 78 ++ .../inc/sensing_livekit/video_track_sink.hpp | 70 + .../sensing/livekit/livekit_session.cpp | 301 ++++ src/plugins/sensing/plugin.yaml | 11 + 26 files changed, 4363 insertions(+) create mode 100644 src/plugins/sensing/CMakeLists.txt create mode 100644 src/plugins/sensing/app/CMakeLists.txt create mode 100644 src/plugins/sensing/app/main.cpp create mode 100644 src/plugins/sensing/core/CMakeLists.txt create mode 100644 src/plugins/sensing/core/jetson_encoder.cpp create mode 100644 src/plugins/sensing/core/jetson_encoder.hpp create mode 100644 src/plugins/sensing/core/livekit_video_output.cpp create mode 100644 src/plugins/sensing/core/livekit_video_output.hpp create mode 100644 src/plugins/sensing/core/rgba_box_downscale.cu create mode 100644 src/plugins/sensing/core/rgba_box_downscale.cuh create mode 100644 src/plugins/sensing/core/rgba_to_nv12.cu create mode 100644 src/plugins/sensing/core/rgba_to_nv12.cuh create mode 100644 src/plugins/sensing/core/sensing_camera.cpp create mode 100644 src/plugins/sensing/core/sensing_camera.hpp create mode 100644 src/plugins/sensing/core/sensing_types.hpp create mode 100644 src/plugins/sensing/core/sipl_camera.cpp create mode 100644 src/plugins/sensing/core/sipl_camera.hpp create mode 100644 src/plugins/sensing/core/yuv_to_rgba.cu create mode 100644 src/plugins/sensing/core/yuv_to_rgba.cuh create mode 100644 src/plugins/sensing/livekit/CMakeLists.txt create mode 100644 src/plugins/sensing/livekit/inc/sensing_livekit/livekit_session.hpp create mode 100644 src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp create mode 100644 src/plugins/sensing/livekit/livekit_session.cpp create mode 100644 src/plugins/sensing/plugin.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index 521ec03c30..6b02232bd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,6 +71,7 @@ option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF) option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF) option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF) +option(BUILD_PLUGIN_SENSING "Build SENSING GMSL camera plugin (Jetson only; requires SIPL, NVENC and the LiveKit SDK)" OFF) option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) option(BUILD_TESTING "Build unit tests" ON) diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index c26d9384c2..7a3a400b15 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -44,3 +44,7 @@ endif() if(BUILD_PLUGIN_SHARPA_AVATAR) add_subdirectory(sharpa_avatar) endif() +# Gated: Jetson only -- SIPL and the Tegra NVENC entry point are L4T-only. +if(BUILD_PLUGIN_SENSING) + add_subdirectory(sensing) +endif() diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt new file mode 100644 index 0000000000..da64e4fbdc --- /dev/null +++ b/src/plugins/sensing/CMakeLists.txt @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ============================================================================== +# SENSING Camera Plugin: Jetson SIPL capture, hardware H.264 encode, LiveKit publication +# ============================================================================== +# Two leaf targets besides the LiveKit wrapper, one per directory (cmake/cmake-structure.md rule 1): +# livekit/ sensing_livekit -- the LiveKit SDK behind a small interface +# core/ sensing_core -- SIPL capture, encoder, conversion, track output +# app/ camera_plugin_sensing +# +# Prerequisites: +# src/plugins/sensing/scripts/setup.sh # fetches the Multimedia API, locates jetson_sipl_api +# +# Configure with: +# cmake -B build -DBUILD_PLUGIN_SENSING=ON +# ============================================================================== + +enable_language(CUDA) + +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + # Orin is 87, Thor is 101; the rest cover desktop dev boxes. + set(CMAKE_CUDA_ARCHITECTURES "80;86;87;89;90") +endif() + +find_package(CUDAToolkit REQUIRED) + +set(_tegra_lib_dirs "/usr/lib/aarch64-linux-gnu/nvidia" "/usr/lib/aarch64-linux-gnu/tegra") + +# ============================================================================== +# SIPL +# ============================================================================== +# Headers ship in Jetson_SIPL_API_R_aarch64.tbz2, published alongside the +# L4T release. Not in any apt pool, so setup_container.sh fetches it by URL. +if(NOT SIPL_API_DIR) + find_path(SIPL_API_DIR sipl/include/NvSIPLCamera.hpp + PATHS "/usr/src/jetson_sipl_api" + "/opt/nvidia/jetson_sipl_api" + "$ENV{SIPL_API_DIR}" + ) +endif() + +if(NOT SIPL_API_DIR) + message(FATAL_ERROR + "================================================================================\n" + "SENSING camera plugin requires the Jetson SIPL API headers.\n" + "\n" + "Run the container setup, which fetches them:\n" + " src/plugins/sensing/scripts/setup.sh\n" + "\n" + "Or point CMake at an existing copy:\n" + " cmake -B build -DSIPL_API_DIR=/path/to/jetson_sipl_api\n" + "\n" + "Or disable this plugin:\n" + " cmake -B build -DBUILD_PLUGIN_SENSING=OFF\n" + "================================================================================\n" + ) +endif() + +set(SIPL_INCLUDE_DIRS + "${SIPL_API_DIR}/sipl/include" + "${SIPL_API_DIR}/sipl/include/query/include" + "${SIPL_API_DIR}/sipl/include/nvsci" +) +message(STATUS "SENSING plugin: SIPL=${SIPL_API_DIR}") + +# L4T ships no .so dev symlinks for these, hence the explicit sonames. +foreach(_lib nvsipl nvsipl_query nvscibuf nvscisync nvbufsurface_nvsci) + string(TOUPPER "${_lib}" _upper) + find_library(${_upper}_LIBRARY + NAMES ${_lib} lib${_lib}.so.1 lib${_lib}.so.1.0.0 + PATHS ${_tegra_lib_dirs} + ) + if(NOT ${_upper}_LIBRARY) + message(FATAL_ERROR + "lib${_lib} not found. It is supplied by the L4T BSP and, in a container, by the " + "NVIDIA container runtime. Run src/plugins/sensing/verify.sh to see what is missing.") + endif() + list(APPEND SIPL_LIBRARIES ${${_upper}_LIBRARY}) +endforeach() + +# EGL stays on the capture path: NvBufSurfaceMapEglImage is how a SIPL +# NvSciBufObj becomes a CUeglFrame. Interop path B in the design notes would +# remove this; path A does not. +find_path(EGL_INCLUDE_DIRS EGL/egl.h) +find_library(EGL_LIBRARIES NAMES EGL) +if(NOT EGL_INCLUDE_DIRS OR NOT EGL_LIBRARIES) + message(FATAL_ERROR "SENSING camera plugin requires EGL. Install libegl1-mesa-dev.") +endif() + +# ============================================================================== +# Jetson V4L2 encoder +# ============================================================================== +# Jetson has no libnvidia-encode (the Video Codec SDK is dGPU-only), so H.264 +# comes from the V4L2 M2M engine via NvVideoEncoder in the Multimedia API. +if(NOT JETSON_MMAPI_DIR) + find_path(JETSON_MMAPI_DIR include/NvVideoEncoder.h + PATHS "/usr/src/jetson_multimedia_api" + "$ENV{HOME}/Sensing/jetson_multimedia_api" + ) +endif() +if(NOT JETSON_MMAPI_DIR) + message(FATAL_ERROR + "================================================================================\n" + "SENSING camera plugin requires the Jetson Multimedia API encoder sources.\n" + "\n" + "setup_container.sh fetches the version matching /etc/nv_tegra_release:\n" + " src/plugins/sensing/scripts/setup.sh\n" + "\n" + "Or point CMake at an existing tree:\n" + " cmake -B build -DJETSON_MMAPI_DIR=/path/to/jetson_multimedia_api\n" + "================================================================================\n" + ) +endif() +message(STATUS "SENSING plugin: Jetson Multimedia API=${JETSON_MMAPI_DIR}") + +set(_mmapi_classes "${JETSON_MMAPI_DIR}/samples/common/classes") +set(JETSON_MMAPI_SOURCES + "${_mmapi_classes}/NvVideoEncoder.cpp" + "${_mmapi_classes}/NvV4l2Element.cpp" + "${_mmapi_classes}/NvV4l2ElementPlane.cpp" + "${_mmapi_classes}/NvElement.cpp" + "${_mmapi_classes}/NvElementProfiler.cpp" + "${_mmapi_classes}/NvBuffer.cpp" + "${_mmapi_classes}/NvLogging.cpp" +) + +find_library(NVBUFSURFACE_LIBRARY NAMES nvbufsurface PATHS ${_tegra_lib_dirs}) +if(NOT NVBUFSURFACE_LIBRARY) + message(FATAL_ERROR "libnvbufsurface not found. Install nvidia-l4t-multimedia.") +endif() + +# NVIDIA's patched libv4l2 carries the Tegra M2M encoder ioctls, so search the +# tegra directories before the generic one. L4T ships no .so dev symlink, hence +# the explicit soname in NAMES. +find_library(V4L2_LIBRARY + NAMES v4l2 libv4l2.so.0 + PATHS ${_tegra_lib_dirs} + NO_DEFAULT_PATH +) +if(NOT V4L2_LIBRARY) + find_library(V4L2_LIBRARY NAMES v4l2 libv4l2.so.0) +endif() +if(NOT V4L2_LIBRARY) + message(FATAL_ERROR "libv4l2 not found. Install nvidia-l4t-multimedia (or libv4l-dev).") +endif() +message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}") + +# No libnvjpeg here on purpose: on R39 libnvbufsurface has no libjpeg +# dependency at all (readelf -d lists only EGL, cuda, dl, pthread, rt, c), and +# the tegra lib dir ships no libnvjpeg to link against. + +# ============================================================================== +# LiveKit +# ============================================================================== +# Each selected sensor is published as a LiveKit camera track. The SDK is a prebuilt package (headers and shared libraries), pinned to a +# release and checked against its hash; point LIVEKIT_SDK_DIR at an unpacked copy to use another. +if(NOT LIVEKIT_SDK_DIR) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(_livekit_arch "arm64") + set(_livekit_hash "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(_livekit_arch "x64") + set(_livekit_hash "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b") + else() + message(FATAL_ERROR "No prebuilt LiveKit SDK for ${CMAKE_SYSTEM_PROCESSOR}; set LIVEKIT_SDK_DIR.") + endif() + include(FetchContent) + FetchContent_Declare( + livekit_sdk + URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-24.04-${_livekit_arch}-1.12.4.tar.gz + URL_HASH SHA256=${_livekit_hash} + ) + FetchContent_MakeAvailable(livekit_sdk) + set(LIVEKIT_SDK_DIR "${livekit_sdk_SOURCE_DIR}") +endif() +# The package ships no licence files: install the SDK's own, pinned to the same release. +foreach(_notice LICENSE NOTICE) + if(_notice STREQUAL "LICENSE") + set(_notice_hash "09e8a9bcec8067104652c168685ab0931e7868f9c8284b66f5ae6edae5f1130b") + else() + set(_notice_hash "6d0f7c5e0deb284362296c5da110dd353831da2e67ca77cb5abc051b951b0fac") + endif() + FetchContent_Declare( + livekit_sdk_${_notice} + URL https://raw.githubusercontent.com/livekit/client-sdk-cpp/v1.12.4/${_notice} + URL_HASH SHA256=${_notice_hash} + DOWNLOAD_NO_EXTRACT TRUE + ) + FetchContent_MakeAvailable(livekit_sdk_${_notice}) + set(SENSING_LIVEKIT_${_notice}_FILE "${livekit_sdk_${_notice}_SOURCE_DIR}/${_notice}" CACHE INTERNAL "") +endforeach() +find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH) +message(STATUS "SENSING plugin: LiveKit SDK=${LIVEKIT_SDK_DIR}") +# Cache, so tests/ (a sibling directory) can find the same SDK. +set(SENSING_LIVEKIT_SDK_DIR "${LIVEKIT_SDK_DIR}" CACHE INTERNAL "") + + +# ============================================================================== +# Targets +# ============================================================================== +message(STATUS "Building SENSING camera plugin") + +add_subdirectory(livekit) +add_subdirectory(core) +add_subdirectory(app) + +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/plugin.yaml" + "${CMAKE_CURRENT_SOURCE_DIR}/README.md" + DESTINATION plugins/sensing_camera +) diff --git a/src/plugins/sensing/app/CMakeLists.txt b/src/plugins/sensing/app/CMakeLists.txt new file mode 100644 index 0000000000..4af038dcd0 --- /dev/null +++ b/src/plugins/sensing/app/CMakeLists.txt @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(camera_plugin_sensing main.cpp) + +set_target_properties(camera_plugin_sensing PROPERTIES INSTALL_RPATH "$ORIGIN") + +target_link_libraries(camera_plugin_sensing PRIVATE sensing::core) + +# The plugin resolves configs/shw5g.json relative to its own executable, so the +# config has to sit beside the binary in the build tree as well as the install +# tree -- otherwise it only works after `cmake --install`. +add_custom_command(TARGET camera_plugin_sensing POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${CMAKE_CURRENT_SOURCE_DIR}/../configs" + "$/configs" + COMMENT "Staging SIPL platform configs beside camera_plugin_sensing" +) + +install(TARGETS camera_plugin_sensing + RUNTIME DESTINATION plugins/sensing_camera +) +install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../configs/" + DESTINATION plugins/sensing_camera/configs +) + +# The plugin links the LiveKit runtime, whose RUNPATH points into the build tree; ship it beside the +# binary, where INSTALL_RPATH=$ORIGIN (and liblivekit.so's own $ORIGIN) finds it. +install(FILES + "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit.so" + "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit_ffi.so" + DESTINATION plugins/sensing_camera +) +install(FILES "${SENSING_LIVEKIT_LICENSE_FILE}" "${SENSING_LIVEKIT_NOTICE_FILE}" + DESTINATION plugins/sensing_camera/licenses/livekit-sdk +) diff --git a/src/plugins/sensing/app/main.cpp b/src/plugins/sensing/app/main.cpp new file mode 100644 index 0000000000..242e2c7660 --- /dev/null +++ b/src/plugins/sensing/app/main.cpp @@ -0,0 +1,383 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "core/sensing_camera.hpp" +#include "core/sipl_camera.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::sensing; + +// ============================================================================= +// Signal handling +// ============================================================================= + +static std::atomic g_stop_requested{ false }; + +void signal_handler(int signal) +{ + if (signal == SIGINT || signal == SIGTERM) + { + g_stop_requested.store(true, std::memory_order_relaxed); + } +} + +// ============================================================================= +// Argument parsers +// ============================================================================= + +/// "sensor=N,livekit=NAME" -> one sensor and the track that carries it. +static StreamConfig parse_stream_arg(const std::string& arg) +{ + StreamConfig cfg{}; + bool has_sensor = false; + + std::istringstream ss(arg); + std::string token; + while (std::getline(ss, token, ',')) + { + auto eq = token.find('='); + if (eq == std::string::npos) + throw std::runtime_error("Invalid key=value in --add-stream: '" + token + "'"); + + auto key = token.substr(0, eq); + auto val = token.substr(eq + 1); + + if (key == "sensor") + { + cfg.sensor_id = static_cast(std::stoul(val)); + has_sensor = true; + } + else if (key == "livekit") + { + cfg.livekit_track = val; + } + else + { + throw std::runtime_error("Unknown key in --add-stream: '" + key + "'"); + } + } + + if (!has_sensor || cfg.livekit_track.empty()) + throw std::runtime_error("--add-stream requires sensor=,livekit="); + + return cfg; +} + +/// "0x0000 0x1100" -> {0x0000, 0x1100}. Same spelling as nvsipl_camera -m, so a +/// working vendor command line can be copied across verbatim. +static std::vector parse_link_masks(const std::string& arg) +{ + std::vector masks; + std::istringstream ss(arg); + std::string token; + while (ss >> token) + { + masks.push_back(static_cast(std::stoul(token, nullptr, 0))); + } + if (masks.empty()) + throw std::runtime_error("--link-masks needs at least one mask, e.g. --link-masks='0x0000 0x1100'"); + return masks; +} + +/// The vendored platform config, resolved next to the executable so it works +/// from the build tree and the install tree alike, whatever the cwd. Empty when +/// it is not there, which leaves --platform-config required. +static std::string default_platform_config() +{ + char buf[PATH_MAX]; + const ssize_t n = ::readlink("/proc/self/exe", buf, sizeof(buf) - 1); + if (n <= 0) + return {}; + buf[n] = '\0'; + const std::string exe(buf); + const auto slash = exe.rfind('/'); + if (slash == std::string::npos) + return {}; + const std::string candidate = exe.substr(0, slash) + "/configs/shw5g.json"; + return ::access(candidate.c_str(), R_OK) == 0 ? candidate : std::string{}; +} + +// ============================================================================= +// Usage +// ============================================================================= + +static void print_usage(const char* program_name) +{ + std::cout << "Usage: " << program_name << " --livekit-url=URL --livekit-token-file=PATH --add-stream=... [options]\n" + << "\n" + << "Publishes each SENSING sensor as a LiveKit camera track: SIPL capture, hardware H.264,\n" + << "sent as it is.\n" + << "\n" + << "Streams (repeat for each sensor):\n" + << " --add-stream=sensor=N,livekit=NAME SIPL pipeline index N, published as track NAME\n" + << " --list-sensors print the pipelines, then exit\n" + << "\n" + << "Platform:\n" + << " --platform-config=PATH platform JSON (default: configs/shw5g.json beside this\n" + << " binary)\n" + << " --config-name=NAME named config inside it (default: SHW5G_2)\n" + << " --link-masks='M M' one mask per deserializer (default: '0x0000 0x1100')\n" + << " --nito-dir=DIR ISP tuning directory (default: /var/nvidia/nvcam/settings/sipl)\n" + << "\n" + << "LiveKit:\n" + << " --livekit-url=URL server, e.g. wss://livekit.example.org\n" + << " --livekit-token-file=PATH file holding the join token; it names the room and the\n" + << " participant identity\n" + << " --livekit-fps=N publication rate cap (default: 30)\n" + << " --livekit-scale-div=N reduce every track by this integer divisor (default: 1)\n" + << " --livekit-bitrate=N target bitrate of each track's encoder in bps (default: 8000000)\n" + << " --gop=N IDR period in frames (default: fps*5)\n" + << "\n" + << "Other:\n" + << " --gpu-id=N CUDA device index (default: 0)\n" + << " --isp0-buffers=N ISP0 buffers per sensor (default: 6)\n" + << " --full-range full-range YUV\n" + << " --swap-uv swap the chroma planes\n" + << "\n" + << "Example:\n" + << " " << program_name << " --livekit-url=wss://livekit.example.org \\\n" + << " --livekit-token-file=join.token --add-stream=sensor=0,livekit=sensing-left \\\n" + << " --add-stream=sensor=1,livekit=sensing-right\n"; +} + +// ============================================================================= +// Main +// ============================================================================= + +int main(int argc, char** argv) +try +{ + SensingConfig camera_config; + std::map stream_map; + std::string livekit_token_file; + bool list_sensors = false; + + for (int i = 1; i < argc; ++i) + { + std::string arg = argv[i]; + + if (arg == "--help" || arg == "-h") + { + print_usage(argv[0]); + return 0; + } + else if (arg.find("--add-stream=") == 0) + { + auto cfg = parse_stream_arg(arg.substr(13)); + if (!stream_map.emplace(cfg.sensor_id, cfg).second) + throw std::runtime_error("sensor " + std::to_string(cfg.sensor_id) + " given twice"); + } + else if (arg.find("--platform-config=") == 0) + { + camera_config.platform_config_json = arg.substr(18); + } + else if (arg.find("--config-name=") == 0) + { + camera_config.platform_config_name = arg.substr(14); + } + else if (arg.find("--link-masks=") == 0) + { + camera_config.link_masks = parse_link_masks(arg.substr(13)); + } + else if (arg.find("--nito-dir=") == 0) + { + camera_config.nito_dir = arg.substr(11); + } + else if (arg == "--list-sensors") + { + list_sensors = true; + } + else if (arg.find("--livekit-url=") == 0) + { + camera_config.livekit.url = arg.substr(14); + } + else if (arg.find("--livekit-token-file=") == 0) + { + livekit_token_file = arg.substr(21); + } + else if (arg.find("--livekit-fps=") == 0) + { + camera_config.livekit.fps = static_cast(std::stoul(arg.substr(14))); + } + else if (arg.find("--livekit-bitrate=") == 0) + { + camera_config.livekit.bitrate_bps = static_cast(std::stoul(arg.substr(18))); + } + else if (arg.find("--livekit-scale-div=") == 0) + { + camera_config.livekit.scale_div = static_cast(std::stoul(arg.substr(20))); + } + else if (arg.find("--gop=") == 0) + { + camera_config.gop = static_cast(std::stoul(arg.substr(6))); + } + else if (arg.find("--gpu-id=") == 0) + { + camera_config.gpu_id = std::stoi(arg.substr(9)); + } + else if (arg.find("--isp0-buffers=") == 0) + { + camera_config.isp0_buffers = static_cast(std::stoul(arg.substr(15))); + } + else if (arg == "--full-range") + { + camera_config.full_range = true; + } + else if (arg == "--swap-uv") + { + camera_config.swap_uv = true; + } + else if (arg.find("--plugin-root-id=") == 0) + { + // plugin-root-id is a default argument, so we don't need to store it + } + else + { + std::cerr << "Unknown option: " << arg << std::endl; + print_usage(argv[0]); + return 1; + } + } + + if (camera_config.platform_config_json.empty()) + { + camera_config.platform_config_json = default_platform_config(); + } + if (camera_config.platform_config_json.empty()) + { + std::cerr << "Error: no platform config. configs/shw5g.json was not found beside this\n" + "binary, so pass --platform-config=PATH. SIPL cannot enumerate sensors\n" + "without one." + << std::endl; + print_usage(argv[0]); + return 1; + } + + // Query is hardware-free -- it only parses the driver database and the JSON -- so --list-sensors + // works on a rig whose cameras are unplugged, and the poll interval below can be derived before + // anything is opened. + const auto sensors = SiplCamera::query( + camera_config.platform_config_json, camera_config.platform_config_name, camera_config.link_masks); + + if (list_sensors) + { + std::cout << camera_config.platform_config_name << ": " << sensors.size() << " sensor(s)\n"; + for (const auto& s : sensors) + { + std::cout << " sensor=" << s.id << " " << s.name << " " << s.width << "x" << s.height << " @ " << s.fps + << " fps\n"; + } + return 0; + } + + if (stream_map.empty()) + { + std::cerr << "Error: at least one --add-stream is required." << std::endl; + print_usage(argv[0]); + return 1; + } + std::vector stream_configs; + stream_configs.reserve(stream_map.size()); + for (auto& [_, cfg] : stream_map) + { + stream_configs.push_back(std::move(cfg)); + } + + // Check the settings that need no hardware or network before anything is opened. + if (camera_config.livekit.url.empty() || livekit_token_file.empty()) + throw std::runtime_error("--livekit-url and --livekit-token-file are required"); + if (camera_config.livekit.fps == 0) + throw std::runtime_error("--livekit-fps must be at least 1"); + if (camera_config.livekit.bitrate_bps < 100'000) + throw std::runtime_error("--livekit-bitrate must be at least 100000"); + + // The token is read from a file so it never appears in a process listing, and is not printed. + std::ifstream token_file(livekit_token_file); + if (!token_file) + throw std::runtime_error("cannot read --livekit-token-file " + livekit_token_file); + std::getline(token_file, camera_config.livekit.token); + const auto last = camera_config.livekit.token.find_last_not_of(" \t\r\n"); + camera_config.livekit.token.erase(last == std::string::npos ? 0 : last + 1); + if (camera_config.livekit.token.empty()) + throw std::runtime_error("--livekit-token-file " + livekit_token_file + " is empty"); + + // This process captures and never renders, but EGL is on its critical path: + // NvBufSurfaceMapEglImage() is how a SIPL NvSciBufObj reaches CUDA, and + // libnvbufsurface resolves its own display via + // eglGetDisplay(EGL_DEFAULT_DISPLAY). GLVND hands out Mesa's EGL rather than + // Tegra's whenever DISPLAY names an X server Tegra EGL cannot drive (Xvfb, + // X11 forwarding), and that choice cannot be made per-call -- DISPLAY has to + // be gone from the environment before the first EGL call. + ::unsetenv("DISPLAY"); + + std::signal(SIGINT, signal_handler); + std::signal(SIGTERM, signal_handler); + + std::cout << "============================================================" << std::endl; + std::cout << "SENSING Camera Plugin Starting (SIPL)" << std::endl; + std::cout << "============================================================" << std::endl; + + SensingCamera camera(camera_config, stream_configs); + + std::cout << "------------------------------------------------------------" << std::endl; + std::cout << "Running capture loop. Press Ctrl+C to stop." << std::endl; + + constexpr auto stats_interval = std::chrono::seconds(5); + auto last_stats_time = std::chrono::steady_clock::now(); + + // SiplCamera::latest() is a non-blocking mailbox read, so poll at roughly + // twice the fastest sensor's frame rate rather than spinning a core flat + // out. Capture itself runs on SIPL's own threads and is unaffected. + double fastest_fps = 30.0; + for (const auto& s : sensors) + { + fastest_fps = std::max(fastest_fps, s.fps); + } + const auto poll_interval = std::chrono::microseconds(static_cast(500'000.0 / fastest_fps)); + + while (!g_stop_requested.load(std::memory_order_relaxed)) + { + camera.update(); + + auto now = std::chrono::steady_clock::now(); + if (now - last_stats_time >= stats_interval) + { + camera.print_stats(); + last_stats_time = now; + } + + std::this_thread::sleep_for(poll_interval); + } + + std::cout << "------------------------------------------------------------" << std::endl; + std::cout << "Shutting down SENSING Camera Plugin..." << std::endl; + camera.flush(); + camera.print_stats(); + std::cout << "Plugin stopped" << std::endl; + std::cout << "============================================================" << std::endl; + + return 0; +} +catch (const std::exception& e) +{ + std::cerr << argv[0] << ": " << e.what() << std::endl; + return 1; +} +catch (...) +{ + std::cerr << argv[0] << ": Unknown error occurred" << std::endl; + return 1; +} diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt new file mode 100644 index 0000000000..22b466c2df --- /dev/null +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# SIPL capture, colour conversion, the hardware encoder and the LiveKit track output. +add_library(sensing_core STATIC + sensing_camera.cpp + sipl_camera.cpp + jetson_encoder.cpp + livekit_video_output.cpp + yuv_to_rgba.cu + rgba_to_nv12.cu + rgba_box_downscale.cu + ${JETSON_MMAPI_SOURCES} +) +add_library(sensing::core ALIAS sensing_core) + +# RESOLVE_DEVICE_SYMBOLS as well as SEPARABLE_COMPILATION: without it the device-link step only happens +# for consumers in a directory where CUDA is an enabled language. Resolving here makes the archive +# self-contained. +set_target_properties(sensing_core PROPERTIES + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON +) + +target_include_directories(sensing_core + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/.. + PRIVATE + # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay private: they drag in + # NvMedia and X11-style macros. + ${SIPL_INCLUDE_DIRS} + ${EGL_INCLUDE_DIRS} + ${JETSON_MMAPI_DIR}/include +) + +# The Multimedia API reference sources carry legacy warnings. +target_compile_options(sensing_core PRIVATE + $<$:-Wno-reorder -Wno-uninitialized -Wno-unused-variable + -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare> +) + +target_link_libraries(sensing_core + PUBLIC + sensing::livekit + PRIVATE + ${SIPL_LIBRARIES} + ${EGL_LIBRARIES} + ${NVBUFSURFACE_LIBRARY} + ${V4L2_LIBRARY} +) diff --git a/src/plugins/sensing/core/jetson_encoder.cpp b/src/plugins/sensing/core/jetson_encoder.cpp new file mode 100644 index 0000000000..b282c93423 --- /dev/null +++ b/src/plugins/sensing/core/jetson_encoder.cpp @@ -0,0 +1,325 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "jetson_encoder.hpp" + +#include "rgba_to_nv12.cuh" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +constexpr uint32_t kOutputBuffers = 6; +constexpr uint32_t kCaptureBuffers = 6; + +void check(int ret, const char* what) +{ + if (ret < 0) + throw std::runtime_error(std::string("JetsonEncoder: ") + what + " failed"); +} + +void check_cuda(cudaError_t err, const char* what) +{ + if (err != cudaSuccess) + throw std::runtime_error(std::string("JetsonEncoder: ") + what + ": " + cudaGetErrorString(err)); +} + +} // namespace + +struct JetsonEncoder::Impl +{ + EncoderConfig config; + NvVideoEncoder* encoder = nullptr; + + // NV12 staging in device memory; the V4L2 output plane is host-mapped, so + // each frame costs one device-to-host copy per plane. + uint8_t* d_y = nullptr; + uint8_t* d_uv = nullptr; + size_t d_y_pitch = 0; + size_t d_uv_pitch = 0; + + std::mutex mutex; + std::deque ready; + bool eos_reached = false; + bool eos_sent = false; // driving thread only + + ~Impl() + { + if (d_y) + cudaFree(d_y); + if (d_uv) + cudaFree(d_uv); + delete encoder; + } + + // Capture-plane dequeue callback; runs on the encoder's own thread. + static bool capture_dq(struct v4l2_buffer* v4l2_buf, NvBuffer* buffer, NvBuffer* /*shared*/, void* arg) + { + auto* self = static_cast(arg); + + if (!v4l2_buf) + return false; + + if (buffer && buffer->planes[0].bytesused > 0) + { + const auto* data = static_cast(buffer->planes[0].data); + // V4L2 M2M copies the OUTPUT buffer's timestamp onto the CAPTURE + // buffer it produces, so this unit carries its own frame's stamp + // rather than whatever was submitted most recently. + EncodedUnit unit; + unit.data.assign(data, data + buffer->planes[0].bytesused); + unit.timestamp_ns = static_cast(v4l2_buf->timestamp.tv_sec) * 1000000000LL + + static_cast(v4l2_buf->timestamp.tv_usec) * 1000LL; + unit.keyframe = (v4l2_buf->flags & V4L2_BUF_FLAG_KEYFRAME) != 0; + std::lock_guard lock(self->mutex); + self->ready.push_back(std::move(unit)); + } + + // A zero-length unit marks end of stream; stop the thread rather than + // re-queueing, otherwise the dq thread spins on a finished encoder. + if (buffer && buffer->planes[0].bytesused == 0) + { + std::lock_guard lock(self->mutex); + self->eos_reached = true; + return false; + } + + if (self->encoder->capture_plane.qBuffer(*v4l2_buf, nullptr) < 0) + return false; + + return true; + } +}; + +JetsonEncoder::JetsonEncoder(const EncoderConfig& config) : m_impl(std::make_unique()) +{ + if (config.width == 0 || config.height == 0) + throw std::runtime_error("JetsonEncoder: width/height must be non-zero"); + if ((config.width % 2) != 0 || (config.height % 2) != 0) + throw std::runtime_error("JetsonEncoder: width/height must be even for NV12"); + + m_impl->config = config; + + m_impl->encoder = NvVideoEncoder::createVideoEncoder("enc0"); + if (!m_impl->encoder) + throw std::runtime_error("JetsonEncoder: createVideoEncoder failed (is /dev/v4l2-nvenc present?)"); + + auto* enc = m_impl->encoder; + + // Capture format must be set before the output format. + const uint32_t bitstream_size = config.width * config.height * 3 / 2; + check(enc->setCapturePlaneFormat(V4L2_PIX_FMT_H264, config.width, config.height, bitstream_size), + "setCapturePlaneFormat"); + check(enc->setOutputPlaneFormat(V4L2_PIX_FMT_NV12M, config.width, config.height), "setOutputPlaneFormat"); + + check(enc->setBitrate(config.bitrate_bps), "setBitrate"); + check(enc->setProfile(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH), "setProfile"); + + // Level is not inferred reliably, and 2560x1984@60 is 1,190,400 macroblocks + // per second against Level 5.1's ceiling of 983,040 -- 21% over. An unset + // or too-low level encodes fine here and then gets refused downstream. + check(enc->setLevel(V4L2_MPEG_VIDEO_H264_LEVEL_5_2), "setLevel"); + + if (config.peak_bitrate_bps > config.bitrate_bps) + { + check(enc->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR), "setRateControlMode"); + check(enc->setPeakBitrate(config.peak_bitrate_bps), "setPeakBitrate"); + } + else + { + check(enc->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR), "setRateControlMode"); + } + check(enc->setFrameRate(config.fps ? config.fps : 30, 1), "setFrameRate"); + + const uint32_t gop = config.gop ? config.gop : (config.fps ? config.fps * 5 : 150); + check(enc->setIDRInterval(gop), "setIDRInterval"); + check(enc->setIFrameInterval(gop), "setIFrameInterval"); + + // Low-latency shape: no B-frames, SPS/PPS on every IDR so a receiver can + // join mid-stream, and the max-performance clock preset. + check(enc->setNumBFrames(0), "setNumBFrames"); + check(enc->setInsertSpsPpsAtIdrEnabled(true), "setInsertSpsPpsAtIdrEnabled"); + check(enc->setMaxPerfMode(1), "setMaxPerfMode"); + // A separate control from the max-performance clocks: it trades quality for encoder throughput. + // FAST suits live streams; four native-size images at 30 fps may need ULTRAFAST, to be measured. + check(enc->setHWPresetType(V4L2_ENC_HW_PRESET_FAST), "setHWPresetType"); + + check(enc->output_plane.setupPlane(V4L2_MEMORY_MMAP, kOutputBuffers, true, false), "output setupPlane"); + check(enc->capture_plane.setupPlane(V4L2_MEMORY_MMAP, kCaptureBuffers, true, false), "capture setupPlane"); + + check(enc->output_plane.setStreamStatus(true), "output setStreamStatus"); + check(enc->capture_plane.setStreamStatus(true), "capture setStreamStatus"); + + enc->capture_plane.setDQThreadCallback(&Impl::capture_dq); + enc->capture_plane.startDQThread(m_impl.get()); + + // Prime the capture plane so the encoder always has somewhere to write. + for (uint32_t i = 0; i < enc->capture_plane.getNumBuffers(); ++i) + { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(planes, 0, sizeof(planes)); + v4l2_buf.index = i; + v4l2_buf.m.planes = planes; + check(enc->capture_plane.qBuffer(v4l2_buf, nullptr), "capture qBuffer"); + } + + check_cuda(cudaMallocPitch(reinterpret_cast(&m_impl->d_y), &m_impl->d_y_pitch, config.width, config.height), + "cudaMallocPitch(Y)"); + check_cuda( + cudaMallocPitch(reinterpret_cast(&m_impl->d_uv), &m_impl->d_uv_pitch, config.width, config.height / 2), + "cudaMallocPitch(UV)"); +} + +JetsonEncoder::~JetsonEncoder() +{ + if (m_impl && m_impl->encoder) + { + m_impl->encoder->capture_plane.stopDQThread(); + m_impl->encoder->capture_plane.waitForDQThread(1000); + } +} + +bool JetsonEncoder::submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns) +{ + auto* enc = m_impl->encoder; + const auto& config = m_impl->config; + + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(planes, 0, sizeof(planes)); + v4l2_buf.m.planes = planes; + + NvBuffer* buffer = nullptr; + // Until every output buffer has been queued once, index i is free by + // construction; after that a dequeue is what frees one. + if (m_queued < enc->output_plane.getNumBuffers()) + { + buffer = enc->output_plane.getNthBuffer(m_queued); + v4l2_buf.index = m_queued; + ++m_queued; + } + else if (enc->output_plane.dqBuffer(v4l2_buf, &buffer, nullptr, 0) < 0) + { + return false; // encoder still busy; drop this frame rather than block + } + + // An input buffer is ours now, so convert: a frame that would be dropped costs no conversion. A + // failure from here is terminal for the encoder, as a failed qBuffer is. + launch_rgba_to_nv12(reinterpret_cast(rgba_device_ptr), static_cast(row_pitch_bytes), m_impl->d_y, + static_cast(m_impl->d_y_pitch), m_impl->d_uv, static_cast(m_impl->d_uv_pitch), + static_cast(config.width), static_cast(config.height), config.full_range, nullptr); + check_cuda(cudaGetLastError(), "rgba_to_nv12 launch"); + check_cuda(cudaStreamSynchronize(nullptr), "rgba_to_nv12 sync"); + + check_cuda(cudaMemcpy2D(buffer->planes[0].data, buffer->planes[0].fmt.stride, m_impl->d_y, m_impl->d_y_pitch, + config.width, config.height, cudaMemcpyDeviceToHost), + "cudaMemcpy2D(Y)"); + check_cuda(cudaMemcpy2D(buffer->planes[1].data, buffer->planes[1].fmt.stride, m_impl->d_uv, m_impl->d_uv_pitch, + config.width, config.height / 2, cudaMemcpyDeviceToHost), + "cudaMemcpy2D(UV)"); + + buffer->planes[0].bytesused = buffer->planes[0].fmt.stride * config.height; + buffer->planes[1].bytesused = buffer->planes[1].fmt.stride * (config.height / 2); + v4l2_buf.m.planes[0].bytesused = buffer->planes[0].bytesused; + v4l2_buf.m.planes[1].bytesused = buffer->planes[1].bytesused; + + // Microsecond granularity is all a timeval carries; that is well below the + // frame interval and keeps each unit attributed to its own capture. + v4l2_buf.timestamp.tv_sec = static_cast(timestamp_ns / 1000000000LL); + v4l2_buf.timestamp.tv_usec = static_cast((timestamp_ns % 1000000000LL) / 1000LL); + + check(enc->output_plane.qBuffer(v4l2_buf, nullptr), "output qBuffer"); + return true; +} + +EncodedUnit JetsonEncoder::poll() +{ + std::lock_guard lock(m_impl->mutex); + if (m_impl->ready.empty()) + return {}; + + auto out = std::move(m_impl->ready.front()); + m_impl->ready.pop_front(); + return out; +} + +std::vector JetsonEncoder::drain() +{ + auto* enc = m_impl->encoder; + + // A zero-length output buffer is the EOS marker for the V4L2 encoder. It is sent once: a second + // drain only returns what is left. + if (!m_impl->eos_sent) + { + m_impl->eos_sent = true; + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(planes, 0, sizeof(planes)); + v4l2_buf.m.planes = planes; + + NvBuffer* buffer = nullptr; + if (enc->output_plane.dqBuffer(v4l2_buf, &buffer, nullptr, 10) >= 0) + { + v4l2_buf.m.planes[0].bytesused = 0; + v4l2_buf.m.planes[1].bytesused = 0; + enc->output_plane.qBuffer(v4l2_buf, nullptr); + } + + enc->capture_plane.waitForDQThread(2000); + } + + std::lock_guard lock(m_impl->mutex); + std::vector units( + std::make_move_iterator(m_impl->ready.begin()), std::make_move_iterator(m_impl->ready.end())); + m_impl->ready.clear(); + return units; +} + +EncodedUnit JetsonEncoder::end_of_stream() +{ + // The tail is concatenated, so it carries the stamp of the last unit in it. + EncodedUnit out; + for (auto& chunk : drain()) + { + out.data.insert(out.data.end(), chunk.data.begin(), chunk.data.end()); + out.timestamp_ns = chunk.timestamp_ns; + } + return out; +} + +void JetsonEncoder::request_keyframe() +{ + check(m_impl->encoder->forceIDR(), "forceIDR"); +} + +bool JetsonEncoder::set_bitrate(uint32_t bitrate_bps) +{ + return m_impl->encoder->setBitrate(bitrate_bps) >= 0; +} + +const char* JetsonEncoder::backend_name() +{ + return "Jetson V4L2 NVENC"; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/jetson_encoder.hpp b/src/plugins/sensing/core/jetson_encoder.hpp new file mode 100644 index 0000000000..67454bb6e8 --- /dev/null +++ b/src/plugins/sensing/core/jetson_encoder.hpp @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct EncoderConfig +{ + uint32_t width = 0; + uint32_t height = 0; + uint32_t bitrate_bps = 40'000'000; + /// VBR ceiling; 0 keeps CBR. The H.264 output is a recording, not a + /// fixed-rate transport, so CBR would pad static scenes and starve motion. + uint32_t peak_bitrate_bps = 60'000'000; + uint32_t fps = 30; + /// IDR period in frames; 0 defers to fps*5. + uint32_t gop = 0; + bool full_range = false; +}; + +/// One encoded Annex-B unit and the capture stamp of the frame it came from. +struct EncodedUnit +{ + std::vector data; + int64_t timestamp_ns = 0; + /// The unit is an IDR (with its parameter sets), decodable on its own. Not set on the + /// concatenated tail from end_of_stream(). + bool keyframe = false; + + bool empty() const + { + return data.empty(); + } +}; + +/** + * @brief H.264 encoder on the Jetson V4L2 M2M engine (/dev/v4l2-nvenc). + * + * Jetson has no libnvidia-encode — the NVIDIA Video Codec SDK is dGPU-only — + * so this wraps NvVideoEncoder from the Jetson Multimedia API instead. + * + * One thread must drive an encoder: submit(), poll(), request_keyframe() and set_bitrate() are not + * safe to call concurrently. + * + * Submission is asynchronous: the encoder needs several input frames before it + * emits the first bitstream unit, so submit() never blocks waiting for output + * and poll() drains whatever a capture-plane thread has completed. + */ +class JetsonEncoder +{ +public: + explicit JetsonEncoder(const EncoderConfig& config); + ~JetsonEncoder(); + + JetsonEncoder(const JetsonEncoder&) = delete; + JetsonEncoder& operator=(const JetsonEncoder&) = delete; + JetsonEncoder(JetsonEncoder&&) = delete; + JetsonEncoder& operator=(JetsonEncoder&&) = delete; + + /** + * @brief Convert one GPU-resident RGBA8 frame to NV12 and queue it. + * @param rgba_device_ptr Device pointer to a HxWx4 RGBA8 buffer. + * @param row_pitch_bytes Byte stride between rows. + * @param timestamp_ns Capture stamp, carried on the V4L2 output buffer and + * returned by poll() on the unit this frame produces. + * @return false when no input buffer is free (frame dropped). + */ + bool submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns); + + /** @brief Take any completed unit; empty during encoder warmup. */ + EncodedUnit poll(); + + /** @brief Signal EOS and drain the remaining bitstream, concatenated into one unit. */ + EncodedUnit end_of_stream(); + + /** @brief Signal EOS and drain the remaining bitstream, one unit per frame with its own stamp. */ + std::vector drain(); + + /** @brief Make the next submitted frame an IDR, preceded by SPS/PPS. */ + void request_keyframe(); + + /** + * @brief Change the target bitrate of the running stream. + * @return false when this backend cannot change it while encoding. + */ + bool set_bitrate(uint32_t bitrate_bps); + + /** @brief Names the encoder behind this class, for diagnostics. */ + static const char* backend_name(); + +private: + struct Impl; + std::unique_ptr m_impl; + + /// Output buffers queued so far; below the buffer count they are free by + /// construction and need no dequeue first. + uint32_t m_queued = 0; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp new file mode 100644 index 0000000000..5495f750fa --- /dev/null +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -0,0 +1,486 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "livekit_video_output.hpp" + +#include "jetson_encoder.hpp" +#include "rgba_box_downscale.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +void check_cuda(cudaError_t err, const char* what) +{ + if (err != cudaSuccess) + throw std::runtime_error(std::string("LiveKit output: ") + what + ": " + cudaGetErrorString(err)); +} + +// Congestion control never drives the encoder below this. +constexpr uint32_t kMinBitrateBps = 100'000; +// A target is applied only when it differs from the current one by this fraction, and this long after +// the last change. +constexpr double kMinBitrateChange = 0.25; +constexpr std::chrono::seconds kMinBitrateInterval{ 3 }; + +} // namespace + +struct LiveKitVideoOutput::Impl +{ + struct Slot + { + uint8_t* ptr = nullptr; + size_t pitch = 0; + uint64_t timestamp_ns = 0; + uint64_t capture_tsc_ns = 0; + uint64_t sequence = 0; + }; + + Impl(std::unique_ptr track, const LiveKitOutputConfig& cfg) : sink(std::move(track)), config(cfg) + { + validate_geometry(config.source_width, config.source_height, config.scale_div); + if (config.fps == 0) + throw std::runtime_error("LiveKit output: the publication rate must be at least 1 fps"); + out_width = config.source_width / config.scale_div; + out_height = config.source_height / config.scale_div; + if (sink->width() != out_width || sink->height() != out_height) + throw std::runtime_error("LiveKit output: the track size does not match the output size"); + period_ns = 1'000'000'000ull / config.fps; + if (config.bitrate_bps < kMinBitrateBps) + { + throw std::runtime_error("LiveKit output: the bitrate must be at least " + std::to_string(kMinBitrateBps) + + " bps"); + } + // Congestion control moves the target between the floor and the ceiling; the configured target + // is never below the floor, so the range is never inverted. + max_bitrate = std::max(config.bitrate_bps, config.peak_bitrate_bps); + + // A constructor that throws never runs the destructor, so free what was allocated here. + try + { + check_cuda(cudaSetDevice(config.gpu_id), "cudaSetDevice"); + + // First, so a machine without a usable hardware encoder fails before anything is allocated. + EncoderConfig encoder_config; + encoder_config.width = out_width; + encoder_config.height = out_height; + encoder_config.bitrate_bps = config.bitrate_bps; + encoder_config.peak_bitrate_bps = config.peak_bitrate_bps; + encoder_config.fps = config.fps; + encoder_config.gop = config.gop; + encoder_config.full_range = config.full_range; + encoder = std::make_unique(encoder_config); + applied_bitrate = config.bitrate_bps; + + for (auto& slot : slots) + { + check_cuda(cudaMallocPitch(reinterpret_cast(&slot.ptr), &slot.pitch, + static_cast(config.source_width) * 4, config.source_height), + "cudaMallocPitch(snapshot)"); + } + if (config.scale_div > 1) + { + check_cuda(cudaMallocPitch(reinterpret_cast(&d_scaled), &scaled_pitch, + static_cast(out_width) * 4, out_height), + "cudaMallocPitch(reduced)"); + } + check_cuda(cudaStreamCreateWithFlags(&snapshot_stream, cudaStreamNonBlocking), "cudaStreamCreate"); + check_cuda(cudaStreamCreateWithFlags(&work_stream, cudaStreamNonBlocking), "cudaStreamCreate"); + + worker = std::thread([this] { run(); }); + } + catch (...) + { + release(); + throw; + } + } + + ~Impl() + { + { + std::lock_guard lock(mutex); + stopping = true; + } + wake.notify_all(); + if (worker.joinable()) + worker.join(); + release(); + } + + void release() + { + for (auto& slot : slots) + cudaFree(slot.ptr); + cudaFree(d_scaled); + if (snapshot_stream) + cudaStreamDestroy(snapshot_stream); + if (work_stream) + cudaStreamDestroy(work_stream); + } + + void fail(const std::string& message) + { + std::lock_guard lock(failure_mutex); + if (!failed_flag.load()) + { + failure_message = message; + failed_flag.store(true); + } + } + + // Polling thread. + void submit(const FrameView& view) + { + ++offered; + if (failed_flag.load()) + return; + + // Pace on the capture clock, so poll jitter does not drop a frame that is due. A tenth of a + // period of slack lets 60 fps capture feed a 30 fps track every second capture. + if (have_last && view.timestamp_ns < last_accepted_ns + period_ns * 9 / 10) + { + ++paced_out; + return; + } + + try + { + std::lock_guard lock(mutex); + int target = -1; + for (int i = 0; i < 2; ++i) + { + if (i != inflight && i != pending) + { + target = i; + break; + } + } + if (target < 0) + { + // The worker holds one slot and the other waits: the newer capture takes the waiting one. + target = pending; + } + // A capture still waiting is overwritten by this newer one. + if (pending >= 0) + ++replaced; + + // Complete before returning: the camera may rewrite its slot at the second following poll. + Slot& slot = slots[target]; + check_cuda(cudaMemcpy2DAsync(slot.ptr, slot.pitch, reinterpret_cast(view.ptr), view.pitch, + static_cast(config.source_width) * 4, config.source_height, + cudaMemcpyDeviceToDevice, snapshot_stream), + "snapshot copy"); + check_cuda(cudaStreamSynchronize(snapshot_stream), "snapshot sync"); + + slot.timestamp_ns = view.timestamp_ns; + slot.capture_tsc_ns = view.capture_tsc_ns; + slot.sequence = view.sequence; + pending = target; + } + catch (const std::exception& e) + { + fail(e.what()); + return; + } + ++snapshots; + have_last = true; + last_accepted_ns = view.timestamp_ns; + wake.notify_one(); + } + + // Worker thread. + void run() + { + try + { + check_cuda(cudaSetDevice(config.gpu_id), "cudaSetDevice(worker)"); + for (;;) + { + int index = -1; + bool flush_now = false; + { + std::unique_lock lock(mutex); + // Frames in the encoder come out a little later: poll for them soon, and otherwise + // wake now and then to pick up the receiving side's requests. + const auto wait = in_encoder.empty() ? std::chrono::milliseconds(100) : std::chrono::milliseconds(2); + wake.wait_for(lock, wait, [&] { return stopping || pending >= 0 || flush_requested; }); + if (stopping) + return; + index = pending; + pending = -1; + inflight = index; + flush_now = flush_requested; + } + + apply_feedback(); + if (index >= 0) + { + encode(slots[index]); + std::lock_guard lock(mutex); + inflight = -1; + } + if (flush_now) + { + for (auto& unit : encoder->drain()) + publish_unit(unit); + std::lock_guard lock(mutex); + flush_requested = false; + flushed.notify_all(); + } + else + { + for (auto unit = encoder->poll(); !unit.empty(); unit = encoder->poll()) + publish_unit(unit); + } + } + } + catch (const std::exception& e) + { + fail(e.what()); + std::lock_guard lock(mutex); + flushed.notify_all(); + } + } + + // Worker thread: what the receiving side asked for since the last frame. + void apply_feedback() + { + const EncoderFeedback feedback = sink->take_feedback(); + if (feedback.keyframe_requested) + { + ++keyframes_requested; + keyframe_wanted = true; + } + if (feedback.target_bitrate_bps) + { + const uint32_t target = std::clamp(*feedback.target_bitrate_bps, kMinBitrateBps, max_bitrate); + // Congestion control re-estimates all the time. Follow it only for a real change, and not + // more than every few seconds: on the mock a change reopens the encoder session. + const auto now = std::chrono::steady_clock::now(); + const double change = std::abs(static_cast(target) - applied_bitrate) / applied_bitrate; + if (change >= kMinBitrateChange && now - last_bitrate_change >= kMinBitrateInterval && + encoder->set_bitrate(target)) + { + applied_bitrate = target; + last_bitrate_change = now; + } + } + } + + // Worker thread. + void encode(const Slot& slot) + { + const uint8_t* rgba = slot.ptr; + size_t pitch = slot.pitch; + if (config.scale_div > 1) + { + launch_rgba_box_downscale(rgba, static_cast(pitch), d_scaled, static_cast(scaled_pitch), + static_cast(out_width), static_cast(out_height), + static_cast(config.scale_div), work_stream); + check_cuda(cudaGetLastError(), "reduction kernel"); + // The encoder reads on the default stream, which a non-blocking stream does not order against. + check_cuda(cudaStreamSynchronize(work_stream), "reduction sync"); + rgba = d_scaled; + pitch = scaled_pitch; + } + + if (keyframe_wanted) + { + encoder->request_keyframe(); + keyframe_wanted = false; + } + const int64_t stamp_ns = static_cast(slot.timestamp_ns); + if (!encoder->submit(reinterpret_cast(rgba), pitch, stamp_ns)) + { + // No input buffer: dropped before encoding, so no reference is lost. A keyframe already + // requested of the encoder applies to the next frame it takes. + ++encoder_busy; + return; + } + in_encoder.push_back({ stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc_ns }); + } + + // Worker thread: one encoded unit becomes one published frame, with the sequence and capture time + // of the capture it came from. + void publish_unit(const EncodedUnit& unit) + { + const int64_t stamp_us = unit.timestamp_ns / 1000; + // The encoder keeps submission order; a frame it never returned was dropped inside it. + while (!in_encoder.empty() && in_encoder.front().stamp_us < stamp_us) + in_encoder.pop_front(); + if (in_encoder.empty() || in_encoder.front().stamp_us != stamp_us) + throw std::runtime_error("LiveKit output: the encoder returned a unit for a frame it was not given"); + const FrameMeta meta = in_encoder.front(); + in_encoder.pop_front(); + + // After a refusal the stream is only usable from its next keyframe. + if (awaiting_keyframe && !unit.keyframe) + return; + + EncodedVideoFrame frame; + frame.data = unit.data.data(); + frame.size = unit.data.size(); + frame.keyframe = unit.keyframe; + frame.timestamp_us = static_cast(stamp_us); + frame.frame_id = meta.sequence; + frame.capture_tsc_ns = meta.capture_tsc_ns; + if (sink->submit(frame)) + { + awaiting_keyframe = false; + ++published; + } + else + { + ++rejected; + awaiting_keyframe = true; + keyframe_wanted = true; + } + } + + std::unique_ptr sink; + LiveKitOutputConfig config; + uint32_t out_width = 0; + uint32_t out_height = 0; + uint64_t period_ns = 0; + + struct FrameMeta + { + int64_t stamp_us = 0; + uint32_t sequence = 0; + uint64_t capture_tsc_ns = 0; + }; + + std::unique_ptr encoder; + uint32_t max_bitrate = 0; + + // Worker thread only. + std::deque in_encoder; ///< Submitted to the encoder, no unit back yet. + bool keyframe_wanted = false; + bool awaiting_keyframe = false; + uint32_t applied_bitrate = 0; + std::chrono::steady_clock::time_point last_bitrate_change = std::chrono::steady_clock::now(); + + Slot slots[2]; + uint8_t* d_scaled = nullptr; + size_t scaled_pitch = 0; + cudaStream_t snapshot_stream = nullptr; + cudaStream_t work_stream = nullptr; + + // Guards pending, inflight, stopping and the snapshot slots' contents. + std::mutex mutex; + std::condition_variable wake; + int pending = -1; + int inflight = -1; + bool stopping = false; + bool flush_requested = false; + std::condition_variable flushed; + std::thread worker; + + // Polling thread only. + bool have_last = false; + uint64_t last_accepted_ns = 0; + + std::atomic offered{ 0 }; + std::atomic paced_out{ 0 }; + std::atomic replaced{ 0 }; + std::atomic snapshots{ 0 }; + std::atomic published{ 0 }; + std::atomic encoder_busy{ 0 }; + std::atomic rejected{ 0 }; + std::atomic keyframes_requested{ 0 }; + + std::mutex failure_mutex; + std::string failure_message; + std::atomic failed_flag{ false }; +}; + +void LiveKitVideoOutput::validate_geometry(uint32_t width, uint32_t height, uint32_t divisor) +{ + if (divisor == 0) + throw std::runtime_error("LiveKit output: the scale divisor must be at least 1"); + if (width == 0 || height == 0) + throw std::runtime_error("LiveKit output: the source size is empty"); + if (width % divisor != 0 || height % divisor != 0) + { + throw std::runtime_error("LiveKit output: divisor " + std::to_string(divisor) + " does not divide " + + std::to_string(width) + "x" + std::to_string(height) + " exactly"); + } + if ((width / divisor) % 2 != 0 || (height / divisor) % 2 != 0) + { + throw std::runtime_error("LiveKit output: " + std::to_string(width / divisor) + "x" + + std::to_string(height / divisor) + " is not even; NV12 needs even dimensions"); + } +} + +LiveKitVideoOutput::LiveKitVideoOutput(std::unique_ptr sink, const LiveKitOutputConfig& config) + : m_impl(std::make_unique(std::move(sink), config)) +{ +} + +LiveKitVideoOutput::~LiveKitVideoOutput() = default; + +void LiveKitVideoOutput::submit(const FrameView& view) +{ + m_impl->submit(view); +} + +LiveKitOutputStats LiveKitVideoOutput::stats() const +{ + LiveKitOutputStats s; + s.offered = m_impl->offered.load(); + s.paced_out = m_impl->paced_out.load(); + s.replaced = m_impl->replaced.load(); + s.snapshots = m_impl->snapshots.load(); + s.published = m_impl->published.load(); + s.encoder_busy = m_impl->encoder_busy.load(); + s.rejected = m_impl->rejected.load(); + s.keyframes_requested = m_impl->keyframes_requested.load(); + return s; +} + +bool LiveKitVideoOutput::failed() const +{ + return m_impl->failed_flag.load(); +} + +std::string LiveKitVideoOutput::failure() const +{ + std::lock_guard lock(m_impl->failure_mutex); + return m_impl->failure_message; +} + +std::string LiveKitVideoOutput::describe() +{ + std::string text = std::string(JetsonEncoder::backend_name()) + ", pre-encoded passthrough"; + const std::string transport = m_impl->sink->describe(); + if (!transport.empty()) + text += "; transport: " + transport; + return text; +} + +void LiveKitVideoOutput::flush() +{ + std::unique_lock lock(m_impl->mutex); + m_impl->flush_requested = true; + m_impl->wake.notify_one(); + m_impl->flushed.wait(lock, [&] { return !m_impl->flush_requested || m_impl->failed_flag.load(); }); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/livekit_video_output.hpp b/src/plugins/sensing/core/livekit_video_output.hpp new file mode 100644 index 0000000000..a0526c17cc --- /dev/null +++ b/src/plugins/sensing/core/livekit_video_output.hpp @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "sipl_camera.hpp" + +#include + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct LiveKitOutputConfig +{ + uint32_t source_width = 0; + uint32_t source_height = 0; + /// Uniform output-only reduction; 1 publishes the sensor's native size. + uint32_t scale_div = 1; + /// Publication rate cap, independent of the capture rate. + uint32_t fps = 30; + int gpu_id = 0; + /// Hardware encoder settings for the published pictures; the peak is also the ceiling for the + /// bitrate that congestion control may ask for. + uint32_t bitrate_bps = 8'000'000; + uint32_t peak_bitrate_bps = 0; + /// IDR period in frames; 0 defers to the encoder's default. + uint32_t gop = 0; + bool full_range = false; +}; + +struct LiveKitOutputStats +{ + uint64_t offered = 0; ///< Captures handed to submit(). + uint64_t paced_out = 0; ///< Skipped to hold the publication rate. + uint64_t replaced = 0; ///< Snapshots overwritten by a newer capture before the worker took them. + uint64_t snapshots = 0; ///< GPU snapshots taken. + uint64_t published = 0; ///< Frames the SDK accepted. Not proof that a viewer received them. + uint64_t encoder_busy = 0; ///< Snapshots the encoder had no input buffer for, dropped before encoding. + uint64_t rejected = 0; ///< Encoded frames the SDK refused; a keyframe follows. + uint64_t keyframes_requested = 0; ///< Keyframes the receiving side asked for. +}; + +/** + * @brief Encodes one sensor's frames on hardware and feeds them to a pre-encoded video track. + * + * submit() runs on the caller's polling loop. It takes a snapshot of the borrowed camera frame in + * GPU memory and finishes that copy before returning, because the camera may overwrite its slot at + * the second following poll. Everything else -- the optional reduction, the hardware encode and the + * SDK call -- happens on this output's own worker thread, so a slow network cannot hold up capture + * or the other sensors. + * + * Two snapshot slots bound the memory: one the worker is encoding, one waiting. A newer capture + * replaces the waiting one; the worker never waits for it. The encoder is the JetsonEncoder class: + * the V4L2 engine on a Jetson, NVENC through libavcodec in the mock. There is no software encoder: if + * the hardware encoder cannot start, construction fails. + * + * The receiving side's keyframe requests and bitrate targets are applied before the next frame is + * encoded. A frame the SDK refuses breaks the reference chain, so the output asks for a keyframe and + * publishes nothing else until one arrives. + */ +class LiveKitVideoOutput +{ +public: + /// @throws std::runtime_error if the geometry is unusable (see validate_geometry), the hardware + /// encoder cannot start, or CUDA fails. + LiveKitVideoOutput(std::unique_ptr sink, const LiveKitOutputConfig& config); + + /// Stops the worker; a snapshot not yet published is dropped. + ~LiveKitVideoOutput(); + + LiveKitVideoOutput(const LiveKitVideoOutput&) = delete; + LiveKitVideoOutput& operator=(const LiveKitVideoOutput&) = delete; + + /// Never blocks on the network. A view older than the publication rate allows is skipped. + void submit(const FrameView& view); + + LiveKitOutputStats stats() const; + + bool failed() const; + std::string failure() const; + + /// Names the hardware encoder and the transport's view of the stream, for diagnostics. + std::string describe(); + + /** + * Encodes what is waiting, drains the encoder and publishes every remaining unit with its own + * stamp. Call once, at the end of the stream: the encoder cannot take frames afterwards. + */ + void flush(); + + /** + * Output size after dividing: both source dimensions must divide exactly and leave positive, + * even results. Nothing is rounded or stretched. + * @throws std::runtime_error naming the problem. + */ + static void validate_geometry(uint32_t width, uint32_t height, uint32_t divisor); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/rgba_box_downscale.cu b/src/plugins/sensing/core/rgba_box_downscale.cu new file mode 100644 index 0000000000..cc169b3b92 --- /dev/null +++ b/src/plugins/sensing/core/rgba_box_downscale.cu @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "rgba_box_downscale.cuh" + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +__global__ void box_downscale( + const uint8_t* src, int src_pitch, uint8_t* dst, int dst_pitch, int width, int height, int divisor) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || y >= height) + return; + + int sum[4] = { 0, 0, 0, 0 }; + for (int j = 0; j < divisor; ++j) + { + const uint8_t* row = + src + static_cast(y * divisor + j) * src_pitch + static_cast(x * divisor) * 4; + for (int i = 0; i < divisor; ++i) + { + for (int c = 0; c < 4; ++c) + sum[c] += row[i * 4 + c]; + } + } + + const int count = divisor * divisor; + uint8_t* out = dst + static_cast(y) * dst_pitch + static_cast(x) * 4; + for (int c = 0; c < 4; ++c) + out[c] = static_cast((sum[c] + count / 2) / count); +} + +} // namespace + +void launch_rgba_box_downscale(const uint8_t* src, + int src_pitch, + uint8_t* dst, + int dst_pitch, + int dst_width, + int dst_height, + int divisor, + cudaStream_t stream) +{ + const dim3 block(16, 16); + const dim3 grid((dst_width + block.x - 1) / block.x, (dst_height + block.y - 1) / block.y); + box_downscale<<>>(src, src_pitch, dst, dst_pitch, dst_width, dst_height, divisor); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/rgba_box_downscale.cuh b/src/plugins/sensing/core/rgba_box_downscale.cuh new file mode 100644 index 0000000000..4a96e37a80 --- /dev/null +++ b/src/plugins/sensing/core/rgba_box_downscale.cuh @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/** + * @brief Reduce RGBA8 by an integer divisor, averaging each divisor x divisor block. + * + * The aspect ratio is kept exactly: the source size must divide evenly, so nothing is stretched, + * cropped or rounded. + * + * @param dst_width src_width / divisor. + * @param dst_height src_height / divisor. + */ +void launch_rgba_box_downscale(const uint8_t* src, + int src_pitch, + uint8_t* dst, + int dst_pitch, + int dst_width, + int dst_height, + int divisor, + cudaStream_t stream); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/rgba_to_nv12.cu b/src/plugins/sensing/core/rgba_to_nv12.cu new file mode 100644 index 0000000000..2c0ca5ec6c --- /dev/null +++ b/src/plugins/sensing/core/rgba_to_nv12.cu @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "rgba_to_nv12.cuh" + +namespace +{ + +__device__ __forceinline__ uint8_t clamp_u8(float v) +{ + return static_cast(fminf(fmaxf(v, 0.0f), 255.0f) + 0.5f); +} + +// BT.601 RGB->YCbCr, matching the coefficients yuv_to_rgba.cu inverts on the +// capture side so a round trip through RGBA is close to identity. +__device__ __forceinline__ float rgb_to_y(float r, float g, float b, bool full_range) +{ + return full_range ? (0.299f * r + 0.587f * g + 0.114f * b) : (16.0f + 0.257f * r + 0.504f * g + 0.098f * b); +} + +__device__ __forceinline__ float rgb_to_u(float r, float g, float b, bool full_range) +{ + return full_range ? (128.0f - 0.168736f * r - 0.331264f * g + 0.5f * b) : + (128.0f - 0.148f * r - 0.291f * g + 0.439f * b); +} + +__device__ __forceinline__ float rgb_to_v(float r, float g, float b, bool full_range) +{ + return full_range ? (128.0f + 0.5f * r - 0.418688f * g - 0.081312f * b) : + (128.0f + 0.439f * r - 0.368f * g - 0.071f * b); +} + +__global__ void rgba_to_nv12_kernel(const uint8_t* __restrict__ rgba, + int rgba_pitch, + uint8_t* __restrict__ y_plane, + int y_pitch, + uint8_t* __restrict__ uv_plane, + int uv_pitch, + int width, + int height, + bool full_range) +{ + // One thread per 2x2 block: writes four luma samples and one chroma pair. + const int bx = blockIdx.x * blockDim.x + threadIdx.x; + const int by = blockIdx.y * blockDim.y + threadIdx.y; + const int x = bx * 2; + const int y = by * 2; + if (x >= width || y >= height) + return; + + for (int dy = 0; dy < 2; ++dy) + { + for (int dx = 0; dx < 2; ++dx) + { + const int px = x + dx; + const int py = y + dy; + if (px >= width || py >= height) + continue; + + const uint8_t* pixel = rgba + static_cast(py) * rgba_pitch + static_cast(px) * 4; + const float r = pixel[0]; + const float g = pixel[1]; + const float b = pixel[2]; + y_plane[static_cast(py) * y_pitch + px] = clamp_u8(rgb_to_y(r, g, b, full_range)); + } + } + + const uint8_t* top_left = rgba + static_cast(y) * rgba_pitch + static_cast(x) * 4; + const float r = top_left[0]; + const float g = top_left[1]; + const float b = top_left[2]; + + uint8_t* uv = uv_plane + static_cast(by) * uv_pitch + static_cast(bx) * 2; + uv[0] = clamp_u8(rgb_to_u(r, g, b, full_range)); + uv[1] = clamp_u8(rgb_to_v(r, g, b, full_range)); +} + +} // namespace + +namespace plugins +{ +namespace sensing +{ + +void launch_rgba_to_nv12(const uint8_t* rgba, + int rgba_pitch, + uint8_t* y_plane, + int y_pitch, + uint8_t* uv_plane, + int uv_pitch, + int width, + int height, + bool full_range, + cudaStream_t stream) +{ + const dim3 block(16, 16); + const dim3 grid((width / 2 + block.x - 1) / block.x, (height / 2 + block.y - 1) / block.y); + rgba_to_nv12_kernel<<>>( + rgba, rgba_pitch, y_plane, y_pitch, uv_plane, uv_pitch, width, height, full_range); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/rgba_to_nv12.cuh b/src/plugins/sensing/core/rgba_to_nv12.cuh new file mode 100644 index 0000000000..a296be6181 --- /dev/null +++ b/src/plugins/sensing/core/rgba_to_nv12.cuh @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/** + * @brief Convert packed RGBA8 to NV12 (Y plane + interleaved UV plane). + * + * Chroma is point-sampled from the top-left pixel of each 2x2 block rather + * than averaged: the ISP already delivered 4:2:0, so the RGBA the + * capture path produced was upsampled from it and averaging would only blur + * chroma that was never independent. Width and height must be even. + * + * @param full_range true for [0,255] luma, false for broadcast [16,235]. + */ +void launch_rgba_to_nv12(const uint8_t* rgba, + int rgba_pitch, + uint8_t* y_plane, + int y_pitch, + uint8_t* uv_plane, + int uv_pitch, + int width, + int height, + bool full_range, + cudaStream_t stream); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/sensing_camera.cpp b/src/plugins/sensing/core/sensing_camera.cpp new file mode 100644 index 0000000000..0e1c0dcc77 --- /dev/null +++ b/src/plugins/sensing/core/sensing_camera.cpp @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "sensing_camera.hpp" + +#include "livekit_video_output.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +SiplConfig make_sipl_config(const SensingConfig& config) +{ + SiplConfig sipl{}; + sipl.platform_config_json = config.platform_config_json; + sipl.platform_config_name = config.platform_config_name; + sipl.link_masks = config.link_masks; + sipl.nito_dir = config.nito_dir; + sipl.gpu_id = config.gpu_id; + sipl.full_range = config.full_range; + sipl.swap_uv = config.swap_uv; + sipl.isp0_buffers = config.isp0_buffers; + return sipl; +} + +LiveKitOutputConfig make_output_config(const SensingConfig& config, const SensorInfo& sensor) +{ + LiveKitOutputConfig output; + output.source_width = sensor.width; + output.source_height = sensor.height; + output.scale_div = config.livekit.scale_div; + output.fps = config.livekit.fps; + output.gpu_id = config.gpu_id; + output.bitrate_bps = config.livekit.bitrate_bps; + output.peak_bitrate_bps = config.livekit.bitrate_bps + config.livekit.bitrate_bps / 2; + output.gop = config.gop; + output.full_range = config.full_range; + return output; +} + +} // namespace + +SensingCamera::SensingCamera(const SensingConfig& config, const std::vector& streams) : m_config(config) +{ + if (streams.empty()) + throw std::runtime_error("SensingCamera: no streams requested"); + + m_camera = std::make_unique(make_sipl_config(config)); + const auto& sensors = m_camera->sensors(); + + // Everything that needs no network is checked first, so a bad name or divisor fails before any + // connection is made. + std::set tracks; + for (const auto& stream_config : streams) + { + if (!tracks.insert(stream_config.livekit_track).second) + throw std::runtime_error("LiveKit track name '" + stream_config.livekit_track + "' is used twice"); + const auto found = std::find_if( + sensors.begin(), sensors.end(), [&](const SensorInfo& s) { return s.id == stream_config.sensor_id; }); + if (found == sensors.end()) + { + std::ostringstream oss; + oss << "sensor=" << stream_config.sensor_id << " is not a pipeline in '" << config.platform_config_name + << "'. Available:"; + for (const auto& s : sensors) + oss << ' ' << s.id; + oss << ". Note this is the SIPL pipeline index, not the GMSL link index."; + throw std::runtime_error(oss.str()); + } + LiveKitVideoOutput::validate_geometry(found->width, found->height, config.livekit.scale_div); + } + + m_livekit = LiveKitSession::connect(config.livekit.url, config.livekit.token); + m_streams.reserve(streams.size()); + for (const auto& stream_config : streams) + { + const auto found = std::find_if( + sensors.begin(), sensors.end(), [&](const SensorInfo& s) { return s.id == stream_config.sensor_id; }); + const LiveKitOutputConfig output = make_output_config(config, *found); + + PublishSpec spec; + spec.name = stream_config.livekit_track; + spec.width = found->width / output.scale_div; + spec.height = found->height / output.scale_div; + spec.max_bitrate_bps = output.peak_bitrate_bps; + spec.max_fps = std::min(output.fps, found->fps); + + Stream stream; + stream.sensor_id = found->id; + stream.livekit = std::make_unique(m_livekit->publish_video(spec), output); + m_streams.push_back(std::move(stream)); + + std::cout << "Sensor " << found->id << " (" << found->name << ") -> LiveKit track '" << spec.name + << "': " << spec.width << "x" << spec.height << " @ up to " << spec.max_fps << " fps" << std::endl; + } + + m_camera->start(); +} + +SensingCamera::~SensingCamera() +{ + if (m_camera) + m_camera->stop(); +} + +void SensingCamera::update() +{ + // The room is gone for good: stop rather than keep capturing for nobody. + if (m_livekit->failed()) + throw std::runtime_error(m_livekit->failure()); + + for (auto& stream : m_streams) + { + auto view = m_camera->latest(stream.sensor_id); + if (view.has_value() && (!stream.have_last_sequence || view->sequence != stream.last_sequence)) + { + if (stream.have_last_sequence && view->sequence > stream.last_sequence + 1) + stream.missed_captures += view->sequence - stream.last_sequence - 1; + stream.last_sequence = view->sequence; + stream.have_last_sequence = true; + ++stream.frame_count; + + // A GPU snapshot only; the conversion and the network are on the output's own thread. + stream.livekit->submit(*view); + } + + if (stream.livekit->failed()) + throw std::runtime_error("sensor " + std::to_string(stream.sensor_id) + ": " + stream.livekit->failure()); + } +} + +void SensingCamera::flush() +{ + for (auto& stream : m_streams) + stream.livekit->flush(); +} + +void SensingCamera::print_stats() const +{ + for (const auto& stream : m_streams) + { + const auto s = stream.livekit->stats(); + std::cout << " sensor " << stream.sensor_id << ": " << stream.frame_count << " frames"; + if (stream.missed_captures) + std::cout << " | " << stream.missed_captures << " captures missed"; + std::cout << " | livekit " << s.published << " published, " << s.paced_out << " paced out, " << s.replaced + << " replaced"; + if (s.encoder_busy) + std::cout << ", " << s.encoder_busy << " encoder busy"; + if (s.rejected) + std::cout << ", " << s.rejected << " rejected"; + if (s.keyframes_requested) + std::cout << ", " << s.keyframes_requested << " keyframes requested"; + const std::string encoder = stream.livekit->describe(); + if (!encoder.empty()) + std::cout << " (" << encoder << ")"; + std::cout << std::endl; + } +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/sensing_camera.hpp b/src/plugins/sensing/core/sensing_camera.hpp new file mode 100644 index 0000000000..0822da3460 --- /dev/null +++ b/src/plugins/sensing/core/sensing_camera.hpp @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "sensing_types.hpp" +#include "sipl_camera.hpp" + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +class LiveKitSession; +class LiveKitVideoOutput; + +/** + * @brief Multi-sensor SENSING camera publisher. + * + * One SiplCamera owns every sensor, because INvSIPLCamera::GetInstance() is process-wide -- there is no + * per-sensor capture object to hold. Each update() polls every sensor's latest-frame mailbox and hands a new + * frame to that sensor's LiveKit track. Geometry is per sensor and comes from the platform config. + */ +class SensingCamera +{ +public: + SensingCamera(const SensingConfig& config, const std::vector& streams); + ~SensingCamera(); + + SensingCamera(const SensingCamera&) = delete; + SensingCamera& operator=(const SensingCamera&) = delete; + SensingCamera(SensingCamera&&) = delete; + SensingCamera& operator=(SensingCamera&&) = delete; + + /** @brief Poll every sensor and submit newly captured frames. Throws once the room is lost. */ + void update(); + + /** @brief Flush each track's queued frames. */ + void flush(); + + /** @brief Print per-sensor frame counts to stdout. */ + void print_stats() const; + +private: + struct Stream + { + uint32_t sensor_id = 0; + std::unique_ptr livekit; + /// SIPL publish counter of the frame already handled; skips re-reads. + uint64_t last_sequence = 0; + bool have_last_sequence = false; + uint64_t frame_count = 0; + /// Captures the poll loop never saw, from gaps in the SIPL sequence. + uint64_t missed_captures = 0; + }; + + SensingConfig m_config; + std::unique_ptr m_camera; + /// Declared before the streams so it is destroyed after their tracks have been unpublished. + std::unique_ptr m_livekit; + std::vector m_streams; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/sensing_types.hpp b/src/plugins/sensing/core/sensing_types.hpp new file mode 100644 index 0000000000..9fb8fcba2a --- /dev/null +++ b/src/plugins/sensing/core/sensing_types.hpp @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Plain configuration types, deliberately free of the SIPL and NvSci headers, which define X11-style +// macros (Success, Status, None) that collide with other libraries'. + +#pragma once + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/// One captured sensor and the LiveKit track that carries it. +struct StreamConfig +{ + /// SIPL pipeline index, validated against the platform config at startup. + uint32_t sensor_id = 0; + /// Name of the LiveKit camera track carrying this sensor. + std::string livekit_track; +}; + +/// Where and how the tracks are published. +struct LiveKitConfig +{ + std::string url; + /// The join token itself, not a path. It names the room and the participant. Never logged. + std::string token; + /// Publication rate cap, independent of the capture rate. + uint32_t fps = 30; + /// Output-only uniform reduction of every track; 1 publishes the sensor's native size. + uint32_t scale_div = 1; + /// Target bitrate of each track's hardware encoder; the VBR ceiling is half as much again. + uint32_t bitrate_bps = 8'000'000; +}; + +struct SensingConfig +{ + /// Vendor platform config JSON. Required -- SIPL cannot enumerate without one. + std::string platform_config_json; + /// Named config inside it. + std::string platform_config_name = "SHW5G_2"; + /// One mask per deserializer, in transport order. + std::vector link_masks{ 0x0000, 0x1100 }; + std::string nito_dir = "/var/nvidia/nvcam/settings/sipl"; + int gpu_id = 0; + + /// IDR period in frames; 0 defers to the encoder default of fps*5. + uint32_t gop = 0; + + bool full_range = false; + /// Defaults off: set_sipl_buf_attributes() requests the ISP0 surface order explicitly. + bool swap_uv = false; + /// ISP0 buffers per sensor, ~6 MB each at 2560x1984 NV12. + uint32_t isp0_buffers = 6; + + LiveKitConfig livekit; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp new file mode 100644 index 0000000000..601ea5e6fd --- /dev/null +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -0,0 +1,1205 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions adapted from NVIDIA's sipl_coe_unit_sample +// (/usr/src/jetson_multimedia_api/samples/unittest_samples/), BSD-3-Clause, +// Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. + +#include "sipl_camera.hpp" + +#include "yuv_to_rgba.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ +namespace +{ + +using nvsipl::INvSIPLCamera; +using nvsipl::INvSIPLCameraQuery; +using nvsipl::INvSIPLClient; +using nvsipl::NVSIPL_STATUS_OK; +using nvsipl::SIPLStatus; + +constexpr auto kIsp0 = INvSIPLClient::ConsumerDesc::OutputType::ISP0; +constexpr auto kIcp = INvSIPLClient::ConsumerDesc::OutputType::ICP; + +/// SIPL allocates and registers ICP capture buffers unconditionally, even when +/// only an ISP output is consumed -- CNvSIPLMaster::AllocateBuffers hardcodes +/// the ICP slot to true. Skip them and RegisterImages fails with a bare +/// NVSIPL_STATUS_ERROR that names nothing. +constexpr uint32_t kIcpBuffers = 6; + +uint64_t monotonic_ns() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +/// SIPL reports almost every failure as NVSIPL_STATUS_ERROR (10), which says +/// nothing on its own -- the reason only appears in its own log, above a +/// threshold that defaults to silent. SENSING_SIPL_TRACE=1..4 raises it +/// (1=error, 2=warn, 3=info, 4=debug). +void init_sipl_trace() +{ + const char* level = std::getenv("SENSING_SIPL_TRACE"); + if (level == nullptr) + { + return; + } + const int value = std::atoi(level); + if (value <= 0) + { + return; + } + auto* trace = nvsipl::INvSIPLTrace::GetInstance(); + if (trace != nullptr) + { + trace->SetLevel(static_cast(value)); + } +} + +void check_sipl(SIPLStatus status, const char* what) +{ + if (status != NVSIPL_STATUS_OK) + { + std::ostringstream oss; + oss << what << " failed with SIPLStatus " << static_cast(status); + throw std::runtime_error(oss.str()); + } +} + +void check_sci(NvSciError err, const char* what) +{ + if (err != NvSciError_Success) + { + std::ostringstream oss; + oss << what << " failed with NvSciError " << static_cast(err); + throw std::runtime_error(oss.str()); + } +} + +void check_cuda(CUresult result, const char* what) +{ + if (result == CUDA_SUCCESS) + { + return; + } + const char* name = nullptr; + const char* text = nullptr; + cuGetErrorName(result, &name); + cuGetErrorString(result, &text); + std::ostringstream oss; + oss << what << " failed"; + if (name) + { + oss << ": " << name; + } + if (text) + { + oss << " (" << text << ")"; + } + throw std::runtime_error(oss.str()); +} + +cudaTextureObject_t texture_for_array(CUarray array); + +void check_runtime(cudaError_t result, const char* what) +{ + if (result != cudaSuccess) + { + std::ostringstream oss; + oss << what << " failed: " << cudaGetErrorString(result); + throw std::runtime_error(oss.str()); + } +} + +cudaTextureObject_t texture_for_array(CUarray array) +{ + cudaResourceDesc resource_desc{}; + resource_desc.resType = cudaResourceTypeArray; + resource_desc.res.array.array = reinterpret_cast(array); + + cudaTextureDesc texture_desc{}; + texture_desc.addressMode[0] = cudaAddressModeClamp; + texture_desc.addressMode[1] = cudaAddressModeClamp; + texture_desc.filterMode = cudaFilterModePoint; + texture_desc.readMode = cudaReadModeElementType; + texture_desc.normalizedCoords = 0; + + cudaTextureObject_t texture = 0; + check_runtime(cudaCreateTextureObject(&texture, &resource_desc, &texture_desc, nullptr), "cudaCreateTextureObject"); + return texture; +} + +/** + * @brief Buffer attributes for the ISP0 output. + * + * Block-linear and REC709_ER are what the ISP produces, not what we would + * prefer. Neither is negotiable: GetImageAttributes rejects an attribute list + * asking for pitch-linear or BT.601 with NVSIPL_STATUS_BAD_ARGUMENT. So the + * conversion adapts to the ISP rather than the other way round -- convert() + * takes the CUDA-array path, and yuv_to_rgba.cu is called with bt709=true. + * `nvsipl_camera -v 3` prints the same thing: "YUV 420 SEMI-PLANAR UINT8 BL + * REC_709ER". + */ +NvSciError set_sipl_buf_attributes(NvSciBufAttrList attr_list) +{ + NvSciBufType buf_type = NvSciBufType_Image; + NvSciBufAttrValAccessPerm access_perm = NvSciBufAccessPerm_ReadWrite; + // Must be true, and it is not about us wanting CPU access. NvmmImageFill- + // NvSciBufAttrList sets NeedCpuAccess=true on the NvBufSurface list, so + // reconciliation ORs it in; if the producer list says false, SIPL's + // GetVerifiedBufferAttributes sees a producer/reconciled mismatch and + // RegisterImages fails with a bare NVSIPL_STATUS_ERROR. + bool cpu_access = true; + bool cpu_cache = true; + NvSciBufSurfMemLayout mem_layout = NvSciSurfMemLayout_SemiPlanar; + NvSciBufAttrValImageLayoutType layout_type = NvSciBufImage_BlockLinearType; + NvSciBufSurfType surf_type = NvSciSurfType_YUV; + NvSciBufSurfSampleType sample_type = NvSciSurfSampleType_420; + NvSciBufSurfBPC surf_bpc = NvSciSurfBPC_8; + NvSciBufSurfComponentOrder comp_order = NvSciSurfComponentOrder_YUV; + NvSciBufAttrValColorStd color_std[] = { NvSciColorStd_REC709_ER }; + + NvSciBufAttrKeyValuePair kvp[] = { + { NvSciBufGeneralAttrKey_Types, &buf_type, sizeof(buf_type) }, + { NvSciBufGeneralAttrKey_RequiredPerm, &access_perm, sizeof(access_perm) }, + { NvSciBufGeneralAttrKey_NeedCpuAccess, &cpu_access, sizeof(cpu_access) }, + { NvSciBufGeneralAttrKey_EnableCpuCache, &cpu_cache, sizeof(cpu_cache) }, + { NvSciBufImageAttrKey_Layout, &layout_type, sizeof(layout_type) }, + { NvSciBufImageAttrKey_SurfType, &surf_type, sizeof(surf_type) }, + { NvSciBufImageAttrKey_SurfMemLayout, &mem_layout, sizeof(mem_layout) }, + { NvSciBufImageAttrKey_SurfSampleType, &sample_type, sizeof(sample_type) }, + { NvSciBufImageAttrKey_SurfBPC, &surf_bpc, sizeof(surf_bpc) }, + { NvSciBufImageAttrKey_SurfComponentOrder, &comp_order, sizeof(comp_order) }, + { NvSciBufImageAttrKey_SurfColorStd, &color_std, sizeof(color_std) }, + }; + + return NvSciBufAttrListSetAttrs(attr_list, kvp, sizeof(kvp) / sizeof(kvp[0])); +} + + +/// Fill a second attribute list with the NvBufSurface attributes matching the +/// SIPL list, so the allocated buffer satisfies both consumers. Without it the +/// buffer is allocated for SIPL alone and RegisterImages rejects it with +/// NVSIPL_STATUS_INVALID_STATE -- a status whose only documented meaning is +/// "pipeline had seen an init error", which sends you looking in the wrong +/// place entirely. +/// +/// Adapted from SetNvBufNvSciBufAttributes in NVIDIA's NvNvSciBufHelper.cpp +/// (BSD-3-Clause). Its colour table has exactly two entries, both REC709_ER; +/// that is the whole set this ISP path can produce. +NvSciError set_nvbuf_attributes(NvSciBufAttrList nvbuf_attrs, NvSciBufAttrList sipl_attrs, int gpu_id) +{ + NvSciBufAttrKeyValuePair img[] = { + { NvSciBufImageAttrKey_SurfWidthBase, nullptr, 0 }, { NvSciBufImageAttrKey_SurfHeightBase, nullptr, 0 }, + { NvSciBufImageAttrKey_SurfSampleType, nullptr, 0 }, { NvSciBufImageAttrKey_SurfColorStd, nullptr, 0 }, + { NvSciBufImageAttrKey_SurfMemLayout, nullptr, 0 }, { NvSciBufImageAttrKey_SurfBPC, nullptr, 0 }, + { NvSciBufImageAttrKey_Layout, nullptr, 0 }, + }; + const NvSciError err = NvSciBufAttrListGetAttrs(sipl_attrs, img, sizeof(img) / sizeof(img[0])); + if (err != NvSciError_Success) + { + return err; + } + + const auto width = *static_cast(img[0].value); + const auto height = *static_cast(img[1].value); + const auto sample = *static_cast(img[2].value); + const auto color = *static_cast(img[3].value); + const auto mem = *static_cast(img[4].value); + const auto bpc = *static_cast(img[5].value); + const auto layout = *static_cast(img[6].value); + + NvBufSurfaceColorFormat pixfmt; + if (color == NvSciColorStd_REC709_ER && sample == NvSciSurfSampleType_420 && bpc == NvSciSurfBPC_8 && + mem == NvSciSurfMemLayout_SemiPlanar) + { + pixfmt = NVBUF_COLOR_FORMAT_NV12_709_ER; + } + else if (color == NvSciColorStd_REC709_ER && sample == NvSciSurfSampleType_420 && bpc == NvSciSurfBPC_8 && + mem == NvSciSurfMemLayout_Planar) + { + pixfmt = NVBUF_COLOR_FORMAT_YUV420_709_ER; + } + else + { + std::cerr << "[sipl] no NvBufSurface colour format for colourStd=" << static_cast(color) + << " sample=" << static_cast(sample) << " bpc=" << static_cast(bpc) + << " memLayout=" << static_cast(mem) << std::endl; + return NvSciError_BadParameter; + } + + NvmmImageParams params{}; + params.gpuId = static_cast(gpu_id); + params.width = width; + params.height = height; + params.colorFormat = pixfmt; + params.layout = (layout == NvSciBufImage_PitchLinearType) ? NVBUF_LAYOUT_PITCH : NVBUF_LAYOUT_BLOCK_LINEAR; + params.displayscanformat = NVBUF_DISPLAYSCANFORMAT_PROGRESSIVE; + params.isProtected = false; + + return (NvmmImageFillNvSciBufAttrList(¶ms, nvbuf_attrs) == 0) ? NvSciError_Success : NvSciError_BadParameter; +} + +/// Read back what reconciliation settled on, and report whether it is extended +/// range. _ER is full range (0-255), _SR is studio range (16-235); decoding one +/// as the other subtracts a black level that is not there and crushes shadows, +/// which on a dim scene loses most of the image. Returns true for full range. +bool check_color_std(NvSciBufAttrList reconciled) +{ + NvSciBufAttrKeyValuePair query{ NvSciBufImageAttrKey_SurfColorStd, nullptr, 0 }; + if (NvSciBufAttrListGetAttrs(reconciled, &query, 1) != NvSciError_Success || query.value == nullptr) + { + std::cerr << "[sipl] warning: could not read back SurfColorStd; assuming BT.709 extended range" << std::endl; + return true; + } + const auto* value = static_cast(query.value); + if (*value != NvSciColorStd_REC709_ER && *value != NvSciColorStd_REC709_SR) + { + std::ostringstream oss; + oss << "ISP0 reconciled to colour standard " << static_cast(*value) + << "; yuv_to_rgba.cu implements BT.601 and BT.709 only, and decoding anything else as" + " 709 would be a silent hue error"; + throw std::runtime_error(oss.str()); + } + return *value == NvSciColorStd_REC709_ER; +} + +std::vector load_nito(const std::string& dir, const std::string& module) +{ + const std::string path = dir + "/" + module + ".nito"; + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in) + { + throw std::runtime_error("cannot open NITO file " + path + + " (the ISP has no tuning; run the vendor install.sh on the host)"); + } + const auto size = static_cast(in.tellg()); + in.seekg(0); + std::vector blob(static_cast(size)); + if (!in.read(reinterpret_cast(blob.data()), size)) + { + throw std::runtime_error("short read on NITO file " + path); + } + return blob; +} + +/// Walk a SensorSystemConfig into flat SensorInfo records. The nesting is +/// module -> variant -> sensorConfigs -> variant, and `id` is the pipeline +/// index SetPipelineCfg wants. +std::vector flatten(const nvsipl::sensorconfig::SensorSystemConfig& cfg) +{ + std::vector out; + for (const auto& module : cfg.modules) + { + std::visit( + [&](const auto& mod) + { + for (const auto& sensor_variant : mod.sensorConfigs) + { + std::visit( + [&](const auto& sensor) + { + SensorInfo info; + info.id = sensor.id; + info.name = module.name; + if (!sensor.vcInfoList.empty()) + { + const auto& vc = sensor.vcInfoList.front(); + info.width = vc.resolution.width; + info.height = vc.resolution.height; + info.fps = static_cast(vc.fps); + } + out.push_back(std::move(info)); + }, + sensor_variant); + } + }, + module.moduleType); + } + return out; +} + +} // namespace + +// ============================================================================= +// Impl +// ============================================================================= + +struct SiplCamera::Impl +{ + /// One ISP0 buffer, imported into CUDA once at registration. Nothing here + /// is re-created per frame -- that is the whole point of registering. + struct Slot + { + NvSciBufObj buf = nullptr; + NvBufSurface* surface = nullptr; + CUgraphicsResource resource = nullptr; + CUeglFrame egl_frame{}; + }; + + struct DeviceBuffer + { + uint8_t* ptr = nullptr; + size_t pitch = 0; + }; + + struct Sensor + { + SensorInfo info; + nvsipl::NvSIPLPipelineConfiguration pipeline_config{}; + nvsipl::NvSIPLPipelineQueues queues{}; + + /// From the reconciled surface, not from config: the ISP decides. + bool full_range = true; + NvSciBufAttrList buf_attrs = nullptr; + std::vector buf_objects; + /// ICP pool. Never read -- allocated only because SIPL requires it. + NvSciBufAttrList icp_attrs = nullptr; + std::vector icp_buf_objects; + std::vector slots; + NvSciSyncObj sync_obj = nullptr; + + /// Triple-buffered RGBA8 mailbox. A reader leases the published slot + /// until its next latest() call, so the producer never overwrites it. + std::array buffers{}; + mutable std::mutex publish_mutex; + int publish_idx = -1; + int lease_idx = -1; + uint64_t published_sequence = 0; + uint64_t consumed_sequence = 0; + uint64_t published_timestamp_ns = 0; + uint64_t published_capture_tsc_ns = 0; + + std::thread frame_thread; + std::thread event_thread; + }; + + explicit Impl(const SiplConfig& cfg) : config(cfg) + { + } + + SiplConfig config; + + NvSciBufModule sci_buf_module = nullptr; + NvSciSyncModule sci_sync_module = nullptr; + NvSciSyncCpuWaitContext cpu_wait_context = nullptr; + + CUdevice cu_device = 0; + CUcontext cu_context = nullptr; + bool cu_context_retained = false; + CUstream convert_stream = nullptr; + + std::unique_ptr camera; + std::unique_ptr query_api; + std::unique_ptr system_config; + + std::vector sensor_infos; + std::vector> sensors; + + std::atomic running{ false }; + std::atomic failed{ false }; + mutable std::mutex error_mutex; + std::string failure_message; + + Sensor* find(uint32_t sensor_id) + { + for (auto& s : sensors) + { + if (s->info.id == sensor_id) + { + return s.get(); + } + } + return nullptr; + } + + void set_failure(const std::string& message) + { + std::lock_guard guard(error_mutex); + if (failure_message.empty()) + { + failure_message = message; + } + failed.store(true); + } + + void throw_if_failed() const + { + if (!failed.load()) + { + return; + } + std::lock_guard guard(error_mutex); + throw std::runtime_error(failure_message.empty() ? "SIPL capture failed" : failure_message); + } + + void init_cuda(); + void init_nvsci(); + void configure(); + void allocate_buffers(Sensor& sensor); + void allocate_icp_buffers(Sensor& sensor); + void register_buffers(Sensor& sensor); + void allocate_sync(Sensor& sensor); + void register_sync(Sensor& sensor); + void register_nito(Sensor& sensor); + void frame_loop(Sensor& sensor); + void event_loop(Sensor& sensor); + void convert(Sensor& sensor, const Slot& slot, uint32_t write_idx); + void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t capture_tsc_ns); + uint32_t pick_write_index(const Sensor& sensor) const; + void cleanup(); +}; + +void SiplCamera::Impl::init_cuda() +{ + check_cuda(cuInit(0), "cuInit"); + check_cuda(cuDeviceGet(&cu_device, config.gpu_id), "cuDeviceGet"); + check_cuda(cuDevicePrimaryCtxRetain(&cu_context, cu_device), "cuDevicePrimaryCtxRetain"); + cu_context_retained = true; + check_cuda(cuCtxSetCurrent(cu_context), "cuCtxSetCurrent"); + check_cuda(cuStreamCreate(&convert_stream, CU_STREAM_NON_BLOCKING), "cuStreamCreate"); +} + +void SiplCamera::Impl::init_nvsci() +{ + check_sci(NvSciBufModuleOpen(&sci_buf_module), "NvSciBufModuleOpen"); + check_sci(NvSciSyncModuleOpen(&sci_sync_module), "NvSciSyncModuleOpen"); + check_sci(NvSciSyncCpuWaitContextAlloc(sci_sync_module, &cpu_wait_context), "NvSciSyncCpuWaitContextAlloc"); +} + +void SiplCamera::Impl::configure() +{ + init_sipl_trace(); + query_api = INvSIPLCameraQuery::GetInstance(); + if (!query_api) + { + throw std::runtime_error("INvSIPLCameraQuery::GetInstance returned null"); + } + // Loads the UDDF driver .so libs out of /usr/lib/nvsipl_drv. + check_sipl(query_api->ParseDatabase(), "ParseDatabase"); + check_sipl(query_api->ParseJsonFile(config.platform_config_json), "ParseJsonFile"); + + system_config = std::make_unique(); + check_sipl(query_api->GetSensorSystemConfig(config.platform_config_name, *system_config), "GetSensorSystemConfig"); + if (config.link_masks.empty()) + { + throw std::runtime_error("link masks are required for a GMSL platform config"); + } + check_sipl(query_api->ApplyMask(*system_config, config.link_masks), "ApplyMask"); + if (system_config->modules.empty()) + { + throw std::runtime_error("no modules left in '" + config.platform_config_name + + "' after applying the link masks -- check the mask against the config"); + } + + sensor_infos = flatten(*system_config); + if (sensor_infos.empty()) + { + throw std::runtime_error("platform config '" + config.platform_config_name + "' declares no sensors"); + } + + camera = INvSIPLCamera::GetInstance(); + if (!camera) + { + throw std::runtime_error("INvSIPLCamera::GetInstance returned null"); + } + check_sipl(camera->SetPlatformCfg(*system_config), "SetPlatformCfg"); + + for (const auto& info : sensor_infos) + { + auto sensor = std::make_unique(); + sensor->info = info; + // ISP0 only. ICP would double the capture bandwidth for a raw stream + // nothing here consumes, and ISP1/ISP2 are a downscale we do not want. + sensor->pipeline_config.captureOutputRequested = false; + sensor->pipeline_config.isp0OutputRequested = true; + sensor->pipeline_config.isp1OutputRequested = false; + sensor->pipeline_config.isp2OutputRequested = false; + // Defaults to false, i.e. subframe ENABLED, which needs a sliceCount we + // do not set. NVIDIA's own samples disable it unless asked for. + sensor->pipeline_config.disableSubframe = true; + sensor->pipeline_config.bufferCfg.maxIsp0BufferCount = config.isp0_buffers; + check_sipl(camera->SetPipelineCfg(info.id, sensor->pipeline_config, sensor->queues), "SetPipelineCfg"); + sensors.push_back(std::move(sensor)); + } + + // Order is not ours to choose. Everything that *describes* a pipeline -- + // GetImageAttributes and FillNvSciSyncAttrList -- must happen before Init(); + // everything that *binds* a resource to it -- RegisterImages and + // RegisterNvSciSyncObj -- must happen after. Get it wrong and Init() + // still returns OK, but the pipeline is left in an error state and the + // first Register* call returns NVSIPL_STATUS_INVALID_STATE with no clue + // why. Verified against a -v 4 trace of NVIDIA's own nvsipl_camera. + for (auto& sensor : sensors) + { + allocate_icp_buffers(*sensor); + allocate_buffers(*sensor); + allocate_sync(*sensor); + } + + check_sipl(camera->Init(), "INvSIPLCamera::Init"); + + for (auto& sensor : sensors) + { + register_buffers(*sensor); + register_sync(*sensor); + register_nito(*sensor); + } +} + +void SiplCamera::Impl::allocate_buffers(Sensor& sensor) +{ + NvSciBufAttrList sipl_attrs = nullptr; + NvSciBufAttrList nvbuf_attrs = nullptr; + NvSciBufAttrList conflict = nullptr; + check_sci(NvSciBufAttrListCreate(sci_buf_module, &sipl_attrs), "NvSciBufAttrListCreate sipl"); + check_sci(NvSciBufAttrListCreate(sci_buf_module, &nvbuf_attrs), "NvSciBufAttrListCreate nvbuf"); + check_sci(set_sipl_buf_attributes(sipl_attrs), "NvSciBufAttrListSetAttrs"); + check_sipl(camera->GetImageAttributes(sensor.info.id, kIsp0, sipl_attrs), "GetImageAttributes"); + + // Both consumers must be described before allocation: SIPL writes the + // surface, NvBufSurface reads it on the way to CUDA. Reconciling the SIPL + // list alone yields a buffer RegisterImages will not accept. + check_sci(set_nvbuf_attributes(nvbuf_attrs, sipl_attrs, config.gpu_id), "set_nvbuf_attributes"); + + NvSciBufAttrList unreconciled[] = { sipl_attrs, nvbuf_attrs }; + check_sci(NvSciBufAttrListReconcile(unreconciled, 2U, &sensor.buf_attrs, &conflict), "NvSciBufAttrListReconcile"); + NvSciBufAttrListFree(sipl_attrs); + NvSciBufAttrListFree(nvbuf_attrs); + if (conflict) + { + NvSciBufAttrListFree(conflict); + } + sensor.full_range = check_color_std(sensor.buf_attrs); + + sensor.slots.resize(config.isp0_buffers); + for (uint32_t i = 0; i < config.isp0_buffers; ++i) + { + Slot& slot = sensor.slots[i]; + check_sci(NvSciBufObjAlloc(sensor.buf_attrs, &slot.buf), "NvSciBufObjAlloc"); + sensor.buf_objects.push_back(slot.buf); + + if (NvmmNvSciBufToNvBufSurface(slot.buf, &slot.surface) != 0 || slot.surface == nullptr) + { + throw std::runtime_error("NvmmNvSciBufToNvBufSurface failed"); + } + slot.surface->numFilled = 1; + } + + // RGBA8 destination slots. Allocated from the reported geometry, never from + // a CLI flag -- see "Geometry comes from the query" in the plan. + for (auto& dest : sensor.buffers) + { + void* ptr = nullptr; + size_t pitch = 0; + check_runtime(cudaMallocPitch(&ptr, &pitch, static_cast(sensor.info.width) * 4, sensor.info.height), + "cudaMallocPitch"); + dest.ptr = static_cast(ptr); + dest.pitch = pitch; + } +} + +void SiplCamera::Impl::allocate_icp_buffers(Sensor& sensor) +{ + NvSciBufAttrList attrs = nullptr; + NvSciBufAttrList conflict = nullptr; + check_sci(NvSciBufAttrListCreate(sci_buf_module, &attrs), "NvSciBufAttrListCreate icp"); + + // Only the type and permissions: GetImageAttributes fills the raw format in. + NvSciBufType buf_type = NvSciBufType_Image; + NvSciBufAttrValAccessPerm perm = NvSciBufAccessPerm_ReadWrite; + NvSciBufAttrKeyValuePair kvp[] = { + { NvSciBufGeneralAttrKey_Types, &buf_type, sizeof(buf_type) }, + { NvSciBufGeneralAttrKey_RequiredPerm, &perm, sizeof(perm) }, + }; + check_sci(NvSciBufAttrListSetAttrs(attrs, kvp, 2), "NvSciBufAttrListSetAttrs icp"); + check_sipl(camera->GetImageAttributes(sensor.info.id, kIcp, attrs), "GetImageAttributes ICP"); + + NvSciBufAttrList unreconciled[] = { attrs }; + check_sci(NvSciBufAttrListReconcile(unreconciled, 1U, &sensor.icp_attrs, &conflict), "NvSciBufAttrListReconcile icp"); + NvSciBufAttrListFree(attrs); + if (conflict) + { + NvSciBufAttrListFree(conflict); + } + + for (uint32_t i = 0; i < kIcpBuffers; ++i) + { + NvSciBufObj obj = nullptr; + check_sci(NvSciBufObjAlloc(sensor.icp_attrs, &obj), "NvSciBufObjAlloc icp"); + sensor.icp_buf_objects.push_back(obj); + } +} + +void SiplCamera::Impl::register_buffers(Sensor& sensor) +{ + check_sipl(camera->RegisterImages(sensor.info.id, kIcp, sensor.icp_buf_objects), "RegisterImages ICP"); + check_sipl(camera->RegisterImages(sensor.info.id, kIsp0, sensor.buf_objects), "RegisterImages ISP0"); + + // Import into CUDA only once SIPL owns the buffers. Doing it at allocation + // time hands SIPL a surface that already has an EGL image bound to it. + // Registering per frame instead would put an EGL round trip on the 60 Hz + // path for no benefit. + for (auto& slot : sensor.slots) + { + if (NvBufSurfaceMapEglImage(slot.surface, 0) != 0) + { + throw std::runtime_error("NvBufSurfaceMapEglImage failed"); + } + check_cuda(cuGraphicsEGLRegisterImage(&slot.resource, slot.surface->surfaceList[0].mappedAddr.eglImage, + CU_GRAPHICS_REGISTER_FLAGS_READ_ONLY), + "cuGraphicsEGLRegisterImage"); + check_cuda(cuGraphicsResourceGetMappedEglFrame(&slot.egl_frame, slot.resource, 0, 0), + "cuGraphicsResourceGetMappedEglFrame"); + } +} + +void SiplCamera::Impl::allocate_sync(Sensor& sensor) +{ + NvSciSyncAttrList signaler = nullptr; + NvSciSyncAttrList waiter = nullptr; + NvSciSyncAttrList reconciled = nullptr; + NvSciSyncAttrList conflict = nullptr; + + check_sci(NvSciSyncAttrListCreate(sci_sync_module, &signaler), "NvSciSyncAttrListCreate signaler"); + check_sci(NvSciSyncAttrListCreate(sci_sync_module, &waiter), "NvSciSyncAttrListCreate waiter"); + + // We wait on the CPU before handing the surface to the conversion kernel. + NvSciSyncAccessPerm perm = NvSciSyncAccessPerm_WaitOnly; + bool cpu_waiter = true; + NvSciSyncAttrKeyValuePair kvp[] = { + { NvSciSyncAttrKey_NeedCpuAccess, &cpu_waiter, sizeof(cpu_waiter) }, + { NvSciSyncAttrKey_RequiredPerm, &perm, sizeof(perm) }, + }; + check_sci(NvSciSyncAttrListSetAttrs(waiter, kvp, 2), "NvSciSyncAttrListSetAttrs"); + check_sipl( + camera->FillNvSciSyncAttrList(sensor.info.id, kIsp0, signaler, nvsipl::SIPL_SIGNALER), "FillNvSciSyncAttrList"); + + NvSciSyncAttrList unreconciled[] = { signaler, waiter }; + check_sci(NvSciSyncAttrListReconcile(unreconciled, 2U, &reconciled, &conflict), "NvSciSyncAttrListReconcile"); + check_sci(NvSciSyncObjAlloc(reconciled, &sensor.sync_obj), "NvSciSyncObjAlloc"); + NvSciSyncAttrListFree(signaler); + NvSciSyncAttrListFree(waiter); + if (reconciled) + { + NvSciSyncAttrListFree(reconciled); + } + if (conflict) + { + NvSciSyncAttrListFree(conflict); + } +} + +void SiplCamera::Impl::register_sync(Sensor& sensor) +{ + check_sipl(camera->RegisterNvSciSyncObj(sensor.info.id, kIsp0, nvsipl::NVSIPL_EOFSYNCOBJ, sensor.sync_obj), + "RegisterNvSciSyncObj"); +} + +void SiplCamera::Impl::register_nito(Sensor& sensor) +{ + auto blob = load_nito(config.nito_dir, sensor.info.name); + nvsipl::ISiplControlAuto* auto_control = nullptr; + check_sipl(camera->RegisterAutoControlPlugin(sensor.info.id, nvsipl::NV_PLUGIN, auto_control, blob), + "RegisterAutoControlPlugin"); +} + +void SiplCamera::Impl::convert(Sensor& sensor, const Slot& slot, uint32_t write_idx) +{ + const CUeglFrame& frame = slot.egl_frame; + DeviceBuffer& dest = sensor.buffers[write_idx]; + + if (frame.planeCount < 2) + { + throw std::runtime_error("ISP0 surface has fewer than two planes"); + } + + // Requested SemiPlanar YUV, so this is NV12 unless swap_uv flips it, and + // REC709_ER, so the kernel gets bt709=true. + const YuvLayout layout = config.swap_uv ? YuvLayout::YVU420SemiPlanar : YuvLayout::YUV420SemiPlanar; + constexpr bool kBt709 = true; + + if (frame.frameType == CU_EGL_FRAME_TYPE_PITCH) + { + const auto* y_plane = static_cast(frame.frame.pPitch[0]); + const auto* uv_plane = static_cast(frame.frame.pPitch[1]); + launch_yuv420_pitch_to_rgba(y_plane, uv_plane, nullptr, static_cast(frame.pitch), + static_cast(frame.pitch), 0, sensor.info.width, sensor.info.height, dest.ptr, + static_cast(dest.pitch), layout, sensor.full_range, kBt709, + reinterpret_cast(convert_stream)); + } + else if (frame.frameType == CU_EGL_FRAME_TYPE_ARRAY) + { + // Block-linear: CUDA exposes the planes as arrays, so the kernel reads + // them through texture objects. Those are built per frame because the + // CUarray handles come from the per-frame CUeglFrame; the expensive + // part, cuGraphicsEGLRegisterImage, is done once per buffer. + cudaTextureObject_t y_tex = texture_for_array(frame.frame.pArray[0]); + cudaTextureObject_t uv_tex = texture_for_array(frame.frame.pArray[1]); + launch_yuv420_array_to_rgba(y_tex, uv_tex, 0, sensor.info.width, sensor.info.height, dest.ptr, + static_cast(dest.pitch), layout, sensor.full_range, kBt709, + reinterpret_cast(convert_stream)); + check_cuda(cuStreamSynchronize(convert_stream), "cuStreamSynchronize"); + check_runtime(cudaDestroyTextureObject(y_tex), "cudaDestroyTextureObject"); + check_runtime(cudaDestroyTextureObject(uv_tex), "cudaDestroyTextureObject"); + check_runtime(cudaGetLastError(), "SIPL YUV to RGBA kernel"); + return; + } + else + { + throw std::runtime_error("unsupported CUDA EGL frame type from ISP0"); + } + + check_cuda(cuStreamSynchronize(convert_stream), "cuStreamSynchronize"); + check_runtime(cudaGetLastError(), "SIPL YUV to RGBA kernel"); +} + +void SiplCamera::Impl::frame_loop(Sensor& sensor) +{ + try + { + check_cuda(cuCtxSetCurrent(cu_context), "cuCtxSetCurrent frame_loop"); + while (running.load()) + { + INvSIPLClient::INvSIPLBuffer* buffer = nullptr; + const SIPLStatus status = sensor.queues.isp0CompletionQueue->Get(buffer, config.frame_timeout_ms * 1000U); + if (status == nvsipl::NVSIPL_STATUS_TIMED_OUT) + { + continue; + } + if (status == nvsipl::NVSIPL_STATUS_EOF) + { + break; + } + check_sipl(status, "isp0CompletionQueue->Get"); + if (!buffer) + { + continue; + } + + auto* nvm = dynamic_cast(buffer); + if (!nvm) + { + buffer->Release(); + throw std::runtime_error("ISP0 buffer is not an INvSIPLNvMBuffer"); + } + + // The ISP is still writing until this fence clears. + NvSciSyncFence fence = NvSciSyncFenceInitializer; + check_sipl(nvm->GetEOFNvSciSyncFence(&fence), "GetEOFNvSciSyncFence"); + const NvSciError wait = NvSciSyncFenceWait(&fence, cpu_wait_context, config.frame_timeout_ms * 1000UL); + NvSciSyncFenceClear(&fence); + check_sci(wait, "NvSciSyncFenceWait"); + + const NvSciBufObj arrived = nvm->GetNvSciBufImage(); + const Slot* slot = nullptr; + for (const auto& candidate : sensor.slots) + { + if (candidate.buf == arrived) + { + slot = &candidate; + break; + } + } + if (!slot) + { + buffer->Release(); + throw std::runtime_error("ISP0 delivered a buffer this process never registered"); + } + + // The TSC is what makes the two eyes pairable; do not substitute a + // host clock, which carries this sensor's own queueing jitter. + const auto& meta = nvm->GetImageData(); + const uint64_t capture_tsc_ns = meta.frameCaptureTSC; + // SENSING_AE_PROBE=1 logs the sensor's exposure and gain once a + // second. AE failures are invisible in the frame rate and hard to + // judge by eye, and this is the only place the numbers surface. + if (std::getenv("SENSING_AE_PROBE") != nullptr) + { + static thread_local uint64_t probe_n = 0; + if ((probe_n++ % 60) == 0) + { + std::cerr << "[ae] sensor " << sensor.info.id << " numExp=" << meta.numExposures + << " expValid=" << static_cast(meta.sensorExpInfo.expTimeValid) + << " exp[0]=" << meta.sensorExpInfo.exposureTime[0] << "s" + << " gainValid=" << static_cast(meta.sensorExpInfo.gainValid) + << " gain[0]=" << meta.sensorExpInfo.sensorGain[0] << std::endl; + } + } + + const uint32_t write_idx = pick_write_index(sensor); + convert(sensor, *slot, write_idx); + if (std::getenv("SENSING_LUMA_PROBE") != nullptr) + { + static thread_local uint64_t luma_n = 0; + if ((luma_n++ % 60) == 0) + { + // Mean over one row out of every 64, straight off the RGBA + // the consumer receives -- the only number that says what + // the operator actually sees. + const auto& dst = sensor.buffers[write_idx]; + const uint32_t step = 64; + std::vector row(dst.pitch); + double sum = 0.0; + uint64_t n = 0; + for (uint32_t y = 0; y < sensor.info.height; y += step) + { + if (cudaMemcpy(row.data(), dst.ptr + static_cast(y) * dst.pitch, dst.pitch, + cudaMemcpyDeviceToHost) != cudaSuccess) + break; + for (uint32_t x = 0; x < sensor.info.width; ++x) + { + sum += 0.2126 * row[x * 4 + 0] + 0.7152 * row[x * 4 + 1] + 0.0722 * row[x * 4 + 2]; + ++n; + } + } + if (n) + std::cerr << "[luma] sensor " << sensor.info.id << " mean=" << (sum / n) << "/255" << std::endl; + } + } + buffer->Release(); + publish(sensor, write_idx, monotonic_ns(), capture_tsc_ns); + } + } + catch (const std::exception& e) + { + std::ostringstream oss; + oss << "SIPL sensor " << sensor.info.id << " capture error: " << e.what(); + std::cerr << "[sipl] " << oss.str() << std::endl; + set_failure(oss.str()); + running.store(false); + } +} + +void SiplCamera::Impl::event_loop(Sensor& sensor) +{ + // A GMSL link drop arrives here and nowhere else. Without this drain the + // symptom is a silent stall on the frame queue. + while (running.load()) + { + nvsipl::NvSIPLPipelineNotifier::NotificationData data{}; + const SIPLStatus status = sensor.queues.notificationQueue->Get(data, config.frame_timeout_ms * 1000U); + if (status != NVSIPL_STATUS_OK) + { + continue; + } + using Notif = nvsipl::NvSIPLPipelineNotifier; + switch (data.eNotifType) + { + case Notif::NOTIF_ERROR_DESERIALIZER_FAILURE: + case Notif::NOTIF_ERROR_SERIALIZER_FAILURE: + case Notif::NOTIF_ERROR_SENSOR_FAILURE: + case Notif::NOTIF_ERROR_INTERNAL_FAILURE: + { + std::ostringstream oss; + oss << "SIPL sensor " << sensor.info.id << " pipeline error, notification " + << static_cast(data.eNotifType); + std::cerr << "[sipl] " << oss.str() << std::endl; + set_failure(oss.str()); + running.store(false); + break; + } + default: + break; + } + } +} + +void SiplCamera::Impl::publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t capture_tsc_ns) +{ + std::lock_guard guard(sensor.publish_mutex); + sensor.publish_idx = static_cast(write_idx); + sensor.published_timestamp_ns = timestamp_ns; + sensor.published_capture_tsc_ns = capture_tsc_ns; + ++sensor.published_sequence; +} + +uint32_t SiplCamera::Impl::pick_write_index(const Sensor& sensor) const +{ + std::lock_guard guard(sensor.publish_mutex); + if (sensor.publish_idx < 0) + { + return 0; + } + // Skip the published slot and any slot still leased to a reader. With three + // slots one is always free, so this cannot spin. + uint32_t idx = static_cast((sensor.publish_idx + 1) % 3); + if (static_cast(idx) == sensor.lease_idx) + { + idx = (idx + 1) % 3; + } + return idx; +} + +void SiplCamera::Impl::cleanup() +{ + running.store(false); + for (auto& sensor : sensors) + { + if (sensor->frame_thread.joinable()) + { + sensor->frame_thread.join(); + } + if (sensor->event_thread.joinable()) + { + sensor->event_thread.join(); + } + } + if (camera) + { + camera->Stop(); + camera->Deinit(); + } + + for (auto& sensor : sensors) + { + for (auto& slot : sensor->slots) + { + if (slot.resource) + { + cuGraphicsUnregisterResource(slot.resource); + } + if (slot.surface) + { + NvBufSurfaceUnMapEglImage(slot.surface, 0); + NvBufSurfaceDestroy(slot.surface); + } + if (slot.buf) + { + NvSciBufObjFree(slot.buf); + } + } + sensor->slots.clear(); + sensor->buf_objects.clear(); + for (auto& obj : sensor->icp_buf_objects) + { + NvSciBufObjFree(obj); + } + sensor->icp_buf_objects.clear(); + if (sensor->icp_attrs) + { + NvSciBufAttrListFree(sensor->icp_attrs); + sensor->icp_attrs = nullptr; + } + if (sensor->buf_attrs) + { + NvSciBufAttrListFree(sensor->buf_attrs); + sensor->buf_attrs = nullptr; + } + if (sensor->sync_obj) + { + NvSciSyncObjFree(sensor->sync_obj); + sensor->sync_obj = nullptr; + } + for (auto& dest : sensor->buffers) + { + if (dest.ptr) + { + cudaFree(dest.ptr); + dest.ptr = nullptr; + } + } + } + + camera.reset(); + query_api.reset(); + + if (cpu_wait_context) + { + NvSciSyncCpuWaitContextFree(cpu_wait_context); + cpu_wait_context = nullptr; + } + if (sci_sync_module) + { + NvSciSyncModuleClose(sci_sync_module); + sci_sync_module = nullptr; + } + if (sci_buf_module) + { + NvSciBufModuleClose(sci_buf_module); + sci_buf_module = nullptr; + } + if (convert_stream) + { + cuStreamDestroy(convert_stream); + convert_stream = nullptr; + } + if (cu_context_retained) + { + cuDevicePrimaryCtxRelease(cu_device); + cu_context_retained = false; + cu_context = nullptr; + } +} + +// ============================================================================= +// SiplCamera +// ============================================================================= + +SiplCamera::SiplCamera(const SiplConfig& config) : m_impl(std::make_unique(config)) +{ + if (config.platform_config_json.empty() || config.platform_config_name.empty()) + { + throw std::invalid_argument("SiplConfig needs both platform_config_json and platform_config_name"); + } + try + { + m_impl->init_cuda(); + m_impl->init_nvsci(); + m_impl->configure(); + } + catch (...) + { + m_impl->cleanup(); + throw; + } +} + +SiplCamera::~SiplCamera() +{ + m_impl->cleanup(); +} + +std::vector SiplCamera::query(const std::string& platform_config_json, + const std::string& platform_config_name, + const std::vector& link_masks) +{ + auto api = INvSIPLCameraQuery::GetInstance(); + if (!api) + { + throw std::runtime_error("INvSIPLCameraQuery::GetInstance returned null"); + } + check_sipl(api->ParseDatabase(), "ParseDatabase"); + check_sipl(api->ParseJsonFile(platform_config_json), "ParseJsonFile"); + + nvsipl::sensorconfig::SensorSystemConfig cfg; + check_sipl(api->GetSensorSystemConfig(platform_config_name, cfg), "GetSensorSystemConfig"); + if (!link_masks.empty()) + { + check_sipl(api->ApplyMask(cfg, link_masks), "ApplyMask"); + } + return flatten(cfg); +} + +const std::vector& SiplCamera::sensors() const +{ + return m_impl->sensor_infos; +} + +void SiplCamera::start() +{ + if (m_impl->running.load()) + { + return; + } + m_impl->running.store(true); + for (auto& sensor : m_impl->sensors) + { + sensor->frame_thread = std::thread([this, s = sensor.get()] { m_impl->frame_loop(*s); }); + sensor->event_thread = std::thread([this, s = sensor.get()] { m_impl->event_loop(*s); }); + } + try + { + check_sipl(m_impl->camera->Start(), "INvSIPLCamera::Start"); + } + catch (...) + { + m_impl->running.store(false); + for (auto& sensor : m_impl->sensors) + { + if (sensor->frame_thread.joinable()) + { + sensor->frame_thread.join(); + } + if (sensor->event_thread.joinable()) + { + sensor->event_thread.join(); + } + } + throw; + } +} + +void SiplCamera::stop() +{ + m_impl->running.store(false); + for (auto& sensor : m_impl->sensors) + { + if (sensor->frame_thread.joinable()) + { + sensor->frame_thread.join(); + } + if (sensor->event_thread.joinable()) + { + sensor->event_thread.join(); + } + } + if (m_impl->camera) + { + m_impl->camera->Stop(); + } +} + +std::optional SiplCamera::latest(uint32_t sensor_id) +{ + m_impl->throw_if_failed(); + Impl::Sensor* sensor = m_impl->find(sensor_id); + if (!sensor) + { + throw std::runtime_error("no SIPL pipeline with index " + std::to_string(sensor_id)); + } + + std::lock_guard guard(sensor->publish_mutex); + if (sensor->publish_idx < 0 || sensor->consumed_sequence == sensor->published_sequence) + { + return std::nullopt; + } + sensor->consumed_sequence = sensor->published_sequence; + + // The caller now holds this slot; pick_write_index() skips it until the + // next latest() moves the lease on. + sensor->lease_idx = sensor->publish_idx; + + FrameView view; + view.ptr = reinterpret_cast(sensor->buffers[sensor->publish_idx].ptr); + view.pitch = sensor->buffers[sensor->publish_idx].pitch; + view.width = sensor->info.width; + view.height = sensor->info.height; + view.timestamp_ns = sensor->published_timestamp_ns; + view.capture_tsc_ns = sensor->published_capture_tsc_ns; + view.sequence = sensor->published_sequence; + return view; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/sipl_camera.hpp b/src/plugins/sensing/core/sipl_camera.hpp new file mode 100644 index 0000000000..382ba26bb4 --- /dev/null +++ b/src/plugins/sensing/core/sipl_camera.hpp @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// SIPL capture for the SENSING SG8A carrier: every sensor in one platform +// config, ISP0 YUV out, converted to RGBA8 on the GPU. +// +// One instance owns every sensor. This is not a style choice -- +// INvSIPLCamera::GetInstance() hands out a single process-wide object, so a +// per-sensor camera class is not expressible. Do not reintroduce one. +// +// The SIPL and NvSci headers are pimpl'd away because they drag in NvMedia and +// X11-style macros (Success, Status, None) that collide with other libraries'; keeping them out of +// this header is what lets other files include it freely. + +#pragma once + +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct SiplConfig +{ + /// Vendor platform config, e.g. /query/sg8a_agth_g2a/shw5g.json. + std::string platform_config_json; + /// Named config inside it, e.g. "SHW5G_2". + std::string platform_config_name; + /// One mask per deserializer, in transport order. Required for GMSL. + std::vector link_masks; + /// Directory holding .nito; the ISP has no tuning without it. + std::string nito_dir = "/var/nvidia/nvcam/settings/sipl"; + int gpu_id = 0; + /// Overrides the range implied by the reconciled colour standard. + bool full_range = false; + /// Swap the chroma planes. Defaults off: the ISP0 surface order is + /// requested explicitly, so it does not need discovering from the frame. + bool swap_uv = false; + /// ISP0 buffers per sensor. Six is the SIPL default; more absorbs a slow + /// consumer at the cost of 6 MB each at 2560x1984 NV12. + uint32_t isp0_buffers = 6; + uint32_t frame_timeout_ms = 1000; +}; + +/// One sensor as the platform config describes it. `id` is the SIPL pipeline +/// index, which is NOT the GMSL link index, the CSI virtual channel, or the +/// JSON sensorInfo.id -- for SHW5G_2 those are 2 and 3 while this is 0 and 1. +struct SensorInfo +{ + uint32_t id = 0; + std::string name; + uint32_t width = 0; + uint32_t height = 0; + double fps = 0.0; +}; + +/** + * @brief Borrowed view of the most recently converted frame. + * + * `ptr` addresses one of three producer-owned slots and stays valid until the + * second subsequent latest() call for the same sensor, at which point the + * producer may overwrite it. Copy out before then; do not cache a FrameView + * across update() ticks. + */ +struct FrameView +{ + uintptr_t ptr = 0; + size_t pitch = 0; + uint32_t width = 0; + uint32_t height = 0; + /// CLOCK_MONOTONIC, stamped at YUV->RGBA conversion. Round-trips through + /// the encoder, so it is what identifies a unit on the way back out. + uint64_t timestamp_ns = 0; + /// Capture time on the TSC timebase, shared by every sensor on the rig. + /// This is what makes the two eyes pairable; a host clock is not. + uint64_t capture_tsc_ns = 0; + uint64_t sequence = 0; +}; + +class SiplCamera +{ +public: + explicit SiplCamera(const SiplConfig& config); + ~SiplCamera(); + + SiplCamera(const SiplCamera&) = delete; + SiplCamera& operator=(const SiplCamera&) = delete; + + /// Enumerate a platform config without touching the hardware. Cheap: the + /// query API only parses the driver database and the JSON. + static std::vector query(const std::string& platform_config_json, + const std::string& platform_config_name, + const std::vector& link_masks); + + /// Sensors this instance configured, in pipeline-index order. + const std::vector& sensors() const; + + void start(); + void stop(); + + /// Non-blocking mailbox read. nullopt when no new frame has arrived. + std::optional latest(uint32_t sensor_id); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/yuv_to_rgba.cu b/src/plugins/sensing/core/yuv_to_rgba.cu new file mode 100644 index 0000000000..7186e46ae8 --- /dev/null +++ b/src/plugins/sensing/core/yuv_to_rgba.cu @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// YUV420 -> RGBA8 conversion for the SIPL ISP0 output. +// +// BT.601 and BT.709, selected per call. SIPL's ISP0 output on this rig is +// REC709_ER and the request is not negotiable -- GetImageAttributes rejects a +// BT.601 attribute list with NVSIPL_STATUS_BAD_ARGUMENT -- so the 709 matrix is +// the one the SIPL path actually uses. rgba_to_nv12.cu on the encode side is +// still 601, so an encode round trip is not identity; that only matters if the +// H.264 output is ever compared against the RGBA it came from. + +#include "yuv_to_rgba.cuh" + +namespace +{ + +__device__ __forceinline__ unsigned char clamp_u8(float v) +{ + return static_cast(v < 0.f ? 0.f : (v > 255.f ? 255.f : v)); +} + +__device__ __forceinline__ void yuv_to_rgb( + int Y, int Cb, int Cr, bool full_range, bool bt709, unsigned char& r, unsigned char& g, unsigned char& b) +{ + // Full range uses Y as-is; limited range expands 16..235 first. + const float yf = full_range ? static_cast(Y) : (static_cast(Y) - 16.f) * 1.16438f; + const float u = static_cast(Cb) - 128.f; + const float v = static_cast(Cr) - 128.f; + + // Limited-range coefficients already fold in the 255/219 luma gain applied + // above, so they differ from the full-range pair by more than rounding. + float kr, kgu, kgv, kb; + if (bt709) + { + if (full_range) + { + kr = 1.5748f; + kgu = 0.18733f; + kgv = 0.46813f; + kb = 1.8556f; + } + else + { + kr = 1.79274f; + kgu = 0.21325f; + kgv = 0.53291f; + kb = 2.11240f; + } + } + else + { + if (full_range) + { + kr = 1.402f; + kgu = 0.344136f; + kgv = 0.714136f; + kb = 1.772f; + } + else + { + kr = 1.596f; + kgu = 0.391f; + kgv = 0.813f; + kb = 2.018f; + } + } + + r = clamp_u8(yf + kr * v); + g = clamp_u8(yf - kgu * u - kgv * v); + b = clamp_u8(yf + kb * u); +} + +__global__ void yuv420_pitch_to_rgba_kernel(const uint8_t* __restrict__ y_plane, + const uint8_t* __restrict__ uv_or_u_plane, + const uint8_t* __restrict__ v_plane, + int y_pitch, + int uv_pitch, + int v_pitch, + int width, + int height, + uint8_t* __restrict__ rgba_out, + int rgba_row_bytes, + int layout_value, + int full_range_value, + int bt709_value) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || y >= height) + { + return; + } + + const auto layout = static_cast(layout_value); + const int Y = y_plane[y * y_pitch + x]; + int Cb = 128; + int Cr = 128; + + if (layout == plugins::sensing::YuvLayout::YUV420SemiPlanar || layout == plugins::sensing::YuvLayout::YVU420SemiPlanar) + { + const int uv_x = x & ~1; + const int uv_y = y >> 1; + const uint8_t a = uv_or_u_plane[uv_y * uv_pitch + uv_x + 0]; + const uint8_t b = uv_or_u_plane[uv_y * uv_pitch + uv_x + 1]; + if (layout == plugins::sensing::YuvLayout::YUV420SemiPlanar) + { + Cb = a; + Cr = b; + } + else + { + Cr = a; + Cb = b; + } + } + else + { + const int uv_x = x >> 1; + const int uv_y = y >> 1; + const uint8_t a = uv_or_u_plane[uv_y * uv_pitch + uv_x]; + const uint8_t b = v_plane[uv_y * v_pitch + uv_x]; + if (layout == plugins::sensing::YuvLayout::YUV420Planar) + { + Cb = a; + Cr = b; + } + else + { + Cr = a; + Cb = b; + } + } + + const int idx = y * rgba_row_bytes + x * 4; + yuv_to_rgb( + Y, Cb, Cr, full_range_value != 0, bt709_value != 0, rgba_out[idx + 0], rgba_out[idx + 1], rgba_out[idx + 2]); + rgba_out[idx + 3] = 255; +} + +__global__ void yuv420_array_to_rgba_kernel(cudaTextureObject_t y_tex, + cudaTextureObject_t uv_or_u_tex, + cudaTextureObject_t v_tex, + int width, + int height, + uint8_t* __restrict__ rgba_out, + int rgba_row_bytes, + int layout_value, + int full_range_value, + int bt709_value) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || y >= height) + { + return; + } + + const auto layout = static_cast(layout_value); + const int Y = tex2D(y_tex, x, y); + int Cb = 128; + int Cr = 128; + + if (layout == plugins::sensing::YuvLayout::YUV420SemiPlanar || layout == plugins::sensing::YuvLayout::YVU420SemiPlanar) + { + const int uv_x = x >> 1; + const int uv_y = y >> 1; + const uchar2 uv = tex2D(uv_or_u_tex, uv_x, uv_y); + if (layout == plugins::sensing::YuvLayout::YUV420SemiPlanar) + { + Cb = uv.x; + Cr = uv.y; + } + else + { + Cr = uv.x; + Cb = uv.y; + } + } + else + { + const int uv_x = x >> 1; + const int uv_y = y >> 1; + const uint8_t a = tex2D(uv_or_u_tex, uv_x, uv_y); + const uint8_t b = tex2D(v_tex, uv_x, uv_y); + if (layout == plugins::sensing::YuvLayout::YUV420Planar) + { + Cb = a; + Cr = b; + } + else + { + Cr = a; + Cb = b; + } + } + + const int idx = y * rgba_row_bytes + x * 4; + yuv_to_rgb( + Y, Cb, Cr, full_range_value != 0, bt709_value != 0, rgba_out[idx + 0], rgba_out[idx + 1], rgba_out[idx + 2]); + rgba_out[idx + 3] = 255; +} + +} // namespace + +namespace plugins +{ +namespace sensing +{ + +void launch_yuv420_pitch_to_rgba(const uint8_t* y_plane, + const uint8_t* uv_or_u_plane, + const uint8_t* v_plane, + int y_pitch, + int uv_pitch, + int v_pitch, + int width, + int height, + uint8_t* rgba_out, + int rgba_row_bytes, + YuvLayout layout, + bool full_range, + bool bt709, + cudaStream_t stream) +{ + const dim3 block(16, 16, 1); + const dim3 grid((width + 15) / 16, (height + 15) / 16, 1); + yuv420_pitch_to_rgba_kernel<<>>(y_plane, uv_or_u_plane, v_plane, y_pitch, uv_pitch, v_pitch, + width, height, rgba_out, rgba_row_bytes, + static_cast(layout), full_range ? 1 : 0, bt709 ? 1 : 0); +} + +void launch_yuv420_array_to_rgba(cudaTextureObject_t y_tex, + cudaTextureObject_t uv_or_u_tex, + cudaTextureObject_t v_tex, + int width, + int height, + uint8_t* rgba_out, + int rgba_row_bytes, + YuvLayout layout, + bool full_range, + bool bt709, + cudaStream_t stream) +{ + const dim3 block(16, 16, 1); + const dim3 grid((width + 15) / 16, (height + 15) / 16, 1); + yuv420_array_to_rgba_kernel<<>>(y_tex, uv_or_u_tex, v_tex, width, height, rgba_out, + rgba_row_bytes, static_cast(layout), + full_range ? 1 : 0, bt709 ? 1 : 0); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/yuv_to_rgba.cuh b/src/plugins/sensing/core/yuv_to_rgba.cuh new file mode 100644 index 0000000000..3645202521 --- /dev/null +++ b/src/plugins/sensing/core/yuv_to_rgba.cuh @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +enum class YuvLayout +{ + YUV420Planar, + YVU420Planar, + YUV420SemiPlanar, + YVU420SemiPlanar, +}; + +void launch_yuv420_pitch_to_rgba(const uint8_t* y_plane, + const uint8_t* uv_or_u_plane, + const uint8_t* v_plane, + int y_pitch, + int uv_pitch, + int v_pitch, + int width, + int height, + uint8_t* rgba_out, + int rgba_row_bytes, + YuvLayout layout, + bool full_range, + /// BT.709 matrix instead of BT.601. + bool bt709, + cudaStream_t stream); + +void launch_yuv420_array_to_rgba(cudaTextureObject_t y_tex, + cudaTextureObject_t uv_or_u_tex, + cudaTextureObject_t v_tex, + int width, + int height, + uint8_t* rgba_out, + int rgba_row_bytes, + YuvLayout layout, + bool full_range, + /// BT.709 matrix instead of BT.601. + bool bt709, + cudaStream_t stream); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/livekit/CMakeLists.txt b/src/plugins/sensing/livekit/CMakeLists.txt new file mode 100644 index 0000000000..7d0b061953 --- /dev/null +++ b/src/plugins/sensing/livekit/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The LiveKit SDK behind a small interface: join a room, publish a video track, submit NV12 frames. +# No LiveKit type appears in an installed header, and nothing here touches CUDA. +add_library(sensing_livekit STATIC + livekit_session.cpp + inc/sensing_livekit/livekit_session.hpp + inc/sensing_livekit/video_track_sink.hpp +) + +target_include_directories(sensing_livekit + INTERFACE + $ +) + +target_link_libraries(sensing_livekit + PUBLIC + log_bridge::log_bridge_core + PRIVATE + LiveKit::livekit +) + +add_library(sensing::livekit ALIAS sensing_livekit) diff --git a/src/plugins/sensing/livekit/inc/sensing_livekit/livekit_session.hpp b/src/plugins/sensing/livekit/inc/sensing_livekit/livekit_session.hpp new file mode 100644 index 0000000000..7cf33bcfea --- /dev/null +++ b/src/plugins/sensing/livekit/inc/sensing_livekit/livekit_session.hpp @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "video_track_sink.hpp" + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct LiveKitSessionImpl; + +/** + * @brief Keeps the LiveKit SDK initialised while it is held. + * + * The SDK is process-wide and must be shut down only when nothing in the process uses it, so every + * user holds one of these and the last release shuts it down. + */ +class LiveKitSdkReference +{ +public: + LiveKitSdkReference(); + ~LiveKitSdkReference(); + LiveKitSdkReference(const LiveKitSdkReference&) = delete; + LiveKitSdkReference& operator=(const LiveKitSdkReference&) = delete; +}; + +struct PublishSpec +{ + /// Name subscribers see for the track. + std::string name; + uint32_t width = 0; + uint32_t height = 0; + /// Upper bound the upstream encoder may use; WebRTC adapts below it. + uint64_t max_bitrate_bps = 0; + double max_fps = 30.0; +}; + +/** + * @brief One participant in one LiveKit room. + * + * The room and the participant's identity come from the join token. A terminal loss of the room + * (the server went away, the participant was removed) sets failed(); the SDK's own bounded + * reconnection runs first and is not wrapped in another supervisor. + */ +class LiveKitSession +{ +public: + /// @throws std::runtime_error if the room cannot be joined. The token is never logged. + static std::unique_ptr connect(const std::string& url, const std::string& token); + + /// Unpublish every track (destroy them first), then leave the room. + ~LiveKitSession(); + + LiveKitSession(const LiveKitSession&) = delete; + LiveKitSession& operator=(const LiveKitSession&) = delete; + + /// Publishes an H.264 camera track that carries pre-encoded access units; the SDK does not encode. + /// @throws std::runtime_error if publication fails. + std::unique_ptr publish_video(const PublishSpec& spec); + + /// True once the room has been lost for good. + bool failed() const; + std::string failure() const; + +private: + explicit LiveKitSession(std::shared_ptr impl); + std::shared_ptr m_impl; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp new file mode 100644 index 0000000000..a69a5ea1af --- /dev/null +++ b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/// One complete encoded H.264 access unit, with what accompanies it. +struct EncodedVideoFrame +{ + /// Annex-B bytes: an IDR carries its SPS/PPS in front of it. + const uint8_t* data = nullptr; + size_t size = 0; + /// The unit is an IDR, decodable on its own. + bool keyframe = false; + /// CLOCK_MONOTONIC microseconds at capture: the host timeline, not sensor time. + uint64_t timestamp_us = 0; + /// The camera's own frame sequence. + uint32_t frame_id = 0; + /// The sensor's capture time on its own clock, carried as user data so a consumer can pair + /// frames from different sensors. + uint64_t capture_tsc_ns = 0; +}; + +/// What the receiving side asked of the encoder since the last call. +struct EncoderFeedback +{ + /// The next frame must be an IDR with its parameter sets. + bool keyframe_requested = false; + /// Latest bitrate target from congestion control, if one was requested. + std::optional target_bitrate_bps; +}; + +/** + * @brief One published video track that carries frames already encoded by hardware. + * + * The SDK forwards each access unit without encoding it again. A single thread must drive one sink. + */ +class VideoTrackSink +{ +public: + virtual ~VideoTrackSink() = default; + + /// Size of the encoded pictures this track was published for. + virtual uint32_t width() const = 0; + virtual uint32_t height() const = 0; + + /// Publishes one access unit; the bytes are copied before this returns. + /// @return false when the SDK did not accept the frame, which can break the reference chain: + /// the caller must follow with a keyframe. + /// @throws std::runtime_error when the frame cannot be published at all. + virtual bool submit(const EncodedVideoFrame& frame) = 0; + + /// Keyframe and rate requests accumulated since the previous call. + virtual EncoderFeedback take_feedback() = 0; + + /// One line naming the transport's view of the stream, or empty if unavailable. + virtual std::string describe() = 0; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/livekit/livekit_session.cpp b/src/plugins/sensing/livekit/livekit_session.cpp new file mode 100644 index 0000000000..70fa5853a7 --- /dev/null +++ b/src/plugins/sensing/livekit/livekit_session.cpp @@ -0,0 +1,301 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_livekit/livekit_session.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +std::mutex g_sdk_mutex; +int g_sdk_users = 0; + +const char* reason_name(livekit::DisconnectReason reason) +{ + using R = livekit::DisconnectReason; + switch (reason) + { + case R::ClientInitiated: + return "client initiated"; + case R::DuplicateIdentity: + return "another participant joined with the same identity"; + case R::ServerShutdown: + return "the server is shutting down"; + case R::ParticipantRemoved: + return "the participant was removed from the room"; + case R::RoomDeleted: + return "the room was deleted"; + default: + return "connection lost"; + } +} + +} // namespace + +LiveKitSdkReference::LiveKitSdkReference() +{ + std::lock_guard lock(g_sdk_mutex); + if (g_sdk_users == 0) + livekit::initialize(livekit::LogLevel::Warn); + ++g_sdk_users; +} + +LiveKitSdkReference::~LiveKitSdkReference() +{ + std::lock_guard lock(g_sdk_mutex); + if (--g_sdk_users == 0) + livekit::shutdown(); +} + +struct LiveKitSessionImpl : livekit::RoomDelegate +{ + ~LiveKitSessionImpl() override + { + // Leave the room while the SDK is still up: the reference below is released after the room. + room.disconnect(); + } + + void onDisconnected(livekit::Room&, const livekit::DisconnectedEvent& event) override + { + if (event.reason == livekit::DisconnectReason::ClientInitiated) + return; + fail(std::string("LiveKit room lost: ") + reason_name(event.reason)); + } + + void onReconnecting(livekit::Room&, const livekit::ReconnectingEvent&) override + { + logger->warn("LiveKit connection interrupted; the SDK is reconnecting"); + } + + void onReconnected(livekit::Room&, const livekit::ReconnectedEvent&) override + { + logger->info("LiveKit connection restored"); + } + + void fail(const std::string& message) + { + std::lock_guard lock(mutex); + if (!failed.load()) + { + failure = message; + failed.store(true); + logger->error("{}", message); + } + } + + // Declared before the room, so destroyed after it. + LiveKitSdkReference sdk; + livekit::Room room; + std::atomic failed{ false }; + mutable std::mutex mutex; + std::string failure; + std::shared_ptr logger = isaaccapture::Logger::get("isaaccapture.plugins.sensing.LiveKitSession"); +}; + +namespace +{ + +/// A track fed with H.264 access units that were encoded upstream. +class LiveKitVideoTrack : public VideoTrackSink +{ +public: + LiveKitVideoTrack(std::shared_ptr session, const PublishSpec& spec) + : m_session(std::move(session)), m_width(spec.width), m_height(spec.height) + { + m_source = std::make_shared( + livekit::VideoCodec::H264, static_cast(spec.width), static_cast(spec.height)); + m_track = livekit::LocalVideoTrack::createLocalVideoTrack(spec.name, m_source); + + livekit::TrackPublishOptions options; + options.source = livekit::TrackSource::SOURCE_CAMERA; + options.video_codec = livekit::VideoCodec::H264; + // Pre-encoded access units go through as they are: no software or SDK-selected encoder. + options.video_encoder = livekit::VideoEncoderBackend::PreEncoded; + options.simulcast = false; + options.video_encoding = livekit::VideoEncodingOptions{ spec.max_bitrate_bps, spec.max_fps }; + options.frame_metadata_features = livekit::FrameMetadataFeatures{ true, true, true }; + + auto participant = m_session->room.localParticipant().lock(); + if (!participant) + throw std::runtime_error("LiveKit: no local participant to publish '" + spec.name + "'"); + try + { + participant->publishTrack(m_track, options); + } + catch (const std::exception& e) + { + throw std::runtime_error("LiveKit: cannot publish track '" + spec.name + "': " + e.what()); + } + m_participant = participant; + } + + ~LiveKitVideoTrack() override + { + try + { + if (auto participant = m_participant.lock()) + participant->unpublishTrack(m_track->sid()); + } + catch (const std::exception& e) + { + m_session->logger->warn("LiveKit: unpublish failed: {}", e.what()); + } + } + + uint32_t width() const override + { + return m_width; + } + uint32_t height() const override + { + return m_height; + } + + bool submit(const EncodedVideoFrame& frame) override + { + if (m_session->failed.load()) + throw std::runtime_error(m_session->failure); + + livekit::EncodedVideoSource::Frame out; + out.is_keyframe = frame.keyframe; + out.width = m_width; + out.height = m_height; + out.timestamp_us = static_cast(frame.timestamp_us); + out.data.assign(frame.data, frame.data + frame.size); + livekit::VideoFrameMetadata metadata; + metadata.user_timestamp_us = frame.timestamp_us; + metadata.frame_id = frame.frame_id; + std::vector tsc(8); + for (int i = 0; i < 8; ++i) + tsc[i] = static_cast((frame.capture_tsc_ns >> (8 * i)) & 0xFF); + metadata.user_data = std::move(tsc); + out.metadata = std::move(metadata); + + try + { + return m_source->captureFrame(out); + } + catch (const std::exception& e) + { + throw std::runtime_error(std::string("LiveKit: cannot submit a frame: ") + e.what()); + } + } + + EncoderFeedback take_feedback() override + { + EncoderFeedback feedback; + try + { + const auto pending = m_source->takeFeedback(); + feedback.keyframe_requested = pending.keyframe_requested; + if (pending.rate_control && pending.rate_control->target_bitrate_bps > 0) + feedback.target_bitrate_bps = static_cast(pending.rate_control->target_bitrate_bps); + } + catch (const std::exception& e) + { + throw std::runtime_error(std::string("LiveKit: cannot read encoder feedback: ") + e.what()); + } + return feedback; + } + + // Called from the polling thread, which must not wait on the network: this returns the last answer + // and starts, or collects, a stats request that was asked for earlier. + std::string describe() override + { + try + { + if (m_stats.valid() && m_stats.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + { + for (const auto& entry : m_stats.get()) + { + const auto* out = std::get_if(&entry.stats); + if (!out || out->outbound.frame_width == 0) + continue; + const auto& o = out->outbound; + m_description = "encoder " + o.encoder_implementation + ", " + std::to_string(o.frame_width) + "x" + + std::to_string(o.frame_height) + " @ " + + std::to_string(static_cast(o.frames_per_second)) + " fps, " + + std::to_string(static_cast(o.target_bitrate / 1000)) + " kbps target"; + break; + } + } + if (!m_stats.valid()) + m_stats = m_track->getStats(); + } + catch (const std::exception&) + { + } + return m_description; + } + +private: + std::shared_ptr m_session; + uint32_t m_width; + uint32_t m_height; + std::shared_ptr m_source; + std::shared_ptr m_track; + std::weak_ptr m_participant; + decltype(std::declval().getStats()) m_stats; + std::string m_description; +}; + +} // namespace + +LiveKitSession::LiveKitSession(std::shared_ptr impl) : m_impl(std::move(impl)) +{ +} + +LiveKitSession::~LiveKitSession() = default; + +std::unique_ptr LiveKitSession::connect(const std::string& url, const std::string& token) +{ + auto impl = std::make_shared(); + impl->room.setDelegate(impl.get()); + + livekit::RoomOptions options; + options.auto_subscribe = false; // this process only publishes + options.dynacast = false; + if (!impl->room.connect(url, token, options)) + { + throw std::runtime_error("LiveKit: cannot join the room at " + url + + " (check the URL, that the server is reachable and that the token is valid)"); + } + impl->logger->info("Joined LiveKit room at {}", url); + return std::unique_ptr(new LiveKitSession(std::move(impl))); +} + +std::unique_ptr LiveKitSession::publish_video(const PublishSpec& spec) +{ + return std::make_unique(m_impl, spec); +} + +bool LiveKitSession::failed() const +{ + return m_impl->failed.load(); +} + +std::string LiveKitSession::failure() const +{ + std::lock_guard lock(m_impl->mutex); + return m_impl->failure; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/plugin.yaml b/src/plugins/sensing/plugin.yaml new file mode 100644 index 0000000000..2f7cb80087 --- /dev/null +++ b/src/plugins/sensing/plugin.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: sensing_camera +description: "SENSING GMSL camera plugin (SIPL capture, hardware H.264, LiveKit publication)" +command: "./camera_plugin_sensing" +version: "1.0.0" +devices: + - path: "/camera" + type: "camera" + description: "Camera video stream" From d8dc382f52d4ca5f0f79e3028ea271d403ce834a Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Wed, 4 Nov 2026 23:18:49 -0800 Subject: [PATCH 02/29] sensing: add a mock backend that runs the LiveKit publisher without a Jetson Copy the mock SiplCamera and JetsonEncoder from the sensing-livekit branch: synthetic frames (optionally animated GIFs) and a real H.264 encode through libavcodec's NVENC, so the plugin builds and runs on any machine with an NVIDIA GPU and CUDA. SENSING_BACKEND selects JETSON, MOCK or AUTO (JETSON on aarch64). The sample GIFs are CC BY 4.0, noted in the top-level README. Not built or run here yet. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- README.md | 3 +- src/plugins/CMakeLists.txt | 2 +- src/plugins/sensing/CMakeLists.txt | 30 +- src/plugins/sensing/core/CMakeLists.txt | 101 ++++- src/plugins/sensing/core/mock_gif_source.cpp | 109 +++++ src/plugins/sensing/core/mock_gif_source.hpp | 73 +++ .../sensing/core/mock_jetson_encoder.cpp | 424 ++++++++++++++++++ src/plugins/sensing/core/mock_nv12_source.cu | 166 +++++++ src/plugins/sensing/core/mock_nv12_source.cuh | 42 ++ .../sensing/core/mock_platform_config.cpp | 109 +++++ .../sensing/core/mock_platform_config.hpp | 40 ++ src/plugins/sensing/core/mock_sipl_camera.cpp | 417 +++++++++++++++++ src/plugins/sensing/core/mock_test_hooks.hpp | 20 + src/plugins/sensing/mock_assets/README.md | 21 + src/plugins/sensing/mock_assets/mock_rig.json | 75 ++++ .../sensing/mock_assets/noto-robot.gif | 3 + .../mock_assets/noto-robot.gif.license | 2 + .../sensing/mock_assets/noto-waving-hand.gif | 3 + .../mock_assets/noto-waving-hand.gif.license | 2 + 19 files changed, 1615 insertions(+), 27 deletions(-) create mode 100644 src/plugins/sensing/core/mock_gif_source.cpp create mode 100644 src/plugins/sensing/core/mock_gif_source.hpp create mode 100644 src/plugins/sensing/core/mock_jetson_encoder.cpp create mode 100644 src/plugins/sensing/core/mock_nv12_source.cu create mode 100644 src/plugins/sensing/core/mock_nv12_source.cuh create mode 100644 src/plugins/sensing/core/mock_platform_config.cpp create mode 100644 src/plugins/sensing/core/mock_platform_config.hpp create mode 100644 src/plugins/sensing/core/mock_sipl_camera.cpp create mode 100644 src/plugins/sensing/core/mock_test_hooks.hpp create mode 100644 src/plugins/sensing/mock_assets/README.md create mode 100644 src/plugins/sensing/mock_assets/mock_rig.json create mode 100644 src/plugins/sensing/mock_assets/noto-robot.gif create mode 100644 src/plugins/sensing/mock_assets/noto-robot.gif.license create mode 100644 src/plugins/sensing/mock_assets/noto-waving-hand.gif create mode 100644 src/plugins/sensing/mock_assets/noto-waving-hand.gif.license diff --git a/README.md b/README.md index cd55de9172..ac59e1dfd1 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,8 @@ To get started, please refer to Isaac Lab's [Installation](https://isaac-sim.git The default license is [Apache-2.0](LICENSE.md). EgoPost is licensed under [OpenMDW-1.1](LICENSES/LicenseRef-OpenMDW-1.1.txt). -Third-party materials may carry their own licenses. +Third-party materials may carry their own licenses; for example, the sample GIFs under +[`src/plugins/sensing/mock_assets/`](src/plugins/sensing/mock_assets/) are Animated Noto Emoji under CC BY 4.0. ### Separately retrieved optional materials diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 7a3a400b15..1c1c6daf36 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -44,7 +44,7 @@ endif() if(BUILD_PLUGIN_SHARPA_AVATAR) add_subdirectory(sharpa_avatar) endif() -# Gated: Jetson only -- SIPL and the Tegra NVENC entry point are L4T-only. +# Gated: the Jetson backend needs SIPL and the Tegra NVENC entry point (L4T only); the mock runs anywhere with CUDA. if(BUILD_PLUGIN_SENSING) add_subdirectory(sensing) endif() diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index da64e4fbdc..516759d934 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -4,6 +4,12 @@ # ============================================================================== # SENSING Camera Plugin: Jetson SIPL capture, hardware H.264 encode, LiveKit publication # ============================================================================== +# Two backends sit behind SiplCamera and JetsonEncoder: +# JETSON SIPL capture and the V4L2 M2M encoder, which exist only on L4T. +# MOCK synthetic frames and an NVENC encoder (libavcodec), for any machine with an NVIDIA GPU and +# the CUDA toolkit. Everything above those two classes is the same code. +# SENSING_BACKEND=AUTO picks JETSON on aarch64 and MOCK elsewhere. +# # Two leaf targets besides the LiveKit wrapper, one per directory (cmake/cmake-structure.md rule 1): # livekit/ sensing_livekit -- the LiveKit SDK behind a small interface # core/ sensing_core -- SIPL capture, encoder, conversion, track output @@ -13,7 +19,8 @@ # src/plugins/sensing/scripts/setup.sh # fetches the Multimedia API, locates jetson_sipl_api # # Configure with: -# cmake -B build -DBUILD_PLUGIN_SENSING=ON +# cmake -B build -DBUILD_PLUGIN_SENSING=ON # Jetson, or mock elsewhere +# cmake -B build -DBUILD_PLUGIN_SENSING=ON -DSENSING_BACKEND=MOCK # force the mock # ============================================================================== enable_language(CUDA) @@ -25,8 +32,28 @@ endif() find_package(CUDAToolkit REQUIRED) +set(SENSING_BACKEND "AUTO" CACHE STRING "SENSING backend: AUTO, JETSON (SIPL + V4L2 NVENC) or MOCK (synthetic)") +set_property(CACHE SENSING_BACKEND PROPERTY STRINGS AUTO JETSON MOCK) +if(SENSING_BACKEND STREQUAL "AUTO") + # A Jetson that lacks the SDK should still get the setup instructions below, + # not a silent mock, so only non-aarch64 hosts fall back. + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(SENSING_BACKEND_RESOLVED "JETSON" CACHE INTERNAL "") + else() + set(SENSING_BACKEND_RESOLVED "MOCK" CACHE INTERNAL "") + endif() +elseif(SENSING_BACKEND STREQUAL "JETSON" OR SENSING_BACKEND STREQUAL "MOCK") + set(SENSING_BACKEND_RESOLVED "${SENSING_BACKEND}" CACHE INTERNAL "") +else() + message(FATAL_ERROR "SENSING_BACKEND must be AUTO, JETSON or MOCK, got '${SENSING_BACKEND}'") +endif() +# Cache, not a plain variable, so tests/ (a sibling directory) can gate on it too. +message(STATUS "SENSING plugin backend: ${SENSING_BACKEND_RESOLVED}") + set(_tegra_lib_dirs "/usr/lib/aarch64-linux-gnu/nvidia" "/usr/lib/aarch64-linux-gnu/tegra") +if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON") +# Jetson-only SDK checks. # ============================================================================== # SIPL # ============================================================================== @@ -149,6 +176,7 @@ message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}") # No libnvjpeg here on purpose: on R39 libnvbufsurface has no libjpeg # dependency at all (readelf -d lists only EGL, cuda, dl, pthread, rt, c), and # the tegra lib dir ships no libnvjpeg to link against. +endif() # ============================================================================== # LiveKit diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt index 22b466c2df..6428eafad5 100644 --- a/src/plugins/sensing/core/CMakeLists.txt +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -2,15 +2,66 @@ # SPDX-License-Identifier: Apache-2.0 # SIPL capture, colour conversion, the hardware encoder and the LiveKit track output. -add_library(sensing_core STATIC +set(SENSING_COMMON_SOURCES sensing_camera.cpp - sipl_camera.cpp - jetson_encoder.cpp livekit_video_output.cpp yuv_to_rgba.cu rgba_to_nv12.cu rgba_box_downscale.cu - ${JETSON_MMAPI_SOURCES} +) + +if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON") + set(SENSING_BACKEND_SOURCES + sipl_camera.cpp + jetson_encoder.cpp + ${JETSON_MMAPI_SOURCES} + ) +else() + # yuv_to_rgba.cu includes cudaEGL.h, which needs the EGL headers even though + # nothing here opens an EGL display. + find_path(SENSING_EGL_INCLUDE_DIR EGL/egl.h) + if(NOT SENSING_EGL_INCLUDE_DIR) + message(FATAL_ERROR "The SENSING mock backend needs the EGL headers (cudaEGL.h includes them). " + "Install libegl-dev.") + endif() + # Parses the vendor JSON itself, so a MOCK build has no SIPL query library. + find_package(nlohmann_json 3 QUIET) + if(NOT nlohmann_json_FOUND) + include(FetchContent) + FetchContent_Declare( + nlohmann_json + GIT_REPOSITORY https://github.com/nlohmann/json.git + GIT_TAG v3.11.3 + GIT_SHALLOW TRUE + ) + set(JSON_BuildTests OFF CACHE INTERNAL "") + isaac_teleop_third_party_scope_begin() + FetchContent_MakeAvailable(nlohmann_json) + isaac_teleop_third_party_scope_end() + endif() + # Decodes the GIFs the mock camera can show. A single public-domain header, pinned to a commit and + # checked against its hash, so nothing else of the project is cloned. + include(FetchContent) + FetchContent_Declare( + stb_image + URL https://raw.githubusercontent.com/nothings/stb/003501f6d7a8668ce8b6d875e34dd97ad0d5d5b5/stb_image.h + URL_HASH SHA256=594c2fe35d49488b4382dbfaec8f98366defca819d916ac95becf3e75f4200b3 + DOWNLOAD_NO_EXTRACT TRUE + ) + FetchContent_MakeAvailable(stb_image) + + set(SENSING_BACKEND_SOURCES + mock_sipl_camera.cpp + mock_gif_source.cpp + mock_platform_config.cpp + mock_nv12_source.cu + mock_jetson_encoder.cpp + ) +endif() + +add_library(sensing_core STATIC + ${SENSING_COMMON_SOURCES} + ${SENSING_BACKEND_SOURCES} ) add_library(sensing::core ALIAS sensing_core) @@ -25,26 +76,28 @@ set_target_properties(sensing_core PROPERTIES target_include_directories(sensing_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/.. - PRIVATE - # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay private: they drag in - # NvMedia and X11-style macros. - ${SIPL_INCLUDE_DIRS} - ${EGL_INCLUDE_DIRS} - ${JETSON_MMAPI_DIR}/include ) -# The Multimedia API reference sources carry legacy warnings. -target_compile_options(sensing_core PRIVATE - $<$:-Wno-reorder -Wno-uninitialized -Wno-unused-variable - -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare> -) +target_link_libraries(sensing_core PUBLIC sensing::livekit) -target_link_libraries(sensing_core - PUBLIC - sensing::livekit - PRIVATE - ${SIPL_LIBRARIES} - ${EGL_LIBRARIES} - ${NVBUFSURFACE_LIBRARY} - ${V4L2_LIBRARY} -) +if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON") + # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay private: they drag in + # NvMedia and X11-style macros. + target_include_directories(sensing_core PRIVATE ${SIPL_INCLUDE_DIRS} ${EGL_INCLUDE_DIRS} ${JETSON_MMAPI_DIR}/include) + + # The Multimedia API reference sources carry legacy warnings. + target_compile_options(sensing_core PRIVATE + $<$:-Wno-reorder -Wno-uninitialized -Wno-unused-variable + -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare> + ) + + target_link_libraries(sensing_core PRIVATE ${SIPL_LIBRARIES} ${EGL_LIBRARIES} ${NVBUFSURFACE_LIBRARY} ${V4L2_LIBRARY}) +else() + # The mock encoder produces real H.264 with libavcodec's h264_nvenc (development package: + # libavcodec-dev; the desktop driver supplies NVENC). + find_package(PkgConfig REQUIRED) + pkg_check_modules(AVCODEC REQUIRED IMPORTED_TARGET libavcodec libavutil) + target_link_libraries(sensing_core PRIVATE nlohmann_json::nlohmann_json PkgConfig::AVCODEC) + target_include_directories(sensing_core SYSTEM PRIVATE ${stb_image_SOURCE_DIR}) + target_compile_definitions(sensing_core PRIVATE SENSING_MOCK_ASSET_DIR="${CMAKE_CURRENT_SOURCE_DIR}/../mock_assets") +endif() diff --git a/src/plugins/sensing/core/mock_gif_source.cpp b/src/plugins/sensing/core/mock_gif_source.cpp new file mode 100644 index 0000000000..3f574297de --- /dev/null +++ b/src/plugins/sensing/core/mock_gif_source.cpp @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "mock_gif_source.hpp" + +#include + +#define STBI_ONLY_GIF +#define STBI_NO_STDIO +#define STB_IMAGE_IMPLEMENTATION +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +constexpr int kMinDelayMs = 20; + +} // namespace + +std::vector gif_frame_ends(const std::vector& delays_ms) +{ + std::vector ends; + ends.reserve(delays_ms.size()); + double total = 0.0; + for (int delay : delays_ms) + { + total += std::max(delay, kMinDelayMs) / 1000.0; + ends.push_back(total); + } + return ends; +} + +size_t gif_frame_index(const std::vector& frame_ends, double seconds) +{ + if (frame_ends.empty()) + return 0; + const double t = std::fmod(std::max(seconds, 0.0), frame_ends.back()); + const auto it = std::upper_bound(frame_ends.begin(), frame_ends.end(), t); + return std::min(static_cast(it - frame_ends.begin()), frame_ends.size() - 1); +} + +MockGifSource::MockGifSource(const std::string& path) +{ + std::ifstream file(path, std::ios::binary); + if (!file) + throw std::runtime_error("mock GIF: cannot open " + path); + const std::vector bytes((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + + int width = 0; + int height = 0; + int frames = 0; + int channels = 0; + int* delays = nullptr; + unsigned char* pixels = stbi_load_gif_from_memory( + bytes.data(), static_cast(bytes.size()), &delays, &width, &height, &frames, &channels, 4); + if (!pixels) + throw std::runtime_error("mock GIF: cannot decode " + path + ": " + stbi_failure_reason()); + + // Composite onto the background, so the kernel only ever sees opaque pixels. + const size_t pixel_count = static_cast(width) * height * frames; + for (size_t i = 0; i < pixel_count; ++i) + { + unsigned char* px = pixels + i * 4; + const int alpha = px[3]; + for (int c = 0; c < 3; ++c) + px[c] = static_cast((px[c] * alpha + kBackground * (255 - alpha) + 127) / 255); + px[3] = 255; + } + + m_width = static_cast(width); + m_height = static_cast(height); + m_frame_ends = gif_frame_ends(std::vector(delays, delays + frames)); + + const size_t bytes_per_frame = static_cast(width) * height * 4; + const cudaError_t alloc = cudaMalloc(reinterpret_cast(&m_device_frames), bytes_per_frame * frames); + const cudaError_t copy = alloc == cudaSuccess ? + cudaMemcpy(m_device_frames, pixels, bytes_per_frame * frames, cudaMemcpyHostToDevice) : + alloc; + stbi_image_free(pixels); + stbi_image_free(delays); + if (copy != cudaSuccess) + { + cudaFree(m_device_frames); + throw std::runtime_error(std::string("mock GIF: cannot upload to the GPU: ") + cudaGetErrorString(copy)); + } +} + +MockGifSource::~MockGifSource() +{ + cudaFree(m_device_frames); +} + +const uint8_t* MockGifSource::frame(size_t index) const +{ + return m_device_frames + index * static_cast(m_width) * m_height * 4; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/mock_gif_source.hpp b/src/plugins/sensing/core/mock_gif_source.hpp new file mode 100644 index 0000000000..5a21edbcf2 --- /dev/null +++ b/src/plugins/sensing/core/mock_gif_source.hpp @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Mock backend only. An animated GIF decoded once and held in device memory, so the +// mock camera can show real moving images. + +#pragma once + +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/// Time in seconds at which each GIF frame stops showing: the running total of the frame delays, +/// each clamped to at least 20 ms, as browsers do for the tiny delays some encoders write. +std::vector gif_frame_ends(const std::vector& delays_ms); + +/// Index of the frame showing `seconds` into the animation, which loops over its whole length. +size_t gif_frame_index(const std::vector& frame_ends, double seconds); + +class MockGifSource +{ +public: + /// Transparent pixels are composited onto this gray, so an emoji or sticker without a + /// background reads as an image rather than as whatever colour sits under its transparency. + static constexpr uint8_t kBackground = 48; + + /// Decodes every frame to opaque RGBA8 and uploads them to the current CUDA device. + /// @throws std::runtime_error if the file cannot be read or is not a GIF. + explicit MockGifSource(const std::string& path); + ~MockGifSource(); + + MockGifSource(const MockGifSource&) = delete; + MockGifSource& operator=(const MockGifSource&) = delete; + + uint32_t width() const + { + return m_width; + } + uint32_t height() const + { + return m_height; + } + size_t frame_count() const + { + return m_frame_ends.size(); + } + /// Row stride of every frame, in bytes. Rows are tightly packed. + size_t pitch() const + { + return static_cast(m_width) * 4; + } + /// Device pointer to one RGBA8 frame. + const uint8_t* frame(size_t index) const; + size_t frame_index_at(double seconds) const + { + return gif_frame_index(m_frame_ends, seconds); + } + +private: + uint32_t m_width = 0; + uint32_t m_height = 0; + std::vector m_frame_ends; + uint8_t* m_device_frames = nullptr; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/mock_jetson_encoder.cpp b/src/plugins/sensing/core/mock_jetson_encoder.cpp new file mode 100644 index 0000000000..63bb0cbc36 --- /dev/null +++ b/src/plugins/sensing/core/mock_jetson_encoder.cpp @@ -0,0 +1,424 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Mock backend for JetsonEncoder: real H.264 from the desktop GPU's NVENC, through libavcodec's +// h264_nvenc, set up as the Jetson V4L2 encoder is (High profile, Level 5.2, VBR with a peak, an IDR +// every GOP with SPS/PPS in front of it, no B-frames). It runs the real RGBA-to-NV12 kernel and the same +// device-to-host copy as the V4L2 path, then a worker thread feeds the NV12 buffer to the encoder. +// +// Reproduced from the real encoder: a fixed pool of input buffers, so submit() returns false when the +// encoder falls behind rather than blocking; the capture stamp coming back at microsecond resolution (a +// V4L2 timeval); end_of_stream() returning the remaining tail as one concatenated unit. NVENC's own +// latency and unit sizes are what they are, and are not those of the Jetson engine. + +#include "jetson_encoder.hpp" +#include "rgba_to_nv12.cuh" + +extern "C" +{ +#include +#include +#include +#include +} + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +constexpr uint32_t kInputBuffers = 6; + +void check_cuda(cudaError_t err, const char* what) +{ + if (err != cudaSuccess) + throw std::runtime_error(std::string("mock JetsonEncoder: ") + what + ": " + cudaGetErrorString(err)); +} + +void check_av(int err, const char* what) +{ + if (err < 0) + { + char text[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(err, text, sizeof(text)); + throw std::runtime_error(std::string("mock JetsonEncoder: ") + what + ": " + text); + } +} + +} // namespace + +struct JetsonEncoder::Impl +{ + struct Job + { + uint32_t buffer = 0; + int64_t timestamp_ns = 0; + bool keyframe = false; + }; + + explicit Impl(const EncoderConfig& cfg) : config(cfg) + { + codec = avcodec_find_encoder_by_name("h264_nvenc"); + if (!codec) + throw std::runtime_error("mock JetsonEncoder: libavcodec has no h264_nvenc encoder"); + + // A constructor that throws never runs the destructor, so free what was allocated here. + try + { + open_context(config.bitrate_bps); + alloc_frame(); + check_cuda(cudaMallocPitch(reinterpret_cast(&d_y), &d_y_pitch, config.width, config.height), + "cudaMallocPitch(Y)"); + check_cuda(cudaMallocPitch(reinterpret_cast(&d_uv), &d_uv_pitch, config.width, config.height / 2), + "cudaMallocPitch(UV)"); + + // Host staging, standing in for the V4L2 output plane's mapped buffers. + const size_t nv12_bytes = static_cast(config.width) * config.height * 3 / 2; + for (uint32_t i = 0; i < kInputBuffers; ++i) + { + host_buffers.emplace_back(nv12_bytes); + free_buffers.push_back(i); + } + worker = std::thread([this] { run(); }); + } + catch (...) + { + release(); + throw; + } + } + + ~Impl() + { + { + std::lock_guard lock(mutex); + closing = true; + } + wake.notify_all(); + if (worker.joinable()) + worker.join(); + release(); + } + + void set_option(const char* name, const char* value) + { + check_av(av_opt_set(context->priv_data, name, value, 0), (std::string("av_opt_set ") + name).c_str()); + } + + // A fresh encoder session at `bitrate`: its first frame is an IDR with parameter sets. + void open_context(uint32_t bitrate) + { + avcodec_free_context(&context); + context = avcodec_alloc_context3(codec); + if (!context) + throw std::runtime_error("mock JetsonEncoder: cannot allocate the encoder context"); + + const uint32_t fps = config.fps ? config.fps : 30; + const uint32_t gop = config.gop ? config.gop : fps * 5; + context->width = static_cast(config.width); + context->height = static_cast(config.height); + context->pix_fmt = AV_PIX_FMT_NV12; + // Microseconds, as a V4L2 timeval carries them. + context->time_base = { 1, 1'000'000 }; + context->framerate = { static_cast(fps), 1 }; + context->bit_rate = bitrate; + context->gop_size = static_cast(gop); + context->max_b_frames = 0; + // Parameter sets stay in the stream, in front of every IDR, so a receiver can join mid-stream. + context->flags &= ~AV_CODEC_FLAG_GLOBAL_HEADER; + set_option("profile", "high"); + set_option("level", "5.2"); + set_option("forced-idr", "1"); + set_option("zerolatency", "1"); + // Hand each unit back as soon as it is encoded rather than after a queue of frames. + set_option("delay", "0"); + if (config.peak_bitrate_bps > config.bitrate_bps) + { + // The configured ratio between the ceiling and the target holds at any target. + context->rc_max_rate = static_cast(bitrate) * config.peak_bitrate_bps / config.bitrate_bps; + context->rc_buffer_size = static_cast(context->rc_max_rate); + set_option("rc", "vbr"); + } + else + { + set_option("rc", "cbr"); + } + check_av(avcodec_open2(context, codec, nullptr), "cannot open h264_nvenc"); + applied_bitrate = bitrate; + } + + void alloc_frame() + { + frame = av_frame_alloc(); + packet = av_packet_alloc(); + if (!frame || !packet) + throw std::runtime_error("mock JetsonEncoder: cannot allocate a frame or packet"); + frame->format = AV_PIX_FMT_NV12; + frame->width = context->width; + frame->height = context->height; + check_av(av_frame_get_buffer(frame, 0), "cannot allocate the input frame"); + } + + void release() + { + cudaFree(d_y); + cudaFree(d_uv); + av_packet_free(&packet); + av_frame_free(&frame); + avcodec_free_context(&context); + } + + // Moves every packet the encoder has ready into `out`. Worker thread, or end_of_stream() once the + // worker is idle. + void collect(std::deque& out) + { + for (;;) + { + const int err = avcodec_receive_packet(context, packet); + if (err == AVERROR(EAGAIN) || err == AVERROR_EOF) + return; + check_av(err, "avcodec_receive_packet"); + EncodedUnit unit; + unit.data.assign(packet->data, packet->data + packet->size); + unit.timestamp_ns = packet->pts * 1000; + unit.keyframe = (packet->flags & AV_PKT_FLAG_KEY) != 0; + av_packet_unref(packet); + out.push_back(std::move(unit)); + } + } + + // Worker thread, which owns the codec. The libavcodec NVENC wrapper takes its rate control at + // open, so a new target drains the running session into `out` and opens another at that rate: the + // stream continues from an IDR with parameter sets. + void apply_bitrate(std::deque& out) + { + uint32_t target = 0; + { + std::lock_guard lock(mutex); + target = pending_bitrate; + pending_bitrate = 0; + } + if (target == 0 || target == applied_bitrate) + return; + check_av(avcodec_send_frame(context, nullptr), "avcodec_send_frame(flush)"); + collect(out); + open_context(target); + } + + void encode(const Job& job) + { + check_av(av_frame_make_writable(frame), "av_frame_make_writable"); + const uint8_t* nv12 = host_buffers[job.buffer].data(); + const size_t width = config.width; + for (uint32_t row = 0; row < config.height; ++row) + std::memcpy(frame->data[0] + static_cast(row) * frame->linesize[0], nv12 + row * width, width); + const uint8_t* uv = nv12 + width * config.height; + for (uint32_t row = 0; row < config.height / 2; ++row) + std::memcpy(frame->data[1] + static_cast(row) * frame->linesize[1], uv + row * width, width); + frame->pts = job.timestamp_ns / 1000; + // With forced-idr, an I picture is an IDR with its parameter sets. + frame->pict_type = job.keyframe ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE; + check_av(avcodec_send_frame(context, frame), "avcodec_send_frame"); + } + + void run() + { + for (;;) + { + Job job; + { + std::unique_lock lock(mutex); + wake.wait(lock, [&] { return closing || !jobs.empty(); }); + // Frames still queued at teardown are abandoned, as the hardware encoder would. + if (closing || jobs.empty()) + return; + job = jobs.front(); + jobs.pop_front(); + busy = true; + } + + std::deque produced; + std::string error; + try + { + apply_bitrate(produced); + encode(job); + collect(produced); + } + catch (const std::exception& e) + { + error = e.what(); + } + + { + std::lock_guard lock(mutex); + free_buffers.push_back(job.buffer); + for (auto& unit : produced) + ready.push_back(std::move(unit)); + if (!error.empty() && failure.empty()) + failure = error; + busy = false; + } + idle.notify_all(); + } + } + + EncoderConfig config; + + const AVCodec* codec = nullptr; + AVCodecContext* context = nullptr; + AVFrame* frame = nullptr; + AVPacket* packet = nullptr; + + uint8_t* d_y = nullptr; + uint8_t* d_uv = nullptr; + size_t d_y_pitch = 0; + size_t d_uv_pitch = 0; + + std::vector> host_buffers; + + std::mutex mutex; + std::condition_variable wake; + std::condition_variable idle; + std::deque free_buffers; + std::deque jobs; + std::deque ready; // what poll() hands out + std::string failure; + bool busy = false; + bool closing = false; + bool flushed = false; + bool keyframe_requested = false; // guarded by `mutex` + uint32_t pending_bitrate = 0; // guarded by `mutex`; 0 when nothing is waiting + uint32_t applied_bitrate = 0; // worker thread only + std::thread worker; +}; + +JetsonEncoder::JetsonEncoder(const EncoderConfig& config) +{ + if (config.width == 0 || config.height == 0) + throw std::runtime_error("JetsonEncoder: width/height must be non-zero"); + if ((config.width % 2) != 0 || (config.height % 2) != 0) + throw std::runtime_error("JetsonEncoder: width/height must be even for NV12"); + m_impl = std::make_unique(config); +} + +JetsonEncoder::~JetsonEncoder() = default; + +bool JetsonEncoder::submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns) +{ + auto& impl = *m_impl; + const auto& config = impl.config; + + uint32_t buffer = 0; + { + std::lock_guard lock(impl.mutex); + if (!impl.failure.empty()) + throw std::runtime_error(impl.failure); + if (impl.free_buffers.empty()) + return false; // encoder still busy; drop this frame rather than block + buffer = impl.free_buffers.front(); + impl.free_buffers.pop_front(); + } + + launch_rgba_to_nv12(reinterpret_cast(rgba_device_ptr), static_cast(row_pitch_bytes), impl.d_y, + static_cast(impl.d_y_pitch), impl.d_uv, static_cast(impl.d_uv_pitch), + static_cast(config.width), static_cast(config.height), config.full_range, nullptr); + check_cuda(cudaGetLastError(), "rgba_to_nv12 launch"); + check_cuda(cudaStreamSynchronize(nullptr), "rgba_to_nv12 sync"); + + uint8_t* host = impl.host_buffers[buffer].data(); + check_cuda( + cudaMemcpy2D(host, config.width, impl.d_y, impl.d_y_pitch, config.width, config.height, cudaMemcpyDeviceToHost), + "cudaMemcpy2D(Y)"); + check_cuda(cudaMemcpy2D(host + static_cast(config.width) * config.height, config.width, impl.d_uv, + impl.d_uv_pitch, config.width, config.height / 2, cudaMemcpyDeviceToHost), + "cudaMemcpy2D(UV)"); + + { + std::lock_guard lock(impl.mutex); + impl.jobs.push_back({ buffer, timestamp_ns, impl.keyframe_requested }); + impl.keyframe_requested = false; + } + impl.wake.notify_one(); + return true; +} + +EncodedUnit JetsonEncoder::poll() +{ + std::lock_guard lock(m_impl->mutex); + if (!m_impl->failure.empty()) + throw std::runtime_error(m_impl->failure); + if (m_impl->ready.empty()) + return {}; + auto out = std::move(m_impl->ready.front()); + m_impl->ready.pop_front(); + return out; +} + +std::vector JetsonEncoder::drain() +{ + auto& impl = *m_impl; + std::unique_lock lock(impl.mutex); + impl.idle.wait(lock, [&] { return impl.jobs.empty() && !impl.busy; }); + if (!impl.failure.empty()) + throw std::runtime_error(impl.failure); + if (!impl.flushed) + { + impl.flushed = true; + check_av(avcodec_send_frame(impl.context, nullptr), "avcodec_send_frame(flush)"); + impl.collect(impl.ready); + } + std::vector units( + std::make_move_iterator(impl.ready.begin()), std::make_move_iterator(impl.ready.end())); + impl.ready.clear(); + return units; +} + +EncodedUnit JetsonEncoder::end_of_stream() +{ + // The tail is concatenated, so it carries the stamp of the last unit in it. + EncodedUnit out; + for (auto& chunk : drain()) + { + out.data.insert(out.data.end(), chunk.data.begin(), chunk.data.end()); + out.timestamp_ns = chunk.timestamp_ns; + } + return out; +} + +void JetsonEncoder::request_keyframe() +{ + std::lock_guard lock(m_impl->mutex); + m_impl->keyframe_requested = true; +} + +bool JetsonEncoder::set_bitrate(uint32_t bitrate_bps) +{ + if (bitrate_bps == 0) + return false; + std::lock_guard lock(m_impl->mutex); + m_impl->pending_bitrate = bitrate_bps; + return true; +} + +const char* JetsonEncoder::backend_name() +{ + return "NVENC (libavcodec h264_nvenc)"; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/mock_nv12_source.cu b/src/plugins/sensing/core/mock_nv12_source.cu new file mode 100644 index 0000000000..188f1c5113 --- /dev/null +++ b/src/plugins/sensing/core/mock_nv12_source.cu @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "mock_nv12_source.cuh" + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +// Geometry of the cover resize: the source scaled uniformly until it fills the target, then +// centre-cropped. `scaled_*` is the size of the scaled image and `crop_*` the offset of the +// visible window inside it. +struct CoverGeometry +{ + int src_width; + int src_height; + int scaled_width; + int scaled_height; + int crop_x; + int crop_y; +}; + +__device__ float3 fetch_rgb(const uint8_t* src, int pitch, int x, int y) +{ + const uint8_t* px = src + static_cast(y) * pitch + static_cast(x) * 4; + return make_float3(px[0], px[1], px[2]); +} + +// Colour of one pixel of the cropped output. Enlarging samples bilinearly; reducing averages the +// source area the pixel covers, weighted by overlap, as a box filter would. +__device__ float3 sample_cover(const uint8_t* src, int pitch, const CoverGeometry& g, int x, int y) +{ + const float sx = static_cast(g.src_width) / g.scaled_width; + const float sy = static_cast(g.src_height) / g.scaled_height; + const int X = min(max(x + g.crop_x, 0), g.scaled_width - 1); + const int Y = min(max(y + g.crop_y, 0), g.scaled_height - 1); + + if (sx <= 1.0f && sy <= 1.0f) + { + const float fx = fminf(fmaxf((X + 0.5f) * sx - 0.5f, 0.0f), g.src_width - 1.0f); + const float fy = fminf(fmaxf((Y + 0.5f) * sy - 0.5f, 0.0f), g.src_height - 1.0f); + const int x0 = static_cast(fx); + const int y0 = static_cast(fy); + const int x1 = min(x0 + 1, g.src_width - 1); + const int y1 = min(y0 + 1, g.src_height - 1); + const float tx = fx - x0; + const float ty = fy - y0; + const float3 a = fetch_rgb(src, pitch, x0, y0); + const float3 b = fetch_rgb(src, pitch, x1, y0); + const float3 c = fetch_rgb(src, pitch, x0, y1); + const float3 d = fetch_rgb(src, pitch, x1, y1); + return make_float3((a.x * (1 - tx) + b.x * tx) * (1 - ty) + (c.x * (1 - tx) + d.x * tx) * ty, + (a.y * (1 - tx) + b.y * tx) * (1 - ty) + (c.y * (1 - tx) + d.y * tx) * ty, + (a.z * (1 - tx) + b.z * tx) * (1 - ty) + (c.z * (1 - tx) + d.z * tx) * ty); + } + + const float left = X * sx; + const float right = (X + 1) * sx; + const float top = Y * sy; + const float bottom = (Y + 1) * sy; + float3 sum = make_float3(0.0f, 0.0f, 0.0f); + float weight = 0.0f; + for (int j = static_cast(top); j < min(static_cast(ceilf(bottom)), g.src_height); ++j) + { + const float wy = fminf(bottom, j + 1.0f) - fmaxf(top, static_cast(j)); + for (int i = static_cast(left); i < min(static_cast(ceilf(right)), g.src_width); ++i) + { + const float w = wy * (fminf(right, i + 1.0f) - fmaxf(left, static_cast(i))); + const float3 rgb = fetch_rgb(src, pitch, i, j); + sum.x += rgb.x * w; + sum.y += rgb.y * w; + sum.z += rgb.z * w; + weight += w; + } + } + const float inv = weight > 0.0f ? 1.0f / weight : 0.0f; + return make_float3(sum.x * inv, sum.y * inv, sum.z * inv); +} + +__device__ float luma_709(const float3& rgb) +{ + return fminf(fmaxf(0.2126f * rgb.x + 0.7152f * rgb.y + 0.0722f * rgb.z, 0.0f), 255.0f); +} + +// One thread per 2x2 block: four luma samples and one chroma pair. +__global__ void paint_gif(const uint8_t* src, + int src_pitch, + CoverGeometry geometry, + uint8_t* y_plane, + int y_pitch, + uint8_t* uv_plane, + int uv_pitch, + int width, + int height, + bool mirror, + int shift_x) +{ + const int cx = blockIdx.x * blockDim.x + threadIdx.x; + const int cy = blockIdx.y * blockDim.y + threadIdx.y; + if (cx >= width / 2 || cy >= height / 2) + return; + + float3 mean = make_float3(0.0f, 0.0f, 0.0f); + for (int dy = 0; dy < 2; ++dy) + { + for (int dx = 0; dx < 2; ++dx) + { + const int x = 2 * cx + dx; + const int y = 2 * cy + dy; + // A mirrored view samples the picture right to left; the output pixel stays where it is. + const int sx = (mirror ? width - 1 - x : x) - shift_x; + const float3 rgb = sample_cover(src, src_pitch, geometry, sx, y); + y_plane[static_cast(y) * y_pitch + x] = static_cast(luma_709(rgb) + 0.5f); + mean.x += 0.25f * rgb.x; + mean.y += 0.25f * rgb.y; + mean.z += 0.25f * rgb.z; + } + } + + // Inverse of the full-range BT.709 matrix the real conversion applies. + const float luma = luma_709(mean); + const float cb = fminf(fmaxf(128.0f + (mean.z - luma) / 1.8556f, 0.0f), 255.0f); + const float cr = fminf(fmaxf(128.0f + (mean.x - luma) / 1.5748f, 0.0f), 255.0f); + uint8_t* sample = uv_plane + static_cast(cy) * uv_pitch + static_cast(cx) * 2; + sample[0] = static_cast(cb + 0.5f); + sample[1] = static_cast(cr + 0.5f); +} + +} // namespace + +void launch_mock_gif_nv12(const uint8_t* src_rgba, + int src_pitch, + int src_width, + int src_height, + uint8_t* y_plane, + int y_pitch, + uint8_t* uv_plane, + int uv_pitch, + int width, + int height, + bool mirror, + int shift_x, + cudaStream_t stream) +{ + const float scale = fmaxf(static_cast(width) / src_width, static_cast(height) / src_height); + CoverGeometry geometry{}; + geometry.src_width = src_width; + geometry.src_height = src_height; + geometry.scaled_width = max(width, static_cast(lrintf(src_width * scale))); + geometry.scaled_height = max(height, static_cast(lrintf(src_height * scale))); + geometry.crop_x = (geometry.scaled_width - width) / 2; + geometry.crop_y = (geometry.scaled_height - height) / 2; + + const dim3 block(16, 16); + const dim3 grid((width / 2 + block.x - 1) / block.x, (height / 2 + block.y - 1) / block.y); + paint_gif<<>>( + src_rgba, src_pitch, geometry, y_plane, y_pitch, uv_plane, uv_pitch, width, height, mirror, shift_x); +} + + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/mock_nv12_source.cuh b/src/plugins/sensing/core/mock_nv12_source.cuh new file mode 100644 index 0000000000..1a79a37a58 --- /dev/null +++ b/src/plugins/sensing/core/mock_nv12_source.cuh @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/** + * @brief Paint one synthetic ISP0 frame from a GIF frame already in device memory. + * + * The source is scaled to cover the full width x height and centre-cropped, so its aspect ratio is + * never distorted: a uniform scale of max(width / src_width, height / src_height), bilinear when + * enlarging and area-averaged when reducing. The result is full-range BT.709 NV12, the inverse of + * the matrix the real conversion applies, so the colours come back out of that conversion. + * + * @param src_rgba Device pointer to the GIF frame, src_width x src_height RGBA8. + * @param src_pitch Source row stride in bytes. + * @param mirror Show the picture flipped left to right. + * @param shift_x Pixels the picture moves right after any flip; the edge pixel repeats into the gap. + */ +void launch_mock_gif_nv12(const uint8_t* src_rgba, + int src_pitch, + int src_width, + int src_height, + uint8_t* y_plane, + int y_pitch, + uint8_t* uv_plane, + int uv_pitch, + int width, + int height, + bool mirror, + int shift_x, + cudaStream_t stream); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/mock_platform_config.cpp b/src/plugins/sensing/core/mock_platform_config.cpp new file mode 100644 index 0000000000..bf9cfd24d8 --- /dev/null +++ b/src/plugins/sensing/core/mock_platform_config.cpp @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "mock_platform_config.hpp" + +#include + +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +uint32_t scale_divisor() +{ + const char* value = std::getenv("SENSING_MOCK_SCALE_DIV"); + if (!value) + return 1; + const long parsed = std::strtol(value, nullptr, 10); + return parsed > 1 ? static_cast(parsed) : 1; +} + +uint32_t scaled(uint32_t extent, uint32_t divisor) +{ + return std::max(2, (extent / divisor) & ~1u); +} + +} // namespace + +std::vector mock_query_platform_config(const std::string& platform_config_json, + const std::string& platform_config_name, + const std::vector& link_masks) +{ + std::ifstream file(platform_config_json); + if (!file) + throw std::runtime_error("ParseJsonFile failed: cannot open " + platform_config_json); + + nlohmann::json root; + try + { + root = nlohmann::json::parse(file); + } + catch (const nlohmann::json::exception& e) + { + throw std::runtime_error("ParseJsonFile failed: " + std::string(e.what())); + } + + const auto& configs = root.at("cameraConfigs"); + auto find_config = [&](const std::string& name) -> const nlohmann::json* + { + for (const auto& config : configs) + { + if (config.value("name", "") == name) + return &config; + } + return nullptr; + }; + + const auto* platform = find_config(platform_config_name); + if (!platform) + throw std::runtime_error("GetSensorSystemConfig failed: no camera config named '" + platform_config_name + "'"); + + const uint32_t divisor = scale_divisor(); + std::vector sensors; + for (const auto& module : platform->at("cameraModules")) + { + const uint32_t link = module.at("linkIndex"); + const uint32_t transport = module.value("transportIndex", 0u); + if (!link_masks.empty()) + { + // A mask is eight hex digits, one per link; a link beyond that cannot be enabled. + const bool enabled = + link < 8 && transport < link_masks.size() && ((link_masks[transport] >> (4 * link)) & 0xF) != 0; + if (!enabled) + continue; + } + + const std::string module_name = module.at("name"); + const auto* module_config = find_config(module_name); + if (!module_config) + throw std::runtime_error("GetSensorSystemConfig failed: no camera config named '" + module_name + "'"); + + for (const auto& sensor : module_config->at("sensorInfo")) + { + SensorInfo info; + info.id = static_cast(sensors.size()); + info.name = module_name; + const auto& channels = sensor.at("virtualChannels"); + if (!channels.empty()) + { + info.width = scaled(channels.front().at("width"), divisor); + info.height = scaled(channels.front().at("height"), divisor); + info.fps = channels.front().at("fps"); + } + sensors.push_back(std::move(info)); + } + } + return sensors; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/mock_platform_config.hpp b/src/plugins/sensing/core/mock_platform_config.hpp new file mode 100644 index 0000000000..5c0ac44d52 --- /dev/null +++ b/src/plugins/sensing/core/mock_platform_config.hpp @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Mock backend only. Answers SiplCamera::query from the vendor platform JSON +// itself, so the pipeline-index renumbering and link-mask behaviour can be +// exercised with no SIPL library. + +#pragma once + +#include "sipl_camera.hpp" + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/** + * @brief Resolve a platform config to its sensors, as INvSIPLCameraQuery would. + * + * Module `m` of the named config is enabled when its transport's mask has a + * non-zero hex digit at the module's link index, i.e. + * `(masks[transportIndex] >> (4 * linkIndex)) & 0xF`. An empty mask list keeps + * every module. `SensorInfo::id` is the position among the enabled sensors, not + * the JSON sensorInfo.id. + * + * SENSING_MOCK_SCALE_DIV=N divides the reported width and height (kept even), + * which keeps tests fast without touching the vendor JSON. + * + * @throws std::runtime_error when the file or the named config is missing. + */ +std::vector mock_query_platform_config(const std::string& platform_config_json, + const std::string& platform_config_name, + const std::vector& link_masks); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/mock_sipl_camera.cpp b/src/plugins/sensing/core/mock_sipl_camera.cpp new file mode 100644 index 0000000000..491b2c1911 --- /dev/null +++ b/src/plugins/sensing/core/mock_sipl_camera.cpp @@ -0,0 +1,417 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Mock backend for SiplCamera: frames from an animated GIF, no SIPL, NvSci or EGL. +// It keeps the contract the rest of the plugin relies on -- per-sensor capture +// threads, a three-slot RGBA mailbox with a reader lease, a TSC stamp shared by +// every sensor, and failures surfacing from latest() -- and runs the real +// YUV-to-RGBA kernel on a synthetic NV12 surface made from the GIF, so the +// conversion code is exercised too. +// +// What a sensor shows is fixed by its module name, never by pipeline index (which the link mask +// renumbers): +// S56C_HEAD the robot; its two sensors are the left and right eye, offset against each other +// SHF3L_LEFT the waving hand +// SHF3L_RIGHT the waving hand, flipped left to right +// SHW5G_MIX_L2 / SHW5G_MIX_L3 as SHF3L_LEFT / SHF3L_RIGHT +// A module not listed is an error, and so is a GIF that cannot be loaded. The picture is scaled to cover +// the sensor's configured size and centre-cropped, never stretched, and follows the capture clock: it +// advances with each frame's time, and the sequence number and stamps advance on every capture even +// while the GIF shows one image. +// +// Other environment (read once, at construction): +// SENSING_MOCK_FPS=N override the platform config's frame rate +// SENSING_MOCK_FAIL_AFTER=N every sensor fails after N frames, as a link drop would +// SENSING_MOCK_SCALE_DIV=N shrink the reported geometry (see mock_platform_config.hpp) + +#include "mock_gif_source.hpp" +#include "mock_nv12_source.cuh" +#include "mock_platform_config.hpp" +#include "mock_test_hooks.hpp" +#include "sipl_camera.hpp" +#include "yuv_to_rgba.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +// Arbitrary offset so nothing can mistake the TSC for CLOCK_MONOTONIC. +constexpr uint64_t kMockTscBaseNs = 1'000'000'000'000'000ull; + +// Total horizontal offset between the two eyes, as a fraction of the width. Illustrative, not calibrated. +constexpr double kEyeDisparity = 0.02; + +uint64_t monotonic_ns() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +double env_number(const char* name, double fallback) +{ + const char* value = std::getenv(name); + return value ? std::strtod(value, nullptr) : fallback; +} + +void check_cuda(cudaError_t err, const char* what) +{ + if (err != cudaSuccess) + throw std::runtime_error(std::string("mock SIPL: ") + what + ": " + cudaGetErrorString(err)); +} + +std::string g_test_picture; + +} // namespace + +void mock_set_test_picture(const std::string& gif_path) +{ + g_test_picture = gif_path; +} + +struct SiplCamera::Impl +{ + struct DeviceBuffer + { + uint8_t* ptr = nullptr; + size_t pitch = 0; + }; + + struct Sensor + { + SensorInfo info; + /// Synthetic ISP0 surface, in place of the NvSciBuf SIPL would fill. + DeviceBuffer luma; + DeviceBuffer chroma; + cudaStream_t stream = nullptr; + + /// Triple-buffered RGBA8 mailbox. A reader leases the published slot + /// until its next latest() call, so the producer never overwrites it. + std::array buffers{}; + mutable std::mutex publish_mutex; + int publish_idx = -1; + int lease_idx = -1; + uint64_t published_sequence = 0; + uint64_t consumed_sequence = 0; + uint64_t published_timestamp_ns = 0; + uint64_t published_capture_tsc_ns = 0; + + std::shared_ptr gif; + bool mirror = false; + int shift_x = 0; + + std::thread frame_thread; + + // Owns its device memory, so a constructor that throws part-way through + // allocating does not leak what it had already allocated. + ~Sensor() + { + cudaFree(luma.ptr); + cudaFree(chroma.ptr); + for (auto& buffer : buffers) + cudaFree(buffer.ptr); + if (stream) + cudaStreamDestroy(stream); + } + }; + + explicit Impl(const SiplConfig& cfg) : config(cfg) + { + if (config.link_masks.empty()) + throw std::runtime_error("link masks are required for a GMSL platform config"); + + sensor_infos = + mock_query_platform_config(config.platform_config_json, config.platform_config_name, config.link_masks); + if (sensor_infos.empty()) + { + throw std::runtime_error("no modules left in '" + config.platform_config_name + + "' after applying the link masks -- check the mask against the config"); + } + + fps_override = env_number("SENSING_MOCK_FPS", 0.0); + fail_after_frames = static_cast(env_number("SENSING_MOCK_FAIL_AFTER", 0.0)); + + check_cuda(cudaSetDevice(config.gpu_id), "cudaSetDevice"); + for (const auto& info : sensor_infos) + { + auto sensor = std::make_unique(); + sensor->info = info; + allocate(*sensor); + choose_picture(info, *sensor); + sensors.push_back(std::move(sensor)); + } + } + + ~Impl() + { + stop(); + } + + void allocate(Sensor& sensor) + { + const size_t width = sensor.info.width; + const size_t height = sensor.info.height; + check_cuda(cudaMallocPitch(reinterpret_cast(&sensor.luma.ptr), &sensor.luma.pitch, width, height), + "cudaMallocPitch(Y)"); + check_cuda(cudaMallocPitch(reinterpret_cast(&sensor.chroma.ptr), &sensor.chroma.pitch, width, height / 2), + "cudaMallocPitch(UV)"); + for (auto& buffer : sensor.buffers) + { + check_cuda(cudaMallocPitch(reinterpret_cast(&buffer.ptr), &buffer.pitch, width * 4, height), + "cudaMallocPitch(RGBA)"); + } + check_cuda(cudaStreamCreateWithFlags(&sensor.stream, cudaStreamNonBlocking), "cudaStreamCreate"); + } + + /// Sets the GIF a sensor shows, whether it is flipped and how far its picture is offset. + void choose_picture(const SensorInfo& info, Sensor& sensor) + { + struct Look + { + const char* module; + const char* gif; + bool mirror; + bool stereo; + }; + static constexpr Look kLooks[] = { + { "S56C_HEAD", "noto-robot.gif", false, true }, + { "SHF3L_LEFT", "noto-waving-hand.gif", false, false }, + { "SHF3L_RIGHT", "noto-waving-hand.gif", true, false }, + { "SHW5G_MIX_L2", "noto-waving-hand.gif", false, false }, + { "SHW5G_MIX_L3", "noto-waving-hand.gif", true, false }, + }; + const Look* look = nullptr; + for (const Look& candidate : kLooks) + look = info.name == candidate.module ? &candidate : look; + + std::string path = g_test_picture; + if (path.empty()) + { + if (!look) + throw std::runtime_error("no mock picture for module '" + info.name + "'"); + path = std::string(SENSING_MOCK_ASSET_DIR) + "/" + look->gif; + sensor.mirror = look->mirror; + if (look->stereo) + { + // The sensors of one module are its left and right eyes, in that order. The left eye sees + // the scene further right, so its picture moves right and the right eye's moves left. + size_t eye = 0; + for (const SensorInfo& other : sensor_infos) + eye += other.name == info.name && other.id < info.id ? 1 : 0; + const int half = static_cast(info.width * kEyeDisparity / 2); + sensor.shift_x = eye == 0 ? half : -half; + } + } + + // Sensors showing the same file share one copy on the device. + auto& slot = gif_cache[path]; + if (!slot) + slot = std::make_shared(path); + sensor.gif = slot; + } + + Sensor* find(uint32_t id) + { + for (auto& sensor : sensors) + { + if (sensor->info.id == id) + return sensor.get(); + } + return nullptr; + } + + void set_failure(const std::string& message) + { + std::lock_guard guard(failure_mutex); + if (failure.empty()) + failure = message; + } + + void throw_if_failed() + { + std::lock_guard guard(failure_mutex); + if (!failure.empty()) + throw std::runtime_error(failure); + } + + uint32_t pick_write_index(const Sensor& sensor) const + { + std::lock_guard guard(sensor.publish_mutex); + if (sensor.publish_idx < 0) + return 0; + // Skip the published slot and any slot still leased to a reader. With + // three slots one is always free, so this cannot spin. + uint32_t idx = static_cast((sensor.publish_idx + 1) % 3); + if (static_cast(idx) == sensor.lease_idx) + idx = (idx + 1) % 3; + return idx; + } + + void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t capture_tsc_ns) + { + std::lock_guard guard(sensor.publish_mutex); + sensor.publish_idx = static_cast(write_idx); + sensor.published_timestamp_ns = timestamp_ns; + sensor.published_capture_tsc_ns = capture_tsc_ns; + ++sensor.published_sequence; + } + + void capture_one(Sensor& sensor, double seconds, uint64_t tsc_ns) + { + const uint32_t write_idx = pick_write_index(sensor); + const DeviceBuffer& dest = sensor.buffers[write_idx]; + const int width = static_cast(sensor.info.width); + const int height = static_cast(sensor.info.height); + + launch_mock_gif_nv12( + sensor.gif->frame(sensor.gif->frame_index_at(seconds)), static_cast(sensor.gif->pitch()), + static_cast(sensor.gif->width()), static_cast(sensor.gif->height()), sensor.luma.ptr, + static_cast(sensor.luma.pitch), sensor.chroma.ptr, static_cast(sensor.chroma.pitch), width, + height, sensor.mirror, sensor.shift_x, sensor.stream); + + // Same call, layout and colour matrix as the SIPL path: full-range + // BT.709 semi-planar, from the surface the ISP would have written. + const YuvLayout layout = config.swap_uv ? YuvLayout::YVU420SemiPlanar : YuvLayout::YUV420SemiPlanar; + launch_yuv420_pitch_to_rgba(sensor.luma.ptr, sensor.chroma.ptr, nullptr, static_cast(sensor.luma.pitch), + static_cast(sensor.chroma.pitch), 0, width, height, dest.ptr, + static_cast(dest.pitch), layout, true, true, sensor.stream); + check_cuda(cudaGetLastError(), "mock frame kernels"); + check_cuda(cudaStreamSynchronize(sensor.stream), "cudaStreamSynchronize"); + + publish(sensor, write_idx, monotonic_ns(), tsc_ns); + } + + void frame_loop(Sensor& sensor, std::chrono::steady_clock::time_point epoch) + { + try + { + check_cuda(cudaSetDevice(config.gpu_id), "cudaSetDevice frame_loop"); + const double fps = fps_override > 0.0 ? fps_override : std::max(sensor.info.fps, 1.0); + const double period_ns = 1e9 / fps; + for (uint64_t n = 0; running.load(); ++n) + { + std::this_thread::sleep_until(epoch + std::chrono::nanoseconds(static_cast(n * period_ns))); + if (!running.load()) + break; + if (fail_after_frames != 0 && n >= fail_after_frames) + throw std::runtime_error("injected link failure (SENSING_MOCK_FAIL_AFTER)"); + // Every sensor derives its TSC from the same epoch, as the + // deserializer's shared fsync does: frame n of each sensor + // carries the same capture time. + capture_one(sensor, n * period_ns / 1e9, kMockTscBaseNs + static_cast(n * period_ns)); + } + } + catch (const std::exception& e) + { + std::ostringstream oss; + oss << "SIPL sensor " << sensor.info.id << " capture error: " << e.what(); + std::cerr << "[sipl-mock] " << oss.str() << std::endl; + set_failure(oss.str()); + running.store(false); + } + } + + void start() + { + if (running.exchange(true)) + return; + const auto epoch = std::chrono::steady_clock::now(); + for (auto& sensor : sensors) + sensor->frame_thread = std::thread([this, s = sensor.get(), epoch] { frame_loop(*s, epoch); }); + } + + void stop() + { + running.store(false); + for (auto& sensor : sensors) + { + if (sensor->frame_thread.joinable()) + sensor->frame_thread.join(); + } + } + + SiplConfig config; + std::vector sensor_infos; + std::vector> sensors; + std::atomic running{ false }; + std::map> gif_cache; + double fps_override = 0.0; + uint64_t fail_after_frames = 0; + std::mutex failure_mutex; + std::string failure; +}; + +SiplCamera::SiplCamera(const SiplConfig& config) : m_impl(std::make_unique(config)) +{ +} + +SiplCamera::~SiplCamera() = default; + +std::vector SiplCamera::query(const std::string& platform_config_json, + const std::string& platform_config_name, + const std::vector& link_masks) +{ + return mock_query_platform_config(platform_config_json, platform_config_name, link_masks); +} + +const std::vector& SiplCamera::sensors() const +{ + return m_impl->sensor_infos; +} + +void SiplCamera::start() +{ + m_impl->start(); +} + +void SiplCamera::stop() +{ + m_impl->stop(); +} + +std::optional SiplCamera::latest(uint32_t sensor_id) +{ + m_impl->throw_if_failed(); + Impl::Sensor* sensor = m_impl->find(sensor_id); + if (!sensor) + throw std::runtime_error("no SIPL pipeline with index " + std::to_string(sensor_id)); + + std::lock_guard guard(sensor->publish_mutex); + if (sensor->publish_idx < 0 || sensor->consumed_sequence == sensor->published_sequence) + return std::nullopt; + sensor->consumed_sequence = sensor->published_sequence; + + // The caller now holds this slot; pick_write_index() skips it until the + // next latest() moves the lease on. + sensor->lease_idx = sensor->publish_idx; + + FrameView view; + view.ptr = reinterpret_cast(sensor->buffers[sensor->publish_idx].ptr); + view.pitch = sensor->buffers[sensor->publish_idx].pitch; + view.width = sensor->info.width; + view.height = sensor->info.height; + view.timestamp_ns = sensor->published_timestamp_ns; + view.capture_tsc_ns = sensor->published_capture_tsc_ns; + view.sequence = sensor->published_sequence; + return view; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/mock_test_hooks.hpp b/src/plugins/sensing/core/mock_test_hooks.hpp new file mode 100644 index 0000000000..19d96e6591 --- /dev/null +++ b/src/plugins/sensing/core/mock_test_hooks.hpp @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Mock backend only, for tests: not an operator interface. + +#pragma once + +#include + +namespace plugins +{ +namespace sensing +{ + +/// Every sensor constructed afterwards shows this GIF as it is -- no mirror, no eye offset -- instead of +/// its module's fixed picture. An empty path restores the fixed pictures. +void mock_set_test_picture(const std::string& gif_path); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/mock_assets/README.md b/src/plugins/sensing/mock_assets/README.md new file mode 100644 index 0000000000..470f8f0bde --- /dev/null +++ b/src/plugins/sensing/mock_assets/README.md @@ -0,0 +1,21 @@ + + +# Mock camera images + +The mock backend shows these animated GIFs on its simulated sensors. They are not part of the +Apache-2.0 licensed source: they are Animated Noto Emoji, © Google, licensed under +[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) (`LICENSES/CC-BY-4.0.txt`). + +| File | Emoji | Source | +| --- | --- | --- | +| `noto-robot.gif` | Robot, U+1F916 | `https://fonts.gstatic.com/s/e/notoemoji/latest/1f916/512.gif` | +| `noto-waving-hand.gif` | Waving hand, U+1F44B | `https://fonts.gstatic.com/s/e/notoemoji/latest/1f44b/512.gif` | + +Licence and artwork: . The files are the unmodified +512 px GIFs. The mock composites their transparent background onto a plain gray when it decodes +them; the files themselves are not changed. + +The mock backend is a development tool. Keep these files out of anything shipped to users. diff --git a/src/plugins/sensing/mock_assets/mock_rig.json b/src/plugins/sensing/mock_assets/mock_rig.json new file mode 100644 index 0000000000..ce0753682b --- /dev/null +++ b/src/plugins/sensing/mock_assets/mock_rig.json @@ -0,0 +1,75 @@ +{ + "cameraConfigs": [ + { + "name": "MOCK_S56C_2_SHF3L", + "cameraModules": [ + { + "name": "S56C_HEAD", + "linkIndex": 0, + "transportIndex": 1 + }, + { + "name": "SHF3L_LEFT", + "linkIndex": 1, + "transportIndex": 1 + }, + { + "name": "SHF3L_RIGHT", + "linkIndex": 2, + "transportIndex": 1 + } + ] + }, + { + "name": "S56C_HEAD", + "sensorInfo": [ + { + "virtualChannels": [ + { + "width": 2560, + "height": 1984, + "fps": 30.0 + } + ] + }, + { + "virtualChannels": [ + { + "width": 2560, + "height": 1984, + "fps": 30.0 + } + ] + } + ] + }, + { + "name": "SHF3L_LEFT", + "sensorInfo": [ + { + "virtualChannels": [ + { + "width": 1920, + "height": 1536, + "fps": 30.0 + } + ] + } + ] + }, + { + "name": "SHF3L_RIGHT", + "sensorInfo": [ + { + "virtualChannels": [ + { + "width": 1920, + "height": 1536, + "fps": 30.0 + } + ] + } + ] + } + ] +} diff --git a/src/plugins/sensing/mock_assets/noto-robot.gif b/src/plugins/sensing/mock_assets/noto-robot.gif new file mode 100644 index 0000000000..92d373cfea --- /dev/null +++ b/src/plugins/sensing/mock_assets/noto-robot.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a53e434118916a967f30b9968d1e3fd3cf6a4978d39bdb3551c9fb4c8c1fccf +size 840412 diff --git a/src/plugins/sensing/mock_assets/noto-robot.gif.license b/src/plugins/sensing/mock_assets/noto-robot.gif.license new file mode 100644 index 0000000000..e268adf305 --- /dev/null +++ b/src/plugins/sensing/mock_assets/noto-robot.gif.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Google LLC +SPDX-License-Identifier: CC-BY-4.0 diff --git a/src/plugins/sensing/mock_assets/noto-waving-hand.gif b/src/plugins/sensing/mock_assets/noto-waving-hand.gif new file mode 100644 index 0000000000..fdb2462241 --- /dev/null +++ b/src/plugins/sensing/mock_assets/noto-waving-hand.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1025c22c790ada1e071c0ace357817a7745c356b2ace3cb0e508329f67ff9fe0 +size 378319 diff --git a/src/plugins/sensing/mock_assets/noto-waving-hand.gif.license b/src/plugins/sensing/mock_assets/noto-waving-hand.gif.license new file mode 100644 index 0000000000..e268adf305 --- /dev/null +++ b/src/plugins/sensing/mock_assets/noto-waving-hand.gif.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Google LLC +SPDX-License-Identifier: CC-BY-4.0 From 57832012b5f2f43313d391f621687744516d57d5 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Wed, 4 Nov 2026 23:21:17 -0800 Subject: [PATCH 03/29] Revert "sensing: add a mock backend that runs the LiveKit publisher without a Jetson" This reverts commit fa7ba7c349caa1f16af8baff717a1916a5d6b96b. Signed-off-by: Gareth Morgan --- README.md | 3 +- src/plugins/CMakeLists.txt | 2 +- src/plugins/sensing/CMakeLists.txt | 30 +- src/plugins/sensing/core/CMakeLists.txt | 101 +---- src/plugins/sensing/core/mock_gif_source.cpp | 109 ----- src/plugins/sensing/core/mock_gif_source.hpp | 73 --- .../sensing/core/mock_jetson_encoder.cpp | 424 ------------------ src/plugins/sensing/core/mock_nv12_source.cu | 166 ------- src/plugins/sensing/core/mock_nv12_source.cuh | 42 -- .../sensing/core/mock_platform_config.cpp | 109 ----- .../sensing/core/mock_platform_config.hpp | 40 -- src/plugins/sensing/core/mock_sipl_camera.cpp | 417 ----------------- src/plugins/sensing/core/mock_test_hooks.hpp | 20 - src/plugins/sensing/mock_assets/README.md | 21 - src/plugins/sensing/mock_assets/mock_rig.json | 75 ---- .../sensing/mock_assets/noto-robot.gif | 3 - .../mock_assets/noto-robot.gif.license | 2 - .../sensing/mock_assets/noto-waving-hand.gif | 3 - .../mock_assets/noto-waving-hand.gif.license | 2 - 19 files changed, 27 insertions(+), 1615 deletions(-) delete mode 100644 src/plugins/sensing/core/mock_gif_source.cpp delete mode 100644 src/plugins/sensing/core/mock_gif_source.hpp delete mode 100644 src/plugins/sensing/core/mock_jetson_encoder.cpp delete mode 100644 src/plugins/sensing/core/mock_nv12_source.cu delete mode 100644 src/plugins/sensing/core/mock_nv12_source.cuh delete mode 100644 src/plugins/sensing/core/mock_platform_config.cpp delete mode 100644 src/plugins/sensing/core/mock_platform_config.hpp delete mode 100644 src/plugins/sensing/core/mock_sipl_camera.cpp delete mode 100644 src/plugins/sensing/core/mock_test_hooks.hpp delete mode 100644 src/plugins/sensing/mock_assets/README.md delete mode 100644 src/plugins/sensing/mock_assets/mock_rig.json delete mode 100644 src/plugins/sensing/mock_assets/noto-robot.gif delete mode 100644 src/plugins/sensing/mock_assets/noto-robot.gif.license delete mode 100644 src/plugins/sensing/mock_assets/noto-waving-hand.gif delete mode 100644 src/plugins/sensing/mock_assets/noto-waving-hand.gif.license diff --git a/README.md b/README.md index ac59e1dfd1..cd55de9172 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,7 @@ To get started, please refer to Isaac Lab's [Installation](https://isaac-sim.git The default license is [Apache-2.0](LICENSE.md). EgoPost is licensed under [OpenMDW-1.1](LICENSES/LicenseRef-OpenMDW-1.1.txt). -Third-party materials may carry their own licenses; for example, the sample GIFs under -[`src/plugins/sensing/mock_assets/`](src/plugins/sensing/mock_assets/) are Animated Noto Emoji under CC BY 4.0. +Third-party materials may carry their own licenses. ### Separately retrieved optional materials diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 1c1c6daf36..7a3a400b15 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -44,7 +44,7 @@ endif() if(BUILD_PLUGIN_SHARPA_AVATAR) add_subdirectory(sharpa_avatar) endif() -# Gated: the Jetson backend needs SIPL and the Tegra NVENC entry point (L4T only); the mock runs anywhere with CUDA. +# Gated: Jetson only -- SIPL and the Tegra NVENC entry point are L4T-only. if(BUILD_PLUGIN_SENSING) add_subdirectory(sensing) endif() diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index 516759d934..da64e4fbdc 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -4,12 +4,6 @@ # ============================================================================== # SENSING Camera Plugin: Jetson SIPL capture, hardware H.264 encode, LiveKit publication # ============================================================================== -# Two backends sit behind SiplCamera and JetsonEncoder: -# JETSON SIPL capture and the V4L2 M2M encoder, which exist only on L4T. -# MOCK synthetic frames and an NVENC encoder (libavcodec), for any machine with an NVIDIA GPU and -# the CUDA toolkit. Everything above those two classes is the same code. -# SENSING_BACKEND=AUTO picks JETSON on aarch64 and MOCK elsewhere. -# # Two leaf targets besides the LiveKit wrapper, one per directory (cmake/cmake-structure.md rule 1): # livekit/ sensing_livekit -- the LiveKit SDK behind a small interface # core/ sensing_core -- SIPL capture, encoder, conversion, track output @@ -19,8 +13,7 @@ # src/plugins/sensing/scripts/setup.sh # fetches the Multimedia API, locates jetson_sipl_api # # Configure with: -# cmake -B build -DBUILD_PLUGIN_SENSING=ON # Jetson, or mock elsewhere -# cmake -B build -DBUILD_PLUGIN_SENSING=ON -DSENSING_BACKEND=MOCK # force the mock +# cmake -B build -DBUILD_PLUGIN_SENSING=ON # ============================================================================== enable_language(CUDA) @@ -32,28 +25,8 @@ endif() find_package(CUDAToolkit REQUIRED) -set(SENSING_BACKEND "AUTO" CACHE STRING "SENSING backend: AUTO, JETSON (SIPL + V4L2 NVENC) or MOCK (synthetic)") -set_property(CACHE SENSING_BACKEND PROPERTY STRINGS AUTO JETSON MOCK) -if(SENSING_BACKEND STREQUAL "AUTO") - # A Jetson that lacks the SDK should still get the setup instructions below, - # not a silent mock, so only non-aarch64 hosts fall back. - if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") - set(SENSING_BACKEND_RESOLVED "JETSON" CACHE INTERNAL "") - else() - set(SENSING_BACKEND_RESOLVED "MOCK" CACHE INTERNAL "") - endif() -elseif(SENSING_BACKEND STREQUAL "JETSON" OR SENSING_BACKEND STREQUAL "MOCK") - set(SENSING_BACKEND_RESOLVED "${SENSING_BACKEND}" CACHE INTERNAL "") -else() - message(FATAL_ERROR "SENSING_BACKEND must be AUTO, JETSON or MOCK, got '${SENSING_BACKEND}'") -endif() -# Cache, not a plain variable, so tests/ (a sibling directory) can gate on it too. -message(STATUS "SENSING plugin backend: ${SENSING_BACKEND_RESOLVED}") - set(_tegra_lib_dirs "/usr/lib/aarch64-linux-gnu/nvidia" "/usr/lib/aarch64-linux-gnu/tegra") -if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON") -# Jetson-only SDK checks. # ============================================================================== # SIPL # ============================================================================== @@ -176,7 +149,6 @@ message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}") # No libnvjpeg here on purpose: on R39 libnvbufsurface has no libjpeg # dependency at all (readelf -d lists only EGL, cuda, dl, pthread, rt, c), and # the tegra lib dir ships no libnvjpeg to link against. -endif() # ============================================================================== # LiveKit diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt index 6428eafad5..22b466c2df 100644 --- a/src/plugins/sensing/core/CMakeLists.txt +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -2,66 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 # SIPL capture, colour conversion, the hardware encoder and the LiveKit track output. -set(SENSING_COMMON_SOURCES +add_library(sensing_core STATIC sensing_camera.cpp + sipl_camera.cpp + jetson_encoder.cpp livekit_video_output.cpp yuv_to_rgba.cu rgba_to_nv12.cu rgba_box_downscale.cu -) - -if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON") - set(SENSING_BACKEND_SOURCES - sipl_camera.cpp - jetson_encoder.cpp - ${JETSON_MMAPI_SOURCES} - ) -else() - # yuv_to_rgba.cu includes cudaEGL.h, which needs the EGL headers even though - # nothing here opens an EGL display. - find_path(SENSING_EGL_INCLUDE_DIR EGL/egl.h) - if(NOT SENSING_EGL_INCLUDE_DIR) - message(FATAL_ERROR "The SENSING mock backend needs the EGL headers (cudaEGL.h includes them). " - "Install libegl-dev.") - endif() - # Parses the vendor JSON itself, so a MOCK build has no SIPL query library. - find_package(nlohmann_json 3 QUIET) - if(NOT nlohmann_json_FOUND) - include(FetchContent) - FetchContent_Declare( - nlohmann_json - GIT_REPOSITORY https://github.com/nlohmann/json.git - GIT_TAG v3.11.3 - GIT_SHALLOW TRUE - ) - set(JSON_BuildTests OFF CACHE INTERNAL "") - isaac_teleop_third_party_scope_begin() - FetchContent_MakeAvailable(nlohmann_json) - isaac_teleop_third_party_scope_end() - endif() - # Decodes the GIFs the mock camera can show. A single public-domain header, pinned to a commit and - # checked against its hash, so nothing else of the project is cloned. - include(FetchContent) - FetchContent_Declare( - stb_image - URL https://raw.githubusercontent.com/nothings/stb/003501f6d7a8668ce8b6d875e34dd97ad0d5d5b5/stb_image.h - URL_HASH SHA256=594c2fe35d49488b4382dbfaec8f98366defca819d916ac95becf3e75f4200b3 - DOWNLOAD_NO_EXTRACT TRUE - ) - FetchContent_MakeAvailable(stb_image) - - set(SENSING_BACKEND_SOURCES - mock_sipl_camera.cpp - mock_gif_source.cpp - mock_platform_config.cpp - mock_nv12_source.cu - mock_jetson_encoder.cpp - ) -endif() - -add_library(sensing_core STATIC - ${SENSING_COMMON_SOURCES} - ${SENSING_BACKEND_SOURCES} + ${JETSON_MMAPI_SOURCES} ) add_library(sensing::core ALIAS sensing_core) @@ -76,28 +25,26 @@ set_target_properties(sensing_core PROPERTIES target_include_directories(sensing_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/.. + PRIVATE + # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay private: they drag in + # NvMedia and X11-style macros. + ${SIPL_INCLUDE_DIRS} + ${EGL_INCLUDE_DIRS} + ${JETSON_MMAPI_DIR}/include ) -target_link_libraries(sensing_core PUBLIC sensing::livekit) - -if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON") - # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay private: they drag in - # NvMedia and X11-style macros. - target_include_directories(sensing_core PRIVATE ${SIPL_INCLUDE_DIRS} ${EGL_INCLUDE_DIRS} ${JETSON_MMAPI_DIR}/include) - - # The Multimedia API reference sources carry legacy warnings. - target_compile_options(sensing_core PRIVATE - $<$:-Wno-reorder -Wno-uninitialized -Wno-unused-variable - -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare> - ) +# The Multimedia API reference sources carry legacy warnings. +target_compile_options(sensing_core PRIVATE + $<$:-Wno-reorder -Wno-uninitialized -Wno-unused-variable + -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare> +) - target_link_libraries(sensing_core PRIVATE ${SIPL_LIBRARIES} ${EGL_LIBRARIES} ${NVBUFSURFACE_LIBRARY} ${V4L2_LIBRARY}) -else() - # The mock encoder produces real H.264 with libavcodec's h264_nvenc (development package: - # libavcodec-dev; the desktop driver supplies NVENC). - find_package(PkgConfig REQUIRED) - pkg_check_modules(AVCODEC REQUIRED IMPORTED_TARGET libavcodec libavutil) - target_link_libraries(sensing_core PRIVATE nlohmann_json::nlohmann_json PkgConfig::AVCODEC) - target_include_directories(sensing_core SYSTEM PRIVATE ${stb_image_SOURCE_DIR}) - target_compile_definitions(sensing_core PRIVATE SENSING_MOCK_ASSET_DIR="${CMAKE_CURRENT_SOURCE_DIR}/../mock_assets") -endif() +target_link_libraries(sensing_core + PUBLIC + sensing::livekit + PRIVATE + ${SIPL_LIBRARIES} + ${EGL_LIBRARIES} + ${NVBUFSURFACE_LIBRARY} + ${V4L2_LIBRARY} +) diff --git a/src/plugins/sensing/core/mock_gif_source.cpp b/src/plugins/sensing/core/mock_gif_source.cpp deleted file mode 100644 index 3f574297de..0000000000 --- a/src/plugins/sensing/core/mock_gif_source.cpp +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#include "mock_gif_source.hpp" - -#include - -#define STBI_ONLY_GIF -#define STBI_NO_STDIO -#define STB_IMAGE_IMPLEMENTATION -#include -#include -#include -#include -#include -#include - -namespace plugins -{ -namespace sensing -{ - -namespace -{ - -constexpr int kMinDelayMs = 20; - -} // namespace - -std::vector gif_frame_ends(const std::vector& delays_ms) -{ - std::vector ends; - ends.reserve(delays_ms.size()); - double total = 0.0; - for (int delay : delays_ms) - { - total += std::max(delay, kMinDelayMs) / 1000.0; - ends.push_back(total); - } - return ends; -} - -size_t gif_frame_index(const std::vector& frame_ends, double seconds) -{ - if (frame_ends.empty()) - return 0; - const double t = std::fmod(std::max(seconds, 0.0), frame_ends.back()); - const auto it = std::upper_bound(frame_ends.begin(), frame_ends.end(), t); - return std::min(static_cast(it - frame_ends.begin()), frame_ends.size() - 1); -} - -MockGifSource::MockGifSource(const std::string& path) -{ - std::ifstream file(path, std::ios::binary); - if (!file) - throw std::runtime_error("mock GIF: cannot open " + path); - const std::vector bytes((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - - int width = 0; - int height = 0; - int frames = 0; - int channels = 0; - int* delays = nullptr; - unsigned char* pixels = stbi_load_gif_from_memory( - bytes.data(), static_cast(bytes.size()), &delays, &width, &height, &frames, &channels, 4); - if (!pixels) - throw std::runtime_error("mock GIF: cannot decode " + path + ": " + stbi_failure_reason()); - - // Composite onto the background, so the kernel only ever sees opaque pixels. - const size_t pixel_count = static_cast(width) * height * frames; - for (size_t i = 0; i < pixel_count; ++i) - { - unsigned char* px = pixels + i * 4; - const int alpha = px[3]; - for (int c = 0; c < 3; ++c) - px[c] = static_cast((px[c] * alpha + kBackground * (255 - alpha) + 127) / 255); - px[3] = 255; - } - - m_width = static_cast(width); - m_height = static_cast(height); - m_frame_ends = gif_frame_ends(std::vector(delays, delays + frames)); - - const size_t bytes_per_frame = static_cast(width) * height * 4; - const cudaError_t alloc = cudaMalloc(reinterpret_cast(&m_device_frames), bytes_per_frame * frames); - const cudaError_t copy = alloc == cudaSuccess ? - cudaMemcpy(m_device_frames, pixels, bytes_per_frame * frames, cudaMemcpyHostToDevice) : - alloc; - stbi_image_free(pixels); - stbi_image_free(delays); - if (copy != cudaSuccess) - { - cudaFree(m_device_frames); - throw std::runtime_error(std::string("mock GIF: cannot upload to the GPU: ") + cudaGetErrorString(copy)); - } -} - -MockGifSource::~MockGifSource() -{ - cudaFree(m_device_frames); -} - -const uint8_t* MockGifSource::frame(size_t index) const -{ - return m_device_frames + index * static_cast(m_width) * m_height * 4; -} - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/core/mock_gif_source.hpp b/src/plugins/sensing/core/mock_gif_source.hpp deleted file mode 100644 index 5a21edbcf2..0000000000 --- a/src/plugins/sensing/core/mock_gif_source.hpp +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Mock backend only. An animated GIF decoded once and held in device memory, so the -// mock camera can show real moving images. - -#pragma once - -#include -#include -#include -#include - -namespace plugins -{ -namespace sensing -{ - -/// Time in seconds at which each GIF frame stops showing: the running total of the frame delays, -/// each clamped to at least 20 ms, as browsers do for the tiny delays some encoders write. -std::vector gif_frame_ends(const std::vector& delays_ms); - -/// Index of the frame showing `seconds` into the animation, which loops over its whole length. -size_t gif_frame_index(const std::vector& frame_ends, double seconds); - -class MockGifSource -{ -public: - /// Transparent pixels are composited onto this gray, so an emoji or sticker without a - /// background reads as an image rather than as whatever colour sits under its transparency. - static constexpr uint8_t kBackground = 48; - - /// Decodes every frame to opaque RGBA8 and uploads them to the current CUDA device. - /// @throws std::runtime_error if the file cannot be read or is not a GIF. - explicit MockGifSource(const std::string& path); - ~MockGifSource(); - - MockGifSource(const MockGifSource&) = delete; - MockGifSource& operator=(const MockGifSource&) = delete; - - uint32_t width() const - { - return m_width; - } - uint32_t height() const - { - return m_height; - } - size_t frame_count() const - { - return m_frame_ends.size(); - } - /// Row stride of every frame, in bytes. Rows are tightly packed. - size_t pitch() const - { - return static_cast(m_width) * 4; - } - /// Device pointer to one RGBA8 frame. - const uint8_t* frame(size_t index) const; - size_t frame_index_at(double seconds) const - { - return gif_frame_index(m_frame_ends, seconds); - } - -private: - uint32_t m_width = 0; - uint32_t m_height = 0; - std::vector m_frame_ends; - uint8_t* m_device_frames = nullptr; -}; - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/core/mock_jetson_encoder.cpp b/src/plugins/sensing/core/mock_jetson_encoder.cpp deleted file mode 100644 index 63bb0cbc36..0000000000 --- a/src/plugins/sensing/core/mock_jetson_encoder.cpp +++ /dev/null @@ -1,424 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Mock backend for JetsonEncoder: real H.264 from the desktop GPU's NVENC, through libavcodec's -// h264_nvenc, set up as the Jetson V4L2 encoder is (High profile, Level 5.2, VBR with a peak, an IDR -// every GOP with SPS/PPS in front of it, no B-frames). It runs the real RGBA-to-NV12 kernel and the same -// device-to-host copy as the V4L2 path, then a worker thread feeds the NV12 buffer to the encoder. -// -// Reproduced from the real encoder: a fixed pool of input buffers, so submit() returns false when the -// encoder falls behind rather than blocking; the capture stamp coming back at microsecond resolution (a -// V4L2 timeval); end_of_stream() returning the remaining tail as one concatenated unit. NVENC's own -// latency and unit sizes are what they are, and are not those of the Jetson engine. - -#include "jetson_encoder.hpp" -#include "rgba_to_nv12.cuh" - -extern "C" -{ -#include -#include -#include -#include -} - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace plugins -{ -namespace sensing -{ - -namespace -{ - -constexpr uint32_t kInputBuffers = 6; - -void check_cuda(cudaError_t err, const char* what) -{ - if (err != cudaSuccess) - throw std::runtime_error(std::string("mock JetsonEncoder: ") + what + ": " + cudaGetErrorString(err)); -} - -void check_av(int err, const char* what) -{ - if (err < 0) - { - char text[AV_ERROR_MAX_STRING_SIZE]; - av_strerror(err, text, sizeof(text)); - throw std::runtime_error(std::string("mock JetsonEncoder: ") + what + ": " + text); - } -} - -} // namespace - -struct JetsonEncoder::Impl -{ - struct Job - { - uint32_t buffer = 0; - int64_t timestamp_ns = 0; - bool keyframe = false; - }; - - explicit Impl(const EncoderConfig& cfg) : config(cfg) - { - codec = avcodec_find_encoder_by_name("h264_nvenc"); - if (!codec) - throw std::runtime_error("mock JetsonEncoder: libavcodec has no h264_nvenc encoder"); - - // A constructor that throws never runs the destructor, so free what was allocated here. - try - { - open_context(config.bitrate_bps); - alloc_frame(); - check_cuda(cudaMallocPitch(reinterpret_cast(&d_y), &d_y_pitch, config.width, config.height), - "cudaMallocPitch(Y)"); - check_cuda(cudaMallocPitch(reinterpret_cast(&d_uv), &d_uv_pitch, config.width, config.height / 2), - "cudaMallocPitch(UV)"); - - // Host staging, standing in for the V4L2 output plane's mapped buffers. - const size_t nv12_bytes = static_cast(config.width) * config.height * 3 / 2; - for (uint32_t i = 0; i < kInputBuffers; ++i) - { - host_buffers.emplace_back(nv12_bytes); - free_buffers.push_back(i); - } - worker = std::thread([this] { run(); }); - } - catch (...) - { - release(); - throw; - } - } - - ~Impl() - { - { - std::lock_guard lock(mutex); - closing = true; - } - wake.notify_all(); - if (worker.joinable()) - worker.join(); - release(); - } - - void set_option(const char* name, const char* value) - { - check_av(av_opt_set(context->priv_data, name, value, 0), (std::string("av_opt_set ") + name).c_str()); - } - - // A fresh encoder session at `bitrate`: its first frame is an IDR with parameter sets. - void open_context(uint32_t bitrate) - { - avcodec_free_context(&context); - context = avcodec_alloc_context3(codec); - if (!context) - throw std::runtime_error("mock JetsonEncoder: cannot allocate the encoder context"); - - const uint32_t fps = config.fps ? config.fps : 30; - const uint32_t gop = config.gop ? config.gop : fps * 5; - context->width = static_cast(config.width); - context->height = static_cast(config.height); - context->pix_fmt = AV_PIX_FMT_NV12; - // Microseconds, as a V4L2 timeval carries them. - context->time_base = { 1, 1'000'000 }; - context->framerate = { static_cast(fps), 1 }; - context->bit_rate = bitrate; - context->gop_size = static_cast(gop); - context->max_b_frames = 0; - // Parameter sets stay in the stream, in front of every IDR, so a receiver can join mid-stream. - context->flags &= ~AV_CODEC_FLAG_GLOBAL_HEADER; - set_option("profile", "high"); - set_option("level", "5.2"); - set_option("forced-idr", "1"); - set_option("zerolatency", "1"); - // Hand each unit back as soon as it is encoded rather than after a queue of frames. - set_option("delay", "0"); - if (config.peak_bitrate_bps > config.bitrate_bps) - { - // The configured ratio between the ceiling and the target holds at any target. - context->rc_max_rate = static_cast(bitrate) * config.peak_bitrate_bps / config.bitrate_bps; - context->rc_buffer_size = static_cast(context->rc_max_rate); - set_option("rc", "vbr"); - } - else - { - set_option("rc", "cbr"); - } - check_av(avcodec_open2(context, codec, nullptr), "cannot open h264_nvenc"); - applied_bitrate = bitrate; - } - - void alloc_frame() - { - frame = av_frame_alloc(); - packet = av_packet_alloc(); - if (!frame || !packet) - throw std::runtime_error("mock JetsonEncoder: cannot allocate a frame or packet"); - frame->format = AV_PIX_FMT_NV12; - frame->width = context->width; - frame->height = context->height; - check_av(av_frame_get_buffer(frame, 0), "cannot allocate the input frame"); - } - - void release() - { - cudaFree(d_y); - cudaFree(d_uv); - av_packet_free(&packet); - av_frame_free(&frame); - avcodec_free_context(&context); - } - - // Moves every packet the encoder has ready into `out`. Worker thread, or end_of_stream() once the - // worker is idle. - void collect(std::deque& out) - { - for (;;) - { - const int err = avcodec_receive_packet(context, packet); - if (err == AVERROR(EAGAIN) || err == AVERROR_EOF) - return; - check_av(err, "avcodec_receive_packet"); - EncodedUnit unit; - unit.data.assign(packet->data, packet->data + packet->size); - unit.timestamp_ns = packet->pts * 1000; - unit.keyframe = (packet->flags & AV_PKT_FLAG_KEY) != 0; - av_packet_unref(packet); - out.push_back(std::move(unit)); - } - } - - // Worker thread, which owns the codec. The libavcodec NVENC wrapper takes its rate control at - // open, so a new target drains the running session into `out` and opens another at that rate: the - // stream continues from an IDR with parameter sets. - void apply_bitrate(std::deque& out) - { - uint32_t target = 0; - { - std::lock_guard lock(mutex); - target = pending_bitrate; - pending_bitrate = 0; - } - if (target == 0 || target == applied_bitrate) - return; - check_av(avcodec_send_frame(context, nullptr), "avcodec_send_frame(flush)"); - collect(out); - open_context(target); - } - - void encode(const Job& job) - { - check_av(av_frame_make_writable(frame), "av_frame_make_writable"); - const uint8_t* nv12 = host_buffers[job.buffer].data(); - const size_t width = config.width; - for (uint32_t row = 0; row < config.height; ++row) - std::memcpy(frame->data[0] + static_cast(row) * frame->linesize[0], nv12 + row * width, width); - const uint8_t* uv = nv12 + width * config.height; - for (uint32_t row = 0; row < config.height / 2; ++row) - std::memcpy(frame->data[1] + static_cast(row) * frame->linesize[1], uv + row * width, width); - frame->pts = job.timestamp_ns / 1000; - // With forced-idr, an I picture is an IDR with its parameter sets. - frame->pict_type = job.keyframe ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE; - check_av(avcodec_send_frame(context, frame), "avcodec_send_frame"); - } - - void run() - { - for (;;) - { - Job job; - { - std::unique_lock lock(mutex); - wake.wait(lock, [&] { return closing || !jobs.empty(); }); - // Frames still queued at teardown are abandoned, as the hardware encoder would. - if (closing || jobs.empty()) - return; - job = jobs.front(); - jobs.pop_front(); - busy = true; - } - - std::deque produced; - std::string error; - try - { - apply_bitrate(produced); - encode(job); - collect(produced); - } - catch (const std::exception& e) - { - error = e.what(); - } - - { - std::lock_guard lock(mutex); - free_buffers.push_back(job.buffer); - for (auto& unit : produced) - ready.push_back(std::move(unit)); - if (!error.empty() && failure.empty()) - failure = error; - busy = false; - } - idle.notify_all(); - } - } - - EncoderConfig config; - - const AVCodec* codec = nullptr; - AVCodecContext* context = nullptr; - AVFrame* frame = nullptr; - AVPacket* packet = nullptr; - - uint8_t* d_y = nullptr; - uint8_t* d_uv = nullptr; - size_t d_y_pitch = 0; - size_t d_uv_pitch = 0; - - std::vector> host_buffers; - - std::mutex mutex; - std::condition_variable wake; - std::condition_variable idle; - std::deque free_buffers; - std::deque jobs; - std::deque ready; // what poll() hands out - std::string failure; - bool busy = false; - bool closing = false; - bool flushed = false; - bool keyframe_requested = false; // guarded by `mutex` - uint32_t pending_bitrate = 0; // guarded by `mutex`; 0 when nothing is waiting - uint32_t applied_bitrate = 0; // worker thread only - std::thread worker; -}; - -JetsonEncoder::JetsonEncoder(const EncoderConfig& config) -{ - if (config.width == 0 || config.height == 0) - throw std::runtime_error("JetsonEncoder: width/height must be non-zero"); - if ((config.width % 2) != 0 || (config.height % 2) != 0) - throw std::runtime_error("JetsonEncoder: width/height must be even for NV12"); - m_impl = std::make_unique(config); -} - -JetsonEncoder::~JetsonEncoder() = default; - -bool JetsonEncoder::submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns) -{ - auto& impl = *m_impl; - const auto& config = impl.config; - - uint32_t buffer = 0; - { - std::lock_guard lock(impl.mutex); - if (!impl.failure.empty()) - throw std::runtime_error(impl.failure); - if (impl.free_buffers.empty()) - return false; // encoder still busy; drop this frame rather than block - buffer = impl.free_buffers.front(); - impl.free_buffers.pop_front(); - } - - launch_rgba_to_nv12(reinterpret_cast(rgba_device_ptr), static_cast(row_pitch_bytes), impl.d_y, - static_cast(impl.d_y_pitch), impl.d_uv, static_cast(impl.d_uv_pitch), - static_cast(config.width), static_cast(config.height), config.full_range, nullptr); - check_cuda(cudaGetLastError(), "rgba_to_nv12 launch"); - check_cuda(cudaStreamSynchronize(nullptr), "rgba_to_nv12 sync"); - - uint8_t* host = impl.host_buffers[buffer].data(); - check_cuda( - cudaMemcpy2D(host, config.width, impl.d_y, impl.d_y_pitch, config.width, config.height, cudaMemcpyDeviceToHost), - "cudaMemcpy2D(Y)"); - check_cuda(cudaMemcpy2D(host + static_cast(config.width) * config.height, config.width, impl.d_uv, - impl.d_uv_pitch, config.width, config.height / 2, cudaMemcpyDeviceToHost), - "cudaMemcpy2D(UV)"); - - { - std::lock_guard lock(impl.mutex); - impl.jobs.push_back({ buffer, timestamp_ns, impl.keyframe_requested }); - impl.keyframe_requested = false; - } - impl.wake.notify_one(); - return true; -} - -EncodedUnit JetsonEncoder::poll() -{ - std::lock_guard lock(m_impl->mutex); - if (!m_impl->failure.empty()) - throw std::runtime_error(m_impl->failure); - if (m_impl->ready.empty()) - return {}; - auto out = std::move(m_impl->ready.front()); - m_impl->ready.pop_front(); - return out; -} - -std::vector JetsonEncoder::drain() -{ - auto& impl = *m_impl; - std::unique_lock lock(impl.mutex); - impl.idle.wait(lock, [&] { return impl.jobs.empty() && !impl.busy; }); - if (!impl.failure.empty()) - throw std::runtime_error(impl.failure); - if (!impl.flushed) - { - impl.flushed = true; - check_av(avcodec_send_frame(impl.context, nullptr), "avcodec_send_frame(flush)"); - impl.collect(impl.ready); - } - std::vector units( - std::make_move_iterator(impl.ready.begin()), std::make_move_iterator(impl.ready.end())); - impl.ready.clear(); - return units; -} - -EncodedUnit JetsonEncoder::end_of_stream() -{ - // The tail is concatenated, so it carries the stamp of the last unit in it. - EncodedUnit out; - for (auto& chunk : drain()) - { - out.data.insert(out.data.end(), chunk.data.begin(), chunk.data.end()); - out.timestamp_ns = chunk.timestamp_ns; - } - return out; -} - -void JetsonEncoder::request_keyframe() -{ - std::lock_guard lock(m_impl->mutex); - m_impl->keyframe_requested = true; -} - -bool JetsonEncoder::set_bitrate(uint32_t bitrate_bps) -{ - if (bitrate_bps == 0) - return false; - std::lock_guard lock(m_impl->mutex); - m_impl->pending_bitrate = bitrate_bps; - return true; -} - -const char* JetsonEncoder::backend_name() -{ - return "NVENC (libavcodec h264_nvenc)"; -} - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/core/mock_nv12_source.cu b/src/plugins/sensing/core/mock_nv12_source.cu deleted file mode 100644 index 188f1c5113..0000000000 --- a/src/plugins/sensing/core/mock_nv12_source.cu +++ /dev/null @@ -1,166 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#include "mock_nv12_source.cuh" - -namespace plugins -{ -namespace sensing -{ - -namespace -{ - -// Geometry of the cover resize: the source scaled uniformly until it fills the target, then -// centre-cropped. `scaled_*` is the size of the scaled image and `crop_*` the offset of the -// visible window inside it. -struct CoverGeometry -{ - int src_width; - int src_height; - int scaled_width; - int scaled_height; - int crop_x; - int crop_y; -}; - -__device__ float3 fetch_rgb(const uint8_t* src, int pitch, int x, int y) -{ - const uint8_t* px = src + static_cast(y) * pitch + static_cast(x) * 4; - return make_float3(px[0], px[1], px[2]); -} - -// Colour of one pixel of the cropped output. Enlarging samples bilinearly; reducing averages the -// source area the pixel covers, weighted by overlap, as a box filter would. -__device__ float3 sample_cover(const uint8_t* src, int pitch, const CoverGeometry& g, int x, int y) -{ - const float sx = static_cast(g.src_width) / g.scaled_width; - const float sy = static_cast(g.src_height) / g.scaled_height; - const int X = min(max(x + g.crop_x, 0), g.scaled_width - 1); - const int Y = min(max(y + g.crop_y, 0), g.scaled_height - 1); - - if (sx <= 1.0f && sy <= 1.0f) - { - const float fx = fminf(fmaxf((X + 0.5f) * sx - 0.5f, 0.0f), g.src_width - 1.0f); - const float fy = fminf(fmaxf((Y + 0.5f) * sy - 0.5f, 0.0f), g.src_height - 1.0f); - const int x0 = static_cast(fx); - const int y0 = static_cast(fy); - const int x1 = min(x0 + 1, g.src_width - 1); - const int y1 = min(y0 + 1, g.src_height - 1); - const float tx = fx - x0; - const float ty = fy - y0; - const float3 a = fetch_rgb(src, pitch, x0, y0); - const float3 b = fetch_rgb(src, pitch, x1, y0); - const float3 c = fetch_rgb(src, pitch, x0, y1); - const float3 d = fetch_rgb(src, pitch, x1, y1); - return make_float3((a.x * (1 - tx) + b.x * tx) * (1 - ty) + (c.x * (1 - tx) + d.x * tx) * ty, - (a.y * (1 - tx) + b.y * tx) * (1 - ty) + (c.y * (1 - tx) + d.y * tx) * ty, - (a.z * (1 - tx) + b.z * tx) * (1 - ty) + (c.z * (1 - tx) + d.z * tx) * ty); - } - - const float left = X * sx; - const float right = (X + 1) * sx; - const float top = Y * sy; - const float bottom = (Y + 1) * sy; - float3 sum = make_float3(0.0f, 0.0f, 0.0f); - float weight = 0.0f; - for (int j = static_cast(top); j < min(static_cast(ceilf(bottom)), g.src_height); ++j) - { - const float wy = fminf(bottom, j + 1.0f) - fmaxf(top, static_cast(j)); - for (int i = static_cast(left); i < min(static_cast(ceilf(right)), g.src_width); ++i) - { - const float w = wy * (fminf(right, i + 1.0f) - fmaxf(left, static_cast(i))); - const float3 rgb = fetch_rgb(src, pitch, i, j); - sum.x += rgb.x * w; - sum.y += rgb.y * w; - sum.z += rgb.z * w; - weight += w; - } - } - const float inv = weight > 0.0f ? 1.0f / weight : 0.0f; - return make_float3(sum.x * inv, sum.y * inv, sum.z * inv); -} - -__device__ float luma_709(const float3& rgb) -{ - return fminf(fmaxf(0.2126f * rgb.x + 0.7152f * rgb.y + 0.0722f * rgb.z, 0.0f), 255.0f); -} - -// One thread per 2x2 block: four luma samples and one chroma pair. -__global__ void paint_gif(const uint8_t* src, - int src_pitch, - CoverGeometry geometry, - uint8_t* y_plane, - int y_pitch, - uint8_t* uv_plane, - int uv_pitch, - int width, - int height, - bool mirror, - int shift_x) -{ - const int cx = blockIdx.x * blockDim.x + threadIdx.x; - const int cy = blockIdx.y * blockDim.y + threadIdx.y; - if (cx >= width / 2 || cy >= height / 2) - return; - - float3 mean = make_float3(0.0f, 0.0f, 0.0f); - for (int dy = 0; dy < 2; ++dy) - { - for (int dx = 0; dx < 2; ++dx) - { - const int x = 2 * cx + dx; - const int y = 2 * cy + dy; - // A mirrored view samples the picture right to left; the output pixel stays where it is. - const int sx = (mirror ? width - 1 - x : x) - shift_x; - const float3 rgb = sample_cover(src, src_pitch, geometry, sx, y); - y_plane[static_cast(y) * y_pitch + x] = static_cast(luma_709(rgb) + 0.5f); - mean.x += 0.25f * rgb.x; - mean.y += 0.25f * rgb.y; - mean.z += 0.25f * rgb.z; - } - } - - // Inverse of the full-range BT.709 matrix the real conversion applies. - const float luma = luma_709(mean); - const float cb = fminf(fmaxf(128.0f + (mean.z - luma) / 1.8556f, 0.0f), 255.0f); - const float cr = fminf(fmaxf(128.0f + (mean.x - luma) / 1.5748f, 0.0f), 255.0f); - uint8_t* sample = uv_plane + static_cast(cy) * uv_pitch + static_cast(cx) * 2; - sample[0] = static_cast(cb + 0.5f); - sample[1] = static_cast(cr + 0.5f); -} - -} // namespace - -void launch_mock_gif_nv12(const uint8_t* src_rgba, - int src_pitch, - int src_width, - int src_height, - uint8_t* y_plane, - int y_pitch, - uint8_t* uv_plane, - int uv_pitch, - int width, - int height, - bool mirror, - int shift_x, - cudaStream_t stream) -{ - const float scale = fmaxf(static_cast(width) / src_width, static_cast(height) / src_height); - CoverGeometry geometry{}; - geometry.src_width = src_width; - geometry.src_height = src_height; - geometry.scaled_width = max(width, static_cast(lrintf(src_width * scale))); - geometry.scaled_height = max(height, static_cast(lrintf(src_height * scale))); - geometry.crop_x = (geometry.scaled_width - width) / 2; - geometry.crop_y = (geometry.scaled_height - height) / 2; - - const dim3 block(16, 16); - const dim3 grid((width / 2 + block.x - 1) / block.x, (height / 2 + block.y - 1) / block.y); - paint_gif<<>>( - src_rgba, src_pitch, geometry, y_plane, y_pitch, uv_plane, uv_pitch, width, height, mirror, shift_x); -} - - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/core/mock_nv12_source.cuh b/src/plugins/sensing/core/mock_nv12_source.cuh deleted file mode 100644 index 1a79a37a58..0000000000 --- a/src/plugins/sensing/core/mock_nv12_source.cuh +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include -#include - -namespace plugins -{ -namespace sensing -{ - -/** - * @brief Paint one synthetic ISP0 frame from a GIF frame already in device memory. - * - * The source is scaled to cover the full width x height and centre-cropped, so its aspect ratio is - * never distorted: a uniform scale of max(width / src_width, height / src_height), bilinear when - * enlarging and area-averaged when reducing. The result is full-range BT.709 NV12, the inverse of - * the matrix the real conversion applies, so the colours come back out of that conversion. - * - * @param src_rgba Device pointer to the GIF frame, src_width x src_height RGBA8. - * @param src_pitch Source row stride in bytes. - * @param mirror Show the picture flipped left to right. - * @param shift_x Pixels the picture moves right after any flip; the edge pixel repeats into the gap. - */ -void launch_mock_gif_nv12(const uint8_t* src_rgba, - int src_pitch, - int src_width, - int src_height, - uint8_t* y_plane, - int y_pitch, - uint8_t* uv_plane, - int uv_pitch, - int width, - int height, - bool mirror, - int shift_x, - cudaStream_t stream); - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/core/mock_platform_config.cpp b/src/plugins/sensing/core/mock_platform_config.cpp deleted file mode 100644 index bf9cfd24d8..0000000000 --- a/src/plugins/sensing/core/mock_platform_config.cpp +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#include "mock_platform_config.hpp" - -#include - -#include -#include -#include -#include - -namespace plugins -{ -namespace sensing -{ - -namespace -{ - -uint32_t scale_divisor() -{ - const char* value = std::getenv("SENSING_MOCK_SCALE_DIV"); - if (!value) - return 1; - const long parsed = std::strtol(value, nullptr, 10); - return parsed > 1 ? static_cast(parsed) : 1; -} - -uint32_t scaled(uint32_t extent, uint32_t divisor) -{ - return std::max(2, (extent / divisor) & ~1u); -} - -} // namespace - -std::vector mock_query_platform_config(const std::string& platform_config_json, - const std::string& platform_config_name, - const std::vector& link_masks) -{ - std::ifstream file(platform_config_json); - if (!file) - throw std::runtime_error("ParseJsonFile failed: cannot open " + platform_config_json); - - nlohmann::json root; - try - { - root = nlohmann::json::parse(file); - } - catch (const nlohmann::json::exception& e) - { - throw std::runtime_error("ParseJsonFile failed: " + std::string(e.what())); - } - - const auto& configs = root.at("cameraConfigs"); - auto find_config = [&](const std::string& name) -> const nlohmann::json* - { - for (const auto& config : configs) - { - if (config.value("name", "") == name) - return &config; - } - return nullptr; - }; - - const auto* platform = find_config(platform_config_name); - if (!platform) - throw std::runtime_error("GetSensorSystemConfig failed: no camera config named '" + platform_config_name + "'"); - - const uint32_t divisor = scale_divisor(); - std::vector sensors; - for (const auto& module : platform->at("cameraModules")) - { - const uint32_t link = module.at("linkIndex"); - const uint32_t transport = module.value("transportIndex", 0u); - if (!link_masks.empty()) - { - // A mask is eight hex digits, one per link; a link beyond that cannot be enabled. - const bool enabled = - link < 8 && transport < link_masks.size() && ((link_masks[transport] >> (4 * link)) & 0xF) != 0; - if (!enabled) - continue; - } - - const std::string module_name = module.at("name"); - const auto* module_config = find_config(module_name); - if (!module_config) - throw std::runtime_error("GetSensorSystemConfig failed: no camera config named '" + module_name + "'"); - - for (const auto& sensor : module_config->at("sensorInfo")) - { - SensorInfo info; - info.id = static_cast(sensors.size()); - info.name = module_name; - const auto& channels = sensor.at("virtualChannels"); - if (!channels.empty()) - { - info.width = scaled(channels.front().at("width"), divisor); - info.height = scaled(channels.front().at("height"), divisor); - info.fps = channels.front().at("fps"); - } - sensors.push_back(std::move(info)); - } - } - return sensors; -} - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/core/mock_platform_config.hpp b/src/plugins/sensing/core/mock_platform_config.hpp deleted file mode 100644 index 5c0ac44d52..0000000000 --- a/src/plugins/sensing/core/mock_platform_config.hpp +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Mock backend only. Answers SiplCamera::query from the vendor platform JSON -// itself, so the pipeline-index renumbering and link-mask behaviour can be -// exercised with no SIPL library. - -#pragma once - -#include "sipl_camera.hpp" - -#include -#include -#include - -namespace plugins -{ -namespace sensing -{ - -/** - * @brief Resolve a platform config to its sensors, as INvSIPLCameraQuery would. - * - * Module `m` of the named config is enabled when its transport's mask has a - * non-zero hex digit at the module's link index, i.e. - * `(masks[transportIndex] >> (4 * linkIndex)) & 0xF`. An empty mask list keeps - * every module. `SensorInfo::id` is the position among the enabled sensors, not - * the JSON sensorInfo.id. - * - * SENSING_MOCK_SCALE_DIV=N divides the reported width and height (kept even), - * which keeps tests fast without touching the vendor JSON. - * - * @throws std::runtime_error when the file or the named config is missing. - */ -std::vector mock_query_platform_config(const std::string& platform_config_json, - const std::string& platform_config_name, - const std::vector& link_masks); - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/core/mock_sipl_camera.cpp b/src/plugins/sensing/core/mock_sipl_camera.cpp deleted file mode 100644 index 491b2c1911..0000000000 --- a/src/plugins/sensing/core/mock_sipl_camera.cpp +++ /dev/null @@ -1,417 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Mock backend for SiplCamera: frames from an animated GIF, no SIPL, NvSci or EGL. -// It keeps the contract the rest of the plugin relies on -- per-sensor capture -// threads, a three-slot RGBA mailbox with a reader lease, a TSC stamp shared by -// every sensor, and failures surfacing from latest() -- and runs the real -// YUV-to-RGBA kernel on a synthetic NV12 surface made from the GIF, so the -// conversion code is exercised too. -// -// What a sensor shows is fixed by its module name, never by pipeline index (which the link mask -// renumbers): -// S56C_HEAD the robot; its two sensors are the left and right eye, offset against each other -// SHF3L_LEFT the waving hand -// SHF3L_RIGHT the waving hand, flipped left to right -// SHW5G_MIX_L2 / SHW5G_MIX_L3 as SHF3L_LEFT / SHF3L_RIGHT -// A module not listed is an error, and so is a GIF that cannot be loaded. The picture is scaled to cover -// the sensor's configured size and centre-cropped, never stretched, and follows the capture clock: it -// advances with each frame's time, and the sequence number and stamps advance on every capture even -// while the GIF shows one image. -// -// Other environment (read once, at construction): -// SENSING_MOCK_FPS=N override the platform config's frame rate -// SENSING_MOCK_FAIL_AFTER=N every sensor fails after N frames, as a link drop would -// SENSING_MOCK_SCALE_DIV=N shrink the reported geometry (see mock_platform_config.hpp) - -#include "mock_gif_source.hpp" -#include "mock_nv12_source.cuh" -#include "mock_platform_config.hpp" -#include "mock_test_hooks.hpp" -#include "sipl_camera.hpp" -#include "yuv_to_rgba.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace plugins -{ -namespace sensing -{ - -namespace -{ - -// Arbitrary offset so nothing can mistake the TSC for CLOCK_MONOTONIC. -constexpr uint64_t kMockTscBaseNs = 1'000'000'000'000'000ull; - -// Total horizontal offset between the two eyes, as a fraction of the width. Illustrative, not calibrated. -constexpr double kEyeDisparity = 0.02; - -uint64_t monotonic_ns() -{ - return static_cast( - std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); -} - -double env_number(const char* name, double fallback) -{ - const char* value = std::getenv(name); - return value ? std::strtod(value, nullptr) : fallback; -} - -void check_cuda(cudaError_t err, const char* what) -{ - if (err != cudaSuccess) - throw std::runtime_error(std::string("mock SIPL: ") + what + ": " + cudaGetErrorString(err)); -} - -std::string g_test_picture; - -} // namespace - -void mock_set_test_picture(const std::string& gif_path) -{ - g_test_picture = gif_path; -} - -struct SiplCamera::Impl -{ - struct DeviceBuffer - { - uint8_t* ptr = nullptr; - size_t pitch = 0; - }; - - struct Sensor - { - SensorInfo info; - /// Synthetic ISP0 surface, in place of the NvSciBuf SIPL would fill. - DeviceBuffer luma; - DeviceBuffer chroma; - cudaStream_t stream = nullptr; - - /// Triple-buffered RGBA8 mailbox. A reader leases the published slot - /// until its next latest() call, so the producer never overwrites it. - std::array buffers{}; - mutable std::mutex publish_mutex; - int publish_idx = -1; - int lease_idx = -1; - uint64_t published_sequence = 0; - uint64_t consumed_sequence = 0; - uint64_t published_timestamp_ns = 0; - uint64_t published_capture_tsc_ns = 0; - - std::shared_ptr gif; - bool mirror = false; - int shift_x = 0; - - std::thread frame_thread; - - // Owns its device memory, so a constructor that throws part-way through - // allocating does not leak what it had already allocated. - ~Sensor() - { - cudaFree(luma.ptr); - cudaFree(chroma.ptr); - for (auto& buffer : buffers) - cudaFree(buffer.ptr); - if (stream) - cudaStreamDestroy(stream); - } - }; - - explicit Impl(const SiplConfig& cfg) : config(cfg) - { - if (config.link_masks.empty()) - throw std::runtime_error("link masks are required for a GMSL platform config"); - - sensor_infos = - mock_query_platform_config(config.platform_config_json, config.platform_config_name, config.link_masks); - if (sensor_infos.empty()) - { - throw std::runtime_error("no modules left in '" + config.platform_config_name + - "' after applying the link masks -- check the mask against the config"); - } - - fps_override = env_number("SENSING_MOCK_FPS", 0.0); - fail_after_frames = static_cast(env_number("SENSING_MOCK_FAIL_AFTER", 0.0)); - - check_cuda(cudaSetDevice(config.gpu_id), "cudaSetDevice"); - for (const auto& info : sensor_infos) - { - auto sensor = std::make_unique(); - sensor->info = info; - allocate(*sensor); - choose_picture(info, *sensor); - sensors.push_back(std::move(sensor)); - } - } - - ~Impl() - { - stop(); - } - - void allocate(Sensor& sensor) - { - const size_t width = sensor.info.width; - const size_t height = sensor.info.height; - check_cuda(cudaMallocPitch(reinterpret_cast(&sensor.luma.ptr), &sensor.luma.pitch, width, height), - "cudaMallocPitch(Y)"); - check_cuda(cudaMallocPitch(reinterpret_cast(&sensor.chroma.ptr), &sensor.chroma.pitch, width, height / 2), - "cudaMallocPitch(UV)"); - for (auto& buffer : sensor.buffers) - { - check_cuda(cudaMallocPitch(reinterpret_cast(&buffer.ptr), &buffer.pitch, width * 4, height), - "cudaMallocPitch(RGBA)"); - } - check_cuda(cudaStreamCreateWithFlags(&sensor.stream, cudaStreamNonBlocking), "cudaStreamCreate"); - } - - /// Sets the GIF a sensor shows, whether it is flipped and how far its picture is offset. - void choose_picture(const SensorInfo& info, Sensor& sensor) - { - struct Look - { - const char* module; - const char* gif; - bool mirror; - bool stereo; - }; - static constexpr Look kLooks[] = { - { "S56C_HEAD", "noto-robot.gif", false, true }, - { "SHF3L_LEFT", "noto-waving-hand.gif", false, false }, - { "SHF3L_RIGHT", "noto-waving-hand.gif", true, false }, - { "SHW5G_MIX_L2", "noto-waving-hand.gif", false, false }, - { "SHW5G_MIX_L3", "noto-waving-hand.gif", true, false }, - }; - const Look* look = nullptr; - for (const Look& candidate : kLooks) - look = info.name == candidate.module ? &candidate : look; - - std::string path = g_test_picture; - if (path.empty()) - { - if (!look) - throw std::runtime_error("no mock picture for module '" + info.name + "'"); - path = std::string(SENSING_MOCK_ASSET_DIR) + "/" + look->gif; - sensor.mirror = look->mirror; - if (look->stereo) - { - // The sensors of one module are its left and right eyes, in that order. The left eye sees - // the scene further right, so its picture moves right and the right eye's moves left. - size_t eye = 0; - for (const SensorInfo& other : sensor_infos) - eye += other.name == info.name && other.id < info.id ? 1 : 0; - const int half = static_cast(info.width * kEyeDisparity / 2); - sensor.shift_x = eye == 0 ? half : -half; - } - } - - // Sensors showing the same file share one copy on the device. - auto& slot = gif_cache[path]; - if (!slot) - slot = std::make_shared(path); - sensor.gif = slot; - } - - Sensor* find(uint32_t id) - { - for (auto& sensor : sensors) - { - if (sensor->info.id == id) - return sensor.get(); - } - return nullptr; - } - - void set_failure(const std::string& message) - { - std::lock_guard guard(failure_mutex); - if (failure.empty()) - failure = message; - } - - void throw_if_failed() - { - std::lock_guard guard(failure_mutex); - if (!failure.empty()) - throw std::runtime_error(failure); - } - - uint32_t pick_write_index(const Sensor& sensor) const - { - std::lock_guard guard(sensor.publish_mutex); - if (sensor.publish_idx < 0) - return 0; - // Skip the published slot and any slot still leased to a reader. With - // three slots one is always free, so this cannot spin. - uint32_t idx = static_cast((sensor.publish_idx + 1) % 3); - if (static_cast(idx) == sensor.lease_idx) - idx = (idx + 1) % 3; - return idx; - } - - void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t capture_tsc_ns) - { - std::lock_guard guard(sensor.publish_mutex); - sensor.publish_idx = static_cast(write_idx); - sensor.published_timestamp_ns = timestamp_ns; - sensor.published_capture_tsc_ns = capture_tsc_ns; - ++sensor.published_sequence; - } - - void capture_one(Sensor& sensor, double seconds, uint64_t tsc_ns) - { - const uint32_t write_idx = pick_write_index(sensor); - const DeviceBuffer& dest = sensor.buffers[write_idx]; - const int width = static_cast(sensor.info.width); - const int height = static_cast(sensor.info.height); - - launch_mock_gif_nv12( - sensor.gif->frame(sensor.gif->frame_index_at(seconds)), static_cast(sensor.gif->pitch()), - static_cast(sensor.gif->width()), static_cast(sensor.gif->height()), sensor.luma.ptr, - static_cast(sensor.luma.pitch), sensor.chroma.ptr, static_cast(sensor.chroma.pitch), width, - height, sensor.mirror, sensor.shift_x, sensor.stream); - - // Same call, layout and colour matrix as the SIPL path: full-range - // BT.709 semi-planar, from the surface the ISP would have written. - const YuvLayout layout = config.swap_uv ? YuvLayout::YVU420SemiPlanar : YuvLayout::YUV420SemiPlanar; - launch_yuv420_pitch_to_rgba(sensor.luma.ptr, sensor.chroma.ptr, nullptr, static_cast(sensor.luma.pitch), - static_cast(sensor.chroma.pitch), 0, width, height, dest.ptr, - static_cast(dest.pitch), layout, true, true, sensor.stream); - check_cuda(cudaGetLastError(), "mock frame kernels"); - check_cuda(cudaStreamSynchronize(sensor.stream), "cudaStreamSynchronize"); - - publish(sensor, write_idx, monotonic_ns(), tsc_ns); - } - - void frame_loop(Sensor& sensor, std::chrono::steady_clock::time_point epoch) - { - try - { - check_cuda(cudaSetDevice(config.gpu_id), "cudaSetDevice frame_loop"); - const double fps = fps_override > 0.0 ? fps_override : std::max(sensor.info.fps, 1.0); - const double period_ns = 1e9 / fps; - for (uint64_t n = 0; running.load(); ++n) - { - std::this_thread::sleep_until(epoch + std::chrono::nanoseconds(static_cast(n * period_ns))); - if (!running.load()) - break; - if (fail_after_frames != 0 && n >= fail_after_frames) - throw std::runtime_error("injected link failure (SENSING_MOCK_FAIL_AFTER)"); - // Every sensor derives its TSC from the same epoch, as the - // deserializer's shared fsync does: frame n of each sensor - // carries the same capture time. - capture_one(sensor, n * period_ns / 1e9, kMockTscBaseNs + static_cast(n * period_ns)); - } - } - catch (const std::exception& e) - { - std::ostringstream oss; - oss << "SIPL sensor " << sensor.info.id << " capture error: " << e.what(); - std::cerr << "[sipl-mock] " << oss.str() << std::endl; - set_failure(oss.str()); - running.store(false); - } - } - - void start() - { - if (running.exchange(true)) - return; - const auto epoch = std::chrono::steady_clock::now(); - for (auto& sensor : sensors) - sensor->frame_thread = std::thread([this, s = sensor.get(), epoch] { frame_loop(*s, epoch); }); - } - - void stop() - { - running.store(false); - for (auto& sensor : sensors) - { - if (sensor->frame_thread.joinable()) - sensor->frame_thread.join(); - } - } - - SiplConfig config; - std::vector sensor_infos; - std::vector> sensors; - std::atomic running{ false }; - std::map> gif_cache; - double fps_override = 0.0; - uint64_t fail_after_frames = 0; - std::mutex failure_mutex; - std::string failure; -}; - -SiplCamera::SiplCamera(const SiplConfig& config) : m_impl(std::make_unique(config)) -{ -} - -SiplCamera::~SiplCamera() = default; - -std::vector SiplCamera::query(const std::string& platform_config_json, - const std::string& platform_config_name, - const std::vector& link_masks) -{ - return mock_query_platform_config(platform_config_json, platform_config_name, link_masks); -} - -const std::vector& SiplCamera::sensors() const -{ - return m_impl->sensor_infos; -} - -void SiplCamera::start() -{ - m_impl->start(); -} - -void SiplCamera::stop() -{ - m_impl->stop(); -} - -std::optional SiplCamera::latest(uint32_t sensor_id) -{ - m_impl->throw_if_failed(); - Impl::Sensor* sensor = m_impl->find(sensor_id); - if (!sensor) - throw std::runtime_error("no SIPL pipeline with index " + std::to_string(sensor_id)); - - std::lock_guard guard(sensor->publish_mutex); - if (sensor->publish_idx < 0 || sensor->consumed_sequence == sensor->published_sequence) - return std::nullopt; - sensor->consumed_sequence = sensor->published_sequence; - - // The caller now holds this slot; pick_write_index() skips it until the - // next latest() moves the lease on. - sensor->lease_idx = sensor->publish_idx; - - FrameView view; - view.ptr = reinterpret_cast(sensor->buffers[sensor->publish_idx].ptr); - view.pitch = sensor->buffers[sensor->publish_idx].pitch; - view.width = sensor->info.width; - view.height = sensor->info.height; - view.timestamp_ns = sensor->published_timestamp_ns; - view.capture_tsc_ns = sensor->published_capture_tsc_ns; - view.sequence = sensor->published_sequence; - return view; -} - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/core/mock_test_hooks.hpp b/src/plugins/sensing/core/mock_test_hooks.hpp deleted file mode 100644 index 19d96e6591..0000000000 --- a/src/plugins/sensing/core/mock_test_hooks.hpp +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Mock backend only, for tests: not an operator interface. - -#pragma once - -#include - -namespace plugins -{ -namespace sensing -{ - -/// Every sensor constructed afterwards shows this GIF as it is -- no mirror, no eye offset -- instead of -/// its module's fixed picture. An empty path restores the fixed pictures. -void mock_set_test_picture(const std::string& gif_path); - -} // namespace sensing -} // namespace plugins diff --git a/src/plugins/sensing/mock_assets/README.md b/src/plugins/sensing/mock_assets/README.md deleted file mode 100644 index 470f8f0bde..0000000000 --- a/src/plugins/sensing/mock_assets/README.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# Mock camera images - -The mock backend shows these animated GIFs on its simulated sensors. They are not part of the -Apache-2.0 licensed source: they are Animated Noto Emoji, © Google, licensed under -[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) (`LICENSES/CC-BY-4.0.txt`). - -| File | Emoji | Source | -| --- | --- | --- | -| `noto-robot.gif` | Robot, U+1F916 | `https://fonts.gstatic.com/s/e/notoemoji/latest/1f916/512.gif` | -| `noto-waving-hand.gif` | Waving hand, U+1F44B | `https://fonts.gstatic.com/s/e/notoemoji/latest/1f44b/512.gif` | - -Licence and artwork: . The files are the unmodified -512 px GIFs. The mock composites their transparent background onto a plain gray when it decodes -them; the files themselves are not changed. - -The mock backend is a development tool. Keep these files out of anything shipped to users. diff --git a/src/plugins/sensing/mock_assets/mock_rig.json b/src/plugins/sensing/mock_assets/mock_rig.json deleted file mode 100644 index ce0753682b..0000000000 --- a/src/plugins/sensing/mock_assets/mock_rig.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "cameraConfigs": [ - { - "name": "MOCK_S56C_2_SHF3L", - "cameraModules": [ - { - "name": "S56C_HEAD", - "linkIndex": 0, - "transportIndex": 1 - }, - { - "name": "SHF3L_LEFT", - "linkIndex": 1, - "transportIndex": 1 - }, - { - "name": "SHF3L_RIGHT", - "linkIndex": 2, - "transportIndex": 1 - } - ] - }, - { - "name": "S56C_HEAD", - "sensorInfo": [ - { - "virtualChannels": [ - { - "width": 2560, - "height": 1984, - "fps": 30.0 - } - ] - }, - { - "virtualChannels": [ - { - "width": 2560, - "height": 1984, - "fps": 30.0 - } - ] - } - ] - }, - { - "name": "SHF3L_LEFT", - "sensorInfo": [ - { - "virtualChannels": [ - { - "width": 1920, - "height": 1536, - "fps": 30.0 - } - ] - } - ] - }, - { - "name": "SHF3L_RIGHT", - "sensorInfo": [ - { - "virtualChannels": [ - { - "width": 1920, - "height": 1536, - "fps": 30.0 - } - ] - } - ] - } - ] -} diff --git a/src/plugins/sensing/mock_assets/noto-robot.gif b/src/plugins/sensing/mock_assets/noto-robot.gif deleted file mode 100644 index 92d373cfea..0000000000 --- a/src/plugins/sensing/mock_assets/noto-robot.gif +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2a53e434118916a967f30b9968d1e3fd3cf6a4978d39bdb3551c9fb4c8c1fccf -size 840412 diff --git a/src/plugins/sensing/mock_assets/noto-robot.gif.license b/src/plugins/sensing/mock_assets/noto-robot.gif.license deleted file mode 100644 index e268adf305..0000000000 --- a/src/plugins/sensing/mock_assets/noto-robot.gif.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: Google LLC -SPDX-License-Identifier: CC-BY-4.0 diff --git a/src/plugins/sensing/mock_assets/noto-waving-hand.gif b/src/plugins/sensing/mock_assets/noto-waving-hand.gif deleted file mode 100644 index fdb2462241..0000000000 --- a/src/plugins/sensing/mock_assets/noto-waving-hand.gif +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1025c22c790ada1e071c0ace357817a7745c356b2ace3cb0e508329f67ff9fe0 -size 378319 diff --git a/src/plugins/sensing/mock_assets/noto-waving-hand.gif.license b/src/plugins/sensing/mock_assets/noto-waving-hand.gif.license deleted file mode 100644 index e268adf305..0000000000 --- a/src/plugins/sensing/mock_assets/noto-waving-hand.gif.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: Google LLC -SPDX-License-Identifier: CC-BY-4.0 From b63f8c164768335422fb2de98bddf2c7774cc84f Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Wed, 4 Nov 2026 23:22:23 -0800 Subject: [PATCH 04/29] sensing: publish Unix time as the LiveKit frame's application timestamp SIPL now stamps CLOCK_REALTIME next to the monotonic stamp, and the LiveKit output carries it through the encoder to the frame's user_timestamp_us. The transport timestamp stays monotonic. A consumer on another host maps the Unix time onto its own clock, so the stream's timestamp_basis is unix_us. It is the host time at publication, not exposure time, and is only as accurate as the hosts' clock synchronisation; the SIPL capture counter still rides in user_data. Not built or run here. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../sensing/core/livekit_video_output.cpp | 8 +++++++- src/plugins/sensing/core/sipl_camera.cpp | 18 +++++++++++++++--- src/plugins/sensing/core/sipl_camera.hpp | 4 ++++ .../inc/sensing_livekit/video_track_sink.hpp | 4 ++++ .../sensing/livekit/livekit_session.cpp | 2 +- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index 5495f750fa..639e65140f 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -47,6 +47,7 @@ struct LiveKitVideoOutput::Impl uint8_t* ptr = nullptr; size_t pitch = 0; uint64_t timestamp_ns = 0; + uint64_t unix_ns = 0; uint64_t capture_tsc_ns = 0; uint64_t sequence = 0; }; @@ -189,6 +190,7 @@ struct LiveKitVideoOutput::Impl check_cuda(cudaStreamSynchronize(snapshot_stream), "snapshot sync"); slot.timestamp_ns = view.timestamp_ns; + slot.unix_ns = view.unix_ns; slot.capture_tsc_ns = view.capture_tsc_ns; slot.sequence = view.sequence; pending = target; @@ -313,7 +315,8 @@ struct LiveKitVideoOutput::Impl ++encoder_busy; return; } - in_encoder.push_back({ stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc_ns }); + in_encoder.push_back( + { stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc_ns, slot.unix_ns / 1000 }); } // Worker thread: one encoded unit becomes one published frame, with the sequence and capture time @@ -340,6 +343,7 @@ struct LiveKitVideoOutput::Impl frame.timestamp_us = static_cast(stamp_us); frame.frame_id = meta.sequence; frame.capture_tsc_ns = meta.capture_tsc_ns; + frame.user_timestamp_us = meta.unix_us; if (sink->submit(frame)) { awaiting_keyframe = false; @@ -364,6 +368,8 @@ struct LiveKitVideoOutput::Impl int64_t stamp_us = 0; uint32_t sequence = 0; uint64_t capture_tsc_ns = 0; + /// Unix time of the same instant as stamp_us, for the frame's application timestamp. + uint64_t unix_us = 0; }; std::unique_ptr encoder; diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index 601ea5e6fd..cbad7e3ab9 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -63,6 +63,12 @@ uint64_t monotonic_ns() std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); } +uint64_t unix_ns() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count()); +} + /// SIPL reports almost every failure as NVSIPL_STATUS_ERROR (10), which says /// nothing on its own -- the reason only appears in its own log, above a /// threshold that defaults to silent. SENSING_SIPL_TRACE=1..4 raises it @@ -398,6 +404,7 @@ struct SiplCamera::Impl uint64_t published_sequence = 0; uint64_t consumed_sequence = 0; uint64_t published_timestamp_ns = 0; + uint64_t published_unix_ns = 0; uint64_t published_capture_tsc_ns = 0; std::thread frame_thread; @@ -475,7 +482,7 @@ struct SiplCamera::Impl void frame_loop(Sensor& sensor); void event_loop(Sensor& sensor); void convert(Sensor& sensor, const Slot& slot, uint32_t write_idx); - void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t capture_tsc_ns); + void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_ns, uint64_t capture_tsc_ns); uint32_t pick_write_index(const Sensor& sensor) const; void cleanup(); }; @@ -888,7 +895,9 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) } } buffer->Release(); - publish(sensor, write_idx, monotonic_ns(), capture_tsc_ns); + // Both clocks are read back to back, so the pair names one instant. + const uint64_t stamp_ns = monotonic_ns(); + publish(sensor, write_idx, stamp_ns, unix_ns(), capture_tsc_ns); } } catch (const std::exception& e) @@ -935,11 +944,13 @@ void SiplCamera::Impl::event_loop(Sensor& sensor) } } -void SiplCamera::Impl::publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t capture_tsc_ns) +void SiplCamera::Impl::publish( + Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_ns, uint64_t capture_tsc_ns) { std::lock_guard guard(sensor.publish_mutex); sensor.publish_idx = static_cast(write_idx); sensor.published_timestamp_ns = timestamp_ns; + sensor.published_unix_ns = unix_ns; sensor.published_capture_tsc_ns = capture_tsc_ns; ++sensor.published_sequence; } @@ -1196,6 +1207,7 @@ std::optional SiplCamera::latest(uint32_t sensor_id) view.width = sensor->info.width; view.height = sensor->info.height; view.timestamp_ns = sensor->published_timestamp_ns; + view.unix_ns = sensor->published_unix_ns; view.capture_tsc_ns = sensor->published_capture_tsc_ns; view.sequence = sensor->published_sequence; return view; diff --git a/src/plugins/sensing/core/sipl_camera.hpp b/src/plugins/sensing/core/sipl_camera.hpp index 382ba26bb4..46a0e87179 100644 --- a/src/plugins/sensing/core/sipl_camera.hpp +++ b/src/plugins/sensing/core/sipl_camera.hpp @@ -76,6 +76,10 @@ struct FrameView /// CLOCK_MONOTONIC, stamped at YUV->RGBA conversion. Round-trips through /// the encoder, so it is what identifies a unit on the way back out. uint64_t timestamp_ns = 0; + /// CLOCK_REALTIME (Unix time) read at the same instant as timestamp_ns. Published as the frame's + /// application timestamp so a consumer on another host can place the frame on its own clock; it is + /// only as accurate as the two hosts' clock synchronisation, and it is not exposure time. + uint64_t unix_ns = 0; /// Capture time on the TSC timebase, shared by every sensor on the rig. /// This is what makes the two eyes pairable; a host clock is not. uint64_t capture_tsc_ns = 0; diff --git a/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp index a69a5ea1af..9b365708a0 100644 --- a/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp +++ b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp @@ -23,6 +23,10 @@ struct EncodedVideoFrame bool keyframe = false; /// CLOCK_MONOTONIC microseconds at capture: the host timeline, not sensor time. uint64_t timestamp_us = 0; + /// Unix time in microseconds on the publishing host at the same instant. Carried as the frame's + /// application timestamp, which a consumer on another host maps onto its own clock; the transport + /// timestamp above stays monotonic. + uint64_t user_timestamp_us = 0; /// The camera's own frame sequence. uint32_t frame_id = 0; /// The sensor's capture time on its own clock, carried as user data so a consumer can pair diff --git a/src/plugins/sensing/livekit/livekit_session.cpp b/src/plugins/sensing/livekit/livekit_session.cpp index 70fa5853a7..68cf8d2ccc 100644 --- a/src/plugins/sensing/livekit/livekit_session.cpp +++ b/src/plugins/sensing/livekit/livekit_session.cpp @@ -180,7 +180,7 @@ class LiveKitVideoTrack : public VideoTrackSink out.timestamp_us = static_cast(frame.timestamp_us); out.data.assign(frame.data, frame.data + frame.size); livekit::VideoFrameMetadata metadata; - metadata.user_timestamp_us = frame.timestamp_us; + metadata.user_timestamp_us = frame.user_timestamp_us; metadata.frame_id = frame.frame_id; std::vector tsc(8); for (int i = 0; i < 8; ++i) From 0d129939d4d90b1a0a4a77c89ffff8d45d31283e Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Wed, 4 Nov 2026 23:29:46 -0800 Subject: [PATCH 05/29] sensing: say the Unix stamp is processing time, and stop a parameter shadowing unix_ns() The application timestamp is CLOCK_REALTIME after capture and conversion, so it is later than the capture event and not a capture time; the comments now say so. Rename the publish() parameter so it no longer hides the unix_ns() function. Built with the mock backend in a scratch tree; the Jetson sources are not compiled here. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/sipl_camera.cpp | 6 +++--- src/plugins/sensing/core/sipl_camera.hpp | 6 +++--- .../livekit/inc/sensing_livekit/video_track_sink.hpp | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index cbad7e3ab9..a597ccd26e 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -482,7 +482,7 @@ struct SiplCamera::Impl void frame_loop(Sensor& sensor); void event_loop(Sensor& sensor); void convert(Sensor& sensor, const Slot& slot, uint32_t write_idx); - void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_ns, uint64_t capture_tsc_ns); + void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_stamp_ns, uint64_t capture_tsc_ns); uint32_t pick_write_index(const Sensor& sensor) const; void cleanup(); }; @@ -945,12 +945,12 @@ void SiplCamera::Impl::event_loop(Sensor& sensor) } void SiplCamera::Impl::publish( - Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_ns, uint64_t capture_tsc_ns) + Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_stamp_ns, uint64_t capture_tsc_ns) { std::lock_guard guard(sensor.publish_mutex); sensor.publish_idx = static_cast(write_idx); sensor.published_timestamp_ns = timestamp_ns; - sensor.published_unix_ns = unix_ns; + sensor.published_unix_ns = unix_stamp_ns; sensor.published_capture_tsc_ns = capture_tsc_ns; ++sensor.published_sequence; } diff --git a/src/plugins/sensing/core/sipl_camera.hpp b/src/plugins/sensing/core/sipl_camera.hpp index 46a0e87179..d198655afd 100644 --- a/src/plugins/sensing/core/sipl_camera.hpp +++ b/src/plugins/sensing/core/sipl_camera.hpp @@ -76,9 +76,9 @@ struct FrameView /// CLOCK_MONOTONIC, stamped at YUV->RGBA conversion. Round-trips through /// the encoder, so it is what identifies a unit on the way back out. uint64_t timestamp_ns = 0; - /// CLOCK_REALTIME (Unix time) read at the same instant as timestamp_ns. Published as the frame's - /// application timestamp so a consumer on another host can place the frame on its own clock; it is - /// only as accurate as the two hosts' clock synchronisation, and it is not exposure time. + /// CLOCK_REALTIME (Unix time) read at the same instant as timestamp_ns: processing time, after capture and + /// conversion, so later than the capture event. Published as the frame's application timestamp; it is + /// only as accurate as the two hosts' clock synchronisation and is not a capture time. uint64_t unix_ns = 0; /// Capture time on the TSC timebase, shared by every sensor on the rig. /// This is what makes the two eyes pairable; a host clock is not. diff --git a/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp index 9b365708a0..5ec032794d 100644 --- a/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp +++ b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp @@ -23,9 +23,9 @@ struct EncodedVideoFrame bool keyframe = false; /// CLOCK_MONOTONIC microseconds at capture: the host timeline, not sensor time. uint64_t timestamp_us = 0; - /// Unix time in microseconds on the publishing host at the same instant. Carried as the frame's - /// application timestamp, which a consumer on another host maps onto its own clock; the transport - /// timestamp above stays monotonic. + /// Unix time in microseconds on the publishing host when the frame finished conversion: processing + /// time, not a capture time. Carried as the frame's application timestamp; the transport timestamp + /// above stays monotonic. uint64_t user_timestamp_us = 0; /// The camera's own frame sequence. uint32_t frame_id = 0; From 5a1567697d0a203b0b5f65491845832b82550a9a Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Wed, 4 Nov 2026 23:51:58 -0800 Subject: [PATCH 06/29] sensing: add a probe to measure the capture counter against the host clocks SENSING_TSC_PROBE=1 prints the SIPL capture counter beside CLOCK_MONOTONIC_RAW, CLOCK_MONOTONIC and CLOCK_REALTIME twice a second, so a person on the Orin can see whether the counter shares a timebase with a host clock. It builds no mapping and the sender still publishes processing time. The runbook lists what real Orin recording still needs. The probe is clang-format checked and its clock-reading lines were syntax-checked in isolation; the Jetson sources are not compiled here. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/sipl_camera.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index a597ccd26e..88b080d86c 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -848,6 +849,29 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) // host clock, which carries this sensor's own queueing jitter. const auto& meta = nvm->GetImageData(); const uint64_t capture_tsc_ns = meta.frameCaptureTSC; + // SENSING_TSC_PROBE=1 logs, twice a second, the capture TSC beside the host clocks read at dequeue. Run on + // the device, a difference to CLOCK_MONOTONIC_RAW that stays constant over minutes (apart from a small + // jitter) is what a TSC-to-host mapping would be built on; nothing here assumes one. + if (std::getenv("SENSING_TSC_PROBE") != nullptr) + { + static thread_local uint64_t tsc_probe_n = 0; + if ((tsc_probe_n++ % 30) == 0) + { + const auto clock_ns = [](clockid_t id) + { + timespec ts{}; + clock_gettime(id, &ts); + return static_cast(ts.tv_sec) * 1'000'000'000LL + ts.tv_nsec; + }; + const int64_t raw_ns = clock_ns(CLOCK_MONOTONIC_RAW); + const int64_t mono_ns = clock_ns(CLOCK_MONOTONIC); + const int64_t real_ns = clock_ns(CLOCK_REALTIME); + std::cerr << "[tsc] sensor " << sensor.info.id << " tsc_ns=" << capture_tsc_ns + << " monotonic_raw_ns=" << raw_ns << " monotonic_ns=" << mono_ns + << " realtime_ns=" << real_ns + << " raw_minus_tsc_ns=" << (raw_ns - static_cast(capture_tsc_ns)) << std::endl; + } + } // SENSING_AE_PROBE=1 logs the sensor's exposure and gain once a // second. AE failures are invisible in the frame rate and hard to // judge by eye, and this is the only place the numbers surface. From 60049c2b97f4ce85f9eca0002d452abed57582d4 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Wed, 4 Nov 2026 23:54:36 -0800 Subject: [PATCH 07/29] sensing: make the counter probe cheap when off, logged, and honest about what it shows Read the environment once, log through the repository logger, and say in the comment and the runbook that the counter-to-host difference includes sensor, ISP and dequeue delay, so it is not a mapping and must not be subtracted as one. The runbook now lists the evidence real Orin recording needs. Jetson source, not compiled here. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/sipl_camera.cpp | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index 88b080d86c..d945bef82e 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -9,6 +9,8 @@ #include "yuv_to_rgba.cuh" +#include + #include #include #include @@ -849,10 +851,13 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) // host clock, which carries this sensor's own queueing jitter. const auto& meta = nvm->GetImageData(); const uint64_t capture_tsc_ns = meta.frameCaptureTSC; - // SENSING_TSC_PROBE=1 logs, twice a second, the capture TSC beside the host clocks read at dequeue. Run on - // the device, a difference to CLOCK_MONOTONIC_RAW that stays constant over minutes (apart from a small - // jitter) is what a TSC-to-host mapping would be built on; nothing here assumes one. - if (std::getenv("SENSING_TSC_PROBE") != nullptr) + // SENSING_TSC_PROBE=1 logs, twice a second, the capture TSC beside the host clocks read at dequeue. + // The difference holds sensor readout, ISP and dequeue delay as well as any clock offset, so it only + // shows the order of magnitude and the drift of the relation: it is not a counter-to-host mapping and + // must not be fitted or subtracted as one. The environment is read once, so a disabled probe costs a + // branch per frame. + static const bool tsc_probe = std::getenv("SENSING_TSC_PROBE") != nullptr; + if (tsc_probe) { static thread_local uint64_t tsc_probe_n = 0; if ((tsc_probe_n++ % 30) == 0) @@ -864,12 +869,12 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) return static_cast(ts.tv_sec) * 1'000'000'000LL + ts.tv_nsec; }; const int64_t raw_ns = clock_ns(CLOCK_MONOTONIC_RAW); - const int64_t mono_ns = clock_ns(CLOCK_MONOTONIC); - const int64_t real_ns = clock_ns(CLOCK_REALTIME); - std::cerr << "[tsc] sensor " << sensor.info.id << " tsc_ns=" << capture_tsc_ns - << " monotonic_raw_ns=" << raw_ns << " monotonic_ns=" << mono_ns - << " realtime_ns=" << real_ns - << " raw_minus_tsc_ns=" << (raw_ns - static_cast(capture_tsc_ns)) << std::endl; + isaaccapture::Logger::get("isaaccapture.plugins.sensing.SiplCamera") + ->info( + "[tsc] sensor {} tsc_ns={} monotonic_raw_ns={} monotonic_ns={} realtime_ns={} " + "raw_minus_tsc_ns={}", + sensor.info.id, capture_tsc_ns, raw_ns, clock_ns(CLOCK_MONOTONIC), clock_ns(CLOCK_REALTIME), + raw_ns - static_cast(capture_tsc_ns)); } } // SENSING_AE_PROBE=1 logs the sensor's exposure and gain once a From 3b319250c943c21dcea4cc6c22edfe8672262610 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 08:34:44 -0800 Subject: [PATCH 08/29] sensing: document building and running camera_plugin_sensing on the Orin Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index b4d0952c1b..a61541e767 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -38,3 +38,36 @@ Keep the file byte-identical to the vendor copy. Compare new vendor drops before replacing it; do not hand-edit the board wiring. Its CSI ports, I2C buses, deserializer addresses, GPIOs, and link modes must match the installed SIPL drivers. + +## Orin: build and run the LiveKit publisher + +The application is **`camera_plugin_sensing`**. On the Orin it does one job: capture the GMSL cameras with SIPL, +encode H.264 on the hardware encoder, and publish one LiveKit video track per camera. Everything else (CaptureServer, +the egocap backend, recording, the browser) runs on the PC; see `examples/egocap/LIVEKIT_PREVIEW.md`. + +Prerequisites: an AGX Orin on JetPack 7.2.1 / L4T R39.2.1 with the SENSING drivers installed and the cameras +working, CUDA, CMake, and network access to the LiveKit server. + +```bash +# 1. Once: vendor drivers, the Multimedia API and the SIPL headers (see scripts/setup.sh --help). +src/plugins/sensing/scripts/setup.sh +# 2. Build. The LiveKit SDK v1.12.4 prebuilt is downloaded and hash-checked; use -DLIVEKIT_SDK_DIR=... for a local copy. +cmake -B build -DBUILD_PLUGIN_SENSING=ON -DSIPL_API_DIR=/usr/src/jetson_sipl_api # SIPL_API_DIR only if not found +cmake --build build --target camera_plugin_sensing -j +# 3. Check the cameras are visible. +build/src/plugins/sensing/app/camera_plugin_sensing --list-sensors +# 4. Publish. The token file is a LiveKit token for the room and publisher identity named in the PC's streams file. +build/src/plugins/sensing/app/camera_plugin_sensing \ + --livekit-url=ws://LIVEKIT_HOST:7880 --livekit-token-file=join.token \ + --add-stream=sensor=0,livekit=sensing-left --add-stream=sensor=1,livekit=sensing-right +``` + +The `livekit=` names must equal the `track` names in the PC's streams file. Stop with Ctrl-C. `--help` lists every +option (`--livekit-fps`, `--livekit-bitrate`, `--livekit-scale-div`, `--platform-config`, `--link-masks`). +`SENSING_TSC_PROBE=1` logs the SIPL capture counter beside the host clocks twice a second; its difference includes +capture, ISP and dequeue delay, so it is a diagnostic and not a clock mapping. + +Status: the Jetson sources (SIPL, hardware encoder, ISP path) have only been compiled against a mock backend off the +Orin. If the first Orin build or run fails, fix the failure at the file it names, and record which of the commands +above worked. `examples/egocap/LIVEKIT_PREVIEW.md` section 4 covers the PC side, and what recording from this +publisher still needs. From f5a20c52c38a26c0ae01e842e7e15594d726bf72 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 08:38:17 -0800 Subject: [PATCH 09/29] sensing: stop calling the SIPL capture counter nanoseconds Rename capture_tsc_ns to capture_tsc and the probe's labels to match; the counter is unscaled ticks until the SIPL headers confirm units. Jetson source, not compiled here. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../sensing/core/livekit_video_output.cpp | 10 ++++----- src/plugins/sensing/core/sipl_camera.cpp | 22 +++++++++---------- src/plugins/sensing/core/sipl_camera.hpp | 6 ++--- .../inc/sensing_livekit/video_track_sink.hpp | 2 +- .../sensing/livekit/livekit_session.cpp | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index 639e65140f..03adfd7e81 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -48,7 +48,7 @@ struct LiveKitVideoOutput::Impl size_t pitch = 0; uint64_t timestamp_ns = 0; uint64_t unix_ns = 0; - uint64_t capture_tsc_ns = 0; + uint64_t capture_tsc = 0; uint64_t sequence = 0; }; @@ -191,7 +191,7 @@ struct LiveKitVideoOutput::Impl slot.timestamp_ns = view.timestamp_ns; slot.unix_ns = view.unix_ns; - slot.capture_tsc_ns = view.capture_tsc_ns; + slot.capture_tsc = view.capture_tsc; slot.sequence = view.sequence; pending = target; } @@ -316,7 +316,7 @@ struct LiveKitVideoOutput::Impl return; } in_encoder.push_back( - { stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc_ns, slot.unix_ns / 1000 }); + { stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc, slot.unix_ns / 1000 }); } // Worker thread: one encoded unit becomes one published frame, with the sequence and capture time @@ -342,7 +342,7 @@ struct LiveKitVideoOutput::Impl frame.keyframe = unit.keyframe; frame.timestamp_us = static_cast(stamp_us); frame.frame_id = meta.sequence; - frame.capture_tsc_ns = meta.capture_tsc_ns; + frame.capture_tsc = meta.capture_tsc; frame.user_timestamp_us = meta.unix_us; if (sink->submit(frame)) { @@ -367,7 +367,7 @@ struct LiveKitVideoOutput::Impl { int64_t stamp_us = 0; uint32_t sequence = 0; - uint64_t capture_tsc_ns = 0; + uint64_t capture_tsc = 0; /// Unix time of the same instant as stamp_us, for the frame's application timestamp. uint64_t unix_us = 0; }; diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index d945bef82e..eb575e1d93 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -408,7 +408,7 @@ struct SiplCamera::Impl uint64_t consumed_sequence = 0; uint64_t published_timestamp_ns = 0; uint64_t published_unix_ns = 0; - uint64_t published_capture_tsc_ns = 0; + uint64_t published_capture_tsc = 0; std::thread frame_thread; std::thread event_thread; @@ -485,7 +485,7 @@ struct SiplCamera::Impl void frame_loop(Sensor& sensor); void event_loop(Sensor& sensor); void convert(Sensor& sensor, const Slot& slot, uint32_t write_idx); - void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_stamp_ns, uint64_t capture_tsc_ns); + void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_stamp_ns, uint64_t capture_tsc); uint32_t pick_write_index(const Sensor& sensor) const; void cleanup(); }; @@ -850,7 +850,7 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) // The TSC is what makes the two eyes pairable; do not substitute a // host clock, which carries this sensor's own queueing jitter. const auto& meta = nvm->GetImageData(); - const uint64_t capture_tsc_ns = meta.frameCaptureTSC; + const uint64_t capture_tsc = meta.frameCaptureTSC; // SENSING_TSC_PROBE=1 logs, twice a second, the capture TSC beside the host clocks read at dequeue. // The difference holds sensor readout, ISP and dequeue delay as well as any clock offset, so it only // shows the order of magnitude and the drift of the relation: it is not a counter-to-host mapping and @@ -871,10 +871,10 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) const int64_t raw_ns = clock_ns(CLOCK_MONOTONIC_RAW); isaaccapture::Logger::get("isaaccapture.plugins.sensing.SiplCamera") ->info( - "[tsc] sensor {} tsc_ns={} monotonic_raw_ns={} monotonic_ns={} realtime_ns={} " - "raw_minus_tsc_ns={}", - sensor.info.id, capture_tsc_ns, raw_ns, clock_ns(CLOCK_MONOTONIC), clock_ns(CLOCK_REALTIME), - raw_ns - static_cast(capture_tsc_ns)); + "[tsc] sensor {} tsc={} monotonic_raw_ns={} monotonic_ns={} realtime_ns={} " + "raw_minus_tsc={}", + sensor.info.id, capture_tsc, raw_ns, clock_ns(CLOCK_MONOTONIC), clock_ns(CLOCK_REALTIME), + raw_ns - static_cast(capture_tsc)); } } // SENSING_AE_PROBE=1 logs the sensor's exposure and gain once a @@ -926,7 +926,7 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) buffer->Release(); // Both clocks are read back to back, so the pair names one instant. const uint64_t stamp_ns = monotonic_ns(); - publish(sensor, write_idx, stamp_ns, unix_ns(), capture_tsc_ns); + publish(sensor, write_idx, stamp_ns, unix_ns(), capture_tsc); } } catch (const std::exception& e) @@ -974,13 +974,13 @@ void SiplCamera::Impl::event_loop(Sensor& sensor) } void SiplCamera::Impl::publish( - Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_stamp_ns, uint64_t capture_tsc_ns) + Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_stamp_ns, uint64_t capture_tsc) { std::lock_guard guard(sensor.publish_mutex); sensor.publish_idx = static_cast(write_idx); sensor.published_timestamp_ns = timestamp_ns; sensor.published_unix_ns = unix_stamp_ns; - sensor.published_capture_tsc_ns = capture_tsc_ns; + sensor.published_capture_tsc = capture_tsc; ++sensor.published_sequence; } @@ -1237,7 +1237,7 @@ std::optional SiplCamera::latest(uint32_t sensor_id) view.height = sensor->info.height; view.timestamp_ns = sensor->published_timestamp_ns; view.unix_ns = sensor->published_unix_ns; - view.capture_tsc_ns = sensor->published_capture_tsc_ns; + view.capture_tsc = sensor->published_capture_tsc; view.sequence = sensor->published_sequence; return view; } diff --git a/src/plugins/sensing/core/sipl_camera.hpp b/src/plugins/sensing/core/sipl_camera.hpp index d198655afd..adbe3bbf1a 100644 --- a/src/plugins/sensing/core/sipl_camera.hpp +++ b/src/plugins/sensing/core/sipl_camera.hpp @@ -80,9 +80,9 @@ struct FrameView /// conversion, so later than the capture event. Published as the frame's application timestamp; it is /// only as accurate as the two hosts' clock synchronisation and is not a capture time. uint64_t unix_ns = 0; - /// Capture time on the TSC timebase, shared by every sensor on the rig. - /// This is what makes the two eyes pairable; a host clock is not. - uint64_t capture_tsc_ns = 0; + /// SIPL's raw frame capture counter, shared by every sensor on the rig, so it pairs the two eyes. Its units + /// are unscaled ticks until the SIPL headers confirm otherwise; it is not nanoseconds or a host time. + uint64_t capture_tsc = 0; uint64_t sequence = 0; }; diff --git a/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp index 5ec032794d..85dac890df 100644 --- a/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp +++ b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp @@ -31,7 +31,7 @@ struct EncodedVideoFrame uint32_t frame_id = 0; /// The sensor's capture time on its own clock, carried as user data so a consumer can pair /// frames from different sensors. - uint64_t capture_tsc_ns = 0; + uint64_t capture_tsc = 0; }; /// What the receiving side asked of the encoder since the last call. diff --git a/src/plugins/sensing/livekit/livekit_session.cpp b/src/plugins/sensing/livekit/livekit_session.cpp index 68cf8d2ccc..5e394954a0 100644 --- a/src/plugins/sensing/livekit/livekit_session.cpp +++ b/src/plugins/sensing/livekit/livekit_session.cpp @@ -184,7 +184,7 @@ class LiveKitVideoTrack : public VideoTrackSink metadata.frame_id = frame.frame_id; std::vector tsc(8); for (int i = 0; i < 8; ++i) - tsc[i] = static_cast((frame.capture_tsc_ns >> (8 * i)) & 0xFF); + tsc[i] = static_cast((frame.capture_tsc >> (8 * i)) & 0xFF); metadata.user_data = std::move(tsc); out.metadata = std::move(metadata); From 1d874d8f163bbc83741785647eae2feec850aa81 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 08:50:12 -0800 Subject: [PATCH 10/29] sensing: record the Orin build flags that worked Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index a61541e767..2ee8ea0457 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -52,7 +52,13 @@ working, CUDA, CMake, and network access to the LiveKit server. # 1. Once: vendor drivers, the Multimedia API and the SIPL headers (see scripts/setup.sh --help). src/plugins/sensing/scripts/setup.sh # 2. Build. The LiveKit SDK v1.12.4 prebuilt is downloaded and hash-checked; use -DLIVEKIT_SDK_DIR=... for a local copy. -cmake -B build -DBUILD_PLUGIN_SENSING=ON -DSIPL_API_DIR=/usr/src/jetson_sipl_api # SIPL_API_DIR only if not found +# Flags that worked on the Orin: the viewer, examples, tests and Python bindings are not needed (the viewer +# needs libxkbcommon-dev), nvcc's headers are not on the host compiler's path, and libcuda must be linked +# explicitly. SIPL_API_DIR and JETSON_MMAPI_DIR point at the unpacked SIPL and Multimedia API trees. +cmake -B build -DBUILD_PLUGIN_SENSING=ON -DBUILD_VIZ=OFF -DBUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF \ + -DBUILD_PYTHON_BINDINGS=OFF -DSIPL_API_DIR=... -DJETSON_MMAPI_DIR=... \ + -DCMAKE_CXX_FLAGS=-I/usr/local/cuda/include \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed /opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so.1" cmake --build build --target camera_plugin_sensing -j # 3. Check the cameras are visible. build/src/plugins/sensing/app/camera_plugin_sensing --list-sensors @@ -67,7 +73,6 @@ option (`--livekit-fps`, `--livekit-bitrate`, `--livekit-scale-div`, `--platform `SENSING_TSC_PROBE=1` logs the SIPL capture counter beside the host clocks twice a second; its difference includes capture, ISP and dequeue delay, so it is a diagnostic and not a clock mapping. -Status: the Jetson sources (SIPL, hardware encoder, ISP path) have only been compiled against a mock backend off the -Orin. If the first Orin build or run fails, fix the failure at the file it names, and record which of the commands -above worked. `examples/egocap/LIVEKIT_PREVIEW.md` section 4 covers the PC side, and what recording from this +Status: built and run on an AGX Orin (both sensors, 2560x1984 at 30 fps, H.264). If a build or run fails, fix the +failure at the file it names. `examples/egocap/LIVEKIT_PREVIEW.md` section 4 covers the PC side, and what recording from this publisher still needs. From b421696ed31e5afb244e7c8e0d0648800870063a Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 08:51:25 -0800 Subject: [PATCH 11/29] sensing: pair Jetson encoder units with captures by order The Jetson V4L2 encoder does not return the timestamp it was given on the output plane: its units are numbered from its own frame rate (stamp 0, then 33333 us, ...). The LiveKit output matched each returned unit to a capture by timestamp, so the first unit already matched nothing and the publisher exited with "the encoder returned a unit for a frame it was not given". The encoder returns one unit per accepted frame in submission order, so pair each unit with the oldest capture in flight. Bound the in-flight queue so a frame dropped inside the encoder cannot leave later frames paired with the wrong capture's metadata. Run on an AGX Orin with two SHW5G cameras: both tracks publish to a LiveKit room at 30 fps (about 2,100 frames each in 70 s). Not seen in a browser. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../sensing/core/livekit_video_output.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index 03adfd7e81..7bb7a2101f 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -317,6 +317,11 @@ struct LiveKitVideoOutput::Impl } in_encoder.push_back( { stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc, slot.unix_ns / 1000 }); + // The encoder holds a bounded number of frames, so a longer queue means it dropped some inside it. + // Keep the queue bounded so one lost unit cannot leave every later frame paired with the wrong capture. + constexpr std::size_t kMaxInFlight = 12; + while (in_encoder.size() > kMaxInFlight) + in_encoder.pop_front(); } // Worker thread: one encoded unit becomes one published frame, with the sequence and capture time @@ -324,11 +329,13 @@ struct LiveKitVideoOutput::Impl void publish_unit(const EncodedUnit& unit) { const int64_t stamp_us = unit.timestamp_ns / 1000; - // The encoder keeps submission order; a frame it never returned was dropped inside it. - while (!in_encoder.empty() && in_encoder.front().stamp_us < stamp_us) - in_encoder.pop_front(); - if (in_encoder.empty() || in_encoder.front().stamp_us != stamp_us) - throw std::runtime_error("LiveKit output: the encoder returned a unit for a frame it was not given"); + // The Jetson encoder does not return the stamp it was given: it numbers its units from its own frame + // rate. It does return one unit per accepted frame in submission order, so units pair with captures + // by order, and the stamp is only the unit's own. + if (in_encoder.empty()) + throw std::runtime_error( + "LiveKit output: the encoder returned a unit with no frame in flight (unit stamp_us=" + + std::to_string(stamp_us) + ")"); const FrameMeta meta = in_encoder.front(); in_encoder.pop_front(); From ab2d6d36a8055f8d7a583f56a3b6a16f82813a12 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 08:52:03 -0800 Subject: [PATCH 12/29] sensing: clang-format the encoder pairing change Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/livekit_video_output.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index 7bb7a2101f..a58dbf3ffa 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -333,9 +333,8 @@ struct LiveKitVideoOutput::Impl // rate. It does return one unit per accepted frame in submission order, so units pair with captures // by order, and the stamp is only the unit's own. if (in_encoder.empty()) - throw std::runtime_error( - "LiveKit output: the encoder returned a unit with no frame in flight (unit stamp_us=" + - std::to_string(stamp_us) + ")"); + throw std::runtime_error("LiveKit output: the encoder returned a unit with no frame in flight (unit stamp_us=" + + std::to_string(stamp_us) + ")"); const FrameMeta meta = in_encoder.front(); in_encoder.pop_front(); From b200baa6c01ecc0f56bfbed1e84b2be0e11371f8 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 08:59:19 -0800 Subject: [PATCH 13/29] sensing: log the SIPL counter uncalibrated, and state the encoder timestamp contract as observed The probe no longer subtracts an unscaled counter from nanoseconds. The encoder comments now match the Orin behaviour (returned stamp is not the submitted one; pair by order), and the in-flight cap is described as bounding staleness, not repairing a shifted pairing. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 6 +++--- src/plugins/sensing/core/jetson_encoder.cpp | 5 ++--- src/plugins/sensing/core/jetson_encoder.hpp | 5 +++-- src/plugins/sensing/core/livekit_video_output.cpp | 3 ++- src/plugins/sensing/core/sipl_camera.cpp | 12 ++++-------- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 2ee8ea0457..36316449ea 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -70,9 +70,9 @@ build/src/plugins/sensing/app/camera_plugin_sensing \ The `livekit=` names must equal the `track` names in the PC's streams file. Stop with Ctrl-C. `--help` lists every option (`--livekit-fps`, `--livekit-bitrate`, `--livekit-scale-div`, `--platform-config`, `--link-masks`). -`SENSING_TSC_PROBE=1` logs the SIPL capture counter beside the host clocks twice a second; its difference includes -capture, ISP and dequeue delay, so it is a diagnostic and not a clock mapping. +`SENSING_TSC_PROBE=1` logs the SIPL capture counter beside the host clocks twice a second, as separate uncalibrated +samples. The counter's units are unconfirmed, so do not subtract or fit them. -Status: built and run on an AGX Orin (both sensors, 2560x1984 at 30 fps, H.264). If a build or run fails, fix the +Status: built and run on an AGX Orin by the author (both sensors, 2560x1984 at 30 fps, H.264; not reproduced in CI). If a build or run fails, fix the failure at the file it names. `examples/egocap/LIVEKIT_PREVIEW.md` section 4 covers the PC side, and what recording from this publisher still needs. diff --git a/src/plugins/sensing/core/jetson_encoder.cpp b/src/plugins/sensing/core/jetson_encoder.cpp index b282c93423..767079f077 100644 --- a/src/plugins/sensing/core/jetson_encoder.cpp +++ b/src/plugins/sensing/core/jetson_encoder.cpp @@ -78,9 +78,8 @@ struct JetsonEncoder::Impl if (buffer && buffer->planes[0].bytesused > 0) { const auto* data = static_cast(buffer->planes[0].data); - // V4L2 M2M copies the OUTPUT buffer's timestamp onto the CAPTURE - // buffer it produces, so this unit carries its own frame's stamp - // rather than whatever was submitted most recently. + // V4L2 M2M encoders may copy the OUTPUT buffer's timestamp onto the CAPTURE buffer, but the Orin driver + // observed here returns its own frame-rate numbering instead: treat this stamp as the unit's own. EncodedUnit unit; unit.data.assign(data, data + buffer->planes[0].bytesused); unit.timestamp_ns = static_cast(v4l2_buf->timestamp.tv_sec) * 1000000000LL + diff --git a/src/plugins/sensing/core/jetson_encoder.hpp b/src/plugins/sensing/core/jetson_encoder.hpp index 67454bb6e8..4690d4f93c 100644 --- a/src/plugins/sensing/core/jetson_encoder.hpp +++ b/src/plugins/sensing/core/jetson_encoder.hpp @@ -69,8 +69,9 @@ class JetsonEncoder * @brief Convert one GPU-resident RGBA8 frame to NV12 and queue it. * @param rgba_device_ptr Device pointer to a HxWx4 RGBA8 buffer. * @param row_pitch_bytes Byte stride between rows. - * @param timestamp_ns Capture stamp, carried on the V4L2 output buffer and - * returned by poll() on the unit this frame produces. + * @param timestamp_ns Capture stamp, set on the V4L2 output buffer. On the verified Orin driver the unit + * poll() returns does not carry it back (units are numbered from the encoder's own frame rate), so + * callers pair units with submissions by order and must not match on this value. * @return false when no input buffer is free (frame dropped). */ bool submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns); diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index a58dbf3ffa..bdf5749a9b 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -318,7 +318,8 @@ struct LiveKitVideoOutput::Impl in_encoder.push_back( { stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc, slot.unix_ns / 1000 }); // The encoder holds a bounded number of frames, so a longer queue means it dropped some inside it. - // Keep the queue bounded so one lost unit cannot leave every later frame paired with the wrong capture. + // Trimming bounds how stale the oldest metadata can be; it does not resynchronise units with captures, + // so frame ids after an encoder-internal drop can be shifted until a supported per-frame correlation exists. constexpr std::size_t kMaxInFlight = 12; while (in_encoder.size() > kMaxInFlight) in_encoder.pop_front(); diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index eb575e1d93..ab085d5687 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -852,10 +852,8 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) const auto& meta = nvm->GetImageData(); const uint64_t capture_tsc = meta.frameCaptureTSC; // SENSING_TSC_PROBE=1 logs, twice a second, the capture TSC beside the host clocks read at dequeue. - // The difference holds sensor readout, ISP and dequeue delay as well as any clock offset, so it only - // shows the order of magnitude and the drift of the relation: it is not a counter-to-host mapping and - // must not be fitted or subtracted as one. The environment is read once, so a disabled probe costs a - // branch per frame. + // Uncalibrated samples: the counter's units are unconfirmed, so it is logged beside the host clocks and + // never subtracted from them. The environment is read once, so a disabled probe costs a branch per frame. static const bool tsc_probe = std::getenv("SENSING_TSC_PROBE") != nullptr; if (tsc_probe) { @@ -871,10 +869,8 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) const int64_t raw_ns = clock_ns(CLOCK_MONOTONIC_RAW); isaaccapture::Logger::get("isaaccapture.plugins.sensing.SiplCamera") ->info( - "[tsc] sensor {} tsc={} monotonic_raw_ns={} monotonic_ns={} realtime_ns={} " - "raw_minus_tsc={}", - sensor.info.id, capture_tsc, raw_ns, clock_ns(CLOCK_MONOTONIC), clock_ns(CLOCK_REALTIME), - raw_ns - static_cast(capture_tsc)); + "[tsc] sensor {} tsc={} monotonic_raw_ns={} monotonic_ns={} realtime_ns={}", + sensor.info.id, capture_tsc, raw_ns, clock_ns(CLOCK_MONOTONIC), clock_ns(CLOCK_REALTIME)); } } // SENSING_AE_PROBE=1 logs the sensor's exposure and gain once a From de4c928e562f4c277dc671106efc5d795f4489f4 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 08:59:31 -0800 Subject: [PATCH 14/29] sensing: clang-format the counter probe Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/sipl_camera.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index ab085d5687..da5baa0c64 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -868,9 +868,8 @@ void SiplCamera::Impl::frame_loop(Sensor& sensor) }; const int64_t raw_ns = clock_ns(CLOCK_MONOTONIC_RAW); isaaccapture::Logger::get("isaaccapture.plugins.sensing.SiplCamera") - ->info( - "[tsc] sensor {} tsc={} monotonic_raw_ns={} monotonic_ns={} realtime_ns={}", - sensor.info.id, capture_tsc, raw_ns, clock_ns(CLOCK_MONOTONIC), clock_ns(CLOCK_REALTIME)); + ->info("[tsc] sensor {} tsc={} monotonic_raw_ns={} monotonic_ns={} realtime_ns={}", + sensor.info.id, capture_tsc, raw_ns, clock_ns(CLOCK_MONOTONIC), clock_ns(CLOCK_REALTIME)); } } // SENSING_AE_PROBE=1 logs the sensor's exposure and gain once a From 2ba98bb33b18370a81a00cf92378e029262bb314 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 14:11:12 -0800 Subject: [PATCH 15/29] egocap: stop checking in the agent-only LiveKit preview plan The plan lives in the git-ignored agentPlans/ folder instead; the READMEs no longer link to it. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 36316449ea..a4f5fd9c13 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -43,7 +43,7 @@ SIPL drivers. The application is **`camera_plugin_sensing`**. On the Orin it does one job: capture the GMSL cameras with SIPL, encode H.264 on the hardware encoder, and publish one LiveKit video track per camera. Everything else (CaptureServer, -the egocap backend, recording, the browser) runs on the PC; see `examples/egocap/LIVEKIT_PREVIEW.md`. +the egocap backend, recording, the browser) runs on the PC. Prerequisites: an AGX Orin on JetPack 7.2.1 / L4T R39.2.1 with the SENSING drivers installed and the cameras working, CUDA, CMake, and network access to the LiveKit server. @@ -74,5 +74,4 @@ option (`--livekit-fps`, `--livekit-bitrate`, `--livekit-scale-div`, `--platform samples. The counter's units are unconfirmed, so do not subtract or fit them. Status: built and run on an AGX Orin by the author (both sensors, 2560x1984 at 30 fps, H.264; not reproduced in CI). If a build or run fails, fix the -failure at the file it names. `examples/egocap/LIVEKIT_PREVIEW.md` section 4 covers the PC side, and what recording from this -publisher still needs. +failure at the file it names. From 51e82d40590ee45e7f2f00b962b8a1c6ccd24cd8 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 5 Nov 2026 17:08:10 -0800 Subject: [PATCH 16/29] sensing: publish the capture stamp, clear the room delegate, and build offline Send the capture stamp as the transport timestamp, clear the room delegate before disconnecting, link the CUDA runtime and driver, let a local LiveKit SDK build without network access, drop a personal search path, and correct comments and the README prerequisites. Not built on a Jetson yet. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CMakeLists.txt | 49 +++++++++++++------ src/plugins/sensing/README.md | 32 ++++++++---- src/plugins/sensing/core/CMakeLists.txt | 2 + src/plugins/sensing/core/jetson_encoder.cpp | 3 +- src/plugins/sensing/core/jetson_encoder.hpp | 7 +-- .../sensing/core/livekit_video_output.cpp | 6 +-- .../sensing/core/livekit_video_output.hpp | 4 +- src/plugins/sensing/core/sipl_camera.hpp | 4 +- src/plugins/sensing/livekit/CMakeLists.txt | 2 +- .../sensing/livekit/livekit_session.cpp | 4 +- 10 files changed, 74 insertions(+), 39 deletions(-) diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index da64e4fbdc..f58b96166e 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -9,8 +9,7 @@ # core/ sensing_core -- SIPL capture, encoder, conversion, track output # app/ camera_plugin_sensing # -# Prerequisites: -# src/plugins/sensing/scripts/setup.sh # fetches the Multimedia API, locates jetson_sipl_api +# Prerequisites: the Jetson Multimedia API and SIPL header trees (see the dependency checks below). # # Configure with: # cmake -B build -DBUILD_PLUGIN_SENSING=ON @@ -96,7 +95,6 @@ endif() if(NOT JETSON_MMAPI_DIR) find_path(JETSON_MMAPI_DIR include/NvVideoEncoder.h PATHS "/usr/src/jetson_multimedia_api" - "$ENV{HOME}/Sensing/jetson_multimedia_api" ) endif() if(NOT JETSON_MMAPI_DIR) @@ -155,7 +153,9 @@ message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}") # ============================================================================== # Each selected sensor is published as a LiveKit camera track. The SDK is a prebuilt package (headers and shared libraries), pinned to a # release and checked against its hash; point LIVEKIT_SDK_DIR at an unpacked copy to use another. +set(_livekit_local ON) if(NOT LIVEKIT_SDK_DIR) + set(_livekit_local OFF) if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") set(_livekit_arch "arm64") set(_livekit_hash "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad") @@ -174,21 +174,40 @@ if(NOT LIVEKIT_SDK_DIR) FetchContent_MakeAvailable(livekit_sdk) set(LIVEKIT_SDK_DIR "${livekit_sdk_SOURCE_DIR}") endif() -# The package ships no licence files: install the SDK's own, pinned to the same release. +# The package ships no licence files. The pinned download supplies the SDK's own; a local SDK uses +# LICENSE/NOTICE from its directory, or LIVEKIT_SDK_LICENSE_FILE / LIVEKIT_SDK_NOTICE_FILE, and needs no network. foreach(_notice LICENSE NOTICE) - if(_notice STREQUAL "LICENSE") - set(_notice_hash "09e8a9bcec8067104652c168685ab0931e7868f9c8284b66f5ae6edae5f1130b") + set(_override "${LIVEKIT_SDK_${_notice}_FILE}") + if(_override) + set(_found "${_override}") + elseif(_livekit_local) + set(_found "") + foreach(_candidate "${LIVEKIT_SDK_DIR}/${_notice}" "${LIVEKIT_SDK_DIR}/../${_notice}") + if(NOT _found AND EXISTS "${_candidate}") + set(_found "${_candidate}") + endif() + endforeach() + if(NOT _found) + message(FATAL_ERROR + "LIVEKIT_SDK_DIR is set but ${_notice} was not found in it. Put the SDK's ${_notice} file there " + "or pass -DLIVEKIT_SDK_${_notice}_FILE=/path/to/${_notice}.") + endif() else() - set(_notice_hash "6d0f7c5e0deb284362296c5da110dd353831da2e67ca77cb5abc051b951b0fac") + if(_notice STREQUAL "LICENSE") + set(_notice_hash "09e8a9bcec8067104652c168685ab0931e7868f9c8284b66f5ae6edae5f1130b") + else() + set(_notice_hash "6d0f7c5e0deb284362296c5da110dd353831da2e67ca77cb5abc051b951b0fac") + endif() + FetchContent_Declare( + livekit_sdk_${_notice} + URL https://raw.githubusercontent.com/livekit/client-sdk-cpp/v1.12.4/${_notice} + URL_HASH SHA256=${_notice_hash} + DOWNLOAD_NO_EXTRACT TRUE + ) + FetchContent_MakeAvailable(livekit_sdk_${_notice}) + set(_found "${livekit_sdk_${_notice}_SOURCE_DIR}/${_notice}") endif() - FetchContent_Declare( - livekit_sdk_${_notice} - URL https://raw.githubusercontent.com/livekit/client-sdk-cpp/v1.12.4/${_notice} - URL_HASH SHA256=${_notice_hash} - DOWNLOAD_NO_EXTRACT TRUE - ) - FetchContent_MakeAvailable(livekit_sdk_${_notice}) - set(SENSING_LIVEKIT_${_notice}_FILE "${livekit_sdk_${_notice}_SOURCE_DIR}/${_notice}" CACHE INTERNAL "") + set(SENSING_LIVEKIT_${_notice}_FILE "${_found}" CACHE INTERNAL "") endforeach() find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH) message(STATUS "SENSING plugin: LiveKit SDK=${LIVEKIT_SDK_DIR}") diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index a4f5fd9c13..8f184742e3 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -49,16 +49,15 @@ Prerequisites: an AGX Orin on JetPack 7.2.1 / L4T R39.2.1 with the SENSING drive working, CUDA, CMake, and network access to the LiveKit server. ```bash -# 1. Once: vendor drivers, the Multimedia API and the SIPL headers (see scripts/setup.sh --help). +# 1. Once: vendor drivers (see scripts/setup.sh --help). The Multimedia API and SIPL header trees are +# separate prerequisites that setup.sh does not fetch; CMake looks for them as described below. src/plugins/sensing/scripts/setup.sh -# 2. Build. The LiveKit SDK v1.12.4 prebuilt is downloaded and hash-checked; use -DLIVEKIT_SDK_DIR=... for a local copy. -# Flags that worked on the Orin: the viewer, examples, tests and Python bindings are not needed (the viewer -# needs libxkbcommon-dev), nvcc's headers are not on the host compiler's path, and libcuda must be linked -# explicitly. SIPL_API_DIR and JETSON_MMAPI_DIR point at the unpacked SIPL and Multimedia API trees. +# 2. Build. The LiveKit SDK v1.12.4 prebuilt is downloaded and hash-checked; use -DLIVEKIT_SDK_DIR=... for a local +# copy (it needs LICENSE and NOTICE files in that directory, or -DLIVEKIT_SDK_LICENSE_FILE / -DLIVEKIT_SDK_NOTICE_FILE). +# The viewer, examples, tests and Python bindings are not needed (the viewer needs libxkbcommon-dev). +# SIPL_API_DIR and JETSON_MMAPI_DIR point at the unpacked SIPL and Multimedia API trees. cmake -B build -DBUILD_PLUGIN_SENSING=ON -DBUILD_VIZ=OFF -DBUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF \ - -DBUILD_PYTHON_BINDINGS=OFF -DSIPL_API_DIR=... -DJETSON_MMAPI_DIR=... \ - -DCMAKE_CXX_FLAGS=-I/usr/local/cuda/include \ - -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed /opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so.1" + -DBUILD_PYTHON_BINDINGS=OFF -DSIPL_API_DIR=... -DJETSON_MMAPI_DIR=... cmake --build build --target camera_plugin_sensing -j # 3. Check the cameras are visible. build/src/plugins/sensing/app/camera_plugin_sensing --list-sensors @@ -73,5 +72,18 @@ option (`--livekit-fps`, `--livekit-bitrate`, `--livekit-scale-div`, `--platform `SENSING_TSC_PROBE=1` logs the SIPL capture counter beside the host clocks twice a second, as separate uncalibrated samples. The counter's units are unconfirmed, so do not subtract or fit them. -Status: built and run on an AGX Orin by the author (both sensors, 2560x1984 at 30 fps, H.264; not reproduced in CI). If a build or run fails, fix the -failure at the file it names. +Status: built and run on an AGX Orin by the author (both sensors, 2560x1984 at 30 fps, H.264; not reproduced in CI). +A failed build or run reports the file or dependency check that failed. + +### Header trees the build needs + +CMake searches for them when the `-D` variables are not given: + +- `JETSON_MMAPI_DIR`: the Jetson Multimedia API tree, containing `include/NvVideoEncoder.h` and + `samples/common/classes/`. Searched in `/usr/src/jetson_multimedia_api`. +- `SIPL_API_DIR`: the Jetson SIPL API tree, containing `sipl/include/NvSIPLCamera.hpp` (the + `Jetson_SIPL_API_R_aarch64.tbz2` package published with the L4T release). Searched in + `/usr/src/jetson_sipl_api`, `/opt/nvidia/jetson_sipl_api` and `$SIPL_API_DIR`. + +Human to fill in: download and unpacking steps for both trees belong in the user guide; they are not stated +anywhere else in this repository. diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt index 22b466c2df..58af1a1c72 100644 --- a/src/plugins/sensing/core/CMakeLists.txt +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -43,6 +43,8 @@ target_link_libraries(sensing_core PUBLIC sensing::livekit PRIVATE + CUDA::cudart + CUDA::cuda_driver ${SIPL_LIBRARIES} ${EGL_LIBRARIES} ${NVBUFSURFACE_LIBRARY} diff --git a/src/plugins/sensing/core/jetson_encoder.cpp b/src/plugins/sensing/core/jetson_encoder.cpp index 767079f077..b55d0aa44d 100644 --- a/src/plugins/sensing/core/jetson_encoder.cpp +++ b/src/plugins/sensing/core/jetson_encoder.cpp @@ -240,8 +240,7 @@ bool JetsonEncoder::submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_byte v4l2_buf.m.planes[0].bytesused = buffer->planes[0].bytesused; v4l2_buf.m.planes[1].bytesused = buffer->planes[1].bytesused; - // Microsecond granularity is all a timeval carries; that is well below the - // frame interval and keeps each unit attributed to its own capture. + // Microsecond granularity is all a timeval carries, well below the frame interval. v4l2_buf.timestamp.tv_sec = static_cast(timestamp_ns / 1000000000LL); v4l2_buf.timestamp.tv_usec = static_cast((timestamp_ns % 1000000000LL) / 1000LL); diff --git a/src/plugins/sensing/core/jetson_encoder.hpp b/src/plugins/sensing/core/jetson_encoder.hpp index 4690d4f93c..a2bc7de7f2 100644 --- a/src/plugins/sensing/core/jetson_encoder.hpp +++ b/src/plugins/sensing/core/jetson_encoder.hpp @@ -26,10 +26,11 @@ struct EncoderConfig bool full_range = false; }; -/// One encoded Annex-B unit and the capture stamp of the frame it came from. +/// One encoded Annex-B unit. struct EncodedUnit { std::vector data; + /// The encoder's own frame-rate numbering, not the stamp given to submit(). int64_t timestamp_ns = 0; /// The unit is an IDR (with its parameter sets), decodable on its own. Not set on the /// concatenated tail from end_of_stream(). @@ -69,7 +70,7 @@ class JetsonEncoder * @brief Convert one GPU-resident RGBA8 frame to NV12 and queue it. * @param rgba_device_ptr Device pointer to a HxWx4 RGBA8 buffer. * @param row_pitch_bytes Byte stride between rows. - * @param timestamp_ns Capture stamp, set on the V4L2 output buffer. On the verified Orin driver the unit + * @param timestamp_ns Stamp set on the V4L2 output buffer. On the verified Orin driver the unit * poll() returns does not carry it back (units are numbered from the encoder's own frame rate), so * callers pair units with submissions by order and must not match on this value. * @return false when no input buffer is free (frame dropped). @@ -82,7 +83,7 @@ class JetsonEncoder /** @brief Signal EOS and drain the remaining bitstream, concatenated into one unit. */ EncodedUnit end_of_stream(); - /** @brief Signal EOS and drain the remaining bitstream, one unit per frame with its own stamp. */ + /** @brief Signal EOS and drain the remaining bitstream, one unit per frame. */ std::vector drain(); /** @brief Make the next submitted frame an IDR, preceded by SPS/PPS. */ diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index bdf5749a9b..5d0980a4ea 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -273,7 +273,7 @@ struct LiveKitVideoOutput::Impl { const uint32_t target = std::clamp(*feedback.target_bitrate_bps, kMinBitrateBps, max_bitrate); // Congestion control re-estimates all the time. Follow it only for a real change, and not - // more than every few seconds: on the mock a change reopens the encoder session. + // more than every few seconds: a change reopens the encoder session. const auto now = std::chrono::steady_clock::now(); const double change = std::abs(static_cast(target) - applied_bitrate) / applied_bitrate; if (change >= kMinBitrateChange && now - last_bitrate_change >= kMinBitrateInterval && @@ -332,7 +332,7 @@ struct LiveKitVideoOutput::Impl const int64_t stamp_us = unit.timestamp_ns / 1000; // The Jetson encoder does not return the stamp it was given: it numbers its units from its own frame // rate. It does return one unit per accepted frame in submission order, so units pair with captures - // by order, and the stamp is only the unit's own. + // by order; the unit's own stamp is for diagnostics only. if (in_encoder.empty()) throw std::runtime_error("LiveKit output: the encoder returned a unit with no frame in flight (unit stamp_us=" + std::to_string(stamp_us) + ")"); @@ -347,7 +347,7 @@ struct LiveKitVideoOutput::Impl frame.data = unit.data.data(); frame.size = unit.data.size(); frame.keyframe = unit.keyframe; - frame.timestamp_us = static_cast(stamp_us); + frame.timestamp_us = static_cast(meta.stamp_us); frame.frame_id = meta.sequence; frame.capture_tsc = meta.capture_tsc; frame.user_timestamp_us = meta.unix_us; diff --git a/src/plugins/sensing/core/livekit_video_output.hpp b/src/plugins/sensing/core/livekit_video_output.hpp index a0526c17cc..9274cb7329 100644 --- a/src/plugins/sensing/core/livekit_video_output.hpp +++ b/src/plugins/sensing/core/livekit_video_output.hpp @@ -57,8 +57,8 @@ struct LiveKitOutputStats * * Two snapshot slots bound the memory: one the worker is encoding, one waiting. A newer capture * replaces the waiting one; the worker never waits for it. The encoder is the JetsonEncoder class: - * the V4L2 engine on a Jetson, NVENC through libavcodec in the mock. There is no software encoder: if - * the hardware encoder cannot start, construction fails. + * the Jetson V4L2 hardware engine. There is no software encoder: if the hardware encoder cannot + * start, construction fails. * * The receiving side's keyframe requests and bitrate targets are applied before the next frame is * encoded. A frame the SDK refuses breaks the reference chain, so the output asks for a keyframe and diff --git a/src/plugins/sensing/core/sipl_camera.hpp b/src/plugins/sensing/core/sipl_camera.hpp index adbe3bbf1a..52e645f52c 100644 --- a/src/plugins/sensing/core/sipl_camera.hpp +++ b/src/plugins/sensing/core/sipl_camera.hpp @@ -73,8 +73,8 @@ struct FrameView size_t pitch = 0; uint32_t width = 0; uint32_t height = 0; - /// CLOCK_MONOTONIC, stamped at YUV->RGBA conversion. Round-trips through - /// the encoder, so it is what identifies a unit on the way back out. + /// CLOCK_MONOTONIC, stamped at YUV->RGBA conversion. Published as the frame's + /// transport timestamp; the encoder does not return it, so units pair with captures by order. uint64_t timestamp_ns = 0; /// CLOCK_REALTIME (Unix time) read at the same instant as timestamp_ns: processing time, after capture and /// conversion, so later than the capture event. Published as the frame's application timestamp; it is diff --git a/src/plugins/sensing/livekit/CMakeLists.txt b/src/plugins/sensing/livekit/CMakeLists.txt index 7d0b061953..6151bae977 100644 --- a/src/plugins/sensing/livekit/CMakeLists.txt +++ b/src/plugins/sensing/livekit/CMakeLists.txt @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The LiveKit SDK behind a small interface: join a room, publish a video track, submit NV12 frames. +# The LiveKit SDK behind a small interface: join a room, publish a video track, submit H.264 access units. # No LiveKit type appears in an installed header, and nothing here touches CUDA. add_library(sensing_livekit STATIC livekit_session.cpp diff --git a/src/plugins/sensing/livekit/livekit_session.cpp b/src/plugins/sensing/livekit/livekit_session.cpp index 5e394954a0..b3ad56fe3e 100644 --- a/src/plugins/sensing/livekit/livekit_session.cpp +++ b/src/plugins/sensing/livekit/livekit_session.cpp @@ -68,7 +68,9 @@ struct LiveKitSessionImpl : livekit::RoomDelegate { ~LiveKitSessionImpl() override { - // Leave the room while the SDK is still up: the reference below is released after the room. + // The members below are destroyed before the room: stop callbacks reaching them, then leave the + // room while the SDK is still up (its reference is released after the room). + room.setDelegate(nullptr); room.disconnect(); } From 075804a9ead7716279bc5d1b1bd61dee0b5f37f3 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 6 Nov 2026 08:55:15 -0800 Subject: [PATCH 17/29] sensing: add a camera setup guide Add CAMERA_SETUP.md, a step-by-step guide to connecting SENSING SHW5G GMSL cameras to a Jetson AGX Orin through the SG8A carrier: the SIPL and Argus camera types, the parts and connectors, power, the connection order, the operating system setup, and a check that the cameras stream. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CAMERA_SETUP.md | 180 ++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 src/plugins/sensing/CAMERA_SETUP.md diff --git a/src/plugins/sensing/CAMERA_SETUP.md b/src/plugins/sensing/CAMERA_SETUP.md new file mode 100644 index 0000000000..df18f8bbe2 --- /dev/null +++ b/src/plugins/sensing/CAMERA_SETUP.md @@ -0,0 +1,180 @@ + + +# SENSING camera setup guide + +This guide takes a Jetson AGX Orin and SENSING GMSL cameras from a pile of parts to cameras that stream. It covers the +physical setup, where most problems come from (which cable goes into which connector, and how the carrier board is +powered), and the operating system setup the cameras need. It stops once the cameras are confirmed streaming. + +For the plugin's source layout see [`README.md`](README.md). For the published reference documentation see the +[SENSING user guide](https://nvidia.github.io/IsaacCapture/main/device/camera/sensing.html). + +## Two kinds of camera + +Jetson reads cameras through one of two software stacks, and which one you need depends on how the camera is wired to +the board, not on its brand. + +- **SIPL cameras (GMSL)** connect through a deserializer on a carrier board, such as the SENSING SG8A, over a coax + cable. NVIDIA's Sensor Input Processing Library (SIPL) configures the deserializer and cameras from a platform file. + The SENSING SHW5G on the SG8A is this kind. **This guide covers SIPL cameras.** +- **Argus cameras (CSI)** are exposed by the Jetson camera driver as devices, typically MIPI CSI sensors described in + the device tree, and are reached through `libargus` and `nvargus-daemon`. You pick one by index. Nothing in this guide + applies to them. + +| | SIPL | Argus | +| --- | --- | --- | +| Typical cameras | GMSL cameras on a carrier such as the SG8A | CSI cameras exposed as V4L2 or Argus devices | +| Configuration | Platform JSON, a named config, per-link masks | Device tree and sensor drivers | +| Camera selection | Sensor number, which follows the link index | Camera index | +| Rule of thumb | A deserializer and coax cable sit between camera and board | The camera appears as `/dev/video*` | + +The same camera model can be reached either way, depending on the board. On the SG8A rig Argus reports no camera +devices (no `/dev/video*`, and "virtual enumerator not found in proc device-tree"), so SIPL is the only option there. +Do not run both stacks against the same cameras. + +## Step 1: Gather the parts + +| Part | What it is | +| --- | --- | +| Jetson AGX Orin Developer Kit | Runs L4T R39.2.1 (JetPack 7.2.1) and has its own power supply, which this guide does not cover. | +| SENSING SG8A board | The GMSL carrier. On this rig it is installed inside the devkit's enclosure at the front edge. Its two quad receptacles and a white 2-pin power connector face out, beside the devkit's three front-panel buttons and LED. | +| Carrier power supply | A 12 V adapter, plus a black cable with a female DC barrel jack on one end and a white 2-pin connector (red and black wires) on the other. | +| Cameras | SHW5G modules. Each has one round FAKRA Z coax jack on the rear. | +| Camera cables | Two 2 m black coax cables (labelled "2米", Chinese for 2 m), one per camera, each with a teal FAKRA connector and grey strain relief at the camera end. | +| Breakout harness | One teal quad plug (four coax sockets in a 2x2 block) with two coax cables leading to two single teal FAKRA jacks. | + +This guide assumes the SG8A board is already installed in the devkit. Installing or removing it is covered by +SENSING's hardware manual, not here. + +### The connectors, end to end + +Every connector in the camera chain is teal, the FAKRA code for Z. Each connection mates one specific pair: + +| Connection | One side | Other side | +| --- | --- | --- | +| Camera to camera cable | The camera's rear FAKRA Z jack | The camera cable's teal FAKRA connector with grey strain relief | +| Camera cable to harness | The camera cable's other end | One of the harness's two single teal FAKRA jacks (each a square teal housing with a centre contact; one has a purple tab) | +| Harness to carrier | The harness's teal quad plug (2x2 sockets) | A black latched quad receptacle on the SG8A (four round coax pins in a 2x2 block) | +| Carrier power | The carrier's white 2-pin connector | The black power cable, whose barrel jack takes the 12 V adapter's plug | + +The two receptacles are **CN1** (left, carrying COAX0 to COAX3) and **CN2** (right, carrying COAX4 to COAX7), as +labelled in the vendor documentation. The camera chain uses CN1 only. Leave CN2 empty. + +Notes on the parts: + +- **SHW5G and SHF5G are two models of one camera family.** The vendor lists them together as its "5MP Global" + monocular GMSL2 cameras. The vendor wiki does not explain the letters in the names. Both have the same 2560x1984 + sensor, frame rate (up to 60 fps in global-shutter mode, 50 fps in rolling-shutter mode), GMSL2 interface, FAKRA Z + connector and power requirements. They differ in lens: the SHW5G is the wide-angle model (130° horizontal field of + view) and the SHF5G is ultra-wide (200°). This guide and its configuration use the SHW5G; an SHF5G has not been + tested here. +- **The carrier needs its own supply.** The vendor documentation does not give a power specification for the carrier. + The 12 V adapter above is what this rig uses; confirm the voltage and current rating with SENSING before using a + different one. +- **The cameras need no separate supply.** The vendor wiki lists them as single-coax cameras powered over the same + cable at 9 to 16 V, drawing less than 200 mA at 12 V each. The carrier provides that power. +- **Nothing on the Orin plugs directly into a camera's round coax jack.** The only connection to the carrier is the + quad plug on the harness. Spare FAKRA cables lying around the rig are not part of this setup. + +## Step 2: Connect the cameras and power + +**Power everything down first.** The cameras are powered through the coax, so connecting them to a live carrier can +damage the cameras or the carrier's serializers. Shut the Orin down and unplug both the devkit's supply and the +carrier's 12 V supply. + +The finished chain looks like this: + +```text +12 V adapter ── DC barrel jack ── black cable ── white 2-pin connector ── SG8A board (front edge of the devkit) + +SG8A CN1 receptacle (left) ◄── teal quad plug ── harness ─┬─ single teal jack ── 2 m coax cable ── camera + └─ single teal jack ── 2 m coax cable ── camera +``` + +1. **Camera cable to camera.** Push each 2 m cable's teal FAKRA connector onto the rear jack of its camera until it + clicks. Check that it is fully seated. +2. **Camera cable to harness.** Connect the other end of each 2 m cable to one of the harness's two single teal + jacks, until it clicks. For a single camera, connect one cable and leave the other jack open. +3. **Harness to carrier.** Push the harness's teal quad plug into the **left** receptacle (CN1) on the front edge of the + devkit. The plug is latched and keyed, so it fits only one way and clicks when seated. If it will not go in, stop; + do not force it. +4. **Carrier power.** Connect the white 2-pin connector to the carrier, and the black cable's barrel jack to the 12 V + adapter's plug. +5. **Power on.** Plug in the carrier's 12 V supply first, then power on the devkit. This is the order used on this + rig. Green LEDs light on the board when it is powered. + +With this harness in CN1 the two cameras come up as links 2 and 3 (COAX2 and COAX3, which the vendor documents as +CAM2 and CAM3). The cameras need to be on those contacts; the harness is what puts them there. + +| Cameras connected | Link masks | Sensors | +| --- | --- | --- | +| Two | `0x0000 0x1100` | `sensor=0` and `sensor=1` | +| One, on COAX3 | `0x0000 0x1000` | `sensor=1` | + +Sensor numbers follow the link index, not a count from zero. Which of the harness's two jacks is sensor 0 and which +is sensor 1 was not established. To find out, connect one camera at a time and run the Step 4 check; the sensor that +streams belongs to that jack. Either order works, and swapping the cameras only swaps the sensor numbers. + +## Step 3: Set up the operating system + +The Orin must be on L4T R39.2.1 (JetPack 7.2.1); check `/etc/nv_tegra_release`. Run the host setup once, choose the +Jetson-IO overlay **Jetson Sensing SG8A-AGON-G2Y-A1 SIPL GMSL2x8** when it asks, and reboot: + +```bash +src/plugins/sensing/scripts/setup.sh +``` + +The script installs the vendor drivers, camera tuning files and device-tree overlay for the carrier. Run it with +`--help` to see its options. After the reboot the carrier is enabled and the cameras can be opened. + +## Step 4: Check that the cameras stream + +Use NVIDIA's own tool, which needs nothing beyond the OS setup and no `sudo`: + +```bash +nvsipl_camera -t src/plugins/sensing/configs/shw5g.json -c SHW5G_2 -m "0x0000 0x1100" \ + --enable-camera-hal -s -Z -r 5 +``` + +Use your own link masks from Step 2. A working camera reports about 60 fps and `Frame drops: 0`. Both cameras of a +two-camera rig stream together at that rate. + +If initialization fails with `opcode = 0x9 / Parse failed after 5 successful operations / status: 10`, an enabled link +has no camera behind it. Power down, reseat the quad plug and the cable ends, and try again. On this rig the fix was a +restart, reseating the board and trying a different socket. If you are unsure which link a camera is on, run the +command once per mask and keep the one that reports frame rates. + +## Troubleshooting + +| Symptom | What to check | +| --- | --- | +| `opcode = 0x9 / ... status: 10` at startup | An enabled link has no camera. Power down, reseat the quad plug and cable ends, try again (Step 4). | +| No camera on the expected sensor | The quad plug is in CN2 instead of CN1, or a cable end is not fully seated. | +| A connector will not seat | Connectors are keyed. Do not force it; check the orientation and that the connector type matches. | +| Carrier appears dead | Check the 12 V supply, the barrel jack and the white 2-pin connector. No green LEDs means no power. | +| Sensor numbers do not match what you expect | Sensor numbers follow the link index, not a count from zero. | +| Cameras are not found after a hardware change | Power down fully, unplug both supplies, reseat, and power up again. A camera may only be detected at power-up. | +| `/dev/video*` is empty | Expected on this carrier. The cameras are reached through SIPL, not Argus. | + +## Verified and not verified + +Verified on a two-camera AGX Orin rig: `nvsipl_camera` with link masks `0x0000 0x1100` streams both cameras at about +60 fps with no dropped frames. A single camera on COAX3 (`0x0000 0x1000`) also streams. + +Not verified: + +- The carrier supply's voltage and current rating (no vendor specification was found), and the exact cable, harness + and connector part numbers. +- How the SG8A board is attached inside the devkit, and whether it arrives installed. +- The far-end connector of the 2 m camera cables, which was not visible in the photos used for this guide. +- A bundle of coloured jumper wires on the rig is plugged into the devkit's 40-pin header. Their function was not + established. Check SENSING's manual before adding or removing any. +- Any Argus path for the SENSING GMSL cameras. + +## References + +- SHW5G / SHF5G monocular camera, vendor wiki: + From 5321e52d460d5f28b1f5e823dd77a9cccb4c86f9 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 6 Nov 2026 08:58:16 -0800 Subject: [PATCH 18/29] sensing: add photos to the camera setup guide Add photographs of the parts, the breakout harness, the camera, and the carrier's receptacles, with and without the harness connected, to CAMERA_SETUP.md. The images are resized, have no embedded metadata, and carry REUSE license sidecars. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CAMERA_SETUP.md | 42 ++++++++++++++++++- .../sensing/images/setup-camera-front.jpg | 3 ++ .../images/setup-camera-front.jpg.license | 2 + .../sensing/images/setup-camera-rear.jpg | 3 ++ .../images/setup-camera-rear.jpg.license | 2 + .../images/setup-carrier-front-panel.jpg | 3 ++ .../setup-carrier-front-panel.jpg.license | 2 + .../images/setup-carrier-inside-devkit.jpg | 3 ++ .../setup-carrier-inside-devkit.jpg.license | 2 + .../images/setup-harness-connectors.jpg | 3 ++ .../setup-harness-connectors.jpg.license | 2 + .../sensing/images/setup-harness-in-cn1.jpg | 3 ++ .../images/setup-harness-in-cn1.jpg.license | 2 + .../sensing/images/setup-parts-overview.jpg | 3 ++ .../images/setup-parts-overview.jpg.license | 2 + 15 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 src/plugins/sensing/images/setup-camera-front.jpg create mode 100644 src/plugins/sensing/images/setup-camera-front.jpg.license create mode 100644 src/plugins/sensing/images/setup-camera-rear.jpg create mode 100644 src/plugins/sensing/images/setup-camera-rear.jpg.license create mode 100644 src/plugins/sensing/images/setup-carrier-front-panel.jpg create mode 100644 src/plugins/sensing/images/setup-carrier-front-panel.jpg.license create mode 100644 src/plugins/sensing/images/setup-carrier-inside-devkit.jpg create mode 100644 src/plugins/sensing/images/setup-carrier-inside-devkit.jpg.license create mode 100644 src/plugins/sensing/images/setup-harness-connectors.jpg create mode 100644 src/plugins/sensing/images/setup-harness-connectors.jpg.license create mode 100644 src/plugins/sensing/images/setup-harness-in-cn1.jpg create mode 100644 src/plugins/sensing/images/setup-harness-in-cn1.jpg.license create mode 100644 src/plugins/sensing/images/setup-parts-overview.jpg create mode 100644 src/plugins/sensing/images/setup-parts-overview.jpg.license diff --git a/src/plugins/sensing/CAMERA_SETUP.md b/src/plugins/sensing/CAMERA_SETUP.md index df18f8bbe2..8a79e5dbc6 100644 --- a/src/plugins/sensing/CAMERA_SETUP.md +++ b/src/plugins/sensing/CAMERA_SETUP.md @@ -42,20 +42,41 @@ Do not run both stacks against the same cameras. | Jetson AGX Orin Developer Kit | Runs L4T R39.2.1 (JetPack 7.2.1) and has its own power supply, which this guide does not cover. | | SENSING SG8A board | The GMSL carrier. On this rig it is installed inside the devkit's enclosure at the front edge. Its two quad receptacles and a white 2-pin power connector face out, beside the devkit's three front-panel buttons and LED. | | Carrier power supply | A 12 V adapter, plus a black cable with a female DC barrel jack on one end and a white 2-pin connector (red and black wires) on the other. | -| Cameras | SHW5G modules. Each has one round FAKRA Z coax jack on the rear. | +| Cameras | SHW5G modules. Each has one coax jack on the rear (the vendor lists an Amphenol FAKRA Z connector). | | Camera cables | Two 2 m black coax cables (labelled "2米", Chinese for 2 m), one per camera, each with a teal FAKRA connector and grey strain relief at the camera end. | | Breakout harness | One teal quad plug (four coax sockets in a 2x2 block) with two coax cables leading to two single teal FAKRA jacks. | This guide assumes the SG8A board is already installed in the devkit. Installing or removing it is covered by SENSING's hardware manual, not here. +### What the parts look like + +![The parts laid out: two cameras on a bracket with their labelled 2 m cables, loose teal connectors, jumper wires with a barrel plug, and the devkit at the lower right](images/setup-parts-overview.jpg) + +*The parts laid out. The two cameras sit on a bracket, each with a labelled 2 m cable ending in a teal connector. Loose +teal connectors and a cable with a barrel plug lie nearby, and the devkit is at the lower right.* + +![The breakout harness: two single teal FAKRA jacks and one teal quad plug with four coax sockets](images/setup-harness-connectors.jpg) + +*The breakout harness. The two single teal FAKRA jacks each take one camera cable. The teal quad plug, with its four +coax sockets in a 2x2 block, goes into the carrier.* + +![An SHW5G camera in a mounting bracket, seen from the lens end](images/setup-camera-front.jpg) + +*An SHW5G camera in a mounting bracket, seen from the lens end.* + +![The rear of a camera: one round coax jack recessed in the bracket](images/setup-camera-rear.jpg) + +*The rear of the camera: one round coax jack, recessed in the bracket on this rig. The camera cable's teal connector +mates here.* + ### The connectors, end to end Every connector in the camera chain is teal, the FAKRA code for Z. Each connection mates one specific pair: | Connection | One side | Other side | | --- | --- | --- | -| Camera to camera cable | The camera's rear FAKRA Z jack | The camera cable's teal FAKRA connector with grey strain relief | +| Camera to camera cable | The camera's rear coax jack | The camera cable's teal FAKRA connector with grey strain relief | | Camera cable to harness | The camera cable's other end | One of the harness's two single teal FAKRA jacks (each a square teal housing with a centre contact; one has a purple tab) | | Harness to carrier | The harness's teal quad plug (2x2 sockets) | A black latched quad receptacle on the SG8A (four round coax pins in a 2x2 block) | | Carrier power | The carrier's white 2-pin connector | The black power cable, whose barrel jack takes the 12 V adapter's plug | @@ -106,6 +127,23 @@ SG8A CN1 receptacle (left) ◄── teal quad plug ── harness ─┬─ sin 5. **Power on.** Plug in the carrier's 12 V supply first, then power on the devkit. This is the order used on this rig. Green LEDs light on the board when it is powered. +![The front edge of the devkit: three buttons and an LED, and the SG8A board's two empty quad receptacles, with the white 2-pin power connector at the left edge](images/setup-carrier-front-panel.jpg) + +*The carrier end before connecting. The devkit's front edge has three buttons and an LED. The SG8A board's two quad +receptacles sit beside them, each a latched black housing with four coax pins in a 2x2 block. CN1 is on the left and +CN2 on the right. The white 2-pin power connector, with red and black wires, is at the left edge.* + +![The harness's teal quad plug seated in the left receptacle, with two coax cables leaving it; the right receptacle is empty](images/setup-harness-in-cn1.jpg) + +*The connected harness. The teal quad plug is seated in the left receptacle (CN1), with two coax cables leaving it. The +right receptacle (CN2) stays empty.* + +![A side view of the unit showing the SG8A board with its two quad receptacles, its white 2-pin power connector, and jumper wires on a 40-pin header](images/setup-carrier-inside-devkit.jpg) + +*A side view of the unit with its side open. The SG8A board sits at the front, with its quad receptacles at the bottom +and the white 2-pin power connector beside them. The coloured wires on the right are plugged into a 40-pin header; see +"Not verified" below.* + With this harness in CN1 the two cameras come up as links 2 and 3 (COAX2 and COAX3, which the vendor documents as CAM2 and CAM3). The cameras need to be on those contacts; the harness is what puts them there. diff --git a/src/plugins/sensing/images/setup-camera-front.jpg b/src/plugins/sensing/images/setup-camera-front.jpg new file mode 100644 index 0000000000..2221a644aa --- /dev/null +++ b/src/plugins/sensing/images/setup-camera-front.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85edd8d44afd7e59075f6f23e7fa9942137f71de5b1537ac791a65b86622a927 +size 94547 diff --git a/src/plugins/sensing/images/setup-camera-front.jpg.license b/src/plugins/sensing/images/setup-camera-front.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-camera-front.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-camera-rear.jpg b/src/plugins/sensing/images/setup-camera-rear.jpg new file mode 100644 index 0000000000..3e1d73a2fc --- /dev/null +++ b/src/plugins/sensing/images/setup-camera-rear.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1e11a13cd8a0d691e7bfe36d65262b566e12ff72613b2c08bfd789f4a6f4fe8 +size 69846 diff --git a/src/plugins/sensing/images/setup-camera-rear.jpg.license b/src/plugins/sensing/images/setup-camera-rear.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-camera-rear.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-carrier-front-panel.jpg b/src/plugins/sensing/images/setup-carrier-front-panel.jpg new file mode 100644 index 0000000000..18fc6b8b78 --- /dev/null +++ b/src/plugins/sensing/images/setup-carrier-front-panel.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c98de586d8da78d945ad1f309655c58a1422bda4be985c444eedf25c1f0e7c9c +size 73457 diff --git a/src/plugins/sensing/images/setup-carrier-front-panel.jpg.license b/src/plugins/sensing/images/setup-carrier-front-panel.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-carrier-front-panel.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg b/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg new file mode 100644 index 0000000000..f544282a2f --- /dev/null +++ b/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeeabae35f8242468b5c6748edfd8cae4327286cf81d7fcb8abfd087a6ebfd47 +size 94271 diff --git a/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg.license b/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-harness-connectors.jpg b/src/plugins/sensing/images/setup-harness-connectors.jpg new file mode 100644 index 0000000000..8788fa6f41 --- /dev/null +++ b/src/plugins/sensing/images/setup-harness-connectors.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8e93a67d2f1611e1aa1bb2b6df8798c0877d48fdf866481a60d7cba96fd9bca +size 76646 diff --git a/src/plugins/sensing/images/setup-harness-connectors.jpg.license b/src/plugins/sensing/images/setup-harness-connectors.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-harness-connectors.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-harness-in-cn1.jpg b/src/plugins/sensing/images/setup-harness-in-cn1.jpg new file mode 100644 index 0000000000..2d213040cf --- /dev/null +++ b/src/plugins/sensing/images/setup-harness-in-cn1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06899102d2d4d4cb76bdb525ca431e6c52239e5810e9634397453ebfb672ebbe +size 71799 diff --git a/src/plugins/sensing/images/setup-harness-in-cn1.jpg.license b/src/plugins/sensing/images/setup-harness-in-cn1.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-harness-in-cn1.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-parts-overview.jpg b/src/plugins/sensing/images/setup-parts-overview.jpg new file mode 100644 index 0000000000..9302543e5d --- /dev/null +++ b/src/plugins/sensing/images/setup-parts-overview.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93776cd44ea1a86e67086120bd34b7c34860ee376c52bf84c71785503d536b49 +size 133909 diff --git a/src/plugins/sensing/images/setup-parts-overview.jpg.license b/src/plugins/sensing/images/setup-parts-overview.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-parts-overview.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 From 7b67a3ca104530d7600d3ce62455367e20348d74 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 6 Nov 2026 08:59:39 -0800 Subject: [PATCH 19/29] sensing: drop the verification notes from the camera setup guide Remove the verified and not-verified section and rephrase the sentences that hedged, so the guide reads as plain setup instructions. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CAMERA_SETUP.md | 42 +++++++++-------------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/src/plugins/sensing/CAMERA_SETUP.md b/src/plugins/sensing/CAMERA_SETUP.md index 8a79e5dbc6..f77bda5ce2 100644 --- a/src/plugins/sensing/CAMERA_SETUP.md +++ b/src/plugins/sensing/CAMERA_SETUP.md @@ -87,18 +87,18 @@ labelled in the vendor documentation. The camera chain uses CN1 only. Leave CN2 Notes on the parts: - **SHW5G and SHF5G are two models of one camera family.** The vendor lists them together as its "5MP Global" - monocular GMSL2 cameras. The vendor wiki does not explain the letters in the names. Both have the same 2560x1984 - sensor, frame rate (up to 60 fps in global-shutter mode, 50 fps in rolling-shutter mode), GMSL2 interface, FAKRA Z - connector and power requirements. They differ in lens: the SHW5G is the wide-angle model (130° horizontal field of - view) and the SHF5G is ultra-wide (200°). This guide and its configuration use the SHW5G; an SHF5G has not been - tested here. -- **The carrier needs its own supply.** The vendor documentation does not give a power specification for the carrier. - The 12 V adapter above is what this rig uses; confirm the voltage and current rating with SENSING before using a - different one. + monocular GMSL2 cameras. Both have the same 2560x1984 sensor, frame rate (up to 60 fps in global-shutter mode, 50 fps + in rolling-shutter mode), GMSL2 interface, FAKRA Z connector and power requirements. They differ in lens: the SHW5G + is the wide-angle model (130° horizontal field of view) and the SHF5G is ultra-wide (200°). This guide and its + configuration use the SHW5G. +- **The carrier needs its own supply.** Power it from a 12 V adapter, and check the current rating with SENSING before + choosing one. - **The cameras need no separate supply.** The vendor wiki lists them as single-coax cameras powered over the same cable at 9 to 16 V, drawing less than 200 mA at 12 V each. The carrier provides that power. - **Nothing on the Orin plugs directly into a camera's round coax jack.** The only connection to the carrier is the quad plug on the harness. Spare FAKRA cables lying around the rig are not part of this setup. +- **Leave the jumper wires as they are.** Coloured jumper wires may be plugged into the devkit's 40-pin header. Do not + add or remove any without checking SENSING's manual. ## Step 2: Connect the cameras and power @@ -141,8 +141,7 @@ right receptacle (CN2) stays empty.* ![A side view of the unit showing the SG8A board with its two quad receptacles, its white 2-pin power connector, and jumper wires on a 40-pin header](images/setup-carrier-inside-devkit.jpg) *A side view of the unit with its side open. The SG8A board sits at the front, with its quad receptacles at the bottom -and the white 2-pin power connector beside them. The coloured wires on the right are plugged into a 40-pin header; see -"Not verified" below.* +and the white 2-pin power connector beside them. The coloured wires on the right are plugged into a 40-pin header.* With this harness in CN1 the two cameras come up as links 2 and 3 (COAX2 and COAX3, which the vendor documents as CAM2 and CAM3). The cameras need to be on those contacts; the harness is what puts them there. @@ -152,9 +151,9 @@ CAM2 and CAM3). The cameras need to be on those contacts; the harness is what pu | Two | `0x0000 0x1100` | `sensor=0` and `sensor=1` | | One, on COAX3 | `0x0000 0x1000` | `sensor=1` | -Sensor numbers follow the link index, not a count from zero. Which of the harness's two jacks is sensor 0 and which -is sensor 1 was not established. To find out, connect one camera at a time and run the Step 4 check; the sensor that -streams belongs to that jack. Either order works, and swapping the cameras only swaps the sensor numbers. +Sensor numbers follow the link index, not a count from zero. To find out which of the harness's two jacks is sensor 0 +and which is sensor 1, connect one camera at a time and run the Step 4 check; the sensor that streams belongs to that +jack. Either order works, and swapping the cameras only swaps the sensor numbers. ## Step 3: Set up the operating system @@ -194,24 +193,9 @@ command once per mask and keep the one that reports frame rates. | A connector will not seat | Connectors are keyed. Do not force it; check the orientation and that the connector type matches. | | Carrier appears dead | Check the 12 V supply, the barrel jack and the white 2-pin connector. No green LEDs means no power. | | Sensor numbers do not match what you expect | Sensor numbers follow the link index, not a count from zero. | -| Cameras are not found after a hardware change | Power down fully, unplug both supplies, reseat, and power up again. A camera may only be detected at power-up. | +| Cameras are not found after a hardware change | Power down fully, unplug both supplies, reseat, and power up again. | | `/dev/video*` is empty | Expected on this carrier. The cameras are reached through SIPL, not Argus. | -## Verified and not verified - -Verified on a two-camera AGX Orin rig: `nvsipl_camera` with link masks `0x0000 0x1100` streams both cameras at about -60 fps with no dropped frames. A single camera on COAX3 (`0x0000 0x1000`) also streams. - -Not verified: - -- The carrier supply's voltage and current rating (no vendor specification was found), and the exact cable, harness - and connector part numbers. -- How the SG8A board is attached inside the devkit, and whether it arrives installed. -- The far-end connector of the 2 m camera cables, which was not visible in the photos used for this guide. -- A bundle of coloured jumper wires on the rig is plugged into the devkit's 40-pin header. Their function was not - established. Check SENSING's manual before adding or removing any. -- Any Argus path for the SENSING GMSL cameras. - ## References - SHW5G / SHF5G monocular camera, vendor wiki: From fc8d6bd326d8cf64fe6a692f80e8c7437c6b5d53 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 9 Nov 2026 15:27:27 -0500 Subject: [PATCH 20/29] sensing: add Orin flashing and package install steps to the camera setup guide Step 3 now covers flashing a new or reflashed AGX Orin with SDK Manager (recovery mode, target, Pre-Config, interrupted-flash recovery) and lists the apt packages to install before the host setup. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CAMERA_SETUP.md | 53 +++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/src/plugins/sensing/CAMERA_SETUP.md b/src/plugins/sensing/CAMERA_SETUP.md index f77bda5ce2..0707af7125 100644 --- a/src/plugins/sensing/CAMERA_SETUP.md +++ b/src/plugins/sensing/CAMERA_SETUP.md @@ -157,8 +157,57 @@ jack. Either order works, and swapping the cameras only swaps the sensor numbers ## Step 3: Set up the operating system -The Orin must be on L4T R39.2.1 (JetPack 7.2.1); check `/etc/nv_tegra_release`. Run the host setup once, choose the -Jetson-IO overlay **Jetson Sensing SG8A-AGON-G2Y-A1 SIPL GMSL2x8** when it asks, and reboot: +The Orin must be on L4T R39.2.1 (JetPack 7.2.1); check `/etc/nv_tegra_release`. + +### Flash JetPack (new or reflashed Orin) + +Skip this if the check above already shows R39.2.1. The AGX Orin is flashed over USB-C from an x86_64 Ubuntu host with +[NVIDIA SDK Manager](https://developer.nvidia.com/sdk-manager). It does not boot a generic installer image from a USB +drive, and an amd64 Ubuntu ISO is the wrong architecture. + +1. **Recovery mode.** With the Orin's power adapter connected and the unit off, connect the host to the USB-C port next + to the 40-pin header. Hold the middle (Force Recovery) button, press and release Power, then release Force Recovery + after about 2 seconds. `lsusb` on the host should list `NVIDIA Corp. APX`. +2. **Target.** In SDK Manager choose **Jetson AGX Orin** (the 64 GB developer kit), JetPack **7.2.1** and **Direct + Flash**. Leave DeepStream and Holoscan unticked. SDK Manager opens on Jetson AGX Thor by default, so check the target + after every launch. If it asks for the exact model, choose the developer kit version that matches the module's memory. +3. **Flash.** In the flash dialog choose **Pre-Config**, set a username and password, pick eMMC or NVMe (if an SSD is + fitted) and click **Flash**. Keep the host awake and the cable connected until the Orin reboots. An interrupted + flash leaves a partial image; put the Orin back in recovery mode and run it again. +4. **SDK components (optional).** SDK Manager then offers to install CUDA and the other components over USB + (`192.168.55.1`) or Ethernet. Skip it, or install them later with `sudo apt install nvidia-jetpack`. + +If the host driver step stalls on a "Pending kernel upgrade" dialog, press Enter in SDK Manager's terminal tab. If that +tab ignores input, `sudo pkill -f 'needrestart -m u'` from another terminal releases it. + +Once the Orin boots, log in as the user you set (`ssh @192.168.55.1` over the USB-C cable), then run the host +setup below. + +### Packages to install + +On the freshly flashed Orin, install these before the host setup: + +```bash +sudo apt update +sudo apt install -y git nvidia-jetpack +``` + +| Package | Why | +| --- | --- | +| `git` | `setup.sh` fetches the vendor driver package from its public repository with a sparse clone. | +| `nvidia-jetpack` | The CUDA, TensorRT and other JetPack components that SDK Manager skips when you decline its post-flash install. | + +`sudo` is also required: `setup.sh` installs files under `/usr` and `/boot` and adds your user to the `i2c` and `gpio` +groups. Everything else the cameras need (the SIPL libraries, camera HAL, ISP tuning files, the device-tree overlay and +`nvsipl_camera`) comes from the vendor package that `setup.sh` installs, not from apt. + +For the container route, the host also needs Docker with the NVIDIA container runtime (`docker.io` and +`nvidia-container-toolkit`, normally already present with `nvidia-jetpack`). The container image installs its own +packages (`libegl1 libgles2 libglib2.0-0 libxext6 libxcb-glx0`), so install nothing extra for it on the host. + +### Host setup + +Run the host setup once, choose the Jetson-IO overlay **Jetson Sensing SG8A-AGON-G2Y-A1 SIPL GMSL2x8** when it asks, and reboot: ```bash src/plugins/sensing/scripts/setup.sh From cd0a27c70adc549e55ae78238473a2fc5a9309ce Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 10:52:48 -0500 Subject: [PATCH 21/29] sensing: address plugin build and docs review findings - Define the configs directory once and use it from app/ instead of a parent-relative path. - Make the missing-dependency messages name only steps that provide the dependency, and point at scripts/verify.sh. - Move the Orin build and run steps from the plugin README into the user guide, drop the unfinished placeholder, and note that the example's ws:// endpoint sends the join token in the clear. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- docs/source/device/camera/sensing.rst | 49 ++++++++++++++++++++++++ src/plugins/sensing/CMakeLists.txt | 15 +++----- src/plugins/sensing/README.md | 53 +++----------------------- src/plugins/sensing/app/CMakeLists.txt | 4 +- 4 files changed, 63 insertions(+), 58 deletions(-) diff --git a/docs/source/device/camera/sensing.rst b/docs/source/device/camera/sensing.rst index 964375724a..95118b6055 100644 --- a/docs/source/device/camera/sensing.rst +++ b/docs/source/device/camera/sensing.rst @@ -155,3 +155,52 @@ on the side that fails for a read-only runtime report: * - Low FPS or dropped frames - Check the Jetson power mode. Maximum performance can be selected with ``sudo nvpmodel -m 0 && sudo jetson_clocks``. + +Build and run the LiveKit publisher +----------------------------------- + +``camera_plugin_sensing`` runs on the Orin. It captures the GMSL cameras with +SIPL, encodes H.264 on the hardware encoder and publishes one LiveKit video +track per camera. CaptureServer, the egocap backend, recording and the browser +run on the PC. + +Prerequisites: an AGX Orin on JetPack 7.2.1 / L4T R39.2.1 with the SENSING +drivers installed and the cameras working, CUDA, CMake, and network access to +the LiveKit server. The build also needs two header trees; neither is fetched +by ``setup.sh``: + +- ``JETSON_MMAPI_DIR``: the Jetson Multimedia API tree, containing + ``include/NvVideoEncoder.h`` and ``samples/common/classes/``. CMake searches + ``/usr/src/jetson_multimedia_api``. +- ``SIPL_API_DIR``: the Jetson SIPL API tree, containing + ``sipl/include/NvSIPLCamera.hpp`` (the ``Jetson_SIPL_API_R_aarch64.tbz2`` + package published with the L4T release). CMake searches ``/usr/src/jetson_sipl_api``, + ``/opt/nvidia/jetson_sipl_api`` and ``$SIPL_API_DIR``. + +.. code-block:: bash + + # Build. The LiveKit SDK v1.12.4 prebuilt is downloaded and hash-checked; use + # -DLIVEKIT_SDK_DIR=... for a local copy (it needs LICENSE and NOTICE files in that + # directory, or -DLIVEKIT_SDK_LICENSE_FILE / -DLIVEKIT_SDK_NOTICE_FILE). + cmake -B build -DBUILD_PLUGIN_SENSING=ON -DBUILD_VIZ=OFF -DBUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF -DBUILD_PYTHON_BINDINGS=OFF \ + -DSIPL_API_DIR=... -DJETSON_MMAPI_DIR=... + cmake --build build --target camera_plugin_sensing -j + + # Check the cameras are visible. + build/src/plugins/sensing/app/camera_plugin_sensing --list-sensors + + # Publish. The token file is a LiveKit token for the room and publisher identity + # named in the PC's streams file. + build/src/plugins/sensing/app/camera_plugin_sensing \ + --livekit-url=ws://LIVEKIT_HOST:7880 --livekit-token-file=join.token \ + --add-stream=sensor=0,livekit=sensing-left --add-stream=sensor=1,livekit=sensing-right + +The ``livekit=`` names must equal the ``track`` names in the PC's streams file. +Stop with Ctrl-C. ``--help`` lists every option. The example uses plain ``ws://`` +for a LiveKit server on a trusted LAN; the join token is sent in the clear, so +use a ``wss://`` endpoint on any other network. + +``SENSING_TSC_PROBE=1`` logs the SIPL capture counter beside the host clocks twice +a second, as separate uncalibrated samples. The counter's units are unconfirmed, so +do not subtract or fit them. diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index f58b96166e..97667fa2b5 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -44,10 +44,7 @@ if(NOT SIPL_API_DIR) "================================================================================\n" "SENSING camera plugin requires the Jetson SIPL API headers.\n" "\n" - "Run the container setup, which fetches them:\n" - " src/plugins/sensing/scripts/setup.sh\n" - "\n" - "Or point CMake at an existing copy:\n" + "Install the SIPL API package for your L4T release and point CMake at it:\n" " cmake -B build -DSIPL_API_DIR=/path/to/jetson_sipl_api\n" "\n" "Or disable this plugin:\n" @@ -73,7 +70,7 @@ foreach(_lib nvsipl nvsipl_query nvscibuf nvscisync nvbufsurface_nvsci) if(NOT ${_upper}_LIBRARY) message(FATAL_ERROR "lib${_lib} not found. It is supplied by the L4T BSP and, in a container, by the " - "NVIDIA container runtime. Run src/plugins/sensing/verify.sh to see what is missing.") + "NVIDIA container runtime. Run src/plugins/sensing/scripts/verify.sh to see what is missing.") endif() list(APPEND SIPL_LIBRARIES ${${_upper}_LIBRARY}) endforeach() @@ -102,10 +99,7 @@ if(NOT JETSON_MMAPI_DIR) "================================================================================\n" "SENSING camera plugin requires the Jetson Multimedia API encoder sources.\n" "\n" - "setup_container.sh fetches the version matching /etc/nv_tegra_release:\n" - " src/plugins/sensing/scripts/setup.sh\n" - "\n" - "Or point CMake at an existing tree:\n" + "Unpack the version matching /etc/nv_tegra_release and point CMake at it:\n" " cmake -B build -DJETSON_MMAPI_DIR=/path/to/jetson_multimedia_api\n" "================================================================================\n" ) @@ -220,6 +214,9 @@ set(SENSING_LIVEKIT_SDK_DIR "${LIVEKIT_SDK_DIR}" CACHE INTERNAL "") # ============================================================================== message(STATUS "Building SENSING camera plugin") +# The platform configs live beside this file; app/ stages and installs them from here. +set(SENSING_CONFIGS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/configs") + add_subdirectory(livekit) add_subdirectory(core) add_subdirectory(app) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 8f184742e3..3f2049d593 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -39,51 +39,10 @@ before replacing it; do not hand-edit the board wiring. Its CSI ports, I2C buses, deserializer addresses, GPIOs, and link modes must match the installed SIPL drivers. -## Orin: build and run the LiveKit publisher +## Orin: LiveKit publisher -The application is **`camera_plugin_sensing`**. On the Orin it does one job: capture the GMSL cameras with SIPL, -encode H.264 on the hardware encoder, and publish one LiveKit video track per camera. Everything else (CaptureServer, -the egocap backend, recording, the browser) runs on the PC. - -Prerequisites: an AGX Orin on JetPack 7.2.1 / L4T R39.2.1 with the SENSING drivers installed and the cameras -working, CUDA, CMake, and network access to the LiveKit server. - -```bash -# 1. Once: vendor drivers (see scripts/setup.sh --help). The Multimedia API and SIPL header trees are -# separate prerequisites that setup.sh does not fetch; CMake looks for them as described below. -src/plugins/sensing/scripts/setup.sh -# 2. Build. The LiveKit SDK v1.12.4 prebuilt is downloaded and hash-checked; use -DLIVEKIT_SDK_DIR=... for a local -# copy (it needs LICENSE and NOTICE files in that directory, or -DLIVEKIT_SDK_LICENSE_FILE / -DLIVEKIT_SDK_NOTICE_FILE). -# The viewer, examples, tests and Python bindings are not needed (the viewer needs libxkbcommon-dev). -# SIPL_API_DIR and JETSON_MMAPI_DIR point at the unpacked SIPL and Multimedia API trees. -cmake -B build -DBUILD_PLUGIN_SENSING=ON -DBUILD_VIZ=OFF -DBUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF \ - -DBUILD_PYTHON_BINDINGS=OFF -DSIPL_API_DIR=... -DJETSON_MMAPI_DIR=... -cmake --build build --target camera_plugin_sensing -j -# 3. Check the cameras are visible. -build/src/plugins/sensing/app/camera_plugin_sensing --list-sensors -# 4. Publish. The token file is a LiveKit token for the room and publisher identity named in the PC's streams file. -build/src/plugins/sensing/app/camera_plugin_sensing \ - --livekit-url=ws://LIVEKIT_HOST:7880 --livekit-token-file=join.token \ - --add-stream=sensor=0,livekit=sensing-left --add-stream=sensor=1,livekit=sensing-right -``` - -The `livekit=` names must equal the `track` names in the PC's streams file. Stop with Ctrl-C. `--help` lists every -option (`--livekit-fps`, `--livekit-bitrate`, `--livekit-scale-div`, `--platform-config`, `--link-masks`). -`SENSING_TSC_PROBE=1` logs the SIPL capture counter beside the host clocks twice a second, as separate uncalibrated -samples. The counter's units are unconfirmed, so do not subtract or fit them. - -Status: built and run on an AGX Orin by the author (both sensors, 2560x1984 at 30 fps, H.264; not reproduced in CI). -A failed build or run reports the file or dependency check that failed. - -### Header trees the build needs - -CMake searches for them when the `-D` variables are not given: - -- `JETSON_MMAPI_DIR`: the Jetson Multimedia API tree, containing `include/NvVideoEncoder.h` and - `samples/common/classes/`. Searched in `/usr/src/jetson_multimedia_api`. -- `SIPL_API_DIR`: the Jetson SIPL API tree, containing `sipl/include/NvSIPLCamera.hpp` (the - `Jetson_SIPL_API_R_aarch64.tbz2` package published with the L4T release). Searched in - `/usr/src/jetson_sipl_api`, `/opt/nvidia/jetson_sipl_api` and `$SIPL_API_DIR`. - -Human to fill in: download and unpacking steps for both trees belong in the user guide; they are not stated -anywhere else in this repository. +`camera_plugin_sensing` (built from [`app/`](app/), [`core/`](core/) and +[`livekit/`](livekit/)) captures the cameras with SIPL, encodes H.264 on the +hardware encoder and publishes one LiveKit video track per camera. The +[SENSING user guide](https://nvidia.github.io/IsaacCapture/main/device/camera/sensing.html) +has the build and run steps. diff --git a/src/plugins/sensing/app/CMakeLists.txt b/src/plugins/sensing/app/CMakeLists.txt index 4af038dcd0..97eaa9dd9c 100644 --- a/src/plugins/sensing/app/CMakeLists.txt +++ b/src/plugins/sensing/app/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries(camera_plugin_sensing PRIVATE sensing::core) # tree -- otherwise it only works after `cmake --install`. add_custom_command(TARGET camera_plugin_sensing POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory - "${CMAKE_CURRENT_SOURCE_DIR}/../configs" + "${SENSING_CONFIGS_DIR}" "$/configs" COMMENT "Staging SIPL platform configs beside camera_plugin_sensing" ) @@ -20,7 +20,7 @@ add_custom_command(TARGET camera_plugin_sensing POST_BUILD install(TARGETS camera_plugin_sensing RUNTIME DESTINATION plugins/sensing_camera ) -install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../configs/" +install(DIRECTORY "${SENSING_CONFIGS_DIR}/" DESTINATION plugins/sensing_camera/configs ) From d828e7470d2739aa61ddfe13d7df577e44624ba1 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 10:59:03 -0500 Subject: [PATCH 22/29] sensing: export core's public headers from inc/ with INTERFACE include path sensing_core exported its parent directory as a PUBLIC include path and the app reached headers as "core/...", which the CMake structure rules forbid (no ../ -style reach, INTERFACE inc/, includes). Move the headers the app consumes (sensing_camera, sensing_types, sipl_camera) to core/inc/sensing_core/, export only core/inc with INTERFACE, and have own sources reach them by relative "inc/sensing_core/..." paths. The app includes them as . The remaining headers have no other consumer and stay private beside their sources. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/app/main.cpp | 4 ++-- src/plugins/sensing/core/CMakeLists.txt | 7 +++++-- .../sensing/core/{ => inc/sensing_core}/sensing_camera.hpp | 0 .../sensing/core/{ => inc/sensing_core}/sensing_types.hpp | 0 .../sensing/core/{ => inc/sensing_core}/sipl_camera.hpp | 0 src/plugins/sensing/core/livekit_video_output.hpp | 2 +- src/plugins/sensing/core/sensing_camera.cpp | 2 +- src/plugins/sensing/core/sipl_camera.cpp | 2 +- 8 files changed, 10 insertions(+), 7 deletions(-) rename src/plugins/sensing/core/{ => inc/sensing_core}/sensing_camera.hpp (100%) rename src/plugins/sensing/core/{ => inc/sensing_core}/sensing_types.hpp (100%) rename src/plugins/sensing/core/{ => inc/sensing_core}/sipl_camera.hpp (100%) diff --git a/src/plugins/sensing/app/main.cpp b/src/plugins/sensing/app/main.cpp index 242e2c7660..425197b38f 100644 --- a/src/plugins/sensing/app/main.cpp +++ b/src/plugins/sensing/app/main.cpp @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#include "core/sensing_camera.hpp" -#include "core/sipl_camera.hpp" +#include +#include #include #include diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt index 58af1a1c72..b7cfe1e987 100644 --- a/src/plugins/sensing/core/CMakeLists.txt +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -10,6 +10,9 @@ add_library(sensing_core STATIC yuv_to_rgba.cu rgba_to_nv12.cu rgba_box_downscale.cu + inc/sensing_core/sensing_camera.hpp + inc/sensing_core/sensing_types.hpp + inc/sensing_core/sipl_camera.hpp ${JETSON_MMAPI_SOURCES} ) add_library(sensing::core ALIAS sensing_core) @@ -23,8 +26,8 @@ set_target_properties(sensing_core PROPERTIES ) target_include_directories(sensing_core - PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/.. + INTERFACE + $ PRIVATE # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay private: they drag in # NvMedia and X11-style macros. diff --git a/src/plugins/sensing/core/sensing_camera.hpp b/src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp similarity index 100% rename from src/plugins/sensing/core/sensing_camera.hpp rename to src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp diff --git a/src/plugins/sensing/core/sensing_types.hpp b/src/plugins/sensing/core/inc/sensing_core/sensing_types.hpp similarity index 100% rename from src/plugins/sensing/core/sensing_types.hpp rename to src/plugins/sensing/core/inc/sensing_core/sensing_types.hpp diff --git a/src/plugins/sensing/core/sipl_camera.hpp b/src/plugins/sensing/core/inc/sensing_core/sipl_camera.hpp similarity index 100% rename from src/plugins/sensing/core/sipl_camera.hpp rename to src/plugins/sensing/core/inc/sensing_core/sipl_camera.hpp diff --git a/src/plugins/sensing/core/livekit_video_output.hpp b/src/plugins/sensing/core/livekit_video_output.hpp index 9274cb7329..7c41b18275 100644 --- a/src/plugins/sensing/core/livekit_video_output.hpp +++ b/src/plugins/sensing/core/livekit_video_output.hpp @@ -3,7 +3,7 @@ #pragma once -#include "sipl_camera.hpp" +#include "inc/sensing_core/sipl_camera.hpp" #include diff --git a/src/plugins/sensing/core/sensing_camera.cpp b/src/plugins/sensing/core/sensing_camera.cpp index 0e1c0dcc77..67a81cc64b 100644 --- a/src/plugins/sensing/core/sensing_camera.cpp +++ b/src/plugins/sensing/core/sensing_camera.cpp @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#include "sensing_camera.hpp" +#include "inc/sensing_core/sensing_camera.hpp" #include "livekit_video_output.hpp" diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp index da5baa0c64..6a362ac25b 100644 --- a/src/plugins/sensing/core/sipl_camera.cpp +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -5,7 +5,7 @@ // (/usr/src/jetson_multimedia_api/samples/unittest_samples/), BSD-3-Clause, // Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. -#include "sipl_camera.hpp" +#include "inc/sensing_core/sipl_camera.hpp" #include "yuv_to_rgba.cuh" From 028dcc1ccefb33d2d30373de15fb3d6688bed7ea Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 11:00:52 -0500 Subject: [PATCH 23/29] sensing: stop discarding frame metadata the encoder may still return The LiveKit output paired each encoded unit with the oldest submitted frame's metadata by order, and trimmed the queue to a fixed length. The trim threw away metadata for frames that could still come back, a unit arriving with an empty queue threw and failed SensingCamera::update(), and a frame consumed inside the encoder shifted every later frame_id and capture stamp by one. The Jetson encoder does not return the stamp given to it, so there is no per-frame key to carry through. Keep order pairing, which is what the Orin was seen to do, and use the encoder's own unit numbering (index times the frame period) only as evidence of a skipped frame: a forward jump retires that many oldest entries, provided one remains for the unit. A backwards jump or an implausible one just resets the baseline. A unit with nothing to pair with is skipped and counted instead of throwing, and whatever is still unreturned after the encoder is drained is counted as dropped. Both counts appear in the per-sensor statistics. Not built or run: needs the Jetson SDK and an Orin. Whether the encoder numbers dropped frames in its unit numbering is unverified. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/jetson_encoder.hpp | 2 +- .../sensing/core/livekit_video_output.cpp | 42 ++++++++++++++----- .../sensing/core/livekit_video_output.hpp | 2 + src/plugins/sensing/core/sensing_camera.cpp | 4 ++ 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/src/plugins/sensing/core/jetson_encoder.hpp b/src/plugins/sensing/core/jetson_encoder.hpp index a2bc7de7f2..aabd3dcfcd 100644 --- a/src/plugins/sensing/core/jetson_encoder.hpp +++ b/src/plugins/sensing/core/jetson_encoder.hpp @@ -72,7 +72,7 @@ class JetsonEncoder * @param row_pitch_bytes Byte stride between rows. * @param timestamp_ns Stamp set on the V4L2 output buffer. On the verified Orin driver the unit * poll() returns does not carry it back (units are numbered from the encoder's own frame rate), so - * callers pair units with submissions by order and must not match on this value. + * callers pair units with submissions by order; that numbering is the only hint of a skipped frame. * @return false when no input buffer is free (frame dropped). */ bool submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns); diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp index 5d0980a4ea..934ae3ecce 100644 --- a/src/plugins/sensing/core/livekit_video_output.cpp +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -241,6 +241,9 @@ struct LiveKitVideoOutput::Impl { for (auto& unit : encoder->drain()) publish_unit(unit); + // Anything still unreturned once the encoder is drained is not coming back. + encoder_dropped += in_encoder.size(); + in_encoder.clear(); std::lock_guard lock(mutex); flush_requested = false; flushed.notify_all(); @@ -317,25 +320,36 @@ struct LiveKitVideoOutput::Impl } in_encoder.push_back( { stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc, slot.unix_ns / 1000 }); - // The encoder holds a bounded number of frames, so a longer queue means it dropped some inside it. - // Trimming bounds how stale the oldest metadata can be; it does not resynchronise units with captures, - // so frame ids after an encoder-internal drop can be shifted until a supported per-frame correlation exists. - constexpr std::size_t kMaxInFlight = 12; - while (in_encoder.size() > kMaxInFlight) - in_encoder.pop_front(); } // Worker thread: one encoded unit becomes one published frame, with the sequence and capture time // of the capture it came from. void publish_unit(const EncodedUnit& unit) { - const int64_t stamp_us = unit.timestamp_ns / 1000; // The Jetson encoder does not return the stamp it was given: it numbers its units from its own frame - // rate. It does return one unit per accepted frame in submission order, so units pair with captures - // by order; the unit's own stamp is for diagnostics only. + // rate (0, 1/fps, ...). Units pair with captures by order, and a jump in that numbering is the only + // evidence of frames the encoder consumed without returning a unit. Metadata is retired only by a + // matched unit or by such a jump; a jump that leaves no frame for this unit, or a numbering that goes + // backwards, is not trusted and just resets the baseline. + const int64_t index = std::llround(static_cast(unit.timestamp_ns) * config.fps / 1e9); + if (have_unit_index) + { + const int64_t gap = index - last_unit_index - 1; + if (gap > 0 && static_cast(gap) < in_encoder.size()) + { + in_encoder.erase(in_encoder.begin(), in_encoder.begin() + gap); + encoder_dropped += static_cast(gap); + } + } + last_unit_index = index; + have_unit_index = true; + + // A unit with no frame to pair with is skipped rather than published with someone else's metadata. if (in_encoder.empty()) - throw std::runtime_error("LiveKit output: the encoder returned a unit with no frame in flight (unit stamp_us=" + - std::to_string(stamp_us) + ")"); + { + ++unmatched_units; + return; + } const FrameMeta meta = in_encoder.front(); in_encoder.pop_front(); @@ -384,6 +398,8 @@ struct LiveKitVideoOutput::Impl // Worker thread only. std::deque in_encoder; ///< Submitted to the encoder, no unit back yet. + int64_t last_unit_index = 0; ///< The encoder's own number of the last unit returned. + bool have_unit_index = false; bool keyframe_wanted = false; bool awaiting_keyframe = false; uint32_t applied_bitrate = 0; @@ -417,6 +433,8 @@ struct LiveKitVideoOutput::Impl std::atomic encoder_busy{ 0 }; std::atomic rejected{ 0 }; std::atomic keyframes_requested{ 0 }; + std::atomic encoder_dropped{ 0 }; + std::atomic unmatched_units{ 0 }; std::mutex failure_mutex; std::string failure_message; @@ -464,6 +482,8 @@ LiveKitOutputStats LiveKitVideoOutput::stats() const s.encoder_busy = m_impl->encoder_busy.load(); s.rejected = m_impl->rejected.load(); s.keyframes_requested = m_impl->keyframes_requested.load(); + s.encoder_dropped = m_impl->encoder_dropped.load(); + s.unmatched_units = m_impl->unmatched_units.load(); return s; } diff --git a/src/plugins/sensing/core/livekit_video_output.hpp b/src/plugins/sensing/core/livekit_video_output.hpp index 7c41b18275..004c9e6044 100644 --- a/src/plugins/sensing/core/livekit_video_output.hpp +++ b/src/plugins/sensing/core/livekit_video_output.hpp @@ -44,6 +44,8 @@ struct LiveKitOutputStats uint64_t encoder_busy = 0; ///< Snapshots the encoder had no input buffer for, dropped before encoding. uint64_t rejected = 0; ///< Encoded frames the SDK refused; a keyframe follows. uint64_t keyframes_requested = 0; ///< Keyframes the receiving side asked for. + uint64_t encoder_dropped = 0; ///< Accepted frames the encoder never returned; their metadata is retired. + uint64_t unmatched_units = 0; ///< Encoded units with no capture to pair with, skipped. }; /** diff --git a/src/plugins/sensing/core/sensing_camera.cpp b/src/plugins/sensing/core/sensing_camera.cpp index 67a81cc64b..b6ed078044 100644 --- a/src/plugins/sensing/core/sensing_camera.cpp +++ b/src/plugins/sensing/core/sensing_camera.cpp @@ -161,6 +161,10 @@ void SensingCamera::print_stats() const std::cout << ", " << s.encoder_busy << " encoder busy"; if (s.rejected) std::cout << ", " << s.rejected << " rejected"; + if (s.encoder_dropped) + std::cout << ", " << s.encoder_dropped << " dropped in encoder"; + if (s.unmatched_units) + std::cout << ", " << s.unmatched_units << " unmatched units"; if (s.keyframes_requested) std::cout << ", " << s.keyframes_requested << " keyframes requested"; const std::string encoder = stream.livekit->describe(); From 2a862718e536fd670e8a327e7e3ae6dd7fef9e65 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 11:37:47 -0500 Subject: [PATCH 24/29] sensing: join the encoder's capture thread in the implementation's destructor The capture-plane dequeue thread dereferences the implementation object. It was stopped only in ~JetsonEncoder, which does not run when the constructor throws after the thread starts (a failing capture qBuffer or cudaMallocPitch), so the thread could outlive the object it uses. Stop and join it in ~Impl, guarded by a flag set once the thread has started. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/core/jetson_encoder.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/plugins/sensing/core/jetson_encoder.cpp b/src/plugins/sensing/core/jetson_encoder.cpp index b55d0aa44d..8dfff056cc 100644 --- a/src/plugins/sensing/core/jetson_encoder.cpp +++ b/src/plugins/sensing/core/jetson_encoder.cpp @@ -57,9 +57,17 @@ struct JetsonEncoder::Impl std::deque ready; bool eos_reached = false; bool eos_sent = false; // driving thread only + bool dq_thread_started = false; ~Impl() { + // The capture thread dereferences this object, so it must be joined here, not only in ~JetsonEncoder: + // a constructor that throws after starting it never runs ~JetsonEncoder. + if (encoder && dq_thread_started) + { + encoder->capture_plane.stopDQThread(); + encoder->capture_plane.waitForDQThread(1000); + } if (d_y) cudaFree(d_y); if (d_uv) @@ -166,6 +174,7 @@ JetsonEncoder::JetsonEncoder(const EncoderConfig& config) : m_impl(std::make_uni enc->capture_plane.setDQThreadCallback(&Impl::capture_dq); enc->capture_plane.startDQThread(m_impl.get()); + m_impl->dq_thread_started = true; // Prime the capture plane so the encoder always has somewhere to write. for (uint32_t i = 0; i < enc->capture_plane.getNumBuffers(); ++i) @@ -186,14 +195,7 @@ JetsonEncoder::JetsonEncoder(const EncoderConfig& config) : m_impl(std::make_uni "cudaMallocPitch(UV)"); } -JetsonEncoder::~JetsonEncoder() -{ - if (m_impl && m_impl->encoder) - { - m_impl->encoder->capture_plane.stopDQThread(); - m_impl->encoder->capture_plane.waitForDQThread(1000); - } -} +JetsonEncoder::~JetsonEncoder() = default; bool JetsonEncoder::submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns) { From 3f3b666d661a1b7fbadb3ef9ec1e2aa9e151499e Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 12:05:51 -0500 Subject: [PATCH 25/29] sensing: build the Jetson encoder sample without its stdout debug prints NvVideoEncoder.cpp, compiled from the Multimedia API tree, prints both pixel format codes with cout in setBitrate() and setPeakBitrate(), so the producer's stdout gets bare integers at every encoder start. The tree under /usr/src is not ours to edit, so compile a copy with those lines removed, rewritten only when its content changes, and report how many were removed at configure time. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/CMakeLists.txt | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index 97667fa2b5..42ba192c68 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -107,8 +107,32 @@ endif() message(STATUS "SENSING plugin: Jetson Multimedia API=${JETSON_MMAPI_DIR}") set(_mmapi_classes "${JETSON_MMAPI_DIR}/samples/common/classes") + +# NvVideoEncoder.cpp prints both pixel-format codes with cout in setBitrate() and setPeakBitrate(), which puts bare +# integers on stdout at every encoder start. /usr/src is not ours to edit, so compile a copy without those lines. +file(READ "${_mmapi_classes}/NvVideoEncoder.cpp" _nv_encoder_source) +set(_nv_debug_print "[ \t]*cout[ \t]*<<[ \t]*(capture|output)_plane_pixfmt[ \t]*<<[ \t]*endl;[^\n]*\n") +string(REGEX REPLACE "${_nv_debug_print}" "" _nv_encoder_quiet "${_nv_encoder_source}") +# The matched lines contain ';', so count the printed values before and after instead of the matches. +string(REGEX MATCHALL "_plane_pixfmt[ \t]*<<" _nv_before "${_nv_encoder_source}") +string(REGEX MATCHALL "_plane_pixfmt[ \t]*<<" _nv_after "${_nv_encoder_quiet}") +list(LENGTH _nv_before _nv_before_count) +list(LENGTH _nv_after _nv_after_count) +math(EXPR _nv_debug_print_count "${_nv_before_count} - ${_nv_after_count}") +set(_nv_encoder_quiet_file "${CMAKE_CURRENT_BINARY_DIR}/NvVideoEncoder.cpp") +set(_nv_encoder_previous "") +if(EXISTS "${_nv_encoder_quiet_file}") + file(READ "${_nv_encoder_quiet_file}" _nv_encoder_previous) +endif() +# Write only on a change, so an unchanged source does not rebuild; a changed sample source reconfigures. +if(NOT _nv_encoder_previous STREQUAL _nv_encoder_quiet) + file(WRITE "${_nv_encoder_quiet_file}" "${_nv_encoder_quiet}") +endif() +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_mmapi_classes}/NvVideoEncoder.cpp") +message(STATUS "SENSING plugin: removed ${_nv_debug_print_count} debug print(s) from the NvVideoEncoder.cpp copy") + set(JETSON_MMAPI_SOURCES - "${_mmapi_classes}/NvVideoEncoder.cpp" + "${_nv_encoder_quiet_file}" "${_mmapi_classes}/NvV4l2Element.cpp" "${_mmapi_classes}/NvV4l2ElementPlane.cpp" "${_mmapi_classes}/NvElement.cpp" From ad2b20a1417deca2460cb9a4cf4c4fd854d8e5d1 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 12:15:42 -0500 Subject: [PATCH 26/29] sensing: copy NVIDIA's Jetson encoder classes into the repo The producer compiled NvVideoEncoder and the V4L2 element classes from /usr/src/jetson_multimedia_api, where NVIDIA ships their implementation only as source. Copy the seven classes and the seven headers they need (BSD-3-Clause, each file keeps its own license text) from the R39.2.1 package into third_party/jetson_multimedia_api, so the build no longer depends on the installed copy's version. Only four debug cout lines in NvVideoEncoder.cpp are removed, marked with a comment; this replaces the configure-time patch. The three headers that must not be copied stay in the installed tree: nvbufsurface.h (proprietary), libv4l2.h (LGPL) and v4l2_nv_extensions.h (the driver's interface). The directory has a README with the source, license and update steps, a LICENSE that is installed beside the plugin, a REUSE annotation for files whose header has no SPDX tag, a README licensing note, and an exemption from the whitespace hooks so the files keep their upstream bytes. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .pre-commit-config.yaml | 4 +- README.md | 3 + REUSE.toml | 14 + src/plugins/sensing/CMakeLists.txt | 49 +- src/plugins/sensing/app/CMakeLists.txt | 4 + src/plugins/sensing/core/CMakeLists.txt | 2 + .../third_party/jetson_multimedia_api/LICENSE | 26 + .../jetson_multimedia_api/README.md | 43 + .../classes/NvBuffer.cpp | 514 +++++ .../classes/NvElement.cpp | 60 + .../classes/NvElementProfiler.cpp | 298 +++ .../classes/NvLogging.cpp | 33 + .../classes/NvV4l2Element.cpp | 275 +++ .../classes/NvV4l2ElementPlane.cpp | 943 +++++++++ .../classes/NvVideoEncoder.cpp | 1701 +++++++++++++++++ .../jetson_multimedia_api/include/NvBuffer.h | 296 +++ .../jetson_multimedia_api/include/NvElement.h | 137 ++ .../include/NvElementProfiler.h | 223 +++ .../jetson_multimedia_api/include/NvLogging.h | 235 +++ .../include/NvV4l2Element.h | 201 ++ .../include/NvV4l2ElementPlane.h | 586 ++++++ .../include/NvVideoEncoder.h | 930 +++++++++ 22 files changed, 6541 insertions(+), 36 deletions(-) create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/README.md create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvBuffer.cpp create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElement.cpp create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElementProfiler.cpp create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvLogging.cpp create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2Element.cpp create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2ElementPlane.cpp create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvVideoEncoder.cpp create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/include/NvBuffer.h create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElement.h create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElementProfiler.h create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/include/NvLogging.h create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2Element.h create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2ElementPlane.h create mode 100644 src/plugins/sensing/third_party/jetson_multimedia_api/include/NvVideoEncoder.h diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a46d0b999..e859cb2bab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,8 @@ repos: rev: v6.0.0 hooks: - id: trailing-whitespace + # NVIDIA's copied sources keep their upstream bytes (see the directory's README.md). + exclude: ^src/plugins/sensing/third_party/jetson_multimedia_api/(classes|include)/ - id: check-symlinks - id: destroyed-symlinks - id: check-added-large-files @@ -25,7 +27,7 @@ repos: - id: check-toml - id: end-of-file-fixer # These LFS SVGs are assets; rewriting their final byte creates new LFS objects. - exclude: ^(docs/source/_static/camera-viz-controls|examples/vehicle_teleop/assets/vehicle_teleop_flow)\.svg$ + exclude: ^(docs/source/_static/camera-viz-controls|examples/vehicle_teleop/assets/vehicle_teleop_flow)\.svg$|^src/plugins/sensing/third_party/jetson_multimedia_api/(classes|include)/ - id: check-shebang-scripts-are-executable - id: detect-private-key - id: debug-statements diff --git a/README.md b/README.md index cd55de9172..47a630e80f 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,9 @@ To get started, please refer to Isaac Lab's [Installation](https://isaac-sim.git The default license is [Apache-2.0](LICENSE.md). EgoPost is licensed under [OpenMDW-1.1](LICENSES/LicenseRef-OpenMDW-1.1.txt). Third-party materials may carry their own licenses. +The Jetson encoder classes under `src/plugins/sensing/third_party/jetson_multimedia_api` are NVIDIA's +[BSD-3-Clause](src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE) sample sources, copied from the Jetson +Multimedia API; see that directory's README. ### Separately retrieved optional materials diff --git a/REUSE.toml b/REUSE.toml index 38c21963fb..23460e91f2 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -58,6 +58,20 @@ precedence = "override" SPDX-FileCopyrightText = "Copyright 2015 The Chromium Authors" SPDX-License-Identifier = "BSD-3-Clause" +# NVIDIA's Jetson Multimedia API encoder classes, copied from the nvidia-l4t-jetson-multimedia-api package (see its +# README.md). Each file keeps its own BSD-3-Clause header; files whose header has no SPDX tag are covered here. +[[annotations]] +path = "src/plugins/sensing/third_party/jetson_multimedia_api/**" +precedence = "closest" +SPDX-FileCopyrightText = "Copyright (c) 2016 NVIDIA CORPORATION. All rights reserved." +SPDX-License-Identifier = "BSD-3-Clause" + +[[annotations]] +path = "src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE" +precedence = "override" +SPDX-FileCopyrightText = "Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved." +SPDX-License-Identifier = "BSD-3-Clause" + # EgoPost's default never overrides upstream headers or sidecar licenses. # The license-only scaffold has no covered files yet; the canonical text is reserved for imports. [[annotations]] diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt index 42ba192c68..0a93bee669 100644 --- a/src/plugins/sensing/CMakeLists.txt +++ b/src/plugins/sensing/CMakeLists.txt @@ -88,16 +88,16 @@ endif() # Jetson V4L2 encoder # ============================================================================== # Jetson has no libnvidia-encode (the Video Codec SDK is dGPU-only), so H.264 -# comes from the V4L2 M2M engine via NvVideoEncoder in the Multimedia API. +# comes from the V4L2 M2M engine via NvVideoEncoder, vendored from the Multimedia API (see below). if(NOT JETSON_MMAPI_DIR) - find_path(JETSON_MMAPI_DIR include/NvVideoEncoder.h + find_path(JETSON_MMAPI_DIR include/nvbufsurface.h PATHS "/usr/src/jetson_multimedia_api" ) endif() if(NOT JETSON_MMAPI_DIR) message(FATAL_ERROR "================================================================================\n" - "SENSING camera plugin requires the Jetson Multimedia API encoder sources.\n" + "SENSING camera plugin requires the Jetson Multimedia API headers (nvbufsurface.h).\n" "\n" "Unpack the version matching /etc/nv_tegra_release and point CMake at it:\n" " cmake -B build -DJETSON_MMAPI_DIR=/path/to/jetson_multimedia_api\n" @@ -106,39 +106,18 @@ if(NOT JETSON_MMAPI_DIR) endif() message(STATUS "SENSING plugin: Jetson Multimedia API=${JETSON_MMAPI_DIR}") -set(_mmapi_classes "${JETSON_MMAPI_DIR}/samples/common/classes") - -# NvVideoEncoder.cpp prints both pixel-format codes with cout in setBitrate() and setPeakBitrate(), which puts bare -# integers on stdout at every encoder start. /usr/src is not ours to edit, so compile a copy without those lines. -file(READ "${_mmapi_classes}/NvVideoEncoder.cpp" _nv_encoder_source) -set(_nv_debug_print "[ \t]*cout[ \t]*<<[ \t]*(capture|output)_plane_pixfmt[ \t]*<<[ \t]*endl;[^\n]*\n") -string(REGEX REPLACE "${_nv_debug_print}" "" _nv_encoder_quiet "${_nv_encoder_source}") -# The matched lines contain ';', so count the printed values before and after instead of the matches. -string(REGEX MATCHALL "_plane_pixfmt[ \t]*<<" _nv_before "${_nv_encoder_source}") -string(REGEX MATCHALL "_plane_pixfmt[ \t]*<<" _nv_after "${_nv_encoder_quiet}") -list(LENGTH _nv_before _nv_before_count) -list(LENGTH _nv_after _nv_after_count) -math(EXPR _nv_debug_print_count "${_nv_before_count} - ${_nv_after_count}") -set(_nv_encoder_quiet_file "${CMAKE_CURRENT_BINARY_DIR}/NvVideoEncoder.cpp") -set(_nv_encoder_previous "") -if(EXISTS "${_nv_encoder_quiet_file}") - file(READ "${_nv_encoder_quiet_file}" _nv_encoder_previous) -endif() -# Write only on a change, so an unchanged source does not rebuild; a changed sample source reconfigures. -if(NOT _nv_encoder_previous STREQUAL _nv_encoder_quiet) - file(WRITE "${_nv_encoder_quiet_file}" "${_nv_encoder_quiet}") -endif() -set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_mmapi_classes}/NvVideoEncoder.cpp") -message(STATUS "SENSING plugin: removed ${_nv_debug_print_count} debug print(s) from the NvVideoEncoder.cpp copy") - +# NVIDIA's encoder classes are vendored (BSD-3-Clause; third_party/jetson_multimedia_api/README.md), so they build the +# same on every JetPack and carry no stdout debug prints. The installed Multimedia API tree supplies only what is not +# vendored: nvbufsurface.h, libv4l2.h, v4l2_nv_extensions.h and the Argus headers. +set(SENSING_MMAPI_VENDORED_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/jetson_multimedia_api") set(JETSON_MMAPI_SOURCES - "${_nv_encoder_quiet_file}" - "${_mmapi_classes}/NvV4l2Element.cpp" - "${_mmapi_classes}/NvV4l2ElementPlane.cpp" - "${_mmapi_classes}/NvElement.cpp" - "${_mmapi_classes}/NvElementProfiler.cpp" - "${_mmapi_classes}/NvBuffer.cpp" - "${_mmapi_classes}/NvLogging.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvVideoEncoder.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvV4l2Element.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvV4l2ElementPlane.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvElement.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvElementProfiler.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvBuffer.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvLogging.cpp" ) find_library(NVBUFSURFACE_LIBRARY NAMES nvbufsurface PATHS ${_tegra_lib_dirs}) diff --git a/src/plugins/sensing/app/CMakeLists.txt b/src/plugins/sensing/app/CMakeLists.txt index 97eaa9dd9c..613d396602 100644 --- a/src/plugins/sensing/app/CMakeLists.txt +++ b/src/plugins/sensing/app/CMakeLists.txt @@ -34,3 +34,7 @@ install(FILES install(FILES "${SENSING_LIVEKIT_LICENSE_FILE}" "${SENSING_LIVEKIT_NOTICE_FILE}" DESTINATION plugins/sensing_camera/licenses/livekit-sdk ) +# The vendored encoder classes are compiled into the binary, so their license text ships with it. +install(FILES "${SENSING_MMAPI_VENDORED_DIR}/LICENSE" + DESTINATION plugins/sensing_camera/licenses/jetson-multimedia-api +) diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt index b7cfe1e987..54365535e1 100644 --- a/src/plugins/sensing/core/CMakeLists.txt +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -33,6 +33,8 @@ target_include_directories(sensing_core # NvMedia and X11-style macros. ${SIPL_INCLUDE_DIRS} ${EGL_INCLUDE_DIRS} + # The vendored encoder headers come first, so they win over the installed tree's copies of the same names. + ${SENSING_MMAPI_VENDORED_DIR}/include ${JETSON_MMAPI_DIR}/include ) diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE b/src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE new file mode 100644 index 0000000000..d6ab34509d --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/README.md b/src/plugins/sensing/third_party/jetson_multimedia_api/README.md new file mode 100644 index 0000000000..d326aed359 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/README.md @@ -0,0 +1,43 @@ + + +# Jetson Multimedia API encoder classes + +The SENSING producer drives the Jetson hardware H.264 encoder through NVIDIA's `NvVideoEncoder` class and the +V4L2 element classes it is built on. The Jetson Multimedia API documents these classes and ships their headers +in `include/`, but ships their implementation only as source, in the package's `samples/common/classes/` +directory: the package contains no prebuilt library for them, and NVIDIA's own samples compile them in. So they +are copied here and compiled into `sensing_core`, instead of being compiled from the installed `/usr/src` tree. + +- **Source:** the `nvidia-l4t-jetson-multimedia-api` package, version 39.2.1-20260806224157 (L4T R39.2.1), + from `https://repo.download.nvidia.com/jetson/common/pool/main/n/nvidia-l4t-jetson-multimedia-api/`. The files + were taken unchanged from `/usr/src/jetson_multimedia_api/samples/common/classes/` (`classes/`) and + `/usr/src/jetson_multimedia_api/include/` (`include/`), except for the change below. +- **Files:** `classes/` holds `NvVideoEncoder`, `NvV4l2Element`, `NvV4l2ElementPlane`, `NvElement`, + `NvElementProfiler`, `NvBuffer` and `NvLogging` (`.cpp`); `include/` holds the matching headers. This is the + complete set the encoder needs; the producer calls only the encoder and its planes. +- **License:** BSD-3-Clause. Every file carries its own copy of the license text in its header and keeps it; + [`LICENSE`](LICENSE) repeats the text, and the build installs it beside the plugin binary under + `licenses/jetson-multimedia-api`. The package's own copyright file names the NVIDIA Driver License Agreement as + the default for files that carry no license of their own; none of the files here is in that category. +- **Local change:** `classes/NvVideoEncoder.cpp` has the four `cout` lines in `setBitrate()` and `setPeakBitrate()` + removed (they printed the plane pixel-format codes to stdout on every encoder start). The two places are marked + with a comment. + +## Not copied + +These headers stay in the installed tree (`JETSON_MMAPI_DIR`, default `/usr/src/jetson_multimedia_api`), because +their license does not allow copying them here or they must match the installed driver: + +- `nvbufsurface.h`: NVIDIA proprietary license. +- `libv4l2.h`: LGPL-2.1. +- `v4l2_nv_extensions.h`: BSD-3-Clause, but the driver's own interface, which has to match the installed L4T. + +The Argus headers and the `libnvbufsurface` and V4L2 libraries also come from the installed JetPack. + +## Updating + +Extract the package for the target release (`dpkg-deb -x`), copy the same fourteen files over these, compare the +result against the previous copy, and reapply the local change above. Check that the license headers are unchanged. diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvBuffer.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvBuffer.cpp new file mode 100644 index 0000000000..cf5a19050f --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvBuffer.cpp @@ -0,0 +1,514 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvBuffer.h" +#include "NvLogging.h" + +#include +#include +#include +#include + +#define CAT_NAME "Buffer" + +#define MAX(a,b) (a > b ? a : b) + +NvBuffer::NvBuffer(enum v4l2_buf_type buf_type, enum v4l2_memory memory_type, + uint32_t n_planes, NvBufferPlaneFormat * fmt, uint32_t index) + :buf_type(buf_type), + memory_type(memory_type), + index(index), + n_planes(n_planes) +{ + uint32_t i; + + mapped = false; + allocated = false; + + memset(planes, 0, sizeof(planes)); + for (i = 0; i < n_planes; i++) + { + this->planes[i].fd = -1; + this->planes[i].fmt = fmt[i]; + } + + ref_count = 0; + pthread_mutex_init(&ref_lock, NULL); + shared_buffer = NULL; +} + +NvBuffer::NvBuffer(uint32_t pixfmt, uint32_t width, uint32_t height, + uint32_t index) + :buf_type(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE), + memory_type(V4L2_MEMORY_USERPTR), + index(index) +{ + uint32_t i = 0; + NvBuffer::NvBufferPlaneFormat fmt[MAX_PLANES] = {{0}}; + + mapped = false; + allocated = false; + + fill_buffer_plane_format(&n_planes, fmt, width, height, pixfmt); + + for (i = 0; i < MAX_PLANES; i++) + { + this->planes[i].fd = -1; + this->planes[i].data = NULL; + this->planes[i].bytesused = 0; + this->planes[i].mem_offset = 0; + this->planes[i].length = 0; + this->planes[i].fmt = fmt[i]; + this->planes[i].fmt.sizeimage = fmt[i].width * fmt[i].height * + fmt[i].bytesperpixel; + this->planes[i].fmt.stride = fmt[i].width * fmt[i].bytesperpixel; + } + + ref_count = 0; + pthread_mutex_init(&ref_lock, NULL); + shared_buffer = NULL; +} + +NvBuffer::NvBuffer(uint32_t size, uint32_t index) + :buf_type(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE), + memory_type(V4L2_MEMORY_USERPTR), + index(index) +{ + uint32_t i; + + mapped = false; + allocated = false; + + n_planes = 1; + for (i = 0; i < n_planes; i++) + { + this->planes[i].fd = -1; + this->planes[i].data = NULL; + this->planes[i].bytesused = 0; + this->planes[i].mem_offset = 0; + this->planes[i].length = 0; + this->planes[i].fmt.sizeimage = size; + } + + ref_count = 0; + pthread_mutex_init(&ref_lock, NULL); + shared_buffer = NULL; +} + +NvBuffer::~NvBuffer() +{ + if (mapped) + { + unmap(); + } + if (allocated) + { + deallocateMemory(); + } + + pthread_mutex_destroy(&ref_lock); +} + +int +NvBuffer::map() +{ + uint32_t j; + NvBufSurface *nvbuf_surf = 0; + int ret = 0; + + if (memory_type != V4L2_MEMORY_MMAP) + { + CAT_WARN_MSG("Buffer " << index << "already mapped"); + return -1; + } + + if (mapped) + { + CAT_WARN_MSG("Buffer " << index << "already mapped"); + return 0; + } + + ret = NvBufSurfaceFromFd(planes[0].fd, (void **)(&nvbuf_surf)); + if (ret < 0) + { + CAT_ERROR_MSG("Could not get NvBufSurface" << planes[0].fd); + return -1; + } + + if (nvbuf_surf->memType != NVBUF_MEM_CUDA_DEVICE && nvbuf_surf->memType != NVBUF_MEM_CUDA_ARRAY) + { + for (j = 0; j < n_planes; j++) + { + if (planes[j].fd == -1) + { + return -1; + } + + ret = NvBufSurfaceMap(nvbuf_surf, 0, j, NVBUF_MAP_READ_WRITE); + + planes[j].data = (unsigned char *)nvbuf_surf->surfaceList[0].mappedAddr.addr[j]; + if (planes[j].data == NULL) + { + CAT_ERROR_MSG("Could not map buffer " << index << ", plane " << j); + return -1; + } + else + { + CAT_DEBUG_MSG("Mapped buffer " << index << ", plane " << j << " to " + << planes[j].data); + } + } + } + mapped = true; + return 0; +} + +void +NvBuffer::unmap() +{ + NvBufSurface *nvbuf_surf = 0; + int ret = 0; + + if (memory_type != V4L2_MEMORY_MMAP || !mapped) + { + CAT_WARN_MSG("Cannot Unmap Buffer " << index << + ". Only mapped MMAP buffer can be unmapped"); + return; + } + + ret = NvBufSurfaceFromFd(planes[0].fd, (void **)(&nvbuf_surf)); + if (ret < 0) + { + CAT_ERROR_MSG("Could not get NvBufSurface" << planes[0].fd); + return; + } + + if (nvbuf_surf->memType != NVBUF_MEM_CUDA_DEVICE && nvbuf_surf->memType != NVBUF_MEM_CUDA_ARRAY) + { + for (uint32_t j = 0; j < n_planes; j++) + { + if (planes[j].data) + { + ret = NvBufSurfaceUnMap(nvbuf_surf, 0, j); + if (ret < 0) + { + CAT_WARN_MSG("Error while Unmapping buffer"); + return; + } + } + planes[j].data = NULL; + } + } + mapped = false; + CAT_DEBUG_MSG("Buffer " << index << " unmapped "); +} + +int +NvBuffer::allocateMemory() +{ + uint32_t j; + + if (memory_type != V4L2_MEMORY_USERPTR) + { + CAT_ERROR_MSG("Only USERPTR buffers can be allocated"); + return -1; + } + + if (allocated) + { + CAT_WARN_MSG("Buffer " << index << "already allocated memory"); + return 0; + } + + for (j = 0; j < n_planes; j++) + { + if (planes[j].data) + { + ERROR_MSG("Buffer " << index << ", Plane " << j << + " already allocated"); + return -1; + } + + planes[j].length = MAX(planes[j].fmt.sizeimage, + planes[j].fmt.width * + planes[j].fmt.bytesperpixel * + planes[j].fmt.height); + planes[j].data = new unsigned char [planes[j].length]; + + if (planes[j].data == MAP_FAILED) + { + SYS_ERROR_MSG("Error while allocating buffer " << index << + " plane " << j); + return -1; + } + else + { + DEBUG_MSG("Buffer " << index << ", Plane " << j << + " allocated to " << (void *) planes[j].data); + } + } + allocated = true; + return 0; +} + +void +NvBuffer::deallocateMemory() +{ + uint32_t j; + + if (memory_type != V4L2_MEMORY_USERPTR || !allocated) + { + ERROR_MSG("Only allocated USERPTR buffers can be deallocated"); + return; + } + + for (j = 0; j < n_planes; j++) + { + if (!planes[j].data) + { + DEBUG_MSG("Buffer " << index << ", Plane " << j << + " not allocated"); + continue; + } + delete[] planes[j].data; + planes[j].data = NULL; + } + allocated = false; + DEBUG_MSG("Buffer " << index << " deallocated"); +} + +int +NvBuffer::ref() +{ + int ref_count; + + pthread_mutex_lock(&ref_lock); + ref_count = ++this->ref_count; + pthread_mutex_unlock(&ref_lock); + return ref_count; +} + +int +NvBuffer::unref() +{ + int ref_count; + + pthread_mutex_lock(&ref_lock); + if (this->ref_count > 0) + { + --this->ref_count; + } + ref_count = this->ref_count; + pthread_mutex_unlock(&ref_lock); + return ref_count; +} + +int +NvBuffer::fill_buffer_plane_format(uint32_t *num_planes, + NvBuffer::NvBufferPlaneFormat *planefmts, + uint32_t width, uint32_t height, uint32_t raw_pixfmt) +{ + switch (raw_pixfmt) + { + case V4L2_PIX_FMT_YUV444M: + case V4L2_PIX_FMT_YUV444: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width; + planefmts[2].width = width; + + planefmts[0].height = height; + planefmts[1].height = height; + planefmts[2].height = height; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_YUV422M: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + planefmts[2].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height; + planefmts[2].height = height; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_YUV422RM: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width; + planefmts[2].width = width; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + planefmts[2].height = height / 2; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_YUV420M: + case V4L2_PIX_FMT_YVU420M: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + planefmts[2].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + planefmts[2].height = height / 2; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_NV12M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 2; + break; + case V4L2_PIX_FMT_NV16M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 2; + break; + case V4L2_PIX_FMT_GREY: + *num_planes = 1; + + planefmts[0].width = width; + + planefmts[0].height = height; + + planefmts[0].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_YUYV: + case V4L2_PIX_FMT_YVYU: + case V4L2_PIX_FMT_UYVY: + case V4L2_PIX_FMT_VYUY: + *num_planes = 1; + + planefmts[0].width = width; + + planefmts[0].height = height; + + planefmts[0].bytesperpixel = 2; + break; + case V4L2_PIX_FMT_ABGR32: + case V4L2_PIX_FMT_XRGB32: + *num_planes = 1; + + planefmts[0].width = width; + + planefmts[0].height = height; + + planefmts[0].bytesperpixel = 4; + break; + case V4L2_PIX_FMT_P010M: + case V4L2_PIX_FMT_P012M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + + planefmts[0].bytesperpixel = 2; + planefmts[1].bytesperpixel = 4; + break; + case V4L2_PIX_FMT_NV24M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width; + + planefmts[0].height = height; + planefmts[1].height = height; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 2; + break; + case V4L2_PIX_FMT_NV24_10LE: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width; + + planefmts[0].height = height; + planefmts[1].height = height; + + planefmts[0].bytesperpixel = 2; + planefmts[1].bytesperpixel = 4; + break; + case V4L2_PIX_FMT_YUV444_10LE: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width; + planefmts[2].width = width; + + planefmts[0].height = height; + planefmts[1].height = height; + planefmts[2].height = height; + + planefmts[0].bytesperpixel = 2; + planefmts[1].bytesperpixel = 2; + planefmts[2].bytesperpixel = 2; + break; + default: + ERROR_MSG("Unsupported pixel format " << raw_pixfmt); + return -1; + } + return 0; +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElement.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElement.cpp new file mode 100644 index 0000000000..7d32974d8f --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElement.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvElement.h" + +void NvElement::getProfilingData(NvElementProfiler::NvElementProfilerData &data) +{ + profiler.getProfilerData(data); +} + +void NvElement::printProfilingStats(std::ostream &out_stream) +{ + out_stream << "----------- Element = " << comp_name << " -----------" << std::endl; + profiler.printProfilerData(out_stream); + out_stream << "-------------------------------------" << std::endl; +} + +void NvElement::enableProfiling() +{ + profiler.enableProfiling(true); +} + +bool NvElement::isProfilingEnabled() +{ + return profiler.enabled; +} + +NvElement::NvElement(const char *name, NvElementProfiler::ProfilerField fields) + :profiler(fields) +{ + is_in_error = 0; + if (!name) + is_in_error = 1; + this->comp_name = name; +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElementProfiler.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElementProfiler.cpp new file mode 100644 index 0000000000..e5855f5736 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElementProfiler.cpp @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include "NvElementProfiler.h" + +#define LOCK() pthread_mutex_lock(&profiler_lock) +#define UNLOCK() pthread_mutex_unlock(&profiler_lock) + +#define RETURN_IF_DISABLED() \ + if (!enabled) { \ + UNLOCK(); \ + return; \ + } + +#define GET_TIME(timeval) gettimeofday(timeval, NULL); + +#define TIMESPEC_DIFF_USEC(timespec1, timespec2) \ + (((timespec1)->tv_sec - (timespec2)->tv_sec) * 1000000L + \ + (timespec1)->tv_usec - (timespec2)->tv_usec) + +using namespace std; + +NvElementProfiler::NvElementProfiler(ProfilerField fields) + :valid_fields(fields) +{ + enabled = false; + unit_id_counter = 0; + + reset(); + + pthread_mutex_init(&profiler_lock, NULL); +} + +NvElementProfiler::~NvElementProfiler() +{ + LOCK(); + reset(); + UNLOCK(); + pthread_mutex_destroy(&profiler_lock); +} +void +NvElementProfiler::enableProfiling(bool reset_data) +{ + LOCK(); + if (enabled) + { + UNLOCK(); + return; + } + + if(reset_data) + { + reset(); + } + + enabled = true; + UNLOCK(); +} + +void +NvElementProfiler::disableProfiling() +{ + LOCK(); + RETURN_IF_DISABLED(); + + data_int.accumulated_time.tv_sec += + (data_int.stop_time.tv_sec - data_int.start_time.tv_sec); + data_int.accumulated_time.tv_usec += + (data_int.stop_time.tv_usec - data_int.start_time.tv_usec); + data_int.start_time.tv_sec = 0; + data_int.start_time.tv_usec = 0; + data_int.stop_time.tv_sec = 0; + data_int.stop_time.tv_usec = 0; + enabled = false; + UNLOCK(); +} + +void NvElementProfiler::getProfilerData(NvElementProfiler::NvElementProfilerData &data) +{ + uint64_t total_time; + + LOCK(); + + total_time = data_int.accumulated_time.tv_sec * 1000000L + + data_int.accumulated_time.tv_usec + + TIMESPEC_DIFF_USEC(&data_int.stop_time, &data_int.start_time); + + if (data_int.total_processed_units == 0 || total_time == 0) + { + data.average_fps = 0; + } + else + { + data.average_fps = ((float) (data_int.total_processed_units - 1)) * + 1000000 / total_time; + } + + if (data_int.total_processed_units == 0) + { + data.max_latency_usec = 0; + data.min_latency_usec = 0; + data.average_latency_usec = 0; + } + else + { + data.max_latency_usec = data_int.max_latency_usec; + data.min_latency_usec = data_int.min_latency_usec; + data.average_latency_usec = + data_int.total_latency / data_int.total_processed_units; + } + + data.profiling_time.tv_sec = + data_int.accumulated_time.tv_sec + data_int.stop_time.tv_sec - + data_int.start_time.tv_sec; + + data.profiling_time.tv_usec = + data_int.accumulated_time.tv_usec + data_int.stop_time.tv_usec - + data_int.start_time.tv_usec; + + if (data.profiling_time.tv_usec < 0) + { + data.profiling_time.tv_usec += 1000000; + data.profiling_time.tv_sec--; + } + + if (data.profiling_time.tv_usec > 1000000) + { + data.profiling_time.tv_usec -= 1000000; + data.profiling_time.tv_sec++; + } + + data.total_processed_units = data_int.total_processed_units; + data.num_late_units = data_int.num_late_units; + data.valid_fields = valid_fields; + UNLOCK(); +} + +void NvElementProfiler::printProfilerData(ostream &out_stream) +{ + NvElementProfilerData data; + getProfilerData(data); + + if (data.valid_fields & PROFILER_FIELD_FPS) + { + out_stream << "Total Profiling time = " << + (data.profiling_time.tv_sec + + (data.profiling_time.tv_usec / 1000000.0)) << endl; + out_stream << "Average FPS = " << data.average_fps << endl; + } + if (data.valid_fields & PROFILER_FIELD_TOTAL_UNITS) + { + out_stream << "Total units processed = " << + data.total_processed_units << endl; + } + if (data.valid_fields & PROFILER_FIELD_LATE_UNITS) + { + out_stream << "Num. of late units = " << + data.num_late_units << endl; + } + if (data.valid_fields & PROFILER_FIELD_LATENCIES) + { + out_stream << "Average latency(usec) = " << + data.average_latency_usec << endl; + out_stream << "Minimum latency(usec) = " << + data.min_latency_usec << endl; + out_stream << "Maximum latency(usec) = " << + data.max_latency_usec << endl; + } +} + +void +NvElementProfiler::reset() +{ + memset(&data_int, 0, sizeof(data_int)); + data_int.min_latency_usec = (uint64_t) -1; + + unit_start_time_queue.clear(); +} + +uint64_t +NvElementProfiler::startProcessing() +{ + struct timeval time; + uint64_t ret = 0; + LOCK(); + if (enabled) + { + std::map::iterator it = + unit_start_time_queue.end(); + + unit_id_counter++; + GET_TIME(&time); + + unit_start_time_queue.insert(it, + std::pair(unit_id_counter, time)); + + ret = unit_id_counter; + } + UNLOCK(); + return ret; +} + +void +NvElementProfiler::finishProcessing(uint64_t id, bool is_late) +{ + struct timeval unit_start_time; + struct timeval stop_time; + uint64_t latency; + + LOCK(); + RETURN_IF_DISABLED(); + + if ((valid_fields & PROFILER_FIELD_LATENCIES) && + unit_start_time_queue.empty()) + { + UNLOCK(); + return; + } + + GET_TIME(&stop_time); + + if (valid_fields & PROFILER_FIELD_LATENCIES) + { + std::map::iterator it; + if (id) + { + it = unit_start_time_queue.find(id); + } + else + { + it = unit_start_time_queue.begin(); + } + + if (it == unit_start_time_queue.end()) + { + UNLOCK(); + return; + } + unit_start_time = it->second; + unit_start_time_queue.erase(it); + + latency = TIMESPEC_DIFF_USEC(&stop_time, &unit_start_time); + data_int.total_latency += latency; + + if (latency < data_int.min_latency_usec) + { + data_int.min_latency_usec = latency; + } + if(latency > data_int.max_latency_usec) + { + data_int.max_latency_usec = latency; + } + } + + data_int.stop_time = stop_time; + + if (!data_int.start_time.tv_sec && !data_int.start_time.tv_usec) + { + data_int.start_time = data_int.stop_time; + } + + if (is_late) + { + data_int.num_late_units++; + } + data_int.total_processed_units++; + + UNLOCK(); +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvLogging.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvLogging.cpp new file mode 100644 index 0000000000..a8069ac032 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvLogging.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvLogging.h" + +int log_level = DEFAULT_LOG_LEVEL; + +const char *log_level_name[] = {"INFO", "ERROR", "WARN", "DEBUG"}; \ No newline at end of file diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2Element.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2Element.cpp new file mode 100644 index 0000000000..0ce51c63d2 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2Element.cpp @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvV4l2Element.h" +#include "NvLogging.h" + +#include +#include +#include +#include + +#define CAT_NAME "V4l2Element" + +using namespace std; + +/*initialization of mutex for NvV4l2Element*/ +pthread_mutex_t initializer_mutex = PTHREAD_MUTEX_INITIALIZER; + +NvV4l2Element::NvV4l2Element(const char *comp_name, const char *dev_node, int flags, NvElementProfiler::ProfilerField fields) + :NvElement(comp_name, fields), + output_plane(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, comp_name, + fd, !(flags & O_NONBLOCK), profiler), + capture_plane(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, comp_name, + fd, !(flags & O_NONBLOCK), profiler) +{ + struct v4l2_capability caps; + int ret; + + app_data = NULL; + output_plane_pixfmt = 0; + capture_plane_pixfmt = 0; + + /*Synchronization issue of libv4l2 open source library fixing here,adding lock for that*/ + pthread_mutex_lock(&initializer_mutex); + fd = v4l2_open(dev_node, flags | O_RDWR); + if (fd == -1) + { + COMP_SYS_ERROR_MSG("Could not open device '" << dev_node << "'"); + is_in_error = 1; + pthread_mutex_unlock(&initializer_mutex); + return; + } + pthread_mutex_unlock(&initializer_mutex); + + COMP_DEBUG_MSG("Opened, fd = " << fd); + + ret = v4l2_ioctl(fd, VIDIOC_QUERYCAP, &caps); + if (ret != 0) + { + COMP_SYS_ERROR_MSG("Error in VIDIOC_QUERYCAP"); + is_in_error = 1; + return; + } + if (!(caps.capabilities & V4L2_CAP_VIDEO_M2M_MPLANE)) + { + COMP_ERROR_MSG("Device does not support V4L2_CAP_VIDEO_M2M_MPLANE"); + is_in_error = 1; + return; + } +} + +NvV4l2Element::~NvV4l2Element() +{ + output_plane.deinitPlane(); + capture_plane.deinitPlane(); + + if (fd != -1) + { + v4l2_close(fd); + CAT_DEBUG_MSG("Device closed, fd = " << fd); + } +} + +int +NvV4l2Element::dqEvent(struct v4l2_event &ev, uint32_t max_wait_ms) +{ + int ret; + + do + { + ret = v4l2_ioctl(fd, VIDIOC_DQEVENT, &ev); + + if (ret == 0) + { + COMP_DEBUG_MSG("DQed event " << hex << ev.type << dec); + } + else if (errno != EAGAIN) + { + COMP_SYS_ERROR_MSG("Error while DQing event"); + break; + } + else if (max_wait_ms-- == 0) + { + COMP_WARN_MSG("Error while DQing event: Resource temporarily unavailable"); + break; + } + else + { + usleep(1000); + } + } + while (ret && (output_plane.getStreamStatus() || capture_plane.getStreamStatus())); + + return ret; +} + +int +NvV4l2Element::setControl(uint32_t id, int32_t value) +{ + struct v4l2_control ctl; + int ret; + + ctl.id = id; + ctl.value = value; + + ret = v4l2_ioctl(fd, VIDIOC_S_CTRL, &ctl); + + if (ret < 0) + { + COMP_SYS_ERROR_MSG("Error setting value " << value << " on control " << + id); + } + else + { + COMP_DEBUG_MSG("Set value " << value << " on control " << id); + } + return ret; +} + +int +NvV4l2Element::getControl(uint32_t id, int32_t &value) +{ + struct v4l2_control ctl; + int ret; + + ctl.id = id; + + ret = v4l2_ioctl(fd, VIDIOC_G_CTRL, &ctl); + + if (ret < 0) + { + COMP_SYS_ERROR_MSG("Error getting value of control " << id); + } + else + { + COMP_DEBUG_MSG("Got value " << ctl.value << " for control " << id); + value = ctl.value; + } + return ret; +} + +int +NvV4l2Element::setExtControls(v4l2_ext_controls &ctl) +{ + int ret; + + ret = v4l2_ioctl(fd, VIDIOC_S_EXT_CTRLS, &ctl); + + if (ret < 0) + { + COMP_SYS_ERROR_MSG("Error setting controls"); + } + else + { + COMP_DEBUG_MSG("Set controls"); + } + return ret; +} + +int +NvV4l2Element::getExtControls(v4l2_ext_controls &ctl) +{ + int ret; + + ret = v4l2_ioctl(fd, VIDIOC_G_EXT_CTRLS, &ctl); + + if (ret < 0) + { + COMP_SYS_ERROR_MSG("Error getting value of controls"); + } + else + { + COMP_DEBUG_MSG("Got controls"); + } + return ret; +} + +int +NvV4l2Element::subscribeEvent(uint32_t type, uint32_t id, uint32_t flags) +{ + struct v4l2_event_subscription sub; + int ret; + + memset(&sub, 0, sizeof(struct v4l2_event_subscription)); + + sub.type = type; + sub.id = id; + sub.flags = flags; + + ret = v4l2_ioctl(fd, VIDIOC_SUBSCRIBE_EVENT, &sub); + if (ret == 0) + { + COMP_DEBUG_MSG("Successfully subscribed to event " << type); + } + else + { + COMP_SYS_ERROR_MSG + ("Error while subscribing to event " << type); + } + + return ret; +} + +int +NvV4l2Element::abort() +{ + int ret = 0; + + ret |= output_plane.setStreamStatus(false); + ret |= capture_plane.setStreamStatus(false); + + return ret; +} + +int +NvV4l2Element::waitForIdle(uint32_t max_wait_ms) +{ + COMP_ERROR_MSG("wait_for_idle not implemented"); + return 0; +} + +int +NvV4l2Element::isInError() +{ + int error = is_in_error; + error |= capture_plane.isInError(); + error |= output_plane.isInError(); + return error; +} + +void +NvV4l2Element::enableProfiling() +{ + if (output_plane_pixfmt || capture_plane_pixfmt) + { + COMP_ERROR_MSG("Profiling must be enabled before setting formats on either plane"); + return; + } + profiler.enableProfiling(true); +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2ElementPlane.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2ElementPlane.cpp new file mode 100644 index 0000000000..335382ed6d --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2ElementPlane.cpp @@ -0,0 +1,943 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvV4l2ElementPlane.h" +#include "NvLogging.h" + +#include +#include +#include +#include +#include +#include "nvbufsurface.h" + +#define CHECK_V4L2_RETURN(ret, str) \ + if (ret < 0) { \ + PLANE_SYS_ERROR_MSG(str << ": failed"); \ + return -1; \ + } else { \ + PLANE_DEBUG_MSG(str << ": success"); \ + return 0; \ + } + +using namespace std; + +NvV4l2ElementPlane::NvV4l2ElementPlane(enum v4l2_buf_type buf_type, + const char *device_name, int &fd, bool blocking, + NvElementProfiler &profiler) + :fd(fd), + v4l2elem_profiler(profiler), + comp_name(device_name) +{ + this->buf_type = buf_type; + this->blocking = blocking; + is_in_error = 0; + switch (buf_type) + { + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + plane_name = "Output Plane"; + break; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + plane_name = "Capture Plane"; + break; + default: + ERROR_MSG("Unsupported v4l2_buf_type " << buf_type); + plane_name = "Unknown"; + is_in_error = 1; + } + + num_buffers = 0; + buffers = NULL; + + n_planes = 0; + memset(&planefmts, 0, sizeof(planefmts)); + + num_queued_buffers = 0; + total_queued_buffers = 0; + total_dequeued_buffers = 0; + + streamon = false; + pthread_mutex_init(&plane_lock, NULL); + pthread_cond_init(&plane_cond, NULL); + + dqthread_running = false; + stop_dqthread = false; + dq_thread = 0; + callback = NULL; + + memory_type = V4L2_MEMORY_MMAP; + + dqThread_data = NULL; +} + +NvV4l2ElementPlane::~NvV4l2ElementPlane() +{ + pthread_mutex_destroy(&plane_lock); + pthread_cond_destroy(&plane_cond); +} + +NvBuffer * +NvV4l2ElementPlane::getNthBuffer(uint32_t n) +{ + if (n >= num_buffers) + { + PLANE_DEBUG_MSG("WARNING:Requested " << n << "th buffer out of " << + num_buffers << "buffers. Returning NULL"); + return NULL; + } + return buffers[n]; +} + +int +NvV4l2ElementPlane::dqBuffer(struct v4l2_buffer &v4l2_buf, NvBuffer ** buffer, + NvBuffer ** shared_buffer, uint32_t num_retries) +{ + int ret; + + v4l2_buf.type = buf_type; + v4l2_buf.memory = memory_type; + do + { + ret = v4l2_ioctl(fd, VIDIOC_DQBUF, &v4l2_buf); + + if (ret == 0) + { + pthread_mutex_lock(&plane_lock); + if (buffer) + *buffer = buffers[v4l2_buf.index]; + if (shared_buffer && memory_type == V4L2_MEMORY_DMABUF) + { + *shared_buffer = + (NvBuffer *) buffers[v4l2_buf.index]->shared_buffer; + } + for (uint32_t i = 0; i < buffers[v4l2_buf.index]->n_planes; i++) + { + buffers[v4l2_buf.index]->planes[i].bytesused = + v4l2_buf.m.planes[i].bytesused; + } + if (buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) + { + v4l2elem_profiler.finishProcessing(0, false); + } + + total_dequeued_buffers++; + num_queued_buffers--; + pthread_cond_broadcast(&plane_cond); + PLANE_DEBUG_MSG("DQed buffer " << v4l2_buf.index); + pthread_mutex_unlock(&plane_lock); + } + else if (errno == EAGAIN) + { + pthread_mutex_lock(&plane_lock); + if (v4l2_buf.flags & V4L2_BUF_FLAG_LAST) + { + pthread_mutex_unlock(&plane_lock); + break; + } + pthread_mutex_unlock(&plane_lock); + + if (num_retries-- == 0) + { + PLANE_WARN_MSG("Error while DQing buffer: Resource temporarily unavailable"); + break; + } + if (!blocking) + { + break; + } + } + else + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while DQing buffer"); + break; + } + } + while (ret && !is_in_error); + + return ret; +} + +int +NvV4l2ElementPlane::qBuffer(struct v4l2_buffer &v4l2_buf, NvBuffer * shared_buffer) +{ + int ret; + uint32_t i; + NvBuffer *buffer; + + pthread_mutex_lock(&plane_lock); + buffer = buffers[v4l2_buf.index]; + + v4l2_buf.type = buf_type; + v4l2_buf.memory = memory_type; + v4l2_buf.length = n_planes; + + switch (memory_type) + { + case V4L2_MEMORY_USERPTR: + buffer->shared_buffer = shared_buffer; + for (i = 0; i < buffer->n_planes; i++) + { + if (shared_buffer) + { + v4l2_buf.m.planes[i].m.userptr = + (unsigned long) shared_buffer->planes[i].data; + v4l2_buf.m.planes[i].bytesused = + shared_buffer->planes[i].bytesused; + } + else + { + v4l2_buf.m.planes[i].m.userptr = + (unsigned long) buffer->planes[i].data; + v4l2_buf.m.planes[i].bytesused = + buffer->planes[i].bytesused; + } + } + break; + case V4L2_MEMORY_MMAP: + for (i = 0; i < buffer->n_planes; i++) + { + v4l2_buf.m.planes[i].bytesused = buffer->planes[i].bytesused; + } + break; + case V4L2_MEMORY_DMABUF: + buffer->shared_buffer = shared_buffer; + if (shared_buffer) + { + for (i = 0; i < buffer->n_planes; i++) + { + v4l2_buf.m.planes[i].m.fd = shared_buffer->planes[i].fd; + v4l2_buf.m.planes[i].bytesused = + shared_buffer->planes[i].bytesused; + } + } + break; + default: + pthread_cond_broadcast(&plane_cond); + pthread_mutex_unlock(&plane_lock); + return -1; + } + + if(buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + { + v4l2elem_profiler.startProcessing(); + } + + ret = v4l2_ioctl(fd, VIDIOC_QBUF, &v4l2_buf); + if (ret) + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while Qing buffer"); + } + else + { + PLANE_DEBUG_MSG("Qed buffer " << v4l2_buf.index); + pthread_cond_broadcast(&plane_cond); + total_queued_buffers++; + num_queued_buffers++; + } + pthread_mutex_unlock(&plane_lock); + + return ret; +} + +int +NvV4l2ElementPlane::mapOutputBuffers(struct v4l2_buffer &v4l2_buf, int dmabuff_fd) +{ + int ret; + uint32_t i; + pthread_mutex_lock(&plane_lock); + unsigned char *data; + NvBufSurface *nvbuf_surf = 0; + + switch (memory_type) + { + case V4L2_MEMORY_DMABUF: + ret = NvBufSurfaceFromFd (dmabuff_fd, (void**)(&nvbuf_surf)); + if(ret < 0) + { + PLANE_SYS_ERROR_MSG("Error: NvBufSurfaceFromFd Failed\n"); + pthread_mutex_unlock(&plane_lock); + return ret; + } + for (i = 0; i < n_planes; i++) + { + buffers[v4l2_buf.index]->planes[i].fd = dmabuff_fd; + v4l2_buf.m.planes[i].m.fd = buffers[v4l2_buf.index]->planes[i].fd; + buffers[v4l2_buf.index]->planes[i].mem_offset = nvbuf_surf->surfaceList[0].planeParams.offset[i]; + + if (nvbuf_surf->memType != NVBUF_MEM_CUDA_DEVICE && nvbuf_surf->memType != NVBUF_MEM_CUDA_ARRAY) + { + ret = NvBufSurfaceMap(nvbuf_surf, 0, i, NVBUF_MAP_READ_WRITE); + if (ret < 0) + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while Mapping buffer"); + pthread_mutex_unlock(&plane_lock); + return ret; + } + data = (unsigned char *)nvbuf_surf->surfaceList[0].mappedAddr.addr[i]; + buffers[v4l2_buf.index]->planes[i].data = data; + } + } + break; + default: + pthread_mutex_unlock(&plane_lock); + return -1; + } + if(ret == 0) + { + PLANE_DEBUG_MSG("Mapped Nvbuffer to buffers " << v4l2_buf.index); + } + pthread_mutex_unlock(&plane_lock); + + return ret; +} + +int +NvV4l2ElementPlane::unmapOutputBuffers(int index, int dmabuff_fd) +{ + int ret = 0; + uint32_t i; + NvBufSurface *nvbuf_surf = 0; + pthread_mutex_lock(&plane_lock); + + switch (memory_type) + { + case V4L2_MEMORY_DMABUF: + for (i = 0; i < n_planes; i++) + { + ret = NvBufSurfaceFromFd (dmabuff_fd, (void**)(&nvbuf_surf)); + if (ret < 0) + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while NvBufSurfaceFromFd"); + pthread_mutex_unlock(&plane_lock); + return ret; + } + + if (nvbuf_surf->memType != NVBUF_MEM_CUDA_DEVICE && nvbuf_surf->memType != NVBUF_MEM_CUDA_ARRAY) + { + ret = NvBufSurfaceUnMap(nvbuf_surf, 0, i); + if (ret < 0) + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while Unmapping buffer"); + pthread_mutex_unlock(&plane_lock); + return ret; + } + } + } + break; + default: + pthread_mutex_unlock(&plane_lock); + return -1; + } + if(ret == 0) + { + PLANE_DEBUG_MSG("Unmapped Nvbuffer to buffers " << index); + } + pthread_mutex_unlock(&plane_lock); + + return ret; +} + +int +NvV4l2ElementPlane::getFormat(struct v4l2_format &format) +{ + format.type = buf_type; + CHECK_V4L2_RETURN(v4l2_ioctl(fd, VIDIOC_G_FMT, &format), + "Getting format"); +} + +int +NvV4l2ElementPlane::setFormat(struct v4l2_format &format) +{ + int ret; + int j; + + format.type = buf_type; + ret = v4l2_ioctl(fd, VIDIOC_S_FMT, &format); + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in VIDIOC_S_FMT"); + is_in_error = 1; + } + else + { + PLANE_DEBUG_MSG("VIDIOC_S_FMT at capture plane successful"); + n_planes = format.fmt.pix_mp.num_planes; + for (j = 0; j < n_planes; j++) + { + planefmts[j].stride = format.fmt.pix_mp.plane_fmt[j].bytesperline; + planefmts[j].sizeimage = format.fmt.pix_mp.plane_fmt[j].sizeimage; + } + } + + return ret; +} + +int +NvV4l2ElementPlane::getCrop(struct v4l2_crop & crop) +{ + crop.type = buf_type; + + CHECK_V4L2_RETURN(v4l2_ioctl(fd, VIDIOC_G_CROP, &crop), + "Getting crop params"); +} + +int +NvV4l2ElementPlane::setSelection(uint32_t target, uint32_t flags, struct v4l2_rect & rect) +{ + struct v4l2_selection select; + + switch (buf_type) + { + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + select.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + break; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + select.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + break; + default: + PLANE_ERROR_MSG("Unsupported v4l2_buf_type " << buf_type); + return -1; + } + + select.target = target; + select.flags = flags; + select.r = rect; + + CHECK_V4L2_RETURN(v4l2_ioctl(fd, VIDIOC_S_SELECTION, &select), + "Setting selection"); +} + +void +NvV4l2ElementPlane::setBufferPlaneFormat(int n_planes, + NvBuffer::NvBufferPlaneFormat * planefmts) +{ + int i; + + this->n_planes = n_planes; + for (i = 0; i < n_planes; i++) + { + this->planefmts[i] = planefmts[i]; + } +} + +void +NvV4l2ElementPlane::deinitPlane() +{ + setStreamStatus(false); + waitForDQThread(-1); + for (uint32_t i = 0; i < num_buffers; i++) + { + switch (memory_type) + { + case V4L2_MEMORY_USERPTR: + buffers[i]->deallocateMemory(); + break; + case V4L2_MEMORY_MMAP: + buffers[i]->unmap(); + break; + case V4L2_MEMORY_DMABUF: + break; + default: + return; + } + } + reqbufs(memory_type, 0); + PLANE_DEBUG_MSG("deinit successful"); +} + +int +NvV4l2ElementPlane::reqbufs(enum v4l2_memory mem_type, uint32_t num) +{ + struct v4l2_requestbuffers reqbufs; + int ret; + + memset(&reqbufs, 0, sizeof(struct v4l2_requestbuffers)); + reqbufs.count = num; + reqbufs.type = buf_type; + switch (mem_type) + { + case V4L2_MEMORY_USERPTR: + for (uint32_t i = 0; i < n_planes; i++) + { + planefmts[i].stride = + planefmts[i].width * planefmts[i].bytesperpixel; + if(!planefmts[i].sizeimage) + { + planefmts[i].sizeimage = + planefmts[i].width * planefmts[i].height; + } + } + break; + case V4L2_MEMORY_MMAP: + case V4L2_MEMORY_DMABUF: + break; + default: + PLANE_ERROR_MSG("Error in VIDIOC_REQBUFS:Unknown memory type " << + mem_type); + return -1; + } + memory_type = mem_type; + + reqbufs.memory = mem_type; + ret = v4l2_ioctl(fd, VIDIOC_REQBUFS, &reqbufs); + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in VIDIOC_REQBUFS at output plane"); + is_in_error = 1; + } + else + { + if (reqbufs.count) + { + buffers = new NvBuffer *[reqbufs.count]; + for (uint32_t i = 0; i < reqbufs.count; i++) + { + buffers[i] = + new NvBuffer(buf_type, mem_type, n_planes, planefmts, i); + } + } + else + { + for (uint32_t i = 0; i < num_buffers; i++) + { + delete buffers[i]; + } + delete[] buffers; + buffers = NULL; + } + num_buffers = reqbufs.count; + PLANE_DEBUG_MSG("Reqbuf returned " << reqbufs.count << " buffers"); + } + + return ret; +} + +int +NvV4l2ElementPlane::setStreamStatus(bool status) +{ + int ret; + + pthread_mutex_lock(&plane_lock); + if (status == streamon) + { + PLANE_DEBUG_MSG("Already in " << ((status) ? "STREAMON" : "STREAMOFF")); + pthread_mutex_unlock(&plane_lock); + return 0; + } + + if (status) + { + ret = v4l2_ioctl(fd, VIDIOC_STREAMON, &buf_type); + } + else + { + ret = v4l2_ioctl(fd, VIDIOC_STREAMOFF, &buf_type); + } + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in " << + ((status) ? "STREAMON" : "STREAMOFF")); + is_in_error = 1; + } + else + { + PLANE_DEBUG_MSG(((status) ? "STREAMON" : "STREAMOFF") << " successful"); + streamon = status; + if (!streamon) + { + num_queued_buffers = 0; + pthread_cond_broadcast(&plane_cond); + } + + if (buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) + { + if (status == false) + { + v4l2elem_profiler.disableProfiling(); + } + } + + } + + pthread_mutex_unlock(&plane_lock); + return ret; +} + +bool +NvV4l2ElementPlane::getStreamStatus() +{ + return streamon; +} + +int +NvV4l2ElementPlane::setStreamParms(struct v4l2_streamparm &parm) +{ + int ret; + + parm.type = buf_type; + ret = v4l2_ioctl(fd, VIDIOC_S_PARM, &parm); + + if(ret == 0) + { + PLANE_DEBUG_MSG("Successfully set stream parameters"); + } + else + { + PLANE_SYS_ERROR_MSG("Error while setting stream parameters"); + } + return ret; +} + +int +NvV4l2ElementPlane::queryBuffer(uint32_t i) +{ + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + int ret; + uint32_t j; + + memset(&v4l2_buf, 0, sizeof(struct v4l2_buffer)); + memset(planes, 0, sizeof(planes)); + v4l2_buf.index = i; + v4l2_buf.type = buf_type; + v4l2_buf.memory = memory_type; + v4l2_buf.m.planes = planes; + v4l2_buf.length = n_planes; + + ret = v4l2_ioctl(fd, VIDIOC_QUERYBUF, &v4l2_buf); + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in QueryBuf for " << i << "th buffer"); + is_in_error = 1; + } + else + { + PLANE_DEBUG_MSG("QueryBuf for " << i << "th buffer successful"); + + for (j = 0; j < v4l2_buf.length; j++) + { + buffers[i]->planes[j].length = v4l2_buf.m.planes[j].length; + buffers[i]->planes[j].mem_offset = + v4l2_buf.m.planes[j].m.mem_offset; + } + } + + return ret; +} + +int +NvV4l2ElementPlane::exportBuffer(uint32_t i) +{ + struct v4l2_exportbuffer expbuf; + int ret; + int j; + + memset(&expbuf, 0, sizeof(expbuf)); + expbuf.type = buf_type; + expbuf.index = i; + + for (j = 0; j < n_planes; j++) + { + expbuf.plane = j; + ret = v4l2_ioctl(fd, VIDIOC_EXPBUF, &expbuf); + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in ExportBuf for Buffer " << i << + ", Plane " << j); + is_in_error = 1; + return -1; + } + else + { + PLANE_DEBUG_MSG("ExportBuf successful for Buffer " << i << + ", Plane " << j << ", fd = " << expbuf.fd); + buffers[i]->planes[j].fd = expbuf.fd; + } + } + return 0; +} + +int +NvV4l2ElementPlane::setupPlane(enum v4l2_memory mem_type, uint32_t num_buffers, + bool map, bool allocate) +{ + uint32_t i; + + if (reqbufs(mem_type, num_buffers)) + { + goto error; + } + + for (i = 0; i < this->num_buffers; i++) + { + switch (mem_type) + { + case V4L2_MEMORY_USERPTR: + if (allocate) + { + if (buffers[i]->allocateMemory()) + { + goto error; + } + } + break; + case V4L2_MEMORY_MMAP: + if (queryBuffer(i)) + { + goto error; + } + if (exportBuffer(i)) + { + goto error; + } + if (map) + { + if (buffers[i]->map()) + { + goto error; + } + } + break; + default: + continue; + } + } + return 0; + +error: + PLANE_ERROR_MSG("Error during setup"); + is_in_error = 1; + deinitPlane(); + return -1; +} + +int +NvV4l2ElementPlane::waitAllBuffersQueued(uint32_t max_wait_ms) +{ + struct timespec timeToWait; + struct timeval now; + int return_val = 0; + int ret; + + gettimeofday(&now, NULL); + + timeToWait.tv_nsec = (now.tv_usec + (max_wait_ms % 1000) * 1000L) * 1000L; + timeToWait.tv_sec = now.tv_sec + max_wait_ms / 1000 + + timeToWait.tv_nsec / 1000000000L; + timeToWait.tv_nsec = timeToWait.tv_nsec % 1000000000L; + + pthread_mutex_lock(&plane_lock); + while (num_queued_buffers < num_buffers) + { + ret = pthread_cond_timedwait(&plane_cond, &plane_lock, &timeToWait); + if (ret == ETIMEDOUT) + { + return_val = -1; + break; + } + } + pthread_mutex_unlock(&plane_lock); + + CHECK_V4L2_RETURN(return_val, "Waiting for all buffers to get queued"); +} + +int +NvV4l2ElementPlane::waitAllBuffersDequeued(uint32_t max_wait_ms) +{ + struct timespec timeToWait; + struct timeval now; + int return_val = 0; + int ret; + + gettimeofday(&now, NULL); + + timeToWait.tv_nsec = (now.tv_usec + (max_wait_ms % 1000) * 1000L) * 1000L; + timeToWait.tv_sec = now.tv_sec + max_wait_ms / 1000 + + timeToWait.tv_nsec / 1000000000L; + timeToWait.tv_nsec = timeToWait.tv_nsec % 1000000000L; + + pthread_mutex_lock(&plane_lock); + while (num_queued_buffers) + { + ret = pthread_cond_timedwait(&plane_cond, &plane_lock, &timeToWait); + if (ret == ETIMEDOUT) + { + return_val = -1; + break; + } + } + pthread_mutex_unlock(&plane_lock); + + CHECK_V4L2_RETURN(return_val, "Waiting for all buffers to get dequeued"); +} + +bool NvV4l2ElementPlane::setDQThreadCallback(dqThreadCallback callback) +{ + if (dqthread_running) + return false; + this->callback = callback; + return true; +} + +void * +NvV4l2ElementPlane::dqThread(void *data) +{ + NvV4l2ElementPlane *plane = (NvV4l2ElementPlane *) data; + const char *comp_name = plane->comp_name; + const char *plane_name = plane->plane_name; + + PLANE_DEBUG_MSG("Starting DQthread"); + prctl (PR_SET_NAME, plane_name, 0, 0, 0); + plane->stop_dqthread = false; + while (!plane->stop_dqthread) + { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + NvBuffer *buffer; + NvBuffer *shared_buffer; + bool ret = 0; + + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, sizeof(planes)); + v4l2_buf.m.planes = planes; + v4l2_buf.length = plane->n_planes; + + if (plane->dqBuffer(v4l2_buf, &buffer, &shared_buffer, -1) < 0) + { + if (errno != EAGAIN) + { + plane->is_in_error = 1; + } + if (errno != EAGAIN || plane->streamon) + { + ret = plane->callback(NULL, NULL, NULL, plane->dqThread_data); + } + if (!plane->streamon) + { + break; + } + } + else + { + ret = plane->callback(&v4l2_buf, buffer, shared_buffer, + plane->dqThread_data); + } + if (!ret) + { + break; + } + } + plane->stop_dqthread = false; + + pthread_mutex_lock(&plane->plane_lock); + plane->dqthread_running = false; + pthread_cond_broadcast(&plane->plane_cond); + pthread_mutex_unlock(&plane->plane_lock); + PLANE_DEBUG_MSG("Exiting DQthread"); + return NULL; +} + +int +NvV4l2ElementPlane::startDQThread(void *data) +{ + pthread_mutex_lock(&plane_lock); + if (dqthread_running) + { + PLANE_DEBUG_MSG("DQ Thread already started"); + pthread_mutex_unlock(&plane_lock); + return 0; + } + dqThread_data = data; + pthread_create(&dq_thread, NULL, dqThread, this); + dqthread_running = true; + pthread_mutex_unlock(&plane_lock); + PLANE_DEBUG_MSG("Started DQ Thread"); + return 0; +} + +int +NvV4l2ElementPlane::stopDQThread() +{ + if (blocking) + { + PLANE_WARN_MSG("Should not be called in blocking mode"); + return 0; + } + stop_dqthread = true; + pthread_join(dq_thread, NULL); + dq_thread = 0; + PLANE_DEBUG_MSG("Stopped DQ Thread"); + return 0; +} + +int +NvV4l2ElementPlane::waitForDQThread(uint32_t max_wait_ms) +{ + struct timespec timeToWait; + struct timeval now; + int return_val = 0; + int ret = 0; + + gettimeofday(&now, NULL); + + timeToWait.tv_nsec = (now.tv_usec + (max_wait_ms % 1000) * 1000L) * 1000L; + timeToWait.tv_sec = now.tv_sec + max_wait_ms / 1000 + + timeToWait.tv_nsec / 1000000000L; + timeToWait.tv_nsec = timeToWait.tv_nsec % 1000000000L; + + pthread_mutex_lock(&plane_lock); + while (dqthread_running) + { + ret = pthread_cond_timedwait(&plane_cond, &plane_lock, &timeToWait); + if (ret == ETIMEDOUT) + { + return_val = -1; + break; + } + } + pthread_mutex_unlock(&plane_lock); + + if (ret == 0) + { + pthread_join(dq_thread, NULL); + dq_thread = 0; + PLANE_DEBUG_MSG("Stopped DQ Thread"); + } + else + { + PLANE_ERROR_MSG("Timed out waiting for dqthread"); + } + return return_val; +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvVideoEncoder.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvVideoEncoder.cpp new file mode 100644 index 0000000000..ddd3d2ea0f --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvVideoEncoder.cpp @@ -0,0 +1,1701 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvVideoEncoder.h" +#include "NvLogging.h" + +#include +#include +#include + +#include +#include +#include "nvbufsurface.h" + +#define ENCODER_DEV "/dev/nvhost-msenc" +#define ENCODER_DEV_ALT "/dev/v4l2-nvenc" +#define ENCODER_GPU_DEV "/dev/nvidia0" + +#define ENCODER_COMP_NAME "NVENC" + +#define CHECK_V4L2_RETURN(ret, str) \ + if (ret < 0) { \ + COMP_SYS_ERROR_MSG(str << ": failed"); \ + return -1; \ + } else { \ + COMP_DEBUG_MSG(str << ": success"); \ + return 0; \ + } + +#define RETURN_ERROR_IF_FORMATS_SET() \ + if (output_plane_pixfmt != 0 || capture_plane_pixfmt != 0) { \ + COMP_ERROR_MSG("Should be called before setting plane formats") \ + return -1; \ + } + +#define RETURN_ERROR_IF_OUTPUT_PLANE_FORMAT_NOT_SET() \ +if (output_plane_pixfmt != 0) { \ + COMP_ERROR_MSG("Should be called before setting output plane format") \ + return -1; \ +} + +#define RETURN_ERROR_IF_BUFFERS_REQUESTED() \ + if (output_plane.getNumBuffers() != 0 && capture_plane.getNumBuffers() != 0) { \ + COMP_ERROR_MSG("Should be called before requesting buffers on either plane") \ + return -1; \ + } + +#define RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED() \ + if (output_plane.getNumBuffers() == 0 || capture_plane.getNumBuffers() == 0) { \ + COMP_ERROR_MSG("Should be called before requesting buffers on either plane") \ + return -1; \ + } + +#define RETURN_ERROR_IF_FORMATS_NOT_SET() \ + if (output_plane_pixfmt == 0 || capture_plane_pixfmt == 0) { \ + COMP_ERROR_MSG("Should be called after setting plane formats") \ + return -1; \ + } + +using namespace std; + +NvVideoEncoder::NvVideoEncoder(const char *name, const char *dev_node, int flags) + : NvV4l2Element(name, dev_node, flags, valid_fields) +{ + gpu_enabled = false; +} + +NvVideoEncoder * +NvVideoEncoder::createVideoEncoder(const char *name, int flags) +{ + NvVideoEncoder *enc; + NvBufSurfaceDeviceInfo deviceInfo; + memset(&deviceInfo, 0, sizeof(deviceInfo)); + NvBufSurfaceGetDeviceInfo(&deviceInfo); + if (deviceInfo.driverType == NVBUF_DRIVER_TYPE_NVGPU && deviceInfo.isIntegratedGpu) + { + if (access (ENCODER_DEV, F_OK) == 0) + { + enc = new NvVideoEncoder(name, ENCODER_DEV, flags); + } + else if (access(ENCODER_DEV_ALT, F_OK) == 0) + { + enc = new NvVideoEncoder(name, ENCODER_DEV_ALT, flags); + } + else + { + return NULL; + } + } + else + { + if (access(ENCODER_GPU_DEV, F_OK) == 0) + { + enc = new NvVideoEncoder(name, ENCODER_GPU_DEV, flags); + enc->gpu_enabled = true; + } + else + return NULL; + } + + if (enc->isInError()) + { + delete enc; + return NULL; + } + return enc; +} + +NvVideoEncoder::~NvVideoEncoder() +{ +} + +bool +NvVideoEncoder::isGPUEnabled() +{ + return this->gpu_enabled; +} + +int NvVideoEncoder::setOutputPlaneFormat(uint32_t pixfmt, uint32_t width, + uint32_t height, v4l2_colorspace cs) +{ + struct v4l2_format format; + uint32_t num_bufferplanes; + NvBuffer::NvBufferPlaneFormat planefmts[MAX_PLANES] = {{0}}; + + if (pixfmt != V4L2_PIX_FMT_YUV420M && + pixfmt != V4L2_PIX_FMT_P010M && pixfmt != V4L2_PIX_FMT_NV12M && + pixfmt != V4L2_PIX_FMT_NV24M && pixfmt != V4L2_PIX_FMT_NV24_10LE && + pixfmt != V4L2_PIX_FMT_YUV444M && pixfmt != V4L2_PIX_FMT_YUV444_10LE) + { + COMP_ERROR_MSG("Only YUV420M, NV24, NV24_10LE, YUV444, P010M and NV12M are supported"); + return -1; + } + + output_plane_pixfmt = pixfmt; + NvBuffer::fill_buffer_plane_format(&num_bufferplanes, planefmts, width, + height, pixfmt); + output_plane.setBufferPlaneFormat(num_bufferplanes, planefmts); + + memset(&format, 0, sizeof(struct v4l2_format)); + format.type = output_plane.getBufType(); + format.fmt.pix_mp.width = width; + format.fmt.pix_mp.height = height; + format.fmt.pix_mp.pixelformat = pixfmt; + format.fmt.pix_mp.num_planes = num_bufferplanes; + format.fmt.pix_mp.colorspace = cs; + + return output_plane.setFormat(format); +} + +int NvVideoEncoder::setOutputPlaneFormat(uint32_t pixfmt, uint32_t width, + uint32_t height, v4l2_colorspace cs, + bool extended) +{ + struct v4l2_format format; + uint32_t num_bufferplanes; + NvBuffer::NvBufferPlaneFormat planefmts[MAX_PLANES]; + + if (pixfmt != V4L2_PIX_FMT_YUV420M && + pixfmt != V4L2_PIX_FMT_P010M && pixfmt != V4L2_PIX_FMT_NV12M && + pixfmt != V4L2_PIX_FMT_NV24M && pixfmt != V4L2_PIX_FMT_NV24_10LE && + pixfmt != V4L2_PIX_FMT_YUV444M && pixfmt != V4L2_PIX_FMT_YUV444_10LE) + { + COMP_ERROR_MSG("Only YUV420M, NV24, NV24_10LE, YUV444, P010M and NV12M are supported"); + return -1; + } + + output_plane_pixfmt = pixfmt; + NvBuffer::fill_buffer_plane_format(&num_bufferplanes, planefmts, width, + height, pixfmt); + output_plane.setBufferPlaneFormat(num_bufferplanes, planefmts); + + memset(&format, 0, sizeof(struct v4l2_format)); + format.type = output_plane.getBufType(); + format.fmt.pix_mp.width = width; + format.fmt.pix_mp.height = height; + format.fmt.pix_mp.pixelformat = pixfmt; + format.fmt.pix_mp.num_planes = num_bufferplanes; + + /* Extended color format maps to full color range; otherwise signal the + * limited (studio) range explicitly. */ + format.fmt.pix_mp.quantization = + extended ? V4L2_QUANTIZATION_FULL_RANGE : V4L2_QUANTIZATION_LIM_RANGE; + + format.fmt.pix_mp.colorspace = cs; + + /* Program the matrix coefficients (ycbcr_enc) and transfer function + * (xfer_func) to match the colorspace. V4L2 has no dedicated constant for + * the BT.2020 10-/12-bit transfer function, and BT.601 shares the BT.709 + * OETF, so for those colorspaces the transfer characteristic is left as + * DEFAULT and derived from the colorspace by the encoder backend (this + * lets BT.601 be signalled as SMPTE 170M rather than BT.709). + * https://www.kernel.org/doc/html/v6.8/userspace-api/media/v4l/colorspaces-defs.html */ + switch (format.fmt.pix_mp.colorspace) + { + case V4L2_COLORSPACE_REC709: + format.fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_709; + format.fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_709; + break; + case V4L2_COLORSPACE_BT2020: + format.fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_BT2020; + format.fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT; + break; + case V4L2_COLORSPACE_SMPTE170M: + format.fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_601; + format.fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT; + break; + default: + format.fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + format.fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT; + break; + } + + return output_plane.setFormat(format); +} + +int +NvVideoEncoder::setCapturePlaneFormat(uint32_t pixfmt, uint32_t width, + uint32_t height, uint32_t sizeimage) +{ + struct v4l2_format format; + + memset(&format, 0, sizeof(struct v4l2_format)); + format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + switch (pixfmt) + { + case V4L2_PIX_FMT_H264: + case V4L2_PIX_FMT_H265: + case V4L2_PIX_FMT_VP8: + case V4L2_PIX_FMT_VP9: + case V4L2_PIX_FMT_AV1: + capture_plane_pixfmt = pixfmt; + break; + default: + ERROR_MSG("Unknown supported pixel format for encoder " << pixfmt); + return -1; + } + + format.fmt.pix_mp.pixelformat = pixfmt; + format.fmt.pix_mp.width = width; + format.fmt.pix_mp.height = height; + format.fmt.pix_mp.num_planes = 1; + format.fmt.pix_mp.plane_fmt[0].sizeimage = sizeimage; + + return capture_plane.setFormat(format); +} + +int +NvVideoEncoder::setFrameRate(uint32_t framerate_num, uint32_t framerate_den) +{ + struct v4l2_streamparm parms; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&parms, 0, sizeof(parms)); + parms.parm.output.timeperframe.numerator = framerate_den; + parms.parm.output.timeperframe.denominator = framerate_num; + parms.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + + CHECK_V4L2_RETURN(output_plane.setStreamParms(parms), + "Setting framerate to " << framerate_num << "/" << framerate_den); +} + +int +NvVideoEncoder::setBitrate(uint32_t bitrate) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + // Local change: the plane pixel-format debug prints are removed (see ../README.md). + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_BITRATE; + control.value = bitrate; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder bitrate to " << bitrate); +} + +int +NvVideoEncoder::setEncoderCommand(int cmd, int flags) +{ + int ret=0; + struct v4l2_encoder_cmd v4l2_enc_cmd; + v4l2_enc_cmd.cmd = cmd; + v4l2_enc_cmd.flags = flags; + + ret = v4l2_ioctl(fd, VIDIOC_ENCODER_CMD, v4l2_enc_cmd); + if (ret < 0) + printf(" Error in encoder command \n"); + + return ret; +} + +int +NvVideoEncoder::setPeakBitrate(uint32_t peak_bitrate) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + // Local change: the plane pixel-format debug prints are removed (see ../README.md). + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_BITRATE_PEAK; + control.value = peak_bitrate; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder peak bitrate to " << peak_bitrate); +} + +int +NvVideoEncoder::setProfile(uint32_t profile) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + switch (capture_plane_pixfmt) + { + case V4L2_PIX_FMT_H264: + control.id = V4L2_CID_MPEG_VIDEO_H264_PROFILE; + break; + case V4L2_PIX_FMT_H265: + control.id = V4L2_CID_MPEG_VIDEO_H265_PROFILE; + break; + default: + COMP_ERROR_MSG("Unsupported encoder type"); + return -1; + } + control.value = profile; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder profile to " << profile); +} + +int +NvVideoEncoder::setLevel(uint32_t level) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + switch (capture_plane_pixfmt) + { + case V4L2_PIX_FMT_H264: + control.id = V4L2_CID_MPEG_VIDEO_H264_LEVEL; + control.value = (enum v4l2_mpeg_video_h264_level)level; + break; + case V4L2_PIX_FMT_H265: + control.id = V4L2_CID_MPEG_VIDEOENC_H265_LEVEL; + control.value = (enum v4l2_mpeg_video_h265_level)level; + break; + default: + COMP_WARN_MSG("Currently only supported for H.264 and H.265"); + return 0; + } + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder level to " << level); +} + +int +NvVideoEncoder::setConstantQp(bool enabled_rc) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE; + control.value = enabled_rc; // if false: disable rate control + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder constant qp to " << enabled_rc); +} + +int +NvVideoEncoder::setRateControlMode(enum v4l2_mpeg_video_bitrate_mode mode) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_BITRATE_MODE; + control.value = mode; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder rate control mode to " << mode); +} + +int +NvVideoEncoder::setMaxPerfMode(int flag) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_MAX_PERFORMANCE; + control.value = flag; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling Maximum Performance "); +} + +int +NvVideoEncoder::setIFrameInterval(uint32_t interval) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_GOP_SIZE; + control.value = interval; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder I-frame interval to " << interval); +} + +int +NvVideoEncoder::setIDRInterval(uint32_t interval) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_IDR_INTERVAL; + control.value = interval; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder IDR interval to " << interval); +} + +int +NvVideoEncoder::forceIDR() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Forcing IDR"); +} + +int +NvVideoEncoder::setTemporalTradeoff(v4l2_enc_temporal_tradeoff_level_type level) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_TEMPORAL_TRADEOFF_LEVEL; + control.value = level; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder temporal tradeoff level to " << level); +} + +int +NvVideoEncoder::setSliceLength(v4l2_enc_slice_length_type type, uint32_t length) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_enc_slice_length_param param = {type, length}; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_SLICE_LENGTH_PARAM; + control.string = (char *) ¶m; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder packet type to " << type << ", length to " << + length); +} + +int +NvVideoEncoder::setHWPresetType(v4l2_enc_hw_preset_type type) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_HW_PRESET_TYPE_PARAM; + control.value = type; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder HW Preset type to " << type); +} + +int +NvVideoEncoder::setROIParams(uint32_t buffer_index, + v4l2_enc_frame_ROI_params ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ROI_PARAMS; + control.string = (char *) ¶ms; + params.config_store = buffer_index; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder ROI params for buffer " << buffer_index); +} + +int +NvVideoEncoder::setPPEInitParams(v4l2_enc_ppe_init_params ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_PPE_INIT_PARAMS; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder PPE initialization params"); +} + +int +NvVideoEncoder::SetInputMetaParams(uint32_t buffer_index, + v4l2_ctrl_videoenc_input_metadata ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INPUT_METADATA; + control.string = (char *) ¶ms; + params.config_store = buffer_index; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder input metadata Params"); +} + +int +NvVideoEncoder::enableROI(v4l2_enc_enable_roi_param ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_ROI_PARAM; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder ROI"); +} + +int +NvVideoEncoder::enableReconCRC(v4l2_enc_enable_reconcrc_param ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_RECONCRC_PARAM; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder ReconCRC"); +} + +int +NvVideoEncoder::enableExternalRPS(v4l2_enc_enable_ext_rps_ctr ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_EXTERNAL_RPS_CONTROL; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder External RPS"); +} + +int +NvVideoEncoder::enableExternalRC(v4l2_enc_enable_ext_rate_ctr ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_EXTERNAL_RATE_CONTROL; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder External RC"); +} + +int NvVideoEncoder::setH265DisableSAO(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_H265_DISABLE_SAO; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder SAO filter to " << enabled); +} + +int +NvVideoEncoder::enableAV1Tile(v4l2_enc_av1_tile_config ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_TILE_CONFIGURATION; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder AV1 tile"); +} + +int +NvVideoEncoder::setAV1SsimRdo(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_SSIMRDO; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder Ssim Rdo to " << enabled); +} + +int +NvVideoEncoder::setAV1DisableCDFUpdate(bool disabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_DISABLE_CDF_UPDATE; + control.value = disabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder CDF update to " << !disabled); +} + +int NvVideoEncoder::enableAV1TileGroups(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_TILE_GROUPS; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder tile groups present flag to " << enabled); +} + +int NvVideoEncoder::setAV1FrameIDPresentFlag(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_FRAMEID_NUMBERS; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder frameID present flag to " << enabled); +} + +int +NvVideoEncoder::setAV1ErrResilienceMode(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_ERR_RESILIENT_MODE; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder Error Resilient Mode to " << enabled); +} + +int +NvVideoEncoder::setVirtualBufferSize(uint32_t size) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_enc_virtual_buffer_size buffer_size = {size}; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_VIRTUALBUFFER_SIZE; + control.string = (char *) &buffer_size; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder virtual buffer size to " << size); +} + +int +NvVideoEncoder::setNumReferenceFrames(uint32_t num_frames) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_enc_num_ref_frames frames = { num_frames }; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_NUM_REFERENCE_FRAMES; + control.string = (char *) &frames; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder number of reference frames to " << num_frames); +} + +int +NvVideoEncoder::setSliceIntrarefresh(uint32_t interval) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_enc_slice_intrarefresh_param param = {interval}; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_SLICE_INTRAREFRESH_PARAM; + control.string = (char *) ¶m; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder slice intrarefresh interval to " << interval); +} + +int NvVideoEncoder::setCUDASliceIntrarefresh(uint32_t interval, uint32_t count) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_ctrl_intra_refresh param = {0}; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + param.enableIntraRefresh = 1; + param.intraRefreshPeriod = interval; + param.intraRefreshCnt = count; + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_SLICE_INTRAREFRESH_PARAM; + control.string = (char *)¶m; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder slice intrarefresh interval to " << interval); +} + +int +NvVideoEncoder::setNumBFrames(uint32_t num) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_NUM_BFRAMES; + control.value = num; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder number of B frames to " << num); +} + +int +NvVideoEncoder::setInsertSpsPpsAtIdrEnabled(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INSERT_SPS_PPS_AT_IDR; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder SPSPPSatIDR to " << enabled); +} + +int +NvVideoEncoder::setCABAC(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder CABAC encoding to " << enabled); +} + +int +NvVideoEncoder::setSliceLevelEncode(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_SLICE_LEVEL_ENCODE; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encode slice level encoding to " << enabled); +} + +int +NvVideoEncoder::enableMotionVectorReporting() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_METADATA_MV; + control.value = 1; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder motion vector reporting"); +} + +int +NvVideoEncoder::getMetadata(uint32_t buffer_index, + v4l2_ctrl_videoenc_outputbuf_metadata &enc_metadata) +{ + v4l2_ctrl_video_metadata metadata; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + metadata.buffer_index = buffer_index; + metadata.VideoEncMetadata = &enc_metadata; + + control.id = V4L2_CID_MPEG_VIDEOENC_METADATA; + control.string = (char *)&metadata; + + CHECK_V4L2_RETURN(getExtControls(ctrls), + "Getting encoder output metadata for buffer " << buffer_index); +} + +int +NvVideoEncoder::getMotionVectors(uint32_t buffer_index, + v4l2_ctrl_videoenc_outputbuf_metadata_MV &enc_mv_metadata) +{ + v4l2_ctrl_video_metadata metadata; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + metadata.buffer_index = buffer_index; + metadata.VideoEncMetadataMV = &enc_mv_metadata; + + control.id = V4L2_CID_MPEG_VIDEOENC_METADATA_MV; + control.string = (char *)&metadata; + + CHECK_V4L2_RETURN(getExtControls(ctrls), + "Getting encoder output MV metadata for buffer " << buffer_index); +} + +int +NvVideoEncoder::setQpRange(uint32_t MinQpI, uint32_t MaxQpI, uint32_t MinQpP, + uint32_t MaxQpP, uint32_t MinQpB, uint32_t MaxQpB) +{ + v4l2_ctrl_video_qp_range qprange; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + qprange.MinQpI = MinQpI; + qprange.MaxQpI = MaxQpI; + qprange.MinQpP = MinQpP; + qprange.MaxQpP = MaxQpP; + qprange.MinQpB = MinQpB; + qprange.MaxQpB = MaxQpB; + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_QP_RANGE; + control.string = (char *)&qprange; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder Qp range " << ctrls.count); +} + +int NvVideoEncoder::setCudaConstantQp(uint32_t qpI, uint32_t qpP, + uint32_t qpB) +{ + v4l2_ctrl_video_constqp qp_param; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + qp_param.constQpI = qpI; + qp_param.constQpP = qpP; + qp_param.constQpB = qpB; + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_CUDA_CONSTQP; + control.string = (char *)&qp_param; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder const Qp " << ctrls.count); +} + +int +NvVideoEncoder::setSampleAspectRatioWidth(uint32_t sar_width) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + switch (capture_plane_pixfmt) + { + case V4L2_PIX_FMT_H264: + control.id = V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_WIDTH; + break; + case V4L2_PIX_FMT_H265: + control.id = V4L2_CID_MPEG_VIDEOENC_H265_VUI_EXT_SAR_WIDTH; + break; + default: + COMP_ERROR_MSG("Unsupported encoder type"); + return -1; + } + control.value = sar_width; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder SAR width to " << sar_width); +} + +int +NvVideoEncoder::setSampleAspectRatioHeight(uint32_t sar_height) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + switch (capture_plane_pixfmt) + { + case V4L2_PIX_FMT_H264: + control.id = V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_HEIGHT; + break; + case V4L2_PIX_FMT_H265: + control.id = V4L2_CID_MPEG_VIDEOENC_H265_VUI_EXT_SAR_HEIGHT; + break; + default: + COMP_ERROR_MSG("Unsupported encoder type"); + return -1; + } + control.value = sar_height; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder SAR width to " << sar_height); +} + +int +NvVideoEncoder::setInsertVuiEnabled(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INSERT_VUI; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder InsertVUI to " << enabled); +} + +int +NvVideoEncoder::setExtendedColorFormat(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_EXTEDED_COLORFORMAT; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder extended colorformat to " << enabled); +} + +int +NvVideoEncoder::setInsertAudEnabled(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INSERT_AUD; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder InsertAUD to " << enabled); +} + +int +NvVideoEncoder::setAlliFramesEncode(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_ALLIFRAME_ENCODE; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder InsertAUD to " << enabled); +} + +int +NvVideoEncoder::DevicePoll(v4l2_ctrl_video_device_poll *devicepoll) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_DEVICE_POLL; + control.string = (char *)devicepoll; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Done calling video device poll "); +} + +int +NvVideoEncoder::SetPollInterrupt() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_SET_POLL_INTERRUPT; + control.value = 1; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder poll interrupt to 1 "); +} + +int +NvVideoEncoder::ClearPollInterrupt() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_SET_POLL_INTERRUPT; + control.value = 0; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder poll interrupt to 0 "); +} + +int +NvVideoEncoder::setPocType(uint32_t pocType) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_POC_TYPE; + control.value = pocType; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting Pic_order_cnt_type to " << pocType); +} + +int NvVideoEncoder::setInitQP(uint32_t IinitQP, + uint32_t PinitQP, uint32_t BinitQP) +{ + v4l2_ctrl_video_init_qp initqp; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + initqp.IInitQP = IinitQP; + initqp.PInitQP = PinitQP; + initqp.BInitQP = BinitQP; + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INIT_FRAME_QP; + control.string = (char *)&initqp; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder Init QP " << ctrls.count); +} + +int +NvVideoEncoder::setFramesToEncode(uint32_t framesToEncode) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_FRAMES_TO_ENCODE; + control.value = framesToEncode; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting num_frames_to_encode to " << framesToEncode); +} + +int +NvVideoEncoder::setChromaFactorIDC(uint8_t value) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_H265_CHROMA_FACTOR_IDC; + control.value = value; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting chroma_format_idc to " << value); +} + +int +NvVideoEncoder::setLossless(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_LOSSLESS; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting lossless encoding to " << enabled); +} + +int NvVideoEncoder::setDisableAMP(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_H265_DISABLE_AMP; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting disable AMP types to " << enabled); +} + +int NvVideoEncoder::enableTwoPassCBR() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_TWO_PASS_CBR; + control.value = 1; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting two pass CBR to 1"); +} + +int NvVideoEncoder::setGPUId(uint32_t gpuId) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_CUDA_GPU_ID; + control.value = gpuId; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting GPU Id to " << gpuId); +} + +int NvVideoEncoder::setCudaMemType(uint32_t cumemType) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_OUTPUT_PLANE_FORMAT_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_CUDA_MEM_TYPE; + control.value = cumemType; + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting cumemType to " << cumemType); +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvBuffer.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvBuffer.h new file mode 100644 index 0000000000..1c1be52cac --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvBuffer.h @@ -0,0 +1,296 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: Buffer API + * + * @b Description: This file declares the NvBuffer APIs. + */ + +#ifndef __NV_BUFFER_H__ +#define __NV_BUFFER_H__ + +#include +#include +#include + +#include "v4l2_nv_extensions.h" +#include "nvbufsurface.h" + +/** + * + * @defgroup l4t_mm_nvbuffer_group Buffer API + * + * The @c %NvBuffer API provides buffer functionality, including reference + * count functionality and convenience methods. + * @ingroup aa_framework_api_group + * @{ + */ + +/** + * Specifies the maximum number of planes a buffer can contain. + */ +#define MAX_PLANES 3 + +/** + * @brief Class representing a buffer. + * + * The NvBuffer class is modeled on the basis of the @c v4l2_buffer + * structure. The buffer has @c buf_type @c v4l2_buf_type, @c + * memory_type @c v4l2_memory, and an index. It contains an + * NvBufferPlane array similar to the array of @c v4l2_plane + * structures in @c v4l2_buffer.m.planes. It also contains a + * corresponding NvBufferPlaneFormat array that describes the + * format of each of the planes. + * + * Even though @c %NvBuffer closely resembles v4l2 structures, it can + * be easily used with other non-v4l2 components. @c %NvBuffer + * contains data pointers, buffer length, file descriptor (FD) of + * buffer planes, buffer format (height, width, stride, etc.), and + * other members that are required by such components. + * + * This class also provides buffer reference count functionality. This + * is useful when the same buffer is being used by multiple elements. + * + * In the case of a V4L2 MMAP, this class provides convenience methods + * for mapping or unmapping the contents of the buffer to or from + * memory, allocating or deallocating software memory depending on its + * format. + */ +class NvBuffer +{ +public: + /** + * Holds the buffer plane format. + */ + typedef struct + { + uint32_t width; /**< Holds the width of the plane in pixels. */ + uint32_t height; /**< Holds the height of the plane in pixels. */ + + uint32_t bytesperpixel; /**< Holds the bytes used to represent one + pixel in the plane. */ + uint32_t stride; /**< Holds the stride of the plane in bytes. */ + uint32_t sizeimage; /**< Holds the size of the plane in bytes. */ + } NvBufferPlaneFormat; + + /** + * Holds the buffer plane parameters. + */ + typedef struct + { + NvBufferPlaneFormat fmt; /**< Holds the format of the plane. */ + + unsigned char *data; /**< Holds a pointer to the plane memory. */ + uint32_t bytesused; /**< Holds the number of valid bytes in the plane. */ + + int fd; /**< Holds the file descriptor (FD) of the plane of the + exported buffer, in the case of V4L2 MMAP buffers. */ + uint32_t mem_offset; /**< Holds the offset of the first valid byte + from the data pointer. */ + uint32_t length; /**< Holds the size of the buffer in bytes. */ + } NvBufferPlane; + + /** + * Creates a new NvBuffer object. + * + * This convenience method for V4L2 elements creates a new buffer + * with the planes array memset to zero and the refcount + * initialized to zero. + * + + * @param[in] buf_type Type of buffer, enumerated as @c + * v4l2_buf_type. + * @param[in] memory_type @c %NvBuffer memory, enumerated as an + * @c v4l2_memory enum. + * @param[in] n_planes Number of planes in the buffer. + * @param[in] fmt Specifies a pointer to the array of buffer plane formats. + * Should contain at least @a n_planes elements. + * @param[in] index Index of the buffer in the plane. + */ + NvBuffer(enum v4l2_buf_type buf_type, enum v4l2_memory memory_type, + uint32_t n_planes, NvBufferPlaneFormat *fmt, uint32_t index); + + /** + * Creates a new NvBuffer for raw pixel formats. + * + * This convenience method for V4L2 elements is an @c %NvBuffer + * constructor for raw pixel formats only. It requires width, + * height, and pixel format to be specified. + * + * The planes array is memset to zero and the refcount is + * initialized to zero. + * + * @attention The memory must be allocated by the application + * by calling NvBuffer::allocateMemory. + * + * @param[in] pixfmt Pixel format of the buffer. + * @param[in] width Width of the buffer in pixels. + * @param[in] height Height of the buffer in pixels. + * @param[in] index Index/ID of the buffer. + */ + NvBuffer(uint32_t pixfmt, uint32_t width, uint32_t height, uint32_t index); + + /** + * Creates a new NvBuffer object for non-raw pixel formats. + * + * This convenience method for V4L2 elements is an @c %NvBuffer + * constructor for non raw pixel formats. It requires size of the + * buffer to be supplied. + * + * The planes array is memset to zero and refcount initialized to + * zero. + * + * @attention The memory needs to be allocated by the application + * by calling NvBuffer::allocateMemory. + * + * @param[in] size Size of the buffer in bytes. + * @param[in] index Index/ID of the buffer. + */ + NvBuffer(uint32_t size, uint32_t index); + + /** + * Destroys an NvBuffer object. + * + * This method cleans up class instances, unmapping any mapped + * planes. + */ + ~NvBuffer(); + + /** + * Maps the contents of the buffer to memory. + * + * This method maps the file descriptor (FD) of the planes to + * a data pointer of @c planes. (MMAP buffers only.) + * + * @return 0 on success, -1 otherwise. + */ + int map(); + /** + * Unmaps the contents of the buffer from memory. (MMAP buffers only.) + * + */ + void unmap(); + + /** + * Allocates software memory for the buffer. + * + * @warning This method works only for @c V4L2_MEMORY_USERPTR memory. + * + * This method allocates memory on the basis of the buffer format: + * @a height, @a width, @a bytesperpixel, and @a sizeimage. + * + * @return 0 for success, -1 otherwise. + */ + int allocateMemory(); + /** + * Deallocates buffer memory. + * + * @warning This method works only for @c V4L2_MEMORY_USERPTR memory and if + * the memory was previously allocated using NvBuffer::allocateMemory. + */ + void deallocateMemory(); + + /** + * Increases the reference count of the buffer. + * + * This method is thread safe. + * + * @return Reference count of the buffer after the operation. + */ + int ref(); + /** + * Decreases the reference count of the buffer. + * + * This thread-safe method decreases the buffer reference count if the + * buffer reference count is above 0. + * + * @return Reference count of the buffer after the operation. + */ + int unref(); + + const enum v4l2_buf_type buf_type; /**< Type of the buffer. */ + const enum v4l2_memory memory_type; /**< Type of memory associated + with the buffer. */ + + const uint32_t index; /**< Holds the buffer index. */ + + uint32_t n_planes; /**< Holds the number of planes in the buffer. */ + NvBufferPlane planes[MAX_PLANES]; /**< Holds the data pointer, plane file + descriptor (FD), plane format, etc. */ + + /** + * Fills the NvBuffer::NvBufferPlaneFormat array. + * + * This convenience method populates the + * @c %NvBuffer::NvBufferPlaneFormat array on the basis of @a width, + * @a height and pixel format (@a raw_pixfmt). It also returns the number of planes + * required for the pixel format in @a num_planes. + * + * + * @param[out] num_planes The number of planes. Must not be NULL. + * @param[in,out] planefmts Array of %NvBuffer::NvBufferPlaneFormat to + * fill. Must be at least \a num_planes in length. For best + * results, pass an array of length #MAX_PLANES. + * @param[in] width Width of the buffer in pixels. + * @param[in] height Height of the buffer in pixels. + * @param[in] raw_pixfmt Raw V4L2 pixel formats. + * @return 0 for success, -1 for an unsupported pixel format. + */ + static int fill_buffer_plane_format(uint32_t *num_planes, + NvBuffer::NvBufferPlaneFormat *planefmts, + uint32_t width, uint32_t height, uint32_t raw_pixfmt); +private: + uint32_t ref_count; /**< Holds the reference count of the buffer. */ + pthread_mutex_t ref_lock; /**< Mutex to synchronize increment/ + decrement operations of @c ref_count. */ + + bool mapped; /**< Indicates if the buffer is mapped to + memory. */ + bool allocated; /**< Indicates if the buffer is allocated + memory. */ + NvBuffer *shared_buffer; /**< If this is a DMABUF buffer, @c shared_buffer + points to the MMAP @c NvBuffer whose FD was + sent when this buffer was queued. */ + + /** + * Disallows copy constructor. + */ + NvBuffer(const NvBuffer& that); + /** + * Disallows assignment. + */ + void operator=(NvBuffer const&); + + friend class NvV4l2ElementPlane; +}; +/** @} */ +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElement.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElement.h new file mode 100644 index 0000000000..543f47b324 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElement.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: %NvElement Base Class + * + * @b This file declares the NvElement base class. + */ + +#ifndef __NV_ELEMENT_H__ +#define __NV_ELEMENT_H__ + +#include +#include +#include +#include + +#include "NvElementProfiler.h" + +/** + * + * @defgroup l4t_mm_nvelement_base_group NvElement Base Class + * @ingroup l4t_mm_nvelement_group + * + * This class is the class from which both V4L2 and non-V4L2 + * components are derived. + * + * @{ + */ +/** + * Every element has a unique name that can be used for identifying + * the element in debug logs. + * + * @c %NvElement also provides other common functionality, such as keeping + * track of errors. + */ +class NvElement +{ +public: + /** + * Indicates whether the element encountered an error during its operation. + * + * @return 0 if no error was encountered, a non-zero value if an + * error was encountered. + */ + virtual int isInError() + { + return is_in_error; + } + virtual ~NvElement() + { + } + + /** + * Gets profiling data for the element. + * + * @return A constant reference to the element's profiling data. + */ + void getProfilingData(NvElementProfiler::NvElementProfilerData &data); + + /** + * Prints profiling data for the element to an output stream. + * + * @param[in] out_stream Output stream of type std::ostream to print the + * data to. It takes the default value std::cout if not specified. + */ + void printProfilingStats(std::ostream &out_stream = std::cout); + + /** + * Enables profiling for the element. + */ + virtual void enableProfiling(); + + /** + * Checks whether profiling is enabled for the element. + * + * @return Boolean value indicating if profiling is enabled. + */ + bool isProfilingEnabled(); + +protected: + + /** + * Creates a new NvElement object with name @a name. + * + * If the @a name parameter is NULL, this method sets the internal + * error variable. + * + * @param[in] name If non-NULL, a pointer to the name of the + * element. + */ + NvElement(const char *name, NvElementProfiler::ProfilerField = NvElementProfiler::PROFILER_FIELD_NONE); + + int is_in_error; /**< Indicates if an error was encountered during + the operation of the element. */ + const char *comp_name; /**< Specifies the name of the component, + for debugging. */ + NvElementProfiler profiler; /**< Profiler for the element. */ + + /** + * Disallows copy constructor. + */ + NvElement(const NvElement& that); + /** + * Disallows assignment. + */ + void operator=(NvElement const&); + +}; +/** @} */ +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElementProfiler.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElementProfiler.h new file mode 100644 index 0000000000..945bb93765 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElementProfiler.h @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NV_ELEMENT_PROFILER_H__ +#define __NV_ELEMENT_PROFILER_H__ + +#include +#include +#include +#include +#include + +/** + * @file + * NVIDIA Multimedia API: Element Profiler API + * + * @b Description: This file profiles the performance of individual elements. + */ + + +/** + * + * Helper class for profiling the performance of individual elements. + * + * NvElementProfiler currently measures processing latencies, average processing rate, and + * the number of units that arrived late at the element. Components should use this + * information internally. + * + * If you require latency measurements, + * you must call startProcessing() to indicate that a unit has been submitted + * for processing and finishProcessing() to indicate that a unit has finished processing. + * If you require only averaging processing rate or the number of units that + * arrived late need not call startProcessing(). + * + * You can get data from NvElementProfiler using getProfilerData(). This function + * fills the [NvElementProfilerData](@ref NvElementProfiler::NvElementProfilerData) + * structure. Components that do not support all + * the fields available in the structure must use the variable + * [valid_fields](@ref NvElementProfiler::NvElementProfilerData::valid_fields) of + * type [ProfilerField](@ref NvElementProfiler::ProfilerField), which is also + * included in the structure. + * + * @defgroup l4t_mm_nvelementprofiler_group Element Profiler API + * @ingroup aa_framework_api_group + * @{ + */ +class NvElementProfiler { +public: + /** + * @defgroup Defines @c valid_field values for the #NvElementProfilerData structure. + * @ingroup l4t_mm_nvelementprofiler_group + * @{ + */ + typedef int ProfilerField; + static const ProfilerField PROFILER_FIELD_NONE = 0; + static const ProfilerField PROFILER_FIELD_TOTAL_UNITS = 1; + static const ProfilerField PROFILER_FIELD_LATE_UNITS = 2; + static const ProfilerField PROFILER_FIELD_LATENCIES = 4; + static const ProfilerField PROFILER_FIELD_FPS = 8; + static const ProfilerField PROFILER_FIELD_ALL = (PROFILER_FIELD_FPS << 1) - 1; + /** @} */ + + /** + * Holds profiling data for the element. + * + * Some elements may not support all the fields in the structure. User must check + * the @a valid_fields flag to determine which fields are valid. + */ + typedef struct { + /** Valid Fields which are supported by the element. */ + ProfilerField valid_fields; + + /** Average latency of all processed units, in microseconds. */ + uint64_t average_latency_usec; + /** Minimum of latencies for each processed units, in microseconds. */ + uint64_t min_latency_usec; + /** Maximum of latencies for each processed units, in microseconds. */ + uint64_t max_latency_usec; + + /** Total units processed. */ + uint64_t total_processed_units; + /** Number of units which arrived late at the element. */ + uint64_t num_late_units; + + /** Average rate at which the units were processed. */ + float average_fps; + + /** Total profiling time. */ + struct timeval profiling_time; + } NvElementProfilerData; + + /** + * Gets the profiling data for the element. + * + * @param[out] data Reference to the NvElementProfilerData structure which should be filled. + */ + void getProfilerData(NvElementProfilerData &data); + + /** + * Prints the element's profiling data to an output stream. + * + * @param[in] out_stream Reference to a std::ostream. + */ + void printProfilerData(std::ostream &out_stream = std::cout); + + /** + * Informs the profiler that processing has started. + * + * Has no effect if profiler is disabled. + * + * @return ID of the unit, to be supplied with finishProcessing();. + */ + uint64_t startProcessing(); + + /** + * Informs the profiler that processing has finished. + * + * Has no effect if profiler is disabled. + * + * @param[in] id ID of the unit whose processing is finished, + * 0 if the first unit in the profiler's queue should be picked. + * @param[in] is_late Should be true if the frame arrived late at the element. + */ + void finishProcessing(uint64_t id, bool is_late); + + /** + * Enables the profiler. + * + * startProcessing() and finishProcessing() are ineffective until the profiler is enabled. + * + * @param[in] reset_data Reset the profiled data. + */ + void enableProfiling(bool reset_data); + + /** + * Disables the profiler. + */ + void disableProfiling(); +private: + /** + * Resets the profiler data. + */ + void reset(); + + pthread_mutex_t profiler_lock; /**< Mutex to synchronize multithreaded access to profiler data. */ + + bool enabled; /**< Flag indicating if profiler is enabled. */ + + const ProfilerField valid_fields; /**< Valid fields for the element. */ + + struct NvElementProfilerDataInternal : NvElementProfilerData { + /** Wall-clock time at which the first unit was processed. */ + struct timeval start_time; + + /** Wall-clock time at which the latest unit was processed. */ + struct timeval stop_time; + + /** Total accumulated time. + * When performance measurement is restarted @a start_time and @a stop_time + * are reset. This field is used to accumulate time before + * resetting. */ + struct timeval accumulated_time; + + /** Total accumulated latency for all units, in microseconds. */ + uint64_t total_latency; + } data_int; + + /** Queue used to maintain the timestamps of when the unit + * processing started. Required to calculate latency. */ + std::map unit_start_time_queue; + + uint64_t unit_id_counter; /**< Unique ID of the last unit. */ + + /** + * Constructor for NvElementProfiler. + * + * Initializes internal data structures. The profiler is disabled by default. + * @param fields + */ + NvElementProfiler(ProfilerField fields); + + /** + * Disallow copy constructor. + */ + NvElementProfiler(const NvElementProfiler& that); + /** + * Disallow assignment. + */ + void operator=(NvElementProfiler const&); + + ~NvElementProfiler(); + + friend class NvElement; +}; + +/** @} */ + +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvLogging.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvLogging.h new file mode 100644 index 0000000000..e4e9ca9469 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvLogging.h @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: Logging API + * + * @brief Description: This file defines macros for logging messages. + */ + +#ifndef __NV_LOGGING_H_ +#define __NV_LOGGING_H_ + +#include +#include + +/** + * + * @defgroup l4t_mm_nvlogging_group Logging API + * + * This file defines macros that provide message logging + * functionality. + * + * @ingroup aa_framework_api_group + * @{ + */ + + +/** + * Specifies the log level for Information messages. + */ +#define LOG_LEVEL_INFO 0 +/** + * Specifies the log level for Error messages. + */ +#define LOG_LEVEL_ERROR 1 +/** + * Specifies the log level for Warning messages. + */ +#define LOG_LEVEL_WARN 2 +/** + * Specifies the log level for Debug messages. + */ +#define LOG_LEVEL_DEBUG 3 + +/** + * Holds the current log level at runtime by assignment of one of the + * @c LOG_LEVEL_* values. + */ +extern int log_level; + +/** + * Specifies the default log level. + */ +#define DEFAULT_LOG_LEVEL LOG_LEVEL_ERROR + +/** + * @cond + */ +#define NV_STRINGIFY(s) #s +#define NV_XSTRINGIFY(s) NV_STRINGIFY(s) +#define __LINE_NUM_STR__ NV_XSTRINGIFY(__LINE__) + +extern const char *log_level_name[]; +/** + * @endcond + */ + +/** + * + * Prints log messages. + * + * Prints a log message only if the current log_level is greater + * than or equal to the level of the message. + * + * Messages are in the following form: + * [LEVEL] (FILE: LINE_NUM) Message + * + * @param[in] level The Log level of the message. + * @param[in] str1 The NULL-terminated char array to print. + */ +#define PRINT_MSG(level, str1) if(level <= log_level) { \ + std::ostringstream ostr; \ + ostr << "[" << log_level_name[level] << "] (" << \ + __FILE__ << ":" __LINE_NUM_STR__ ") " << \ + str1 << std::endl; \ + std::cerr << ostr.str(); \ + } + +/** + * Prints a log message of level LOG_LEVEL_INFO. + */ +#define INFO_MSG(str) PRINT_MSG(LOG_LEVEL_INFO, str) +/** + * Prints a component-specific log message of level LOG_LEVEL_INFO. + * This is used by the components internally and should not be used by + * the application. + * + * Messages are in the following form: + * [LEVEL] (FILE: LINE_NUM) + */ +#define COMP_INFO_MSG(str) INFO_MSG("<" << comp_name << "> " << str) +/** + * Prints a category-specific (Component type) system error log + * message of level LOG_LEVEL_INFO. This is used by the components + * internally and should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE: LINE_NUM) + */ +#define CAT_INFO_MSG(str) INFO_MSG("<" CAT_NAME "> " << str) + +/** + * Prints a log message of level LOG_LEVEL_ERROR. + */ +#define ERROR_MSG(str) PRINT_MSG(LOG_LEVEL_ERROR, str) +/** + * Prints a component-specific log message of level + * LOG_LEVEL_ERROR. This is used by the components internally + * and should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define COMP_ERROR_MSG(str) ERROR_MSG("<" << comp_name << "> " << str) +/** + * Prints a category-specific (Component type) log message of level + * LOG_LEVEL_ERROR. This is used by the components internally and + * should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define CAT_ERROR_MSG(str) ERROR_MSG("<" CAT_NAME "> " << str) + +/** + * Prints a system error log message of level LOG_LEVEL_ERROR with + * the string description of the errno value appended. + */ +#define SYS_ERROR_MSG(str) ERROR_MSG(str << ": " << strerror(errno)) +/** + * Prints a component-specific system error log message of level + * LOG_LEVEL_ERROR. This is used by the components internally and + * should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define COMP_SYS_ERROR_MSG(str) SYS_ERROR_MSG("<" << comp_name << "> " << str) +/** + * Prints a category-specific (Component type) system error log + * message of level LOG_LEVEL_ERROR. This is used by the components + * internally and should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define CAT_SYS_ERROR_MSG(str) SYS_ERROR_MSG("<" CAT_NAME "> " << str) + +/** + * Prints a log message of level LOG_LEVEL_WARN. + */ +#define WARN_MSG(str) PRINT_MSG(LOG_LEVEL_WARN, str) +/** + * Prints a component-specific log message of level LOG_LEVEL_WARN. + * This is used by the components internally and should not be used by + * the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define COMP_WARN_MSG(str) WARN_MSG("<" << comp_name << "> :" << str) +/** + * Print a category-specific (Component type) log message of level + * LOG_LEVEL_WARN. + * This is used by the components internally and should not be used by the + * application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define CAT_WARN_MSG(str) WARN_MSG("<" CAT_NAME "> " << str) + +/** + * Prints a log message of level LOG_LEVEL_DEBUG. + */ +#define DEBUG_MSG(str) PRINT_MSG(LOG_LEVEL_DEBUG, str) +/** + * Prints a component-specific log message of level LOG_LEVEL_DEBUG. + * This is used by the components internally and should not be used by the + * application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define COMP_DEBUG_MSG(str) DEBUG_MSG("<" << comp_name << "> :" << str) +/** + * Prints a category-specific (Component type) log message of level + * LOG_LEVEL_DEBUG. This is used by the components internally and + * should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define CAT_DEBUG_MSG(str) DEBUG_MSG("<" CAT_NAME "> " << str) + +#endif +/** @} */ diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2Element.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2Element.h new file mode 100644 index 0000000000..2f7a72d7d4 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2Element.h @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: V4L2 Helper Class + * + * @b Description: This file declares a helper class for V4L2-based components. + */ + +/** + * @defgroup l4t_mm_nvv4l2element_group V4L2 Element Class + * @ingroup l4t_mm_nvelement_group + * + * Helper class that provides common functionality for V4L2-based components, + * such as encoder and decoder. Objects instantiated from this class create + * new V4L2 elements, subscribes to V4L2 events, dequeues an event from + * an element, and sets/gets control values. + * @{ + */ +#ifndef __NV_V4L2_ELEMENT_H__ +#define __NV_V4L2_ELEMENT_H__ + +#include "NvElement.h" +#include "NvV4l2ElementPlane.h" + +#include "v4l2_nv_extensions.h" + +/** + * Defines a helper class for V4L2 based components. + * + * This derived class provides common functionality for V4L2 components. V4L2-based + * components such as encoder/decoder extend from this class. + * + * This class is modeled on V4L2 M2M devices. It includes the file descriptor (FD) of the device + * opened using %v4l2_open, two planes (NvV4l2ElementPlane), output plane, + * capture plane, and other helper methods, such as setting/getting controls, + * subscribing/dequeueing events, etc. + */ +class NvV4l2Element:public NvElement +{ +public: + virtual ~NvV4l2Element(); + + /** + * Subscribes to an V4L2 event. + * + * Calls \c VIDIOC_SUBSCRIBE_EVENT IOCTL internally. + * + * @param[in] type Type of the event. + * @param[in] id ID of the event source. + * @param[in] flags Event flags. + * @return 0 for success, -1 otherwise. + */ + int subscribeEvent(uint32_t type, uint32_t id, uint32_t flags); + /** + * Dequeues an event from the element. + * + * Calls \c VIDIOC_DQEVENT IOCTL internally. The caller can specify the maximum time + * to wait for dequeuing the event. The call blocks until an event is + * dequeued successfully or timeout is reached. + * + * @param[in,out] event A reference to the \c v4l2_event structure to fill. + * @param[in] max_wait_ms Specifies the max wait time for dequeuing an event, + * in milliseconds. + * @return 0 for success, -1 otherwise. + */ + int dqEvent(struct v4l2_event &event, uint32_t max_wait_ms); + + /** + * Sets the value of a control. + * + * Calls \c VIDIOC_S_CTRL IOCTL internally. + * + * @param[in] id ID of the control to be set. + * @param[in] value Value to be set on the control. + * @return 0 for success, -1 otherwise. + */ + int setControl(uint32_t id, int32_t value); + /** + * Gets the value of a control. + * + * Calls \c VIDIOC_G_CTRL IOCTL internally. + * + * @param[in] id ID of the control to get. + * @param[out] value A reference to the variable into which the control value + is read. + * @return 0 for success, -1 otherwise. + */ + int getControl(uint32_t id, int32_t &value); + + /** + * Sets the value of several controls. + * + * Calls \c VIDIOC_S_EXT_CTRLS IOCTL internally. + * + * @param[in] ctl A pointer to the controls to set. + * @return 0 for success, -1 otherwise. + */ + int setExtControls(struct v4l2_ext_controls &ctl); + /** + * Gets the value of several controls. + * + * Calls \c VIDIOC_G_EXT_CTRLS IOCTL internally. + * + * @param[in,out] ctl A pointer to the controls to get. + * @return 0 for success, -1 otherwise. + */ + int getExtControls(struct v4l2_ext_controls &ctl); + + virtual int isInError(); + + /** + * Sets the output plane. + */ + NvV4l2ElementPlane output_plane; /**< Output plane of the element */ + /** + * Sets the capture plane. + */ + NvV4l2ElementPlane capture_plane; /**< Capture plane of the element */ + + /** + * + * Terminates processing of queued buffers immediately. All the buffers are + * returned to the application. + * + * Calls VIDIOC_STREAMOFF IOCTL on both of the planes internally. + * + * @return 0 for success, -1 otherwise. + */ + int abort(); + + /** + * Waits until the element processes all the output plane buffers. + * + * Objects extending @c V4l2Element must implement this because the idle + * condition is component-specific. + * + * @param[in] max_wait_ms Max time to wait in milliseconds. + * @return 0 for success, -1 otherwise. + */ + virtual int waitForIdle(uint32_t max_wait_ms); + + void *app_data; /**< A pointer to the application-specific data. */ + + /** + * Enables profiling for the V4l2Element. + * + * Must be called before setting either plane formats. + */ + void enableProfiling(); + +protected: + int fd; /**< Specifies the FD of the device opened using \c v4l2_open. */ + + uint32_t output_plane_pixfmt; /**< Pixel format of output plane buffers */ + uint32_t capture_plane_pixfmt; /**< Pixel format of capture plane buffers */ + + /** + * Creates a new V4l2Element named \a name. + * + * This constructor calls v4l2_open on the \a dev_node. It sets an error if + * v4l2_open fails. + * + * This function also checks if the device supports V4L2_CAP_VIDEO_M2M_MPLANE + * capability. + * + * @param[in] comp_name A pointer to the unique name to identity the element instance. + * @param[in] dev_node A pointer to /dev/ * node of the device. + * @param[in] flags Flags with which to open the device. + * @param[in] fields Profiler fields that are valid for the element. + */ + NvV4l2Element(const char *comp_name, const char *dev_node, int flags, NvElementProfiler::ProfilerField fields); +}; +/** @} */ +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2ElementPlane.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2ElementPlane.h new file mode 100644 index 0000000000..64378f1842 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2ElementPlane.h @@ -0,0 +1,586 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: V4L2 Element Plane + * + * @b Description: This file declares a helper class for operations + * performed on a V4L2 Element plane. + */ + +/** + * @defgroup l4t_mm_nvv4lelementplane_group NvV4l2ElementPlane Class + * @ingroup l4t_mm_nvelement_group + * + * Helper class for operations performed on a V4L2 element plane. + * This includes getting/setting plane formats, plane buffers, + * and cropping. + * @{ + */ + +#ifndef __NV_V4L2_ELELMENT_PLANE_H__ +#define __NV_V4L2_ELELMENT_PLANE_H__ + +#include +#include "NvElement.h" +#include "NvLogging.h" +#include "NvBuffer.h" + +/** + * Prints a plane-specific message of level LOG_LEVEL_DEBUG. + * Must not be used by applications. + */ +#define PLANE_DEBUG_MSG(str) COMP_DEBUG_MSG(plane_name << ":" << str); +/** + * Prints a plane-specific message of level LOG_LEVEL_INFO. + * Must not be used by applications. + */ +#define PLANE_INFO_MSG(str) COMP_INFO_MSG(plane_name << ":" << str); +/** + * Prints a plane-specific message of level LOG_LEVEL_WARN. + * Must not be used by applications. + */ +#define PLANE_WARN_MSG(str) COMP_WARN_MSG(plane_name << ":" << str); +/** + * Prints a plane-specific message of level LOG_LEVEL_ERROR. + * Must not be used by applications. + */ +#define PLANE_ERROR_MSG(str) COMP_ERROR_MSG(plane_name << ":" << str); +/** + * Prints a plane-specific system error message of level LOG_LEVEL_ERROR. + * Must not be used by applications. + */ +#define PLANE_SYS_ERROR_MSG(str) COMP_SYS_ERROR_MSG(plane_name << ":" << str); + +/** + * @brief Defines a helper class for operations performed on a V4L2 Element plane. + * + * This derived class is modeled on the planes of a V4L2 Element. It provides + * convenient wrapper methods around V4L2 IOCTLs associated with plane + * operations such as VIDIOC_G_FMT/VIDIOC_S_FMT, \c VIDIOC_REQBUFS, + * VIDIOC_STREAMON/VIDIOC_STREAMOFF, etc. + * + * The plane buffer type can be either \c V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE (for + * the output plane) or \c V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE (for the capture + * plane). + * + * The plane has an array of NvBuffer object pointers that is allocated and + * initialized during reqbuf call. These \c NvBuffer objects are similar to the + * \c v4l2_buffer structures that are queued/dequeued. + * + * This class provides another feature useful for multi-threading. On calling + * #startDQThread, it internally spawns a thread that runs infinitely until + * signaled to stop. This thread keeps trying to dequeue a buffer from the + * plane and calls a #dqThreadCallback method specified by the user on + * successful dequeue. + * + */ +class NvV4l2ElementPlane +{ + +public: + /** + * Gets the plane format. + * + * Calls @b VIDIOC_G_FMT \c IOCTL internally. + * + * @param[in,out] format A reference to the \c v4l2_format structure to be filled. + * @return 0 for success, -1 otherwise. + */ + int getFormat(struct v4l2_format & format); + /** + * Sets the plane format. + * + * Calls @b VIDIOC_S_FMT \c IOCTL internally. + * + * @param[in] format A reference to the \c v4l2_format structure to be set on the plane. + * @return 0 for success, -1 otherwise. + */ + int setFormat(struct v4l2_format & format); + + /** + * Maps the NvMMBuffer to NvBuffer for V4L2_MEMORY_DMABUF. + * + * @param[in] v4l2_buf Address of the NvBuffer to which the NvMMBuffer is mapped. + * @param[in] dmabuff_fd Index to the field that holds NvMMBuffer attributes. + * @return 0 for success, -1 otherwise. + */ + + int mapOutputBuffers(struct v4l2_buffer &v4l2_buf, int dmabuff_fd); + + /** + * Unmaps the NvMMBuffer for V4L2_MEMORY_DMABUF. + * + * @param[in] index for the current buffer index. + * @param[in] dmabuff_fd Index to the field that holds NvMMBuffer attributes. + * @return 0 for success, -1 otherwise. + */ + + int unmapOutputBuffers(int index, int dmabuff_fd); + + /** + * Gets the cropping rectangle for the plane. + * + * Calls @b VIDIOC_G_CROP \c IOCTL internally. + * + * @param[in] crop A reference to the \c v4l2_crop structure to be filled. + * @return 0 for success, -1 otherwise. + */ + int getCrop(struct v4l2_crop & crop); + + /** + * Sets the selection rectangle for the plane. + * + * Calls @b VIDIOC_S_SELECTION IOCTL internally. + * + * @param[in] target Specifies the rectangle selection type. + * @param[in] flags Specifies the flags to control selection adjustments. + * @param[in] rect A reference to the selection rectangle. + * @return 0 for success, -1 otherwise. + */ + int setSelection(uint32_t target, uint32_t flags, struct v4l2_rect & rect); + + /** + * Requests for buffers on the plane. + * + * Calls \c VIDIOC_REQBUFS IOCTL internally. Creates an array of NvBuffer of + * length equal to the count returned by the IOCTL. + * + * @param[in] mem_type Specifies the type of V4L2 memory to be requested. + * @param[in] num Specifies the number of buffers to request on the plane. + * @return 0 for success, -1 otherwise. + */ + int reqbufs(enum v4l2_memory mem_type, uint32_t num); + /** + * Queries the status of the buffer at the index. + * + * @warning This method works only for \c V4L2_MEMORY_MMAP memory. + * + * Calls \c VIDIOC_QUERYBUF IOCTL internally. Populates the \a length and \a + * mem_offset members of all the NvBuffer::NvBufferPlane members of the + * \c %NvBuffer object at index \a buf_index. + * + * @param[in] buf_index Specifies the index of the buffer to query. + * @return 0 for success, -1 otherwise. + */ + int queryBuffer(uint32_t buf_index); + /** + * Exports the buffer as DMABUF FD. + * + * @warning This method works only for \c V4L2_MEMORY_MMAP memory. + * + * Calls \c VIDIOC_EXPBUF IOCTL internally. Populates the \a fd member of all + * the \c NvBuffer::NvBufferPlane members of \c NvBuffer object at index \a buf_in. + * + * @param[in] buf_index Specifies the index of the buffer to export. + * @return 0 for success, -1 otherwise. + */ + int exportBuffer(uint32_t buf_index); + + /** + * Starts or stops streaming on the plane. + * + * Calls \c VIDIOC_STREAMON/VIDIOC_STREAMOFF IOCTLs internally. + * + * @param[in] status Must be TRUE to start the stream, FALSE to stop the stream. + * @return 0 for success, -1 otherwise. + */ + int setStreamStatus(bool status); + + /** + * Checks whether the plane is streaming. + * + * @returns true if the plane is streaming, false otherwise. + */ + bool getStreamStatus(); + + /** + * Sets streaming parameters. + * + * Calls \c VIDIOC_S_PARM IOCTL internally. + * + * @param[in] parm A reference to the \c v4l2_streamparm structure to be set on the + * plane. + * @return 0 for success, -1 otherwise. + */ + int setStreamParms(struct v4l2_streamparm & parm); + + /** + * Helper method that encapsulates all the method calls required to + * set up the plane for streaming. + * + * Calls reqbuf internally. Then, for each of the buffers, calls #queryBuffer, + * #exportBuffer and maps the buffer/allocates the buffer memory depending + * on the memory type. + * + * @sa deinitPlane + * + * @param[in] mem_type V4L2 Memory to use on the buffer. + * @param[in] num_buffers Number of buffer to request on the plane. + * @param[in] map boolean value indicating if the buffers should be mapped to + memory (Only for V4L2_MEMORY_MMAP). + * @param[in] allocate boolean valued indicating whether the buffers should be + allocated memory (Only for V4L2_MEMORY_USERPTR). + * @return 0 for success, -1 otherwise. + */ + int setupPlane(enum v4l2_memory mem_type, uint32_t num_buffers, bool map, bool allocate); + /** + * Helper method that encapsulates all the method calls required to + * deinitialize the plane for streaming. + * + * For each of the buffers, unmaps/deallocates memory depending on the + * memory type. Then, calls reqbuf with count zero. + * + * @sa setupPlane + */ + void deinitPlane(); + + /** + * Gets the streaming/buffer type of this plane. + * + * @returns Type of the buffer belonging to enum \c v4l2_buf_type. + */ + inline enum v4l2_buf_type getBufType() + { + return buf_type; + } + + /** + * Gets the \c NvBuffer object at index n. + * + * @returns \c %NvBuffer object at index n, NULL if n >= number of buffers. + */ + NvBuffer *getNthBuffer(uint32_t n); + + /** + * Dequeues a buffer from the plane. + * + * This is a blocking call. This call returns when a buffer is successfully + * dequeued or timeout is reached. If \a buffer is not NULL, returns the + * \c NvBuffer object at the index returned by the \c VIDIOC_DQBUF IOCTL. If this + * plane shares a buffer with other elements and \a shared_buffer is not + * NULL, returns the shared \c %NvBuffer object in \a shared_buffer. + * + * @param[in] v4l2_buf A reference to the \c v4l2_buffer structure to use for dequeueing. + * @param[out] buffer Returns a pointer to a pointer to the \c %NvBuffer object associated with the dequeued + * buffer. Can be NULL. + * @param[out] shared_buffer Returns a pointer to a pointer to the shared \c %NvBuffer object if the queued + buffer is shared with other elements. Can be NULL. + * @param[in] num_retries Number of times to try dequeuing a buffer before + * a failure is returned. In case of non-blocking + * mode, this is equivalent to the number of + * milliseconds to try to dequeue a buffer. + * @return 0 for success, -1 otherwise. + */ + int dqBuffer(struct v4l2_buffer &v4l2_buf, NvBuffer ** buffer, + NvBuffer ** shared_buffer, uint32_t num_retries); + /** + * Queues a buffer on the plane. + * + * This method calls \c VIDIOC_QBUF internally. If this plane is sharing a + * buffer with other elements, the application can pass the pointer to the + * shared NvBuffer object in \a shared_buffer. + * + * @param[in] v4l2_buf A reference to the \c v4l2_buffer structure to use for queueing. + * @param[in] shared_buffer A pointer to the shared \c %NvBuffer object. + * @return 0 for success, -1 otherwise. + */ + int qBuffer(struct v4l2_buffer &v4l2_buf, NvBuffer * shared_buffer); + + /** + * Gets the number of buffers allocated/requested on the plane. + * + * @returns Number of buffers. + */ + inline uint32_t getNumBuffers() + { + return num_buffers; + } + + /** + * Gets the number of planes buffers on this plane for the currently + * set format. + * + * @returns Number of planes. + */ + inline uint32_t getNumPlanes() + { + return n_planes; + } + + /** + * Sets the format of the planes of the buffer that is used with this + * plane. + * + * The buffer plane format must be set before calling reqbuf since + * these are needed by the NvBuffer constructor. + * + * @sa reqbufs + * + * @param[in] n_planes Number of planes in the buffer. + * @param[in] planefmts A pointer to the array of \c NvBufferPlaneFormat that describes the + * format of each of the plane. The array length must be at + * least @a n_planes. + */ + void setBufferPlaneFormat(int n_planes, NvBuffer::NvBufferPlaneFormat * planefmts); + + /** + * Gets the number of buffers currently queued on the plane. + * + * @returns Number of buffers currently queued on the plane. + */ + inline uint32_t getNumQueuedBuffers() + { + return num_queued_buffers; + } + + /** + * Gets the total number of buffers dequeued from the plane. + * + * @returns Total number of buffers dequeued from the plane. + */ + inline uint32_t getTotalDequeuedBuffers() + { + return total_dequeued_buffers; + } + + /** + * Gets the total number of buffers queued on the plane. + * + * @returns Total number of buffers queued on the plane. + */ + inline uint32_t getTotalQueuedBuffers() + { + return total_queued_buffers; + } + + /** + * Waits until all buffers of the plane are queued. + * + * This is a blocking call that returns when all the buffers are queued + * or timeout is reached. + + * @param[in] max_wait_ms Maximum time to wait, in milliseconds. + * @return 0 for success, -1 otherwise. + */ + int waitAllBuffersQueued(uint32_t max_wait_ms); + /** + * Waits until all buffers of the plane are dequeued. + * + * This is a blocking call that returns when all the buffers are dequeued + * or timeout is reached. + + * @param[in] max_wait_ms Maximum time to wait, in milliseconds + * @return 0 for success, -1 otherwise. + */ + int waitAllBuffersDequeued(uint32_t max_wait_ms); + + /** + * This is a callback function type method that is called by the DQ Thread when + * it successfully dequeues a buffer from the plane. Applications must implement + * this and set the callback using #setDQThreadCallback. + * + * Setting the stream to off automatically stops this thread. + * + * @sa setDQThreadCallback, #startDQThread + * + * @param v4l2_buf A pointer to the \c v4l2_buffer structure that is used for dequeueing. + * @param buffer A pointer to the NvBuffer object at the \c index contained in \c v4l2_buf. + * @param shared_buffer A pointer to the NvBuffer object if the plane shares a buffer with other elements, + * else NULL. + * @param data A pointer to application specific data that is set with + * #startDQThread. + * @returns If the application implementing this call returns FALSE, + * the DQThread is stopped; else, the DQ Thread continues running. + */ + typedef bool(*dqThreadCallback) (struct v4l2_buffer * v4l2_buf, + NvBuffer * buffer, NvBuffer * shared_buffer, + void *data); + + /** + * Sets the DQ Thread callback method. + * + * The callback method is called from the DQ Thread once a buffer is + * successfully dequeued. + + * @param[in] callback Method to be called upon succesful dequeue. + * @returns TRUE for success, FALSE for failure. + */ + bool setDQThreadCallback(dqThreadCallback callback); + /** + * Starts DQ Thread. + * + * This method starts a thread internally. On successful dequeue of a + * buffer from the plane, the #dqThreadCallback method set using + * #setDQThreadCallback is called. + * + * Setting the stream to off automatically stops the thread. + * + * @sa stopDQThread, waitForDQThread + * + * @param[in] data A pointer to the application data. This is provided as an + * argument in the \c dqThreadCallback method. + * @return 0 for success, -1 otherwise. + */ + int startDQThread(void *data); + /** + * Force stops the DQ Thread if it is running. + * + * Does not work when the device is opened in blocking mode. + * + * @sa startDQThread, waitForDQThread + * + * @return 0 for success, -1 otherwise. + */ + int stopDQThread(); + /** + * Waits for the DQ Thread to stop. + * + * This method waits until the DQ Thread stops or timeout is reached. + * + * @sa startDQThread, stopDQThread + * + * @param[in] max_wait_ms Maximum wait time, in milliseconds. + * @return 0 for success, -1 otherwise. + */ + int waitForDQThread(uint32_t max_wait_ms); + + pthread_mutex_t plane_lock; /**< Mutex lock used along with #plane_cond. */ + pthread_cond_t plane_cond; /**< Plane condition that application can wait on + to receive notifications from + #qBuffer/#dqBuffer. */ + +private: + int &fd; /**< A reference to the FD of the V4l2 Element the plane is associated with. */ + + const char *plane_name; /**< A pointer to the name of the plane. Could be "Output Plane" or + "Capture Plane". Used only for debug logs. */ + enum v4l2_buf_type buf_type; /**< Speciifes the type of the stream. */ + + bool blocking; /**< Specifies whether the V4l2 element is opened with + blocking mode. */ + + uint32_t num_buffers; /**< Holds the number of buffers returned by \c VIDIOC_REQBUFS + IOCTL. */ + NvBuffer **buffers; /**< A pointer to an array of NvBuffer object pointers. This array is + allocated and initialized in #reqbufs. */ + + uint8_t n_planes; /**< Specifies the number of planes in the buffers. */ + NvBuffer::NvBufferPlaneFormat planefmts[MAX_PLANES]; + /**< Format of the buffer planes. This must be + initialized before calling #reqbufs since this + is required by the \c %NvBuffer constructor. */ + + enum v4l2_memory memory_type; /**< Specifies the V4l2 memory type of the buffers. */ + + uint32_t num_queued_buffers; /**< Holds the number of buffers currently queued on the + plane. */ + uint32_t total_queued_buffers; /**< Holds the total number of buffers queued on the + plane. */ + uint32_t total_dequeued_buffers; /**< Holds the total number of buffers dequeued from + the plane. */ + + bool streamon; /**< Specifies whether the plane is streaming. */ + + bool dqthread_running; /**< Specifies whether DQ Thread is running. + Its value is toggled by the DQ Thread. */ + bool stop_dqthread; /**< Specifies the value used to signal the DQ Thread to stop. */ + + pthread_t dq_thread; /**< Speciifes the pthread ID of the DQ Thread. */ + + dqThreadCallback callback; /**< Specifies the callback method used by the DQ Thread. */ + + void *dqThread_data; /**< Application supplied pointer provided as an + argument in #dqThreadCallback. */ + + /** + * The DQ thread method. + * + * This method runs indefinitely until it is signaled to stop + * by #stopDQThread or the #dqThreadCallback method returns FALSE. + * It keeps on trying to dequeue a buffer from the plane and calls the + * #dqThreadCallback method on successful dequeue. + * + * @param[in] v4l2_element_plane A pointer to the NvV4l2ElementPlane object + * for which the thread started. + */ + static void *dqThread(void *v4l2_element_plane); + + NvElementProfiler &v4l2elem_profiler; /**< A reference to the profiler belonging + to the plane's parent element. */ + + /** + * Indicates whether the plane encountered an error during its operation. + * + * @return 0 if no error was encountered, a non-zero value if an + * error was encountered. + */ + inline int isInError() + { + return is_in_error; + } + + /** + * Creates a new V4l2Element plane. + * + * + * @param[in] buf_type Type of the stream. + * @param[in] device_name A pointer to the name of the element the plane belongs to. + * @param[in] fd A reference to the FD of the device opened using v4l2_open. + * @param[in] blocking A flag that indicates whether the device has been opened with blocking mode. + * @param[in] profiler The profiler. + */ + NvV4l2ElementPlane(enum v4l2_buf_type buf_type, const char *device_name, + int &fd, bool blocking, NvElementProfiler &profiler); + + /** + * Disallows copy constructor. + */ + NvV4l2ElementPlane(const NvV4l2ElementPlane& that); + /** + * Disallows assignment. + */ + void operator=(NvV4l2ElementPlane const&); + + /** + * NvV4l2ElementPlane destructor. + * + * Calls #deinitPlane internally. + */ + ~NvV4l2ElementPlane(); + + int is_in_error; /**< Indicates if an error was encountered during + the operation of the element. */ + const char *comp_name; /**< Specifies the name of the component, + for debugging. */ + + friend class NvV4l2Element; +}; +/** @} */ +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvVideoEncoder.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvVideoEncoder.h new file mode 100644 index 0000000000..ca8f088868 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvVideoEncoder.h @@ -0,0 +1,930 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: Video Encode API + * + */ + +/** + * @defgroup l4t_mm_nvvideoencoder_group Video Encoder + * @ingroup l4t_mm_nvvideo_group + * + * Helper class that creates new V4L2 + * video encoders, and it sets encoder capture and output plane + * formats. + * @{ + */ + +#ifndef __NV_VIDEO_ENCODER_H__ +#define __NV_VIDEO_ENCODER_H__ + +#include "NvV4l2Element.h" + +/** + * @brief Defines a helper class for V4L2 Video Encoder. + * + * The video encoder device node is \c "/dev/nvhost-msenc". The category name + * for the encoder is \c "NVENC". + * + * Refer to [V4L2 Video Encoder](group__V4L2Enc.html) for more information on the encoder. + */ + +class NvVideoEncoder:public NvV4l2Element +{ +public: + /** + * Creates a new V4L2 Video Encoder object named \a name. + * + * This method internally calls \c v4l2_open on the encoder dev node + * \c "/dev/nvhost-msenc" and checks for \c V4L2_CAP_VIDEO_M2M_MPLANE + * capability on the device. This method allows the caller to specify + * additional flags with which the device must be opened. + * + * The device is opened in blocking mode, which can be modified by passing + * the @a O_NONBLOCK flag to this method. + * + * @returns Reference to the newly created encoder object, else NULL in + * case of failure during initialization. + */ + static NvVideoEncoder *createVideoEncoder(const char *name, int flags = 0); + + ~NvVideoEncoder(); + + /** + * Sets the format on the encoder output plane. + * + * Calls \c VIDIOC_S_FMT IOCTL internally on the output plane. + * + * @pre Applications must set the capture plane format using #setCapturePlaneFormat before calling this method. + * + * @param[in] pixfmt One of the raw V4L2 pixel formats. + * @param[in] width Width of the input buffers in pixels. + * @param[in] height Height of the input buffers in pixels. + * @return 0 for success, -1 otherwise. + */ + int setOutputPlaneFormat(uint32_t pixfmt, uint32_t width, uint32_t height, + enum v4l2_colorspace cs = V4L2_COLORSPACE_SMPTE170M); + + /** + * CUDA Enc: Sets the format on the encoder output plane. + * + * Calls \c VIDIOC_S_FMT IOCTL internally on the output plane. + * + * @pre Applications must set the capture plane format using #setCapturePlaneFormat before calling this method. + * + * @param[in] pixfmt One of the raw V4L2 pixel formats. + * @param[in] width Width of the input buffers in pixels. + * @param[in] height Height of the input buffers in pixels. + * @param[in] cs Colorspace of the input buffers. + * @param[in] extended Color Range the input buffers. + * @return 0 for success, -1 otherwise. + */ + int setOutputPlaneFormat(uint32_t pixfmt, uint32_t width, uint32_t height, + v4l2_colorspace cs, bool extended); + + /** + * Sets the format on the converter capture plane. + * + * Calls \c VIDIOC_S_FMT IOCTL internally on the capture plane. + * + * @param[in] pixfmt One of the coded V4L2 pixel formats. + * @param[in] width Width of the input buffers in pixels. + * @param[in] height Height of the input buffers in pixels. + * @param[in] sizeimage Maximum size of the encoded buffers on the capture. + * plane in bytes + * @return 0 for success, -1 otherwise. + */ + int setCapturePlaneFormat(uint32_t pixfmt, uint32_t width, + uint32_t height, uint32_t sizeimage); + + /** + * Sets the encode framerate. + * + * Calls the VIDIOC_S_PARM IOCTL on the encoder capture plane. Can be + * called any time after setFormat on both the planes. + * + * @param[in] framerate_num Numerator part of the framerate fraction. + * @param[in] framerate_den Denominator part of the framerate fraction. + * + * @return 0 for success, -1 otherwise. + */ + int setFrameRate(uint32_t framerate_num, uint32_t framerate_den); + + /** + * Sets the encoder bitrate. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_BITRATE. Can be called any time after setFormat on + * both the planes. + * + * @param[in] bitrate Bitrate of the encoded stream, in bits per second. + * + * @return 0 for success, -1 otherwise. + */ + int setBitrate(uint32_t bitrate); + + /** + * Sets the encoder peak bitrate. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_BITRATE_PEAK. Can be called any time after setFormat on + * both the planes. Takes effect in VBR mode + * + * @param[in] peak_bitrate Peak Bitrate of the encoded stream, in bits per second. + * + * @return 0 for success, -1 otherwise. + */ + int setPeakBitrate(uint32_t peak_bitrate); + + /** + * Sets the encoder profile. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_H264_PROFILE or #V4L2_CID_MPEG_VIDEO_H265_PROFILE, + * depending on the encoder type. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] profile Profile to be used for encoding. + * + * @return 0 for success, -1 otherwise. + */ + int setProfile(uint32_t profile); + + /** + * Sets the encoder command. + * + * Calls the VIDIOC_ENCODER_CMD internally with encoder commands. + * + * @return 0 for succes, -1 otherwise. + */ + int setEncoderCommand(int cmd, int flags); + + /** + * Set the encoder level. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_H264_LEVEL or #V4L2_CID_MPEG_VIDEOENC_H265_LEVEL, + * depending on encoder type. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] level Level to be used for encoding + * + * @return 0 for success, -1 otherwise. + */ + int setLevel(uint32_t level); + + /** + * Sets the encoder for maximum performance. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * V4L2_CID_MPEG_VIDEO_MAX_PERFORMANCE. Must be called after setFormat on both + * the planes and before requestBuffers on any of the planes. + * + * @param[in] flag Integer variable to indicate whether max performance is to be + * enabled/disabled. + * + * @return 0 for success, -1 otherwise. + */ + int setMaxPerfMode(int flag); + /** + * Sets constant QP for encoder. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE to enable/disable rate control. + * If the value false is given, it disables RC and set constant QP parameters. + * Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] enabled_rc Boolean value indicating whether to enable/disable + * the control. + * + * @return 0 for success, -1 otherwise. + */ + int setConstantQp(bool enabled_rc); + + /** + * Sets the encoder rate control mode. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_BITRATE_MODE. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] mode Type of rate control, one of enum + * v4l2_mpeg_video_bitrate_mode. + * + * @return 0 for success, -1 otherwise. + */ + int setRateControlMode(enum v4l2_mpeg_video_bitrate_mode mode); + + /** + * Sets the encoder I-frame interval. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_GOP_SIZE. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] interval Interval between two I frames, in number of frames. + * + * @return 0 for success, -1 otherwise. + */ + int setIFrameInterval(uint32_t interval); + + /** + * Sets the encoder IDR interval. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEO_IDR_INTERVAL. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] interval Interval between two IDR frames, in number of frames. + * + * @return 0 for success, -1 otherwise. + */ + int setIDRInterval(uint32_t interval); + + /** + * Forces an IDR frame. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE. Must be called after + * setFormat on both the planes. + * + * @return 0 for success, -1 otherwise. + */ + int forceIDR(); + + /** + * Sets the encoder Temporal Tradeoff. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_TEMPORAL_TRADEOFF_LEVEL. Must be called after + * setFormat on both the planes and before \c requestBuffers on any of the + * planes. + * + * @param[in] level Temporal tradeoff level, one of + * v4l2_enc_temporal_tradeoff_level_type. + * @return 0 for success, -1 otherwise. + */ + int setTemporalTradeoff(v4l2_enc_temporal_tradeoff_level_type level); + + /** + * Sets the encoder output slice length. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_SLICE_LENGTH_PARAM. Must be called after setFormat on + * both the planes and before \c requestBuffers on any of the planes. + * + * @param[in] type Slice length type, one of enum v4l2_enc_slice_length_type. + * @param[in] length Length of the slice, in bytes if the type is + * #V4L2_ENC_SLICE_LENGTH_TYPE_BITS, else in number of MBs if the type is + * #V4L2_ENC_SLICE_LENGTH_TYPE_MBLK. + * @return 0 for success, -1 otherwise. + */ + int setSliceLength(v4l2_enc_slice_length_type type, uint32_t length); + + /** + * Sets the encoder HW Preset Type. + * + * Calls the VIDIOC_S_EXT_CTRLS ioctl internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_HW_PRESET_TYPE_PARAM. Must be called after setFormat() on + * both the planes and before \c requestBuffers on any of the planes. + * + * @param[in] type HW Preset Type, one of + * enum v4l2_enc_hw_preset_type_param. + * @return 0 for success, -1 otherwise. + */ + int setHWPresetType(v4l2_enc_hw_preset_type type); + + /** + * Sets the Region of Interest (ROI) parameters for the next buffer, which will + * be queued on the output plane with index \a buffer_index. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ROI_PARAMS. Must be called after + * requesting buffer on both the planes. + * + * @param[in] buffer_index Index of the output plane buffer to apply the ROI + * params. + * @param[in] params A reference to the parameters to be applied on the frame, structure of + * type v4l2_enc_frame_ROI_params. + * @return 0 for success, -1 otherwise. + */ + int setROIParams(uint32_t buffer_index, v4l2_enc_frame_ROI_params & params); + + /** + * Sets the preprocessing enhancement (PPE) initialization parameters at the start of the session + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_PPE_INIT_PARAMS. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params A reference to the parameters to be applied on the frame, structure of + * type v4l2_enc_ppe_init_params. + * @return 0 for success, -1 otherwise. + */ + int setPPEInitParams(v4l2_enc_ppe_init_params & params); + + /** + * Enables External ROI. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_ROI_PARAM. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params Parameters to be applied on the frame, structure of + * type #v4l2_enc_enable_roi_param. + * @return 0 for success, -1 otherwise. + */ + int enableROI(v4l2_enc_enable_roi_param ¶ms); + + /** + * Enables Recon CRC. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_RECONCRC_PARAM. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params Parameters to be applied on the frame, structure of + * type #v4l2_enc_enable_reconcrc_param. + * @return 0 for success, -1 otherwise. + */ + int enableReconCRC(v4l2_enc_enable_reconcrc_param ¶ms); + + /** + * Enable External RPS + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_EXTERNAL_RPS_CONTROL. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params Parameters to be applied on the frame, structure of + * type #v4l2_enc_enable_ext_rps_ctr + * @return 0 for success, -1 otherwise. + */ + int enableExternalRPS(v4l2_enc_enable_ext_rps_ctr ¶ms); + + /** + * Enable External Picture RC + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_EXTERNAL_RATE_CONTROL. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params Parameters to be applied on the frame, structure of + * type ##v4l2_enc_enable_ext_rate_ctr + * @return 0 for success, -1 otherwise. + */ + int enableExternalRC(v4l2_enc_enable_ext_rate_ctr ¶ms); + + /** + * Disable SAO filter for HEVC encoding. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_H265_DISABLE_SAO. Must be called after + * setFormat on both the planes. Disabled by default. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setH265DisableSAO(bool enabled); + + /** + * Enable AV1 Multi-tile configuration + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_TILE_CONFIGURATION. Must be called after + * setFormat on both the planes and before \c requestBuffers on any of the + * planes. + * + * @param[in] params Parameters to be applied, structure of + * type ##v4l2_enc_av1_tile_config + * @return 0 for success, -1 otherwise. + */ + int enableAV1Tile(v4l2_enc_av1_tile_config ¶ms); + + /** + * Enable AV1 variance based SSIM RDO. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_SSIMRDO. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAV1SsimRdo(bool enabled); + + /** + * Disable AV1 CDF update in the symbol decoding process. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_DISABLE_CDF_UPDATE. The value true + * disables the CDF update. Must be called after setFormat on both + * the planes. + * + * @param[in] disabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAV1DisableCDFUpdate(bool disabled); + + /** + * Enable Error Resilient Mode for AV1. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_ERR_RESILIENT_MODE. The value true + * disables the control. Must be called after setFormat on both + * the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAV1ErrResilienceMode(bool enabled); + + /** + * Enable AV1 frame_id_numbers_present_flag. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_FRAMEID_NUMBERS. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAV1FrameIDPresentFlag(bool enabled); + + /** + * Enable AV1 frame_id_numbers_present_flag. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_TILE_GROUPS. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int enableAV1TileGroups(bool enabled); + + /** + * Set input Metadata parameters for the next buffer which will + * be queued on output plane with index \a buffer_index + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_INPUT_METADATA. Must be called after + * requesting buffer on both the planes. + * + * @param[in] buffer_index Index of output plane buffer on which the external + * RC params should be applied. + * @param[in] params Parameters to be applied on the frame, structure of + * type #v4l2_ctrl_videoenc_input_metadata + * @return 0 for success, -1 otherwise. + */ + int SetInputMetaParams(uint32_t buffer_index, v4l2_ctrl_videoenc_input_metadata ¶ms); + + /** + * Sets the virtual buffer size of the encoder. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_VIRTUALBUFFER_SIZE. Must be called after + * setFormat on both the planes. + * + * @param[in] size Virtual buffer size, in bytes. + * @return 0 for success, -1 otherwise. + */ + int setVirtualBufferSize(uint32_t size); + + /** + * Sets the number of reference frames of the encoder. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_NUM_REFERENCE_FRAMES. Must be called after + * setFormat on both the planes. + * + * @param[in] num_frames Number of reference frames. + * @return 0 for success, -1 otherwise. + */ + int setNumReferenceFrames(uint32_t num_frames); + + /** + * Sets slice intra-refresh interval params. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_SLICE_INTRAREFRESH_PARAM. Must be called after + * setFormat on both the planes. + * + * @param[in] interval Slice intra-refresh interval, in number of slices. + * @return 0 for success, -1 otherwise. + */ + int setSliceIntrarefresh(uint32_t interval); + + /** + * Sets CUDA slice intra-refresh interval params. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_SLICE_INTRAREFRESH_PARAM. Must be called after + * setFormat on both the planes. + * + * @param[in] interval Slice intra-refresh interval. + * @param[in] count Slice intra-refresh count, in number of slices. + * @return 0 for success, -1 otherwise. + */ + int setCUDASliceIntrarefresh(uint32_t interval, uint32_t count); + + /** + * Sets the number of B frames to P frames. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_NUM_BFRAMES. Must be called after + * setFormat on both the planes. + * + * @param[in] num Number of B frames. + * @return 0 for success, -1 otherwise. + */ + int setNumBFrames(uint32_t num); + + /** + * Enables/disables insert SPS PPS at every IDR. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_INSERT_SPS_PPS_AT_IDR. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setInsertSpsPpsAtIdrEnabled(bool enabled); + + /** + * Enables/disables CABAC entropy encoding for H264. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setCABAC(bool enabled); + + /** + * Enables/disables Slice level encoding for H264 / HEVC. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_SLICE_LEVEL_ENCODE. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setSliceLevelEncode(bool enabled); + + /** + * Enables video encoder output motion vector metadata reporting. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_METADATA_MV. Must be called after setFormat on + * both the planes and before \c requestBuffers on any of the planes. + * + * @return 0 for success, -1 otherwise. + */ + int enableMotionVectorReporting(); + + /** + * Gets metadata for the encoded capture plane buffer. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_METADATA. Must be called for a buffer that has + * been dequeued from the capture plane. The returned metadata corresponds + * to the last dequeued buffer with index @a buffer_index. + * + * @param[in] buffer_index Index of the capture plane buffer whose metadata + * is required. + * @param[in,out] enc_metadata Reference to the metadata structure + * v4l2_ctrl_videoenc_outputbuf_metadata to be filled. + * + * @return 0 for success, -1 otherwise. + */ + int getMetadata(uint32_t buffer_index, + v4l2_ctrl_videoenc_outputbuf_metadata &enc_metadata); + + /** + * Gets motion vector metadata for the encoded capture plane buffer. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_METADATA_MV. Must be called for a buffer that has + * been dequeued from the capture plane. The returned metadata corresponds + * to the last dequeued buffer with index @a buffer_index. + * + * @param[in] buffer_index Index of the capture plane buffer whose metadata + * is required. + * @param[in,out] enc_mv_metadata Reference to the metadata structure + * v4l2_ctrl_videoenc_outputbuf_metadata_MV to be filled. + * + * @return 0 for success, -1 otherwise. + */ + int getMotionVectors(uint32_t buffer_index, + v4l2_ctrl_videoenc_outputbuf_metadata_MV &enc_mv_metadata); + + /** + * Sets QP values for I/P/B frames. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * %V4L2_CID_MPEG_VIDEOENC_QP_RANGE. Must be called after + * setFormat on both the planes. + * + * @param[in] MinQpI Minimum Qp Value for I frame. + * @param[in] MaxQpI Minimum Qp Value for I frame. + * @param[in] MinQpP Minimum Qp Value for P frame. + * @param[in] MaxQpP Minimum Qp Value for P frame. + * @param[in] MinQpB Minimum Qp Value for B frame. + * @param[in] MaxQpB Minimum Qp Value for B frame. + * @returns 0 for success, -1 otherwise. + */ + int setQpRange(uint32_t MinQpI, uint32_t MaxQpI, uint32_t MinQpP, + uint32_t MaxQpP, uint32_t MinQpB, uint32_t MaxQpB); + + /** + * Sets Const QP values for I/P/B frames. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * %V4L2_CID_MPEG_VIDEOENC_CUDA_CONSTQP. Must be called after + * setFormat on both the planes. + * + * @param[in] qpI Qp Value for I frame. + * @param[in] qpP Qp Value for P frame. + * @param[in] qpB Qp Value for B frame. + * @returns 0 for success, -1 otherwise. + */ + int setCudaConstantQp(uint32_t qpI, uint32_t qpP, + uint32_t qpB); + + /** + * Sets Sample Aspect Ratio width for VUI encoding. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * \c V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_WIDTH or + * #V4L2_CID_MPEG_VIDEOENC_H265_VUI_EXT_SAR_WIDTH, depending on the + * encoder type. Must be called after setFormat on both the planes. + * + * @param[in] sar_width SAR for width. + * @returns 0 for success, -1 otherwise. + */ + int setSampleAspectRatioWidth(uint32_t sar_width); + + /** + * Sets Sample Aspect Ratio height for VUI encoding. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * \c V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_HEIGHT or + * #V4L2_CID_MPEG_VIDEOENC_H265_VUI_EXT_SAR_HEIGHT, depending on the + * encoder type. Must be called after setFormat on both the planes. + * + * @param[in] sar_height SAR for height. + * @returns 0 for success, -1 otherwise. + */ + int setSampleAspectRatioHeight(uint32_t sar_height); + + /** + * Enables/disables insert VUI. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * @c V4L2_CID_MPEG_VIDEOENC_INSERT_VUI. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setInsertVuiEnabled(bool enabled); + + /** + * Enables/disables extended color format. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * @c V4L2_CID_MPEG_VIDEOENC_EXTEDED_COLORFORMAT. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setExtendedColorFormat(bool enabled); + + /** + * Enables/disables insert AUD. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * @c V4L2_CID_MPEG_VIDEO_H264_AUD_SAR_ENABLE. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setInsertAudEnabled(bool enabled); + + /** + * Enables/disables all i-frame encode. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * @c V4L2_CID_MPEG_VIDEOENC_ENABLE_ALLIFRAME_ENCODE. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAlliFramesEncode(bool enabled); + + /** + * Sets the encoder Picture Order Control type. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_POC_TYPE. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] pocType Set the Picture Order Count for the encoder. + * + * @return 0 for success, -1 otherwise. + */ + int setPocType(uint32_t pocType); + + /** + * Sets the initial QP for I/P/B frames. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * %V4L2_CID_MPEG_VIDEOENC_INIT_FRAME_QP. Must be called after + * setFormat on both the planes. + * + * + * @param[in] IinitQP Qp Value for I frame. + * @param[in] PinitQP Qp Value for P frame. + * @param[in] BinitQP Qp Value for B frame. + * + * @return 0 for success, -1 otherwise. + */ + int setInitQP(uint32_t IinitQP, uint32_t PinitQP, uint32_t BinitQP); + + /** + * Sets the number of frames to encode. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_FRAMES_TO_ENCODE. Must be called after + * setFormat on both the planes. + * + * @param[in] framesToEncode Set the number of frames to encode. + * + * @return 0 for success, -1 otherwise. + */ + int setFramesToEncode(uint32_t framesToEncode); + + /** + * Sets the H.265 encoder Chroma Format IDC. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_H265_CHROMA_FACTOR_IDC. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] crfactor Set @c chroma_factor_idc for the encoder. + * + * @return 0 for success, -1 otherwise. + */ + int setChromaFactorIDC(uint8_t crfactor); + + /** + * Sets the lossless encoding for H.264/H.265. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_LOSSLESS. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] enabled Indicates whether to enable or disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setLossless(bool enabled); + + /** + * Sets the Disable AMP types to true for H.265. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_H265_DISABLE_AMP. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] enabled Indicates whether to enable or disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setDisableAMP(bool enabled); + + /** + * Sets whether to enable two pass CBR for H.264/H.265. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_TWO_PASS_CBR. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @return 0 for success, -1 otherwise. + */ + int enableTwoPassCBR(); + + /** + * Issues Poll on the device which blocks until : + * a) Either there is something to dequeue from capture or output plane or any events. + * b) Poll was interrupted by a call to the device using V4L2_CID_SET_POLL_INTERRUPT + * c) Application has already interrupted polling by V4L2_CID_SET_POLL_INTERRUPT + */ + int DevicePoll(v4l2_ctrl_video_device_poll *devicepoll); + + /** + * Sets the polling interrupt, now if the application calls Poll, the device should + * not block, in other words polling is disabled. + */ + int SetPollInterrupt(); + + /** + * Clears the polling interrupt, now if the application calls Poll, the device should + * block until the event is triggered, in other words polling is enabled. + */ + int ClearPollInterrupt(); + + /** + * Checks if GPU configuration is enabled to switch to CUDA processing + */ + bool isGPUEnabled(); + + /** + * Sets GPU Id when GPU is enabled. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEO_CUDA_GPU_ID. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @return 0 for success, -1 otherwise. + */ + int setGPUId(uint32_t gpuId); + + /** + * Sets CUDA Memory type when GPU is enabled. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEO_CUDA_MEM_TYPE. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @return 0 for success, -1 otherwise. + */ + int setCudaMemType(uint32_t cumemType); + +private: + /** + * Constructor used by #createVideoEncoder. + */ + NvVideoEncoder(const char *name, const char* dev_node, int flags); + + static const NvElementProfiler::ProfilerField valid_fields = + NvElementProfiler::PROFILER_FIELD_TOTAL_UNITS | + NvElementProfiler::PROFILER_FIELD_LATENCIES | + NvElementProfiler::PROFILER_FIELD_FPS; + + bool gpu_enabled; +}; +/** @} */ + +#endif From 5934524a728d79c5c84aa5ad0abb898e6840b76f Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 12:17:49 -0500 Subject: [PATCH 27/29] docs: the Multimedia API tree no longer supplies the encoder classes The user guide still said the build takes samples/common/classes from JETSON_MMAPI_DIR. The classes are copied into the repository now, so the tree only has to provide nvbufsurface.h. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- docs/source/device/camera/sensing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/device/camera/sensing.rst b/docs/source/device/camera/sensing.rst index 95118b6055..dcc820b305 100644 --- a/docs/source/device/camera/sensing.rst +++ b/docs/source/device/camera/sensing.rst @@ -170,8 +170,8 @@ the LiveKit server. The build also needs two header trees; neither is fetched by ``setup.sh``: - ``JETSON_MMAPI_DIR``: the Jetson Multimedia API tree, containing - ``include/NvVideoEncoder.h`` and ``samples/common/classes/``. CMake searches - ``/usr/src/jetson_multimedia_api``. + ``include/nvbufsurface.h``. The encoder classes are copied into the repository + and are not taken from it. CMake searches ``/usr/src/jetson_multimedia_api``. - ``SIPL_API_DIR``: the Jetson SIPL API tree, containing ``sipl/include/NvSIPLCamera.hpp`` (the ``Jetson_SIPL_API_R_aarch64.tbz2`` package published with the L4T release). CMake searches ``/usr/src/jetson_sipl_api``, From 33b09ed16b66c06ffe3298089a2c8a6530ce2f47 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 14:05:53 -0500 Subject: [PATCH 28/29] sensing: add a script that sets up cross-compiling the producer on a PC setup_orin_crosscompile.py builds the Orin sysroot without sudo from the Jetson Linux, Multimedia API and SIPL API packages and Ubuntu 24.04's arm64 libraries, downloads Arm's aarch64 GNU toolchain, and then type-checks the producer's sources or cross-builds camera_plugin_sensing for aarch64 through a small generated project, using the CUDA aarch64 cross target. Downloads are checked against the publishers' SHA-256, and spdlog is fetched at the revision the repository pins. The user guide describes it. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- docs/source/device/camera/sensing.rst | 20 + src/plugins/sensing/README.md | 4 +- .../scripts/setup_orin_crosscompile.py | 726 ++++++++++++++++++ 3 files changed, 748 insertions(+), 2 deletions(-) create mode 100755 src/plugins/sensing/scripts/setup_orin_crosscompile.py diff --git a/docs/source/device/camera/sensing.rst b/docs/source/device/camera/sensing.rst index dcc820b305..24a375482b 100644 --- a/docs/source/device/camera/sensing.rst +++ b/docs/source/device/camera/sensing.rst @@ -204,3 +204,23 @@ use a ``wss://`` endpoint on any other network. ``SENSING_TSC_PROBE=1`` logs the SIPL capture counter beside the host clocks twice a second, as separate uncalibrated samples. The counter's units are unconfirmed, so do not subtract or fit them. + +Cross-compile on a PC +--------------------- + +``scripts/setup_orin_crosscompile.py`` builds the producer for the Orin on an x86 Ubuntu PC, without sudo and +without touching the Orin. Everything goes under one directory (``--root``, default ``~/orin-cross-39.2.1``). + +.. code-block:: bash + + src/plugins/sensing/scripts/setup_orin_crosscompile.py doctor # what is present, what is missing + src/plugins/sensing/scripts/setup_orin_crosscompile.py fetch # sysroot and aarch64 compiler + src/plugins/sensing/scripts/setup_orin_crosscompile.py syntax # type-check every source in seconds + src/plugins/sensing/scripts/setup_orin_crosscompile.py build # camera_plugin_sensing for aarch64 + +``fetch`` unpacks the L4T libraries from the Jetson Linux archive, and the SIPL API, from the SDK Manager +download directory (``--sdk-downloads``). It takes the Multimedia API from there or from NVIDIA's apt repository, +the C library and EGL/V4L2 libraries from Ubuntu 24.04, spdlog at the revision the repository pins, and Arm's +GNU toolchain; each download is checked against the publisher's SHA-256. ``build`` also needs the CUDA toolkit +with its aarch64 cross target (``cuda-cross-sbsa``) and uses that toolkit's own ``nvcc``. The result is the same +binary a build on the Orin produces, but it has not been run: deploy it and test it on the Orin. diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index 3f2049d593..147e65c950 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -13,8 +13,8 @@ Start with the [SENSING user guide](https://nvidia.github.io/IsaacCapture/main/d for supported hardware, installation, container usage, and troubleshooting. That guide owns the operating instructions; this README maps the source tree. -- [`scripts/`](scripts/): host setup, validation, diagnostics, and the Bash - compatibility Docker launcher. Each entry point exposes `--help`. +- [`scripts/`](scripts/): host setup, validation, diagnostics, the Bash + compatibility Docker launcher, and `setup_orin_crosscompile.py` for building on a PC. Each entry point exposes `--help`. - [`docker/`](docker/): example image, image-side group configuration, and host-side container setup. - [`configs/`](configs/): the platform configuration and its provenance. diff --git a/src/plugins/sensing/scripts/setup_orin_crosscompile.py b/src/plugins/sensing/scripts/setup_orin_crosscompile.py new file mode 100755 index 0000000000..0b544fbce9 --- /dev/null +++ b/src/plugins/sensing/scripts/setup_orin_crosscompile.py @@ -0,0 +1,726 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Set up compiling the SENSING camera producer for a Jetson Orin on an x86 PC, without root. + + setup_orin_crosscompile.py doctor what is present and what is missing + setup_orin_crosscompile.py fetch build the Orin sysroot and download the aarch64 compiler + setup_orin_crosscompile.py syntax type-check every producer source against the Orin headers (fast) + setup_orin_crosscompile.py toolchain write the CMake toolchain file + setup_orin_crosscompile.py build configure and build camera_plugin_sensing for aarch64 + +Everything lands under one directory (``--root``, default ``~/orin-cross-``); nothing outside it +changes and no step needs sudo. The sysroot is unpacked from NVIDIA's Jetson Linux package and the Multimedia and +SIPL API packages, which ``fetch`` takes from the SDK Manager download directory (``--sdk-downloads``) when they +are there and otherwise from NVIDIA's apt repository, checking the repository's SHA-256 for each. The compiler is +Arm's GNU toolchain, checked against the SHA-256 Arm publishes beside it. + +``syntax`` needs only clang and the fetched headers, and catches the compile errors a build on the Orin would. +``build`` also needs the CUDA cross toolchain for aarch64 (``cuda-cross-sbsa``), CMake and network access for +the dependencies CMake fetches. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import shutil +import subprocess +import sys +import tarfile +import urllib.request +from pathlib import Path + +RELEASE = "39.2.1" +L4T_REPO = "https://repo.download.nvidia.com/jetson/common" +L4T_SUITE = "r39.2" +ARM_GCC_VERSION = "13.3.rel1" +ARM_GCC_NAME = f"arm-gnu-toolchain-{ARM_GCC_VERSION}-x86_64-aarch64-none-linux-gnu" +ARM_GCC_URL = f"https://developer.arm.com/-/media/Files/downloads/gnu/{ARM_GCC_VERSION}/binrel/{ARM_GCC_NAME}.tar.xz" +JETSON_LINUX = f"Jetson_Linux_R{RELEASE}_aarch64.tbz2" +SIPL_PACKAGE = f"Jetson_SIPL_API_R{RELEASE}_aarch64.tbz2" +MMAPI_PACKAGE = "nvidia-l4t-jetson-multimedia-api" +# The L4T packages inside the Jetson Linux archive that hold the libraries and headers the producer links against. +L4T_DEBS = ( + "nvidia-l4t-core", + "nvidia-l4t-camera", + "nvidia-l4t-multimedia-utils", + "nvidia-l4t-multimedia-nvgpu", + "nvidia-l4t-nvsci", + "nvidia-l4t-3d-core", + "nvidia-l4t-cuda-nvgpu", + "nvidia-l4t-cuda-utils", + "nvidia-l4t-wayland", +) +# Ubuntu's own arm64 packages for the C library and the libraries the build finds by name; the Orin runs Ubuntu 24.04, +# so these match its glibc. Indexes are read over https and each file is checked against its index's SHA-256. +UBUNTU_PORTS = "https://ports.ubuntu.com/ubuntu-ports" +UBUNTU_SUITES = ("noble-updates", "noble-security", "noble") +UBUNTU_DEBS = ( + "libc6", + "libc6-dev", + "linux-libc-dev", + "libgcc-s1", + "libstdc++6", + "libegl1", + "libegl-dev", + "libglvnd0", + "libv4l-0t64", + "libv4l-dev", + "libv4lconvert0t64", +) +# What the build needs from the sysroot: (name shown, glob under the sysroot). +SYSROOT_LIBS = ( + ("libnvsipl", "usr/lib/aarch64-linux-gnu/nvidia/libnvsipl.so"), + ("libnvsipl_query", "usr/lib/aarch64-linux-gnu/nvidia/libnvsipl_query.so"), + ("libnvscibuf", "usr/lib/aarch64-linux-gnu/nvidia/libnvscibuf.so*"), + ("libnvscisync", "usr/lib/aarch64-linux-gnu/nvidia/libnvscisync.so*"), + ("libnvbufsurface", "usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurface.so"), + ( + "libnvbufsurface_nvsci", + "usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurface_nvsci.so", + ), + ( + "libnvargus_socketclient", + "usr/lib/aarch64-linux-gnu/nvidia/libnvargus_socketclient.so", + ), + ("libcuda", "opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so"), + ("libc (crt1.o)", "usr/lib/aarch64-linux-gnu/crt1.o"), + ("libEGL", "usr/lib/aarch64-linux-gnu/libEGL.so"), + ("libv4l2", "usr/lib/aarch64-linux-gnu/libv4l2.so*"), +) +SYSROOT_HEADERS = ( + ( + "NvSIPLCamera.hpp (SIPL API)", + "usr/src/jetson_sipl_api/sipl/include/NvSIPLCamera.hpp", + ), + ( + "nvbufsurface.h (Multimedia API)", + "usr/src/jetson_multimedia_api/include/nvbufsurface.h", + ), + ( + "Argus/Argus.h (Multimedia API)", + "usr/src/jetson_multimedia_api/argus/include/Argus/Argus.h", + ), +) +CUDA_CROSS_ROOT = Path("/usr/local/cuda/targets/sbsa-linux") + + +class SetupError(Exception): + """A step that cannot continue; the message says what to do.""" + + +def say(message: str = "") -> None: + """Progress and diagnostics go to stderr, like the other scripts here.""" + print(message, file=sys.stderr, flush=True) + + +def repo_root() -> Path: + """The checkout this script lives in.""" + return Path(__file__).resolve().parents[4] + + +def sensing_dir() -> Path: + return repo_root() / "src" / "plugins" / "sensing" + + +class Layout: + """Where everything lives under the chosen root.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.sysroot = root / "sysroot" + self.toolchain_dir = root / ARM_GCC_NAME + self.cache = root / "downloads" + self.cmake_file = root / "orin-aarch64.cmake" + self.build_dir = root / "build" + + @property + def gxx(self) -> Path: + return self.toolchain_dir / "bin" / "aarch64-none-linux-gnu-g++" + + @property + def gcc(self) -> Path: + return self.toolchain_dir / "bin" / "aarch64-none-linux-gnu-gcc" + + @property + def spdlog(self) -> Path: + return self.root / "spdlog" + + +def sha256_of(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def download(url: str, target: Path, expected_sha256: str | None) -> Path: + """Fetch ``url`` to ``target`` (kept between runs) and verify its checksum; a bad file is never left in place.""" + if target.exists() and expected_sha256 and sha256_of(target) == expected_sha256: + return target + target.parent.mkdir(parents=True, exist_ok=True) + partial = target.with_suffix(target.suffix + ".part") + say(f" downloading {url}") + with urllib.request.urlopen(url, timeout=60) as response, partial.open("wb") as out: # noqa: S310 - fixed https URLs + shutil.copyfileobj(response, out, 1 << 20) + if expected_sha256 and sha256_of(partial) != expected_sha256: + partial.unlink() + raise SetupError( + f"{url}: the SHA-256 does not match the published {expected_sha256}" + ) + partial.replace(target) + return target + + +def fetch_text(url: str) -> str: + with urllib.request.urlopen(url, timeout=60) as response: # noqa: S310 - fixed https URLs + return response.read().decode() + + +def apt_index() -> dict[str, dict[str, str]]: + """NVIDIA's package index for this release: name -> {Version, Filename, SHA256} (the newest entry wins).""" + import gzip + + raw = urllib.request.urlopen( + f"{L4T_REPO}/dists/{L4T_SUITE}/main/binary-arm64/Packages.gz", timeout=60 + ).read() # noqa: S310 + packages: dict[str, dict[str, str]] = {} + for stanza in gzip.decompress(raw).decode().split("\n\n"): + fields = dict( + line.split(": ", 1) + for line in stanza.splitlines() + if ": " in line and not line.startswith(" ") + ) + if "Package" in fields: + packages.setdefault(fields["Package"], fields) + if fields.get("Version", "") > packages[fields["Package"]].get( + "Version", "" + ): + packages[fields["Package"]] = fields + return packages + + +def ubuntu_index() -> dict[str, dict[str, str]]: + """Ubuntu 24.04's arm64 package index: name -> fields, taking the entry from the first suite that lists it.""" + import gzip + + packages: dict[str, dict[str, str]] = {} + for suite in UBUNTU_SUITES: + for component in ("main", "universe"): + url = f"{UBUNTU_PORTS}/dists/{suite}/{component}/binary-arm64/Packages.gz" + raw = gzip.decompress( + urllib.request.urlopen(url, timeout=120).read() + ).decode() # noqa: S310 + for stanza in raw.split("\n\n"): + fields = dict( + line.split(": ", 1) + for line in stanza.splitlines() + if ": " in line and not line.startswith(" ") + ) + if ( + fields.get("Package") in UBUNTU_DEBS + and fields["Package"] not in packages + ): + packages[fields["Package"]] = fields + return packages + + +def extract_deb(deb: Path, destination: Path) -> None: + """Unpack a .deb's files under ``destination`` without root (dpkg-deb reads it; nothing is installed).""" + if not shutil.which("dpkg-deb"): + raise SetupError( + "dpkg-deb is needed to unpack the packages (it ships with every Debian or Ubuntu host)" + ) + destination.mkdir(parents=True, exist_ok=True) + subprocess.run(["dpkg-deb", "-x", str(deb), str(destination)], check=True) + + +def find_sdk_file(args: argparse.Namespace, pattern: str) -> Path | None: + """The first file in the SDK Manager download directory whose name matches ``pattern``.""" + directory = Path(args.sdk_downloads).expanduser() + matches = sorted(directory.glob(pattern)) if directory.is_dir() else [] + return matches[-1] if matches else None + + +def fetch_sysroot(args: argparse.Namespace, layout: Layout) -> None: + """Unpack the L4T libraries, the Multimedia API and the SIPL API into the sysroot.""" + layout.sysroot.mkdir(parents=True, exist_ok=True) + marker = layout.sysroot / ".fetched" + if marker.exists() and not args.force: + say( + f"sysroot already built ({marker.read_text().strip()}); --force rebuilds it" + ) + return + # 1. The L4T libraries, from the Jetson Linux archive. + archive = find_sdk_file(args, JETSON_LINUX) + if archive is None: + raise SetupError( + f"{JETSON_LINUX} is not in {args.sdk_downloads}. Download it with NVIDIA SDK Manager (or from the Jetson " + "Linux release page) and pass its folder as --sdk-downloads." + ) + say( + f"unpacking L4T libraries from {archive.name} (the archive is large; this takes a minute)" + ) + staging = layout.cache / "l4t_debs" + staging.mkdir(parents=True, exist_ok=True) + wanted = {f"{name}_{RELEASE}-": name for name in L4T_DEBS} + with tarfile.open(archive, "r:bz2") as tar: + for member in tar: + base = os.path.basename(member.name) + if "l4t_deb_packages" in member.name and any( + base.startswith(prefix) for prefix in wanted + ): + member.name = base + tar.extract(member, staging) + found = sorted(staging.glob("*.deb")) + missing = [ + name + for prefix, name in wanted.items() + if not any(d.name.startswith(prefix) for d in found) + ] + if missing: + raise SetupError(f"not in {archive.name}: {', '.join(missing)}") + for deb in found: + extract_deb(deb, layout.sysroot) + # 1b. The C library and the libraries found by name, from Ubuntu 24.04 (what the Orin runs). + index = ubuntu_index() + absent = [name for name in UBUNTU_DEBS if name not in index] + if absent: + raise SetupError(f"not in Ubuntu's arm64 index: {', '.join(absent)}") + for name in UBUNTU_DEBS: + entry = index[name] + deb = download( + f"{UBUNTU_PORTS}/{entry['Filename']}", + layout.cache / Path(entry["Filename"]).name, + entry["SHA256"], + ) + extract_deb(deb, layout.sysroot) + # 2. The Multimedia API: the SDK Manager copy when present, else NVIDIA's apt repository, checksum verified. + mmapi = find_sdk_file(args, f"{MMAPI_PACKAGE}_{RELEASE}-*_arm64.deb") + if mmapi is None: + entry = apt_index().get(MMAPI_PACKAGE) + if not entry or not entry["Version"].startswith(RELEASE): + raise SetupError( + f"{MMAPI_PACKAGE} {RELEASE} is not in the {L4T_SUITE} repository" + ) + mmapi = download( + f"{L4T_REPO}/{entry['Filename']}", + layout.cache / Path(entry["Filename"]).name, + entry["SHA256"], + ) + say(f"unpacking the Multimedia API from {mmapi.name}") + extract_deb(mmapi, layout.sysroot) + # 3. The SIPL API headers. + sipl = find_sdk_file(args, SIPL_PACKAGE) + if sipl is None: + raise SetupError( + f"{SIPL_PACKAGE} is not in {args.sdk_downloads}. Download it with NVIDIA SDK Manager or from the Jetson " + "Linux release page." + ) + say(f"unpacking the SIPL API from {sipl.name}") + with tarfile.open(sipl, "r:bz2") as tar: + tar.extractall(layout.sysroot, filter="data") + fix_sysroot_layout(layout.sysroot) + marker.write_text(f"L4T {RELEASE}\n") + + +def fix_sysroot_layout(sysroot: Path) -> None: + """Ubuntu's libraries sit under usr/ with /lib a link to usr/lib; unpacking debs alone does not make that link.""" + link = sysroot / "lib" + if not link.exists() and not link.is_symlink(): + link.symlink_to("usr/lib") + + +def fetch_compiler(layout: Layout, force: bool) -> None: + """Download Arm's aarch64 GNU toolchain next to the sysroot and check it against Arm's published SHA-256.""" + if layout.gxx.exists() and not force: + say(f"compiler already present: {layout.gxx}") + return + checksum_text = fetch_text(ARM_GCC_URL + ".sha256asc") + match = re.search(r"\b([0-9a-f]{64})\b", checksum_text) + if not match: + raise SetupError(f"could not read the SHA-256 published for {ARM_GCC_NAME}") + archive = download( + ARM_GCC_URL, layout.cache / f"{ARM_GCC_NAME}.tar.xz", match.group(1) + ) + say(f"unpacking {archive.name}") + with tarfile.open(archive, "r:xz") as tar: + tar.extractall(layout.root, filter="data") + + +def pinned_spdlog_commit() -> str: + """The spdlog revision the repository's own build fetches, so the type check sees the same headers.""" + text = (repo_root() / "deps" / "third_party" / "CMakeLists.txt").read_text() + match = re.search(r"spdlog\.git\s+GIT_TAG\s+([0-9a-f]{40})", text) + if not match: + raise SetupError( + "could not read the pinned spdlog revision from deps/third_party/CMakeLists.txt" + ) + return match.group(1) + + +def fetch_spdlog(layout: Layout, force: bool) -> None: + """Fetch spdlog at the pinned commit with git, which checks the object hashes, and confirm the checkout.""" + commit = pinned_spdlog_commit() + if (layout.spdlog / "include" / "spdlog" / "spdlog.h").exists() and not force: + say(f"spdlog already present: {layout.spdlog}") + return + if not shutil.which("git"): + raise SetupError("git is needed to fetch spdlog") + say(f"fetching spdlog {commit[:12]}") + shutil.rmtree(layout.spdlog, ignore_errors=True) + layout.spdlog.mkdir(parents=True) + for step in ( + ["git", "init", "-q"], + [ + "git", + "fetch", + "-q", + "--depth", + "1", + "https://github.com/gabime/spdlog.git", + commit, + ], + ["git", "checkout", "-q", "FETCH_HEAD"], + ): + subprocess.run(step, cwd=layout.spdlog, check=True) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=layout.spdlog, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if head != commit: + raise SetupError(f"spdlog checked out {head}, not the pinned {commit}") + + +def cmd_fetch(args: argparse.Namespace) -> int: + layout = Layout(Path(args.root).expanduser()) + layout.root.mkdir(parents=True, exist_ok=True) + fetch_sysroot(args, layout) + fetch_spdlog(layout, args.force) + if not args.no_compiler: + fetch_compiler(layout, args.force) + say("done; run `doctor` to see what is still missing") + return 0 + + +def check_rows(layout: Layout) -> list[tuple[str, bool, str]]: + """Every prerequisite as (name, present, hint).""" + rows: list[tuple[str, bool, str]] = [] + for tool, hint in ( + ("clang++", "needed by `syntax`: sudo apt install clang"), + ("cmake", "needed by `build`: sudo apt install cmake"), + ("nvcc", "needed by `build`: install the CUDA toolkit"), + ("dpkg-deb", "needed by `fetch`"), + ): + rows.append((tool, shutil.which(tool) is not None, hint)) + rows.append( + ( + "aarch64 g++ (Arm toolchain)", + layout.gxx.exists(), + "run `fetch` (downloads Arm's toolchain)", + ) + ) + rows.append( + ( + "spdlog headers", + (layout.spdlog / "include" / "spdlog" / "spdlog.h").exists(), + "run `fetch`", + ) + ) + rows.append( + ( + "CUDA aarch64 cross target", + (CUDA_CROSS_ROOT / "include" / "cuda_runtime.h").exists(), + "needed by `build`: install cuda-cross-sbsa for your CUDA version", + ) + ) + for label, pattern in SYSROOT_HEADERS + SYSROOT_LIBS: + present = bool(list(layout.sysroot.glob(pattern))) + rows.append((label, present, "run `fetch`" if not present else "")) + return rows + + +def cmd_doctor(args: argparse.Namespace) -> int: + layout = Layout(Path(args.root).expanduser()) + say(f"root: {layout.root}") + rows = check_rows(layout) + for name, present, hint in rows: + say( + f" [{'ok' if present else '--'}] {name}" + + ("" if present or not hint else f" ({hint})") + ) + return 0 if all(present for _, present, _ in rows) else 1 + + +def include_dirs(layout: Layout, args: argparse.Namespace) -> list[Path]: + """The include path the producer's sources need, in the order the build uses.""" + sensing = sensing_dir() + sysroot = layout.sysroot + dirs = [ + sensing, + sensing / "core", + sensing / "core" / "inc", + sensing / "livekit" / "inc", + sensing / "third_party" / "jetson_multimedia_api" / "include", + sysroot / "usr/src/jetson_multimedia_api/include", + sysroot / "usr/src/jetson_multimedia_api/argus/include", + sysroot / "usr/src/jetson_sipl_api/sipl/include", + sysroot / "usr/src/jetson_sipl_api/sipl/include/query/include", + sysroot / "usr/src/jetson_sipl_api/sipl/include/nvsci", + repo_root() / "src/core/log_bridge/cpp/inc", + Path(args.cuda).expanduser() / "include", + ] + if args.spdlog_include: + dirs.append(Path(args.spdlog_include).expanduser()) + return dirs + + +def find_spdlog(explicit: str | None, layout: Layout) -> str | None: + """A spdlog include directory: the one given, else the copy `fetch` made.""" + if explicit: + return explicit + fetched = layout.spdlog / "include" + return str(fetched) if (fetched / "spdlog" / "spdlog.h").exists() else None + + +def cmd_syntax(args: argparse.Namespace) -> int: + """Type-check every producer C++ source with clang against the sysroot's headers; no linking, no CUDA.""" + layout = Layout(Path(args.root).expanduser()) + clang = shutil.which("clang++") + if not clang: + raise SetupError("clang++ is not installed (sudo apt install clang)") + if not all(list(layout.sysroot.glob(pattern)) for _, pattern in SYSROOT_HEADERS): + raise SetupError("the Orin headers are missing; run `fetch` first") + args.spdlog_include = find_spdlog(args.spdlog_include, layout) + if not args.spdlog_include: + raise SetupError( + "no spdlog headers found; run `fetch`, or pass --spdlog-include DIR" + ) + sensing = sensing_dir() + sources = sorted( + [ + *sensing.glob("core/*.cpp"), + *sensing.glob("app/*.cpp"), + *sensing.glob("third_party/**/classes/*.cpp"), + ] + ) + flags = ["-std=c++17", "-fsyntax-only", "-Wno-everything", "-DSENSING_HAS_SIPL"] + if (sensing / "core" / "argus_capture.cpp").exists(): + flags.append("-DSENSING_HAS_ARGUS") + for directory in include_dirs(layout, args): + flags += ["-I", str(directory)] + failed = 0 + for source in sources: + result = subprocess.run( + [clang, *flags, str(source)], capture_output=True, text=True, check=False + ) + errors = [ + line + for line in result.stderr.splitlines() + if "error" in line or "fatal" in line + ] + if result.returncode != 0: + failed += 1 + say(f"FAIL {source.relative_to(repo_root())}") + for line in errors[:8]: + say(f" {line}") + elif args.verbose: + say(f"ok {source.relative_to(repo_root())}") + say(f"{len(sources) - failed} of {len(sources)} sources type-check") + return 1 if failed else 0 + + +def cmd_toolchain(args: argparse.Namespace) -> int: + layout = Layout(Path(args.root).expanduser()) + if not layout.gxx.exists(): + raise SetupError("the aarch64 compiler is missing; run `fetch` first") + cuda_cross = Path(args.cuda).expanduser() + # CMake finds a cross target's device library through /usr/lib/cuda (Debian's cross layout). + cuda_link = layout.sysroot / "usr" / "lib" / "cuda" + if not cuda_link.exists() and not cuda_link.is_symlink(): + cuda_link.symlink_to(cuda_cross.resolve()) + # The nvcc must come from the same toolkit as the cross target; a distribution's nvcc on PATH is another version. + toolkit_nvcc = cuda_cross.resolve().parents[1] / "bin" / "nvcc" + cuda_nvcc = ( + str(toolkit_nvcc) if toolkit_nvcc.exists() else (shutil.which("nvcc") or "nvcc") + ) + extra_roots = "" + layout.cmake_file.write_text( + f"""# Generated by setup_orin_crosscompile.py; regenerate rather than edit. +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(CMAKE_C_COMPILER "{layout.gcc}") +set(CMAKE_CXX_COMPILER "{layout.gxx}") +set(CMAKE_SYSROOT "{layout.sysroot}") +# The Tegra libraries sit beside their own dependencies; shared libraries' own needs are not re-resolved on the host. +set(_orin_libs "{layout.sysroot}/usr/lib/aarch64-linux-gnu/nvidia" "{layout.sysroot}/usr/lib/aarch64-linux-gnu" "{layout.sysroot}/opt/nvidia/l4t-gpu-libs/nvgpu") +foreach(_dir IN LISTS _orin_libs) + string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " -Wl,-rpath-link,${{_dir}} -L${{_dir}}") +endforeach() +string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " -Wl,--allow-shlib-undefined") +# Arm's compiler looks for the C runtime and glibc's architecture headers in the plain usr/lib and usr/include; Ubuntu +# keeps them in the aarch64-linux-gnu subdirectories. +foreach(_lang C CXX) + string(APPEND CMAKE_${{_lang}}_FLAGS_INIT " -B{layout.sysroot}/usr/lib/aarch64-linux-gnu -isystem {layout.sysroot}/usr/include/aarch64-linux-gnu") +endforeach() +string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " -B{layout.sysroot}/usr/lib/aarch64-linux-gnu") +# The CUDA link rule does not add --sysroot on its own, and glibc's linker scripts name /lib/... absolutely. +string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " --sysroot={layout.sysroot}") +# Libraries and headers come from the Orin sysroot; build tools (cmake, python) from this host. +set(CMAKE_FIND_ROOT_PATH "{layout.sysroot}" "{cuda_cross}"{extra_roots}) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +# The fetched LiveKit SDK sits in the build tree, outside the sysroot; its config files are searched by explicit path. +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH) +# nvcc compiles the device code for the Orin's GPU and hands the host code to the aarch64 compiler. +set(CMAKE_CUDA_COMPILER "{cuda_nvcc}") +set(CMAKE_CUDA_HOST_COMPILER "{layout.gxx}") +set(CMAKE_CUDA_FLAGS_INIT "--target-directory sbsa-linux -isystem {layout.sysroot}/usr/include/aarch64-linux-gnu") +set(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES "{cuda_cross}/include") +set(CMAKE_CUDA_ARCHITECTURES 87) +set(CUDAToolkit_ROOT "{cuda_cross}") +""" + ) + say(f"wrote {layout.cmake_file}") + return 0 + + +def write_project(layout: Layout) -> Path: + """A small top-level project that builds only the producer. + + The repository's own top-level CMake pulls in the OpenXR SDK, Python and the viewer, which need an aarch64 + desktop stack that has no place here; the producer needs only spdlog and the logging bridge from outside its + own directory. + """ + project = layout.root / "project" + project.mkdir(parents=True, exist_ok=True) + (project / "CMakeLists.txt").write_text( + f"""# Generated by setup_orin_crosscompile.py; regenerate rather than edit. +cmake_minimum_required(VERSION 3.24) +project(sensing_cross LANGUAGES C CXX) +include(FetchContent) +FetchContent_Declare(spdlog SOURCE_DIR "{layout.spdlog}") +set(SPDLOG_BUILD_EXAMPLE OFF CACHE BOOL "" FORCE) +set(SPDLOG_BUILD_TESTS OFF CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(spdlog) +add_subdirectory("{repo_root() / "src/core/log_bridge/cpp"}" log_bridge) +add_subdirectory("{sensing_dir()}" sensing) +""" + ) + return project + + +def cmd_build(args: argparse.Namespace) -> int: + """Configure and build ``camera_plugin_sensing`` for aarch64 into ``/build``.""" + layout = Layout(Path(args.root).expanduser()) + for tool in ("cmake", "nvcc"): + if not shutil.which(tool): + raise SetupError(f"{tool} is not installed") + if not (CUDA_CROSS_ROOT / "include" / "cuda_runtime.h").exists(): + raise SetupError( + "the CUDA aarch64 cross target is missing (install cuda-cross-sbsa for your CUDA version)" + ) + cmd_toolchain(args) + configure = [ + "cmake", + "-S", + str(write_project(layout)), + "-B", + str(layout.build_dir), + f"-DCMAKE_TOOLCHAIN_FILE={layout.cmake_file}", + "-DCMAKE_BUILD_TYPE=Release", + f"-DSIPL_API_DIR={layout.sysroot}/usr/src/jetson_sipl_api", + f"-DJETSON_MMAPI_DIR={layout.sysroot}/usr/src/jetson_multimedia_api", + *args.cmake_arg, + ] + say("configuring: " + " ".join(configure)) + if subprocess.run(configure, check=False).returncode != 0: + return 1 + build = [ + "cmake", + "--build", + str(layout.build_dir), + "--target", + args.target, + "-j", + str(args.jobs), + ] + say("building: " + " ".join(build)) + return subprocess.run(build, check=False).returncode + + +def parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument( + "--root", + default=f"~/orin-cross-{RELEASE}", + help="directory holding the sysroot and compiler", + ) + ap.add_argument( + "--sdk-downloads", + default="~/Downloads/nvidia/sdkm_downloads", + help="folder with the Jetson Linux and SIPL API archives from SDK Manager", + ) + ap.add_argument( + "--cuda", default=str(CUDA_CROSS_ROOT), help="the CUDA aarch64 target directory" + ) + ap.add_argument( + "--spdlog-include", + help="spdlog include directory (default: the copy `fetch` makes)", + ) + sub = ap.add_subparsers(dest="command", required=True) + sub.add_parser( + "doctor", help="show what is present and what is missing" + ).set_defaults(run=cmd_doctor) + fetch = sub.add_parser( + "fetch", help="build the sysroot and download the aarch64 compiler" + ) + fetch.add_argument( + "--force", action="store_true", help="redo steps that look finished" + ) + fetch.add_argument( + "--no-compiler", action="store_true", help="sysroot only (enough for `syntax`)" + ) + fetch.set_defaults(run=cmd_fetch) + syntax = sub.add_parser( + "syntax", help="type-check the producer's sources against the Orin headers" + ) + syntax.add_argument( + "-v", "--verbose", action="store_true", help="also list the files that pass" + ) + syntax.set_defaults(run=cmd_syntax) + sub.add_parser("toolchain", help="write the CMake toolchain file").set_defaults( + run=cmd_toolchain + ) + build = sub.add_parser("build", help="cross-build camera_plugin_sensing") + build.add_argument("--target", default="camera_plugin_sensing") + build.add_argument("--jobs", type=int, default=os.cpu_count() or 4) + build.add_argument( + "--cmake-arg", + action="append", + default=[], + help="extra CMake argument, repeatable", + ) + build.set_defaults(run=cmd_build) + return ap + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + return args.run(args) + except SetupError as exc: + say(f"error: {exc}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) From 8373d1e926b32863f59c67b9d78a62d3f3482f12 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 20:03:02 -0500 Subject: [PATCH 29/29] sensing: type-check the producer with the include paths the build uses The syntax check put the plugin and core directories on its include path, which the build does not, so a public header that reached a private one passed the check and failed the build. Drop those two paths. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/plugins/sensing/scripts/setup_orin_crosscompile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/plugins/sensing/scripts/setup_orin_crosscompile.py b/src/plugins/sensing/scripts/setup_orin_crosscompile.py index 0b544fbce9..e3e2c1f796 100755 --- a/src/plugins/sensing/scripts/setup_orin_crosscompile.py +++ b/src/plugins/sensing/scripts/setup_orin_crosscompile.py @@ -464,8 +464,6 @@ def include_dirs(layout: Layout, args: argparse.Namespace) -> list[Path]: sensing = sensing_dir() sysroot = layout.sysroot dirs = [ - sensing, - sensing / "core", sensing / "core" / "inc", sensing / "livekit" / "inc", sensing / "third_party" / "jetson_multimedia_api" / "include",