sensing: Argus capture backend and encoder DMA-BUF input

src/plugins/sensing/AGENTS.md

@@ -21,3 +21,12 @@ Container preview must carry the desktop display and authorization into the imag
 
 Support SSH launch with a local DISPLAY: when XAUTHORITY is unset, check the
 current user's GDM authority file as well as ~/.Xauthority.
+
+A frame lease keeps its capture backend alive, so a held frame is an ownership
+cycle. Every cleanup path, including a failed start,
+must join the worker threads and drop the held frames, not only normal stop.
+A camera session that will not stop ends the process; do not retry in-process.
+Only a camera's owner thread calls its buffer stream; a frame lease posts its
+buffer to that owner's mailbox and makes no SDK call.
+Check an SDK call's return type before testing it for success: Argus
+stopRepeat returns a capture-id range, not a Status.

src/plugins/sensing/CMakeLists.txt

@@ -29,51 +29,55 @@ set(_tegra_lib_dirs "/usr/lib/aarch64-linux-gnu/nvidia" "/usr/lib/aarch64-linux-
 # ==============================================================================
 # SIPL
 # ==============================================================================
-# Headers ship in Jetson_SIPL_API_R<rel>_aarch64.tbz2, published alongside the
-# L4T release. Not in any apt pool, so setup_container.sh fetches it by URL.
-if(NOT SIPL_API_DIR)
-    find_path(SIPL_API_DIR sipl/include/NvSIPLCamera.hpp
-        PATHS "/usr/src/jetson_sipl_api"
-              "/opt/nvidia/jetson_sipl_api"
-              "$ENV{SIPL_API_DIR}"
-    )
-endif()
-
-if(NOT SIPL_API_DIR)
-    message(FATAL_ERROR
-        "================================================================================\n"
-        "SENSING camera plugin requires the Jetson SIPL API headers.\n"
-        "\n"
-        "Install the SIPL API package for your L4T release and point CMake at it:\n"
-        "  cmake -B build -DSIPL_API_DIR=/path/to/jetson_sipl_api\n"
-        "\n"
-        "Or disable this plugin:\n"
-        "  cmake -B build -DBUILD_PLUGIN_SENSING=OFF\n"
-        "================================================================================\n"
-    )
-endif()
+# Off for an Argus-only rig, which needs neither the SIPL headers nor its libraries.
+option(SENSING_WITH_SIPL "Build the SIPL capture backend" ON)
+if(SENSING_WITH_SIPL)
+    # Headers ship in Jetson_SIPL_API_R<rel>_aarch64.tbz2, published alongside the
+    # L4T release. Not in any apt pool, so setup_container.sh fetches it by URL.
+    if(NOT SIPL_API_DIR)
+        find_path(SIPL_API_DIR sipl/include/NvSIPLCamera.hpp
+            PATHS "/usr/src/jetson_sipl_api"
+                  "/opt/nvidia/jetson_sipl_api"
+                  "$ENV{SIPL_API_DIR}"
+        )
+    endif()
 
-set(SIPL_INCLUDE_DIRS
-    "${SIPL_API_DIR}/sipl/include"
-    "${SIPL_API_DIR}/sipl/include/query/include"
-    "${SIPL_API_DIR}/sipl/include/nvsci"
-)
-message(STATUS "SENSING plugin: SIPL=${SIPL_API_DIR}")
-
-# L4T ships no .so dev symlinks for these, hence the explicit sonames.
-foreach(_lib nvsipl nvsipl_query nvscibuf nvscisync nvbufsurface_nvsci)
-    string(TOUPPER "${_lib}" _upper)
-    find_library(${_upper}_LIBRARY
-        NAMES ${_lib} lib${_lib}.so.1 lib${_lib}.so.1.0.0
-        PATHS ${_tegra_lib_dirs}
-    )
-    if(NOT ${_upper}_LIBRARY)
+    if(NOT SIPL_API_DIR)
         message(FATAL_ERROR
-            "lib${_lib} not found. It is supplied by the L4T BSP and, in a container, by the "
-            "NVIDIA container runtime. Run src/plugins/sensing/scripts/verify.sh to see what is missing.")
+            "================================================================================\n"
+            "SENSING camera plugin requires the Jetson SIPL API headers.\n"
+            "\n"
+            "Install the SIPL API package for your L4T release and point CMake at it:\n"
+            "  cmake -B build -DSIPL_API_DIR=/path/to/jetson_sipl_api\n"
+            "\n"
+            "Or disable this plugin:\n"
+            "  cmake -B build -DBUILD_PLUGIN_SENSING=OFF\n"
+            "================================================================================\n"
+        )
     endif()
-    list(APPEND SIPL_LIBRARIES ${${_upper}_LIBRARY})
-endforeach()
+
+    set(SIPL_INCLUDE_DIRS
+        "${SIPL_API_DIR}/sipl/include"
+        "${SIPL_API_DIR}/sipl/include/query/include"
+        "${SIPL_API_DIR}/sipl/include/nvsci"
+    )
+    message(STATUS "SENSING plugin: SIPL=${SIPL_API_DIR}")
+
+    # L4T ships no .so dev symlinks for these, hence the explicit sonames.
+    foreach(_lib nvsipl nvsipl_query nvscibuf nvscisync nvbufsurface_nvsci)
+        string(TOUPPER "${_lib}" _upper)
+        find_library(${_upper}_LIBRARY
+            NAMES ${_lib} lib${_lib}.so.1 lib${_lib}.so.1.0.0
+            PATHS ${_tegra_lib_dirs}
+        )
+        if(NOT ${_upper}_LIBRARY)
+            message(FATAL_ERROR
+                "lib${_lib} not found. It is supplied by the L4T BSP and, in a container, by the "
+                "NVIDIA container runtime. Run src/plugins/sensing/scripts/verify.sh to see what is missing.")
+        endif()
+        list(APPEND SIPL_LIBRARIES ${${_upper}_LIBRARY})
+    endforeach()
+endif()
 
 # EGL stays on the capture path: NvBufSurfaceMapEglImage is how a SIPL
 # NvSciBufObj becomes a CUeglFrame. Interop path B in the design notes would
@@ -85,10 +89,9 @@ if(NOT EGL_INCLUDE_DIRS OR NOT EGL_LIBRARIES)
 endif()
 
 # ==============================================================================
-# Jetson V4L2 encoder
+# Jetson Multimedia API tree
 # ==============================================================================
-# Jetson has no libnvidia-encode (the Video Codec SDK is dGPU-only), so H.264
-# comes from the V4L2 M2M engine via NvVideoEncoder, vendored from the Multimedia API (see below).
+# Found before the Argus block below, whose header search is relative to it.
 if(NOT JETSON_MMAPI_DIR)
     find_path(JETSON_MMAPI_DIR include/nvbufsurface.h
         PATHS "/usr/src/jetson_multimedia_api"
@@ -106,6 +109,35 @@ if(NOT JETSON_MMAPI_DIR)
 endif()
 message(STATUS "SENSING plugin: Jetson Multimedia API=${JETSON_MMAPI_DIR}")
 
+# ==============================================================================
+# Argus (optional)
+# ==============================================================================
+# The Argus backend captures NV12 into DMA-BUFs the hardware encoder reads. It needs the Argus headers from the
+# Multimedia API and libnvargus; none of this is built or run by default.
+option(SENSING_WITH_ARGUS "Build the Argus capture backend (not built or run by the author)" OFF)
+# Argus normally runs through nvargus-daemon, and its client library is libnvargus_socketclient. The single-process
+# libnvargus does not connect to the daemon, so it is an explicit choice.
+option(SENSING_ARGUS_SINGLE_PROCESS "Link single-process libnvargus instead of the nvargus-daemon client" OFF)
+if(SENSING_WITH_ARGUS)
+    if(SENSING_ARGUS_SINGLE_PROCESS)
+        set(_argus_lib_name nvargus)
+    else()
+        set(_argus_lib_name nvargus_socketclient)
+    endif()
+    find_path(ARGUS_INCLUDE_DIR Argus/Argus.h HINTS "${JETSON_MMAPI_DIR}/argus/include")
+    find_library(ARGUS_LIBRARY NAMES ${_argus_lib_name} PATHS ${_tegra_lib_dirs})
+    if(NOT ARGUS_INCLUDE_DIR OR NOT ARGUS_LIBRARY)
+        message(FATAL_ERROR
+            "SENSING_WITH_ARGUS needs the Argus headers (<jetson_multimedia_api>/argus/include) and lib${_argus_lib_name}.")
+    endif()
+    message(STATUS "SENSING plugin: Argus=${ARGUS_INCLUDE_DIR} (${ARGUS_LIBRARY})")
+endif()
+
+# ==============================================================================
+# Jetson V4L2 encoder
+# ==============================================================================
+# Jetson has no libnvidia-encode (the Video Codec SDK is dGPU-only), so H.264
+# comes from the V4L2 M2M engine via NvVideoEncoder, vendored from the Multimedia API (see below).
 # NVIDIA's encoder classes are vendored (BSD-3-Clause; third_party/jetson_multimedia_api/README.md), so they build the
 # same on every JetPack and carry no stdout debug prints. The installed Multimedia API tree supplies only what is not
 # vendored: nvbufsurface.h, libv4l2.h, v4l2_nv_extensions.h and the Argus headers.
@@ -153,19 +185,36 @@ message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}")
 set(_livekit_local ON)
 if(NOT LIVEKIT_SDK_DIR)
     set(_livekit_local OFF)
+    # The prebuilt links against the glibc of the Ubuntu release it was built on: the 24.04 build does not load on
+    # JetPack 6 (Ubuntu 22.04). Pick the release matching the host; LIVEKIT_SDK_UBUNTU overrides the detection.
+    if(NOT LIVEKIT_SDK_UBUNTU)
+        set(LIVEKIT_SDK_UBUNTU "24.04")
+        if(EXISTS /etc/os-release)
+            file(STRINGS /etc/os-release _os_version_id REGEX "^VERSION_ID=")
+            if(_os_version_id MATCHES "\"?22\\.04\"?$")
+                set(LIVEKIT_SDK_UBUNTU "22.04")
+            endif()
+        endif()
+    endif()
     if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
         set(_livekit_arch "arm64")
-        set(_livekit_hash "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad")
+        set(_livekit_hash_24.04 "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad")
+        set(_livekit_hash_22.04 "7a698bbd8f61f019744fd99b3a5d2b6c6118787fd1f81140c9a6fc8732fc87e0")
     elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
         set(_livekit_arch "x64")
-        set(_livekit_hash "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b")
+        set(_livekit_hash_24.04 "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b")
+        set(_livekit_hash_22.04 "9e568ee6b2bb3c7b0d07faba694930767096a14d0e09422ba01f34077ffb03cb")
     else()
         message(FATAL_ERROR "No prebuilt LiveKit SDK for ${CMAKE_SYSTEM_PROCESSOR}; set LIVEKIT_SDK_DIR.")
     endif()
+    if(NOT DEFINED _livekit_hash_${LIVEKIT_SDK_UBUNTU})
+        message(FATAL_ERROR "No prebuilt LiveKit SDK for Ubuntu ${LIVEKIT_SDK_UBUNTU}; set LIVEKIT_SDK_DIR.")
+    endif()
+    set(_livekit_hash "${_livekit_hash_${LIVEKIT_SDK_UBUNTU}}")
     include(FetchContent)
     FetchContent_Declare(
         livekit_sdk
-        URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-24.04-${_livekit_arch}-1.12.4.tar.gz
+        URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-${LIVEKIT_SDK_UBUNTU}-${_livekit_arch}-1.12.4.tar.gz
         URL_HASH SHA256=${_livekit_hash}
     )
     FetchContent_MakeAvailable(livekit_sdk)

src/plugins/sensing/README.md

@@ -47,6 +47,10 @@ hardware encoder and publishes one LiveKit video track per camera. The
 [SENSING user guide](https://nvidia.github.io/IsaacCapture/main/device/camera/sensing.html)
 has the build and run steps.
 
-`--camera-backend=auto|sipl|argus` selects the capture backend once at startup. Only SIPL is built in today: `argus` fails
-with a message saying so, and `auto` picks the only matching backend (and, with several, names the override instead of
-guessing).
+`--camera-backend=auto|sipl|argus` selects the capture backend once at startup: an explicit choice must match or fail
+with its own reason, and `auto` picks the only matching backend (with several it names the override instead of
+guessing). SIPL is built by default (`-DSENSING_WITH_SIPL=OFF` for an Argus-only rig). The Argus backend
+(`-DSENSING_WITH_ARGUS=ON`, Argus device index with `--add-stream=sensor=N,...`, `--argus-mode=K`) captures NV12 into
+DMA-BUFs that the hardware encoder reads without a copy. **The Argus code has never been built or run: there is no
+Argus SDK or Argus camera available to the author.** Argus DMA-BUF input cannot be combined with
+`--livekit-scale-div` above 1. An Argus-only build (`-DSENSING_WITH_SIPL=OFF -DSENSING_WITH_ARGUS=ON`) also builds on JetPack 6.2 / L4T R36.4.3.