diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a46d0b999..e859cb2bab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,8 @@ repos: rev: v6.0.0 hooks: - id: trailing-whitespace + # NVIDIA's copied sources keep their upstream bytes (see the directory's README.md). + exclude: ^src/plugins/sensing/third_party/jetson_multimedia_api/(classes|include)/ - id: check-symlinks - id: destroyed-symlinks - id: check-added-large-files @@ -25,7 +27,7 @@ repos: - id: check-toml - id: end-of-file-fixer # These LFS SVGs are assets; rewriting their final byte creates new LFS objects. - exclude: ^(docs/source/_static/camera-viz-controls|examples/vehicle_teleop/assets/vehicle_teleop_flow)\.svg$ + exclude: ^(docs/source/_static/camera-viz-controls|examples/vehicle_teleop/assets/vehicle_teleop_flow)\.svg$|^src/plugins/sensing/third_party/jetson_multimedia_api/(classes|include)/ - id: check-shebang-scripts-are-executable - id: detect-private-key - id: debug-statements diff --git a/CMakeLists.txt b/CMakeLists.txt index 521ec03c30..6b02232bd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,6 +71,7 @@ option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF) option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF) option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF) +option(BUILD_PLUGIN_SENSING "Build SENSING GMSL camera plugin (Jetson only; requires SIPL, NVENC and the LiveKit SDK)" OFF) option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) option(BUILD_TESTING "Build unit tests" ON) diff --git a/README.md b/README.md index cd55de9172..47a630e80f 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,9 @@ To get started, please refer to Isaac Lab's [Installation](https://isaac-sim.git The default license is [Apache-2.0](LICENSE.md). EgoPost is licensed under [OpenMDW-1.1](LICENSES/LicenseRef-OpenMDW-1.1.txt). Third-party materials may carry their own licenses. +The Jetson encoder classes under `src/plugins/sensing/third_party/jetson_multimedia_api` are NVIDIA's +[BSD-3-Clause](src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE) sample sources, copied from the Jetson +Multimedia API; see that directory's README. ### Separately retrieved optional materials diff --git a/REUSE.toml b/REUSE.toml index 38c21963fb..23460e91f2 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -58,6 +58,20 @@ precedence = "override" SPDX-FileCopyrightText = "Copyright 2015 The Chromium Authors" SPDX-License-Identifier = "BSD-3-Clause" +# NVIDIA's Jetson Multimedia API encoder classes, copied from the nvidia-l4t-jetson-multimedia-api package (see its +# README.md). Each file keeps its own BSD-3-Clause header; files whose header has no SPDX tag are covered here. +[[annotations]] +path = "src/plugins/sensing/third_party/jetson_multimedia_api/**" +precedence = "closest" +SPDX-FileCopyrightText = "Copyright (c) 2016 NVIDIA CORPORATION. All rights reserved." +SPDX-License-Identifier = "BSD-3-Clause" + +[[annotations]] +path = "src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE" +precedence = "override" +SPDX-FileCopyrightText = "Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved." +SPDX-License-Identifier = "BSD-3-Clause" + # EgoPost's default never overrides upstream headers or sidecar licenses. # The license-only scaffold has no covered files yet; the canonical text is reserved for imports. [[annotations]] diff --git a/docs/source/device/camera/sensing.rst b/docs/source/device/camera/sensing.rst index 964375724a..24a375482b 100644 --- a/docs/source/device/camera/sensing.rst +++ b/docs/source/device/camera/sensing.rst @@ -155,3 +155,72 @@ on the side that fails for a read-only runtime report: * - Low FPS or dropped frames - Check the Jetson power mode. Maximum performance can be selected with ``sudo nvpmodel -m 0 && sudo jetson_clocks``. + +Build and run the LiveKit publisher +----------------------------------- + +``camera_plugin_sensing`` runs on the Orin. It captures the GMSL cameras with +SIPL, encodes H.264 on the hardware encoder and publishes one LiveKit video +track per camera. CaptureServer, the egocap backend, recording and the browser +run on the PC. + +Prerequisites: an AGX Orin on JetPack 7.2.1 / L4T R39.2.1 with the SENSING +drivers installed and the cameras working, CUDA, CMake, and network access to +the LiveKit server. The build also needs two header trees; neither is fetched +by ``setup.sh``: + +- ``JETSON_MMAPI_DIR``: the Jetson Multimedia API tree, containing + ``include/nvbufsurface.h``. The encoder classes are copied into the repository + and are not taken from it. CMake searches ``/usr/src/jetson_multimedia_api``. +- ``SIPL_API_DIR``: the Jetson SIPL API tree, containing + ``sipl/include/NvSIPLCamera.hpp`` (the ``Jetson_SIPL_API_R_aarch64.tbz2`` + package published with the L4T release). CMake searches ``/usr/src/jetson_sipl_api``, + ``/opt/nvidia/jetson_sipl_api`` and ``$SIPL_API_DIR``. + +.. code-block:: bash + + # Build. The LiveKit SDK v1.12.4 prebuilt is downloaded and hash-checked; use + # -DLIVEKIT_SDK_DIR=... for a local copy (it needs LICENSE and NOTICE files in that + # directory, or -DLIVEKIT_SDK_LICENSE_FILE / -DLIVEKIT_SDK_NOTICE_FILE). + cmake -B build -DBUILD_PLUGIN_SENSING=ON -DBUILD_VIZ=OFF -DBUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF -DBUILD_PYTHON_BINDINGS=OFF \ + -DSIPL_API_DIR=... -DJETSON_MMAPI_DIR=... + cmake --build build --target camera_plugin_sensing -j + + # Check the cameras are visible. + build/src/plugins/sensing/app/camera_plugin_sensing --list-sensors + + # Publish. The token file is a LiveKit token for the room and publisher identity + # named in the PC's streams file. + build/src/plugins/sensing/app/camera_plugin_sensing \ + --livekit-url=ws://LIVEKIT_HOST:7880 --livekit-token-file=join.token \ + --add-stream=sensor=0,livekit=sensing-left --add-stream=sensor=1,livekit=sensing-right + +The ``livekit=`` names must equal the ``track`` names in the PC's streams file. +Stop with Ctrl-C. ``--help`` lists every option. The example uses plain ``ws://`` +for a LiveKit server on a trusted LAN; the join token is sent in the clear, so +use a ``wss://`` endpoint on any other network. + +``SENSING_TSC_PROBE=1`` logs the SIPL capture counter beside the host clocks twice +a second, as separate uncalibrated samples. The counter's units are unconfirmed, so +do not subtract or fit them. + +Cross-compile on a PC +--------------------- + +``scripts/setup_orin_crosscompile.py`` builds the producer for the Orin on an x86 Ubuntu PC, without sudo and +without touching the Orin. Everything goes under one directory (``--root``, default ``~/orin-cross-39.2.1``). + +.. code-block:: bash + + src/plugins/sensing/scripts/setup_orin_crosscompile.py doctor # what is present, what is missing + src/plugins/sensing/scripts/setup_orin_crosscompile.py fetch # sysroot and aarch64 compiler + src/plugins/sensing/scripts/setup_orin_crosscompile.py syntax # type-check every source in seconds + src/plugins/sensing/scripts/setup_orin_crosscompile.py build # camera_plugin_sensing for aarch64 + +``fetch`` unpacks the L4T libraries from the Jetson Linux archive, and the SIPL API, from the SDK Manager +download directory (``--sdk-downloads``). It takes the Multimedia API from there or from NVIDIA's apt repository, +the C library and EGL/V4L2 libraries from Ubuntu 24.04, spdlog at the revision the repository pins, and Arm's +GNU toolchain; each download is checked against the publisher's SHA-256. ``build`` also needs the CUDA toolkit +with its aarch64 cross target (``cuda-cross-sbsa``) and uses that toolkit's own ``nvcc``. The result is the same +binary a build on the Orin produces, but it has not been run: deploy it and test it on the Orin. diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index c26d9384c2..7a3a400b15 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -44,3 +44,7 @@ endif() if(BUILD_PLUGIN_SHARPA_AVATAR) add_subdirectory(sharpa_avatar) endif() +# Gated: Jetson only -- SIPL and the Tegra NVENC entry point are L4T-only. +if(BUILD_PLUGIN_SENSING) + add_subdirectory(sensing) +endif() diff --git a/src/plugins/sensing/CAMERA_SETUP.md b/src/plugins/sensing/CAMERA_SETUP.md new file mode 100644 index 0000000000..0707af7125 --- /dev/null +++ b/src/plugins/sensing/CAMERA_SETUP.md @@ -0,0 +1,251 @@ + + +# SENSING camera setup guide + +This guide takes a Jetson AGX Orin and SENSING GMSL cameras from a pile of parts to cameras that stream. It covers the +physical setup, where most problems come from (which cable goes into which connector, and how the carrier board is +powered), and the operating system setup the cameras need. It stops once the cameras are confirmed streaming. + +For the plugin's source layout see [`README.md`](README.md). For the published reference documentation see the +[SENSING user guide](https://nvidia.github.io/IsaacCapture/main/device/camera/sensing.html). + +## Two kinds of camera + +Jetson reads cameras through one of two software stacks, and which one you need depends on how the camera is wired to +the board, not on its brand. + +- **SIPL cameras (GMSL)** connect through a deserializer on a carrier board, such as the SENSING SG8A, over a coax + cable. NVIDIA's Sensor Input Processing Library (SIPL) configures the deserializer and cameras from a platform file. + The SENSING SHW5G on the SG8A is this kind. **This guide covers SIPL cameras.** +- **Argus cameras (CSI)** are exposed by the Jetson camera driver as devices, typically MIPI CSI sensors described in + the device tree, and are reached through `libargus` and `nvargus-daemon`. You pick one by index. Nothing in this guide + applies to them. + +| | SIPL | Argus | +| --- | --- | --- | +| Typical cameras | GMSL cameras on a carrier such as the SG8A | CSI cameras exposed as V4L2 or Argus devices | +| Configuration | Platform JSON, a named config, per-link masks | Device tree and sensor drivers | +| Camera selection | Sensor number, which follows the link index | Camera index | +| Rule of thumb | A deserializer and coax cable sit between camera and board | The camera appears as `/dev/video*` | + +The same camera model can be reached either way, depending on the board. On the SG8A rig Argus reports no camera +devices (no `/dev/video*`, and "virtual enumerator not found in proc device-tree"), so SIPL is the only option there. +Do not run both stacks against the same cameras. + +## Step 1: Gather the parts + +| Part | What it is | +| --- | --- | +| Jetson AGX Orin Developer Kit | Runs L4T R39.2.1 (JetPack 7.2.1) and has its own power supply, which this guide does not cover. | +| SENSING SG8A board | The GMSL carrier. On this rig it is installed inside the devkit's enclosure at the front edge. Its two quad receptacles and a white 2-pin power connector face out, beside the devkit's three front-panel buttons and LED. | +| Carrier power supply | A 12 V adapter, plus a black cable with a female DC barrel jack on one end and a white 2-pin connector (red and black wires) on the other. | +| Cameras | SHW5G modules. Each has one coax jack on the rear (the vendor lists an Amphenol FAKRA Z connector). | +| Camera cables | Two 2 m black coax cables (labelled "2米", Chinese for 2 m), one per camera, each with a teal FAKRA connector and grey strain relief at the camera end. | +| Breakout harness | One teal quad plug (four coax sockets in a 2x2 block) with two coax cables leading to two single teal FAKRA jacks. | + +This guide assumes the SG8A board is already installed in the devkit. Installing or removing it is covered by +SENSING's hardware manual, not here. + +### What the parts look like + +![The parts laid out: two cameras on a bracket with their labelled 2 m cables, loose teal connectors, jumper wires with a barrel plug, and the devkit at the lower right](images/setup-parts-overview.jpg) + +*The parts laid out. The two cameras sit on a bracket, each with a labelled 2 m cable ending in a teal connector. Loose +teal connectors and a cable with a barrel plug lie nearby, and the devkit is at the lower right.* + +![The breakout harness: two single teal FAKRA jacks and one teal quad plug with four coax sockets](images/setup-harness-connectors.jpg) + +*The breakout harness. The two single teal FAKRA jacks each take one camera cable. The teal quad plug, with its four +coax sockets in a 2x2 block, goes into the carrier.* + +![An SHW5G camera in a mounting bracket, seen from the lens end](images/setup-camera-front.jpg) + +*An SHW5G camera in a mounting bracket, seen from the lens end.* + +![The rear of a camera: one round coax jack recessed in the bracket](images/setup-camera-rear.jpg) + +*The rear of the camera: one round coax jack, recessed in the bracket on this rig. The camera cable's teal connector +mates here.* + +### The connectors, end to end + +Every connector in the camera chain is teal, the FAKRA code for Z. Each connection mates one specific pair: + +| Connection | One side | Other side | +| --- | --- | --- | +| Camera to camera cable | The camera's rear coax jack | The camera cable's teal FAKRA connector with grey strain relief | +| Camera cable to harness | The camera cable's other end | One of the harness's two single teal FAKRA jacks (each a square teal housing with a centre contact; one has a purple tab) | +| Harness to carrier | The harness's teal quad plug (2x2 sockets) | A black latched quad receptacle on the SG8A (four round coax pins in a 2x2 block) | +| Carrier power | The carrier's white 2-pin connector | The black power cable, whose barrel jack takes the 12 V adapter's plug | + +The two receptacles are **CN1** (left, carrying COAX0 to COAX3) and **CN2** (right, carrying COAX4 to COAX7), as +labelled in the vendor documentation. The camera chain uses CN1 only. Leave CN2 empty. + +Notes on the parts: + +- **SHW5G and SHF5G are two models of one camera family.** The vendor lists them together as its "5MP Global" + monocular GMSL2 cameras. Both have the same 2560x1984 sensor, frame rate (up to 60 fps in global-shutter mode, 50 fps + in rolling-shutter mode), GMSL2 interface, FAKRA Z connector and power requirements. They differ in lens: the SHW5G + is the wide-angle model (130° horizontal field of view) and the SHF5G is ultra-wide (200°). This guide and its + configuration use the SHW5G. +- **The carrier needs its own supply.** Power it from a 12 V adapter, and check the current rating with SENSING before + choosing one. +- **The cameras need no separate supply.** The vendor wiki lists them as single-coax cameras powered over the same + cable at 9 to 16 V, drawing less than 200 mA at 12 V each. The carrier provides that power. +- **Nothing on the Orin plugs directly into a camera's round coax jack.** The only connection to the carrier is the + quad plug on the harness. Spare FAKRA cables lying around the rig are not part of this setup. +- **Leave the jumper wires as they are.** Coloured jumper wires may be plugged into the devkit's 40-pin header. Do not + add or remove any without checking SENSING's manual. + +## Step 2: Connect the cameras and power + +**Power everything down first.** The cameras are powered through the coax, so connecting them to a live carrier can +damage the cameras or the carrier's serializers. Shut the Orin down and unplug both the devkit's supply and the +carrier's 12 V supply. + +The finished chain looks like this: + +```text +12 V adapter ── DC barrel jack ── black cable ── white 2-pin connector ── SG8A board (front edge of the devkit) + +SG8A CN1 receptacle (left) ◄── teal quad plug ── harness ─┬─ single teal jack ── 2 m coax cable ── camera + └─ single teal jack ── 2 m coax cable ── camera +``` + +1. **Camera cable to camera.** Push each 2 m cable's teal FAKRA connector onto the rear jack of its camera until it + clicks. Check that it is fully seated. +2. **Camera cable to harness.** Connect the other end of each 2 m cable to one of the harness's two single teal + jacks, until it clicks. For a single camera, connect one cable and leave the other jack open. +3. **Harness to carrier.** Push the harness's teal quad plug into the **left** receptacle (CN1) on the front edge of the + devkit. The plug is latched and keyed, so it fits only one way and clicks when seated. If it will not go in, stop; + do not force it. +4. **Carrier power.** Connect the white 2-pin connector to the carrier, and the black cable's barrel jack to the 12 V + adapter's plug. +5. **Power on.** Plug in the carrier's 12 V supply first, then power on the devkit. This is the order used on this + rig. Green LEDs light on the board when it is powered. + +![The front edge of the devkit: three buttons and an LED, and the SG8A board's two empty quad receptacles, with the white 2-pin power connector at the left edge](images/setup-carrier-front-panel.jpg) + +*The carrier end before connecting. The devkit's front edge has three buttons and an LED. The SG8A board's two quad +receptacles sit beside them, each a latched black housing with four coax pins in a 2x2 block. CN1 is on the left and +CN2 on the right. The white 2-pin power connector, with red and black wires, is at the left edge.* + +![The harness's teal quad plug seated in the left receptacle, with two coax cables leaving it; the right receptacle is empty](images/setup-harness-in-cn1.jpg) + +*The connected harness. The teal quad plug is seated in the left receptacle (CN1), with two coax cables leaving it. The +right receptacle (CN2) stays empty.* + +![A side view of the unit showing the SG8A board with its two quad receptacles, its white 2-pin power connector, and jumper wires on a 40-pin header](images/setup-carrier-inside-devkit.jpg) + +*A side view of the unit with its side open. The SG8A board sits at the front, with its quad receptacles at the bottom +and the white 2-pin power connector beside them. The coloured wires on the right are plugged into a 40-pin header.* + +With this harness in CN1 the two cameras come up as links 2 and 3 (COAX2 and COAX3, which the vendor documents as +CAM2 and CAM3). The cameras need to be on those contacts; the harness is what puts them there. + +| Cameras connected | Link masks | Sensors | +| --- | --- | --- | +| Two | `0x0000 0x1100` | `sensor=0` and `sensor=1` | +| One, on COAX3 | `0x0000 0x1000` | `sensor=1` | + +Sensor numbers follow the link index, not a count from zero. To find out which of the harness's two jacks is sensor 0 +and which is sensor 1, connect one camera at a time and run the Step 4 check; the sensor that streams belongs to that +jack. Either order works, and swapping the cameras only swaps the sensor numbers. + +## Step 3: Set up the operating system + +The Orin must be on L4T R39.2.1 (JetPack 7.2.1); check `/etc/nv_tegra_release`. + +### Flash JetPack (new or reflashed Orin) + +Skip this if the check above already shows R39.2.1. The AGX Orin is flashed over USB-C from an x86_64 Ubuntu host with +[NVIDIA SDK Manager](https://developer.nvidia.com/sdk-manager). It does not boot a generic installer image from a USB +drive, and an amd64 Ubuntu ISO is the wrong architecture. + +1. **Recovery mode.** With the Orin's power adapter connected and the unit off, connect the host to the USB-C port next + to the 40-pin header. Hold the middle (Force Recovery) button, press and release Power, then release Force Recovery + after about 2 seconds. `lsusb` on the host should list `NVIDIA Corp. APX`. +2. **Target.** In SDK Manager choose **Jetson AGX Orin** (the 64 GB developer kit), JetPack **7.2.1** and **Direct + Flash**. Leave DeepStream and Holoscan unticked. SDK Manager opens on Jetson AGX Thor by default, so check the target + after every launch. If it asks for the exact model, choose the developer kit version that matches the module's memory. +3. **Flash.** In the flash dialog choose **Pre-Config**, set a username and password, pick eMMC or NVMe (if an SSD is + fitted) and click **Flash**. Keep the host awake and the cable connected until the Orin reboots. An interrupted + flash leaves a partial image; put the Orin back in recovery mode and run it again. +4. **SDK components (optional).** SDK Manager then offers to install CUDA and the other components over USB + (`192.168.55.1`) or Ethernet. Skip it, or install them later with `sudo apt install nvidia-jetpack`. + +If the host driver step stalls on a "Pending kernel upgrade" dialog, press Enter in SDK Manager's terminal tab. If that +tab ignores input, `sudo pkill -f 'needrestart -m u'` from another terminal releases it. + +Once the Orin boots, log in as the user you set (`ssh @192.168.55.1` over the USB-C cable), then run the host +setup below. + +### Packages to install + +On the freshly flashed Orin, install these before the host setup: + +```bash +sudo apt update +sudo apt install -y git nvidia-jetpack +``` + +| Package | Why | +| --- | --- | +| `git` | `setup.sh` fetches the vendor driver package from its public repository with a sparse clone. | +| `nvidia-jetpack` | The CUDA, TensorRT and other JetPack components that SDK Manager skips when you decline its post-flash install. | + +`sudo` is also required: `setup.sh` installs files under `/usr` and `/boot` and adds your user to the `i2c` and `gpio` +groups. Everything else the cameras need (the SIPL libraries, camera HAL, ISP tuning files, the device-tree overlay and +`nvsipl_camera`) comes from the vendor package that `setup.sh` installs, not from apt. + +For the container route, the host also needs Docker with the NVIDIA container runtime (`docker.io` and +`nvidia-container-toolkit`, normally already present with `nvidia-jetpack`). The container image installs its own +packages (`libegl1 libgles2 libglib2.0-0 libxext6 libxcb-glx0`), so install nothing extra for it on the host. + +### Host setup + +Run the host setup once, choose the Jetson-IO overlay **Jetson Sensing SG8A-AGON-G2Y-A1 SIPL GMSL2x8** when it asks, and reboot: + +```bash +src/plugins/sensing/scripts/setup.sh +``` + +The script installs the vendor drivers, camera tuning files and device-tree overlay for the carrier. Run it with +`--help` to see its options. After the reboot the carrier is enabled and the cameras can be opened. + +## Step 4: Check that the cameras stream + +Use NVIDIA's own tool, which needs nothing beyond the OS setup and no `sudo`: + +```bash +nvsipl_camera -t src/plugins/sensing/configs/shw5g.json -c SHW5G_2 -m "0x0000 0x1100" \ + --enable-camera-hal -s -Z -r 5 +``` + +Use your own link masks from Step 2. A working camera reports about 60 fps and `Frame drops: 0`. Both cameras of a +two-camera rig stream together at that rate. + +If initialization fails with `opcode = 0x9 / Parse failed after 5 successful operations / status: 10`, an enabled link +has no camera behind it. Power down, reseat the quad plug and the cable ends, and try again. On this rig the fix was a +restart, reseating the board and trying a different socket. If you are unsure which link a camera is on, run the +command once per mask and keep the one that reports frame rates. + +## Troubleshooting + +| Symptom | What to check | +| --- | --- | +| `opcode = 0x9 / ... status: 10` at startup | An enabled link has no camera. Power down, reseat the quad plug and cable ends, try again (Step 4). | +| No camera on the expected sensor | The quad plug is in CN2 instead of CN1, or a cable end is not fully seated. | +| A connector will not seat | Connectors are keyed. Do not force it; check the orientation and that the connector type matches. | +| Carrier appears dead | Check the 12 V supply, the barrel jack and the white 2-pin connector. No green LEDs means no power. | +| Sensor numbers do not match what you expect | Sensor numbers follow the link index, not a count from zero. | +| Cameras are not found after a hardware change | Power down fully, unplug both supplies, reseat, and power up again. | +| `/dev/video*` is empty | Expected on this carrier. The cameras are reached through SIPL, not Argus. | + +## References + +- SHW5G / SHF5G monocular camera, vendor wiki: + diff --git a/src/plugins/sensing/CMakeLists.txt b/src/plugins/sensing/CMakeLists.txt new file mode 100644 index 0000000000..0a93bee669 --- /dev/null +++ b/src/plugins/sensing/CMakeLists.txt @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ============================================================================== +# SENSING Camera Plugin: Jetson SIPL capture, hardware H.264 encode, LiveKit publication +# ============================================================================== +# Two leaf targets besides the LiveKit wrapper, one per directory (cmake/cmake-structure.md rule 1): +# livekit/ sensing_livekit -- the LiveKit SDK behind a small interface +# core/ sensing_core -- SIPL capture, encoder, conversion, track output +# app/ camera_plugin_sensing +# +# Prerequisites: the Jetson Multimedia API and SIPL header trees (see the dependency checks below). +# +# Configure with: +# cmake -B build -DBUILD_PLUGIN_SENSING=ON +# ============================================================================== + +enable_language(CUDA) + +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + # Orin is 87, Thor is 101; the rest cover desktop dev boxes. + set(CMAKE_CUDA_ARCHITECTURES "80;86;87;89;90") +endif() + +find_package(CUDAToolkit REQUIRED) + +set(_tegra_lib_dirs "/usr/lib/aarch64-linux-gnu/nvidia" "/usr/lib/aarch64-linux-gnu/tegra") + +# ============================================================================== +# SIPL +# ============================================================================== +# Headers ship in Jetson_SIPL_API_R_aarch64.tbz2, published alongside the +# L4T release. Not in any apt pool, so setup_container.sh fetches it by URL. +if(NOT SIPL_API_DIR) + find_path(SIPL_API_DIR sipl/include/NvSIPLCamera.hpp + PATHS "/usr/src/jetson_sipl_api" + "/opt/nvidia/jetson_sipl_api" + "$ENV{SIPL_API_DIR}" + ) +endif() + +if(NOT SIPL_API_DIR) + message(FATAL_ERROR + "================================================================================\n" + "SENSING camera plugin requires the Jetson SIPL API headers.\n" + "\n" + "Install the SIPL API package for your L4T release and point CMake at it:\n" + " cmake -B build -DSIPL_API_DIR=/path/to/jetson_sipl_api\n" + "\n" + "Or disable this plugin:\n" + " cmake -B build -DBUILD_PLUGIN_SENSING=OFF\n" + "================================================================================\n" + ) +endif() + +set(SIPL_INCLUDE_DIRS + "${SIPL_API_DIR}/sipl/include" + "${SIPL_API_DIR}/sipl/include/query/include" + "${SIPL_API_DIR}/sipl/include/nvsci" +) +message(STATUS "SENSING plugin: SIPL=${SIPL_API_DIR}") + +# L4T ships no .so dev symlinks for these, hence the explicit sonames. +foreach(_lib nvsipl nvsipl_query nvscibuf nvscisync nvbufsurface_nvsci) + string(TOUPPER "${_lib}" _upper) + find_library(${_upper}_LIBRARY + NAMES ${_lib} lib${_lib}.so.1 lib${_lib}.so.1.0.0 + PATHS ${_tegra_lib_dirs} + ) + if(NOT ${_upper}_LIBRARY) + message(FATAL_ERROR + "lib${_lib} not found. It is supplied by the L4T BSP and, in a container, by the " + "NVIDIA container runtime. Run src/plugins/sensing/scripts/verify.sh to see what is missing.") + endif() + list(APPEND SIPL_LIBRARIES ${${_upper}_LIBRARY}) +endforeach() + +# EGL stays on the capture path: NvBufSurfaceMapEglImage is how a SIPL +# NvSciBufObj becomes a CUeglFrame. Interop path B in the design notes would +# remove this; path A does not. +find_path(EGL_INCLUDE_DIRS EGL/egl.h) +find_library(EGL_LIBRARIES NAMES EGL) +if(NOT EGL_INCLUDE_DIRS OR NOT EGL_LIBRARIES) + message(FATAL_ERROR "SENSING camera plugin requires EGL. Install libegl1-mesa-dev.") +endif() + +# ============================================================================== +# Jetson V4L2 encoder +# ============================================================================== +# Jetson has no libnvidia-encode (the Video Codec SDK is dGPU-only), so H.264 +# comes from the V4L2 M2M engine via NvVideoEncoder, vendored from the Multimedia API (see below). +if(NOT JETSON_MMAPI_DIR) + find_path(JETSON_MMAPI_DIR include/nvbufsurface.h + PATHS "/usr/src/jetson_multimedia_api" + ) +endif() +if(NOT JETSON_MMAPI_DIR) + message(FATAL_ERROR + "================================================================================\n" + "SENSING camera plugin requires the Jetson Multimedia API headers (nvbufsurface.h).\n" + "\n" + "Unpack the version matching /etc/nv_tegra_release and point CMake at it:\n" + " cmake -B build -DJETSON_MMAPI_DIR=/path/to/jetson_multimedia_api\n" + "================================================================================\n" + ) +endif() +message(STATUS "SENSING plugin: Jetson Multimedia API=${JETSON_MMAPI_DIR}") + +# NVIDIA's encoder classes are vendored (BSD-3-Clause; third_party/jetson_multimedia_api/README.md), so they build the +# same on every JetPack and carry no stdout debug prints. The installed Multimedia API tree supplies only what is not +# vendored: nvbufsurface.h, libv4l2.h, v4l2_nv_extensions.h and the Argus headers. +set(SENSING_MMAPI_VENDORED_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/jetson_multimedia_api") +set(JETSON_MMAPI_SOURCES + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvVideoEncoder.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvV4l2Element.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvV4l2ElementPlane.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvElement.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvElementProfiler.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvBuffer.cpp" + "${SENSING_MMAPI_VENDORED_DIR}/classes/NvLogging.cpp" +) + +find_library(NVBUFSURFACE_LIBRARY NAMES nvbufsurface PATHS ${_tegra_lib_dirs}) +if(NOT NVBUFSURFACE_LIBRARY) + message(FATAL_ERROR "libnvbufsurface not found. Install nvidia-l4t-multimedia.") +endif() + +# NVIDIA's patched libv4l2 carries the Tegra M2M encoder ioctls, so search the +# tegra directories before the generic one. L4T ships no .so dev symlink, hence +# the explicit soname in NAMES. +find_library(V4L2_LIBRARY + NAMES v4l2 libv4l2.so.0 + PATHS ${_tegra_lib_dirs} + NO_DEFAULT_PATH +) +if(NOT V4L2_LIBRARY) + find_library(V4L2_LIBRARY NAMES v4l2 libv4l2.so.0) +endif() +if(NOT V4L2_LIBRARY) + message(FATAL_ERROR "libv4l2 not found. Install nvidia-l4t-multimedia (or libv4l-dev).") +endif() +message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}") + +# No libnvjpeg here on purpose: on R39 libnvbufsurface has no libjpeg +# dependency at all (readelf -d lists only EGL, cuda, dl, pthread, rt, c), and +# the tegra lib dir ships no libnvjpeg to link against. + +# ============================================================================== +# LiveKit +# ============================================================================== +# Each selected sensor is published as a LiveKit camera track. The SDK is a prebuilt package (headers and shared libraries), pinned to a +# release and checked against its hash; point LIVEKIT_SDK_DIR at an unpacked copy to use another. +set(_livekit_local ON) +if(NOT LIVEKIT_SDK_DIR) + set(_livekit_local OFF) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(_livekit_arch "arm64") + set(_livekit_hash "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(_livekit_arch "x64") + set(_livekit_hash "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b") + else() + message(FATAL_ERROR "No prebuilt LiveKit SDK for ${CMAKE_SYSTEM_PROCESSOR}; set LIVEKIT_SDK_DIR.") + endif() + include(FetchContent) + FetchContent_Declare( + livekit_sdk + URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-24.04-${_livekit_arch}-1.12.4.tar.gz + URL_HASH SHA256=${_livekit_hash} + ) + FetchContent_MakeAvailable(livekit_sdk) + set(LIVEKIT_SDK_DIR "${livekit_sdk_SOURCE_DIR}") +endif() +# The package ships no licence files. The pinned download supplies the SDK's own; a local SDK uses +# LICENSE/NOTICE from its directory, or LIVEKIT_SDK_LICENSE_FILE / LIVEKIT_SDK_NOTICE_FILE, and needs no network. +foreach(_notice LICENSE NOTICE) + set(_override "${LIVEKIT_SDK_${_notice}_FILE}") + if(_override) + set(_found "${_override}") + elseif(_livekit_local) + set(_found "") + foreach(_candidate "${LIVEKIT_SDK_DIR}/${_notice}" "${LIVEKIT_SDK_DIR}/../${_notice}") + if(NOT _found AND EXISTS "${_candidate}") + set(_found "${_candidate}") + endif() + endforeach() + if(NOT _found) + message(FATAL_ERROR + "LIVEKIT_SDK_DIR is set but ${_notice} was not found in it. Put the SDK's ${_notice} file there " + "or pass -DLIVEKIT_SDK_${_notice}_FILE=/path/to/${_notice}.") + endif() + else() + if(_notice STREQUAL "LICENSE") + set(_notice_hash "09e8a9bcec8067104652c168685ab0931e7868f9c8284b66f5ae6edae5f1130b") + else() + set(_notice_hash "6d0f7c5e0deb284362296c5da110dd353831da2e67ca77cb5abc051b951b0fac") + endif() + FetchContent_Declare( + livekit_sdk_${_notice} + URL https://raw.githubusercontent.com/livekit/client-sdk-cpp/v1.12.4/${_notice} + URL_HASH SHA256=${_notice_hash} + DOWNLOAD_NO_EXTRACT TRUE + ) + FetchContent_MakeAvailable(livekit_sdk_${_notice}) + set(_found "${livekit_sdk_${_notice}_SOURCE_DIR}/${_notice}") + endif() + set(SENSING_LIVEKIT_${_notice}_FILE "${_found}" CACHE INTERNAL "") +endforeach() +find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH) +message(STATUS "SENSING plugin: LiveKit SDK=${LIVEKIT_SDK_DIR}") +# Cache, so tests/ (a sibling directory) can find the same SDK. +set(SENSING_LIVEKIT_SDK_DIR "${LIVEKIT_SDK_DIR}" CACHE INTERNAL "") + + +# ============================================================================== +# Targets +# ============================================================================== +message(STATUS "Building SENSING camera plugin") + +# The platform configs live beside this file; app/ stages and installs them from here. +set(SENSING_CONFIGS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/configs") + +add_subdirectory(livekit) +add_subdirectory(core) +add_subdirectory(app) + +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/plugin.yaml" + "${CMAKE_CURRENT_SOURCE_DIR}/README.md" + DESTINATION plugins/sensing_camera +) diff --git a/src/plugins/sensing/README.md b/src/plugins/sensing/README.md index b4d0952c1b..147e65c950 100644 --- a/src/plugins/sensing/README.md +++ b/src/plugins/sensing/README.md @@ -13,8 +13,8 @@ Start with the [SENSING user guide](https://nvidia.github.io/IsaacCapture/main/d for supported hardware, installation, container usage, and troubleshooting. That guide owns the operating instructions; this README maps the source tree. -- [`scripts/`](scripts/): host setup, validation, diagnostics, and the Bash - compatibility Docker launcher. Each entry point exposes `--help`. +- [`scripts/`](scripts/): host setup, validation, diagnostics, the Bash + compatibility Docker launcher, and `setup_orin_crosscompile.py` for building on a PC. Each entry point exposes `--help`. - [`docker/`](docker/): example image, image-side group configuration, and host-side container setup. - [`configs/`](configs/): the platform configuration and its provenance. @@ -38,3 +38,11 @@ Keep the file byte-identical to the vendor copy. Compare new vendor drops before replacing it; do not hand-edit the board wiring. Its CSI ports, I2C buses, deserializer addresses, GPIOs, and link modes must match the installed SIPL drivers. + +## Orin: LiveKit publisher + +`camera_plugin_sensing` (built from [`app/`](app/), [`core/`](core/) and +[`livekit/`](livekit/)) captures the cameras with SIPL, encodes H.264 on the +hardware encoder and publishes one LiveKit video track per camera. The +[SENSING user guide](https://nvidia.github.io/IsaacCapture/main/device/camera/sensing.html) +has the build and run steps. diff --git a/src/plugins/sensing/app/CMakeLists.txt b/src/plugins/sensing/app/CMakeLists.txt new file mode 100644 index 0000000000..613d396602 --- /dev/null +++ b/src/plugins/sensing/app/CMakeLists.txt @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(camera_plugin_sensing main.cpp) + +set_target_properties(camera_plugin_sensing PROPERTIES INSTALL_RPATH "$ORIGIN") + +target_link_libraries(camera_plugin_sensing PRIVATE sensing::core) + +# The plugin resolves configs/shw5g.json relative to its own executable, so the +# config has to sit beside the binary in the build tree as well as the install +# tree -- otherwise it only works after `cmake --install`. +add_custom_command(TARGET camera_plugin_sensing POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${SENSING_CONFIGS_DIR}" + "$/configs" + COMMENT "Staging SIPL platform configs beside camera_plugin_sensing" +) + +install(TARGETS camera_plugin_sensing + RUNTIME DESTINATION plugins/sensing_camera +) +install(DIRECTORY "${SENSING_CONFIGS_DIR}/" + DESTINATION plugins/sensing_camera/configs +) + +# The plugin links the LiveKit runtime, whose RUNPATH points into the build tree; ship it beside the +# binary, where INSTALL_RPATH=$ORIGIN (and liblivekit.so's own $ORIGIN) finds it. +install(FILES + "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit.so" + "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit_ffi.so" + DESTINATION plugins/sensing_camera +) +install(FILES "${SENSING_LIVEKIT_LICENSE_FILE}" "${SENSING_LIVEKIT_NOTICE_FILE}" + DESTINATION plugins/sensing_camera/licenses/livekit-sdk +) +# The vendored encoder classes are compiled into the binary, so their license text ships with it. +install(FILES "${SENSING_MMAPI_VENDORED_DIR}/LICENSE" + DESTINATION plugins/sensing_camera/licenses/jetson-multimedia-api +) diff --git a/src/plugins/sensing/app/main.cpp b/src/plugins/sensing/app/main.cpp new file mode 100644 index 0000000000..425197b38f --- /dev/null +++ b/src/plugins/sensing/app/main.cpp @@ -0,0 +1,383 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::sensing; + +// ============================================================================= +// Signal handling +// ============================================================================= + +static std::atomic g_stop_requested{ false }; + +void signal_handler(int signal) +{ + if (signal == SIGINT || signal == SIGTERM) + { + g_stop_requested.store(true, std::memory_order_relaxed); + } +} + +// ============================================================================= +// Argument parsers +// ============================================================================= + +/// "sensor=N,livekit=NAME" -> one sensor and the track that carries it. +static StreamConfig parse_stream_arg(const std::string& arg) +{ + StreamConfig cfg{}; + bool has_sensor = false; + + std::istringstream ss(arg); + std::string token; + while (std::getline(ss, token, ',')) + { + auto eq = token.find('='); + if (eq == std::string::npos) + throw std::runtime_error("Invalid key=value in --add-stream: '" + token + "'"); + + auto key = token.substr(0, eq); + auto val = token.substr(eq + 1); + + if (key == "sensor") + { + cfg.sensor_id = static_cast(std::stoul(val)); + has_sensor = true; + } + else if (key == "livekit") + { + cfg.livekit_track = val; + } + else + { + throw std::runtime_error("Unknown key in --add-stream: '" + key + "'"); + } + } + + if (!has_sensor || cfg.livekit_track.empty()) + throw std::runtime_error("--add-stream requires sensor=,livekit="); + + return cfg; +} + +/// "0x0000 0x1100" -> {0x0000, 0x1100}. Same spelling as nvsipl_camera -m, so a +/// working vendor command line can be copied across verbatim. +static std::vector parse_link_masks(const std::string& arg) +{ + std::vector masks; + std::istringstream ss(arg); + std::string token; + while (ss >> token) + { + masks.push_back(static_cast(std::stoul(token, nullptr, 0))); + } + if (masks.empty()) + throw std::runtime_error("--link-masks needs at least one mask, e.g. --link-masks='0x0000 0x1100'"); + return masks; +} + +/// The vendored platform config, resolved next to the executable so it works +/// from the build tree and the install tree alike, whatever the cwd. Empty when +/// it is not there, which leaves --platform-config required. +static std::string default_platform_config() +{ + char buf[PATH_MAX]; + const ssize_t n = ::readlink("/proc/self/exe", buf, sizeof(buf) - 1); + if (n <= 0) + return {}; + buf[n] = '\0'; + const std::string exe(buf); + const auto slash = exe.rfind('/'); + if (slash == std::string::npos) + return {}; + const std::string candidate = exe.substr(0, slash) + "/configs/shw5g.json"; + return ::access(candidate.c_str(), R_OK) == 0 ? candidate : std::string{}; +} + +// ============================================================================= +// Usage +// ============================================================================= + +static void print_usage(const char* program_name) +{ + std::cout << "Usage: " << program_name << " --livekit-url=URL --livekit-token-file=PATH --add-stream=... [options]\n" + << "\n" + << "Publishes each SENSING sensor as a LiveKit camera track: SIPL capture, hardware H.264,\n" + << "sent as it is.\n" + << "\n" + << "Streams (repeat for each sensor):\n" + << " --add-stream=sensor=N,livekit=NAME SIPL pipeline index N, published as track NAME\n" + << " --list-sensors print the pipelines, then exit\n" + << "\n" + << "Platform:\n" + << " --platform-config=PATH platform JSON (default: configs/shw5g.json beside this\n" + << " binary)\n" + << " --config-name=NAME named config inside it (default: SHW5G_2)\n" + << " --link-masks='M M' one mask per deserializer (default: '0x0000 0x1100')\n" + << " --nito-dir=DIR ISP tuning directory (default: /var/nvidia/nvcam/settings/sipl)\n" + << "\n" + << "LiveKit:\n" + << " --livekit-url=URL server, e.g. wss://livekit.example.org\n" + << " --livekit-token-file=PATH file holding the join token; it names the room and the\n" + << " participant identity\n" + << " --livekit-fps=N publication rate cap (default: 30)\n" + << " --livekit-scale-div=N reduce every track by this integer divisor (default: 1)\n" + << " --livekit-bitrate=N target bitrate of each track's encoder in bps (default: 8000000)\n" + << " --gop=N IDR period in frames (default: fps*5)\n" + << "\n" + << "Other:\n" + << " --gpu-id=N CUDA device index (default: 0)\n" + << " --isp0-buffers=N ISP0 buffers per sensor (default: 6)\n" + << " --full-range full-range YUV\n" + << " --swap-uv swap the chroma planes\n" + << "\n" + << "Example:\n" + << " " << program_name << " --livekit-url=wss://livekit.example.org \\\n" + << " --livekit-token-file=join.token --add-stream=sensor=0,livekit=sensing-left \\\n" + << " --add-stream=sensor=1,livekit=sensing-right\n"; +} + +// ============================================================================= +// Main +// ============================================================================= + +int main(int argc, char** argv) +try +{ + SensingConfig camera_config; + std::map stream_map; + std::string livekit_token_file; + bool list_sensors = false; + + for (int i = 1; i < argc; ++i) + { + std::string arg = argv[i]; + + if (arg == "--help" || arg == "-h") + { + print_usage(argv[0]); + return 0; + } + else if (arg.find("--add-stream=") == 0) + { + auto cfg = parse_stream_arg(arg.substr(13)); + if (!stream_map.emplace(cfg.sensor_id, cfg).second) + throw std::runtime_error("sensor " + std::to_string(cfg.sensor_id) + " given twice"); + } + else if (arg.find("--platform-config=") == 0) + { + camera_config.platform_config_json = arg.substr(18); + } + else if (arg.find("--config-name=") == 0) + { + camera_config.platform_config_name = arg.substr(14); + } + else if (arg.find("--link-masks=") == 0) + { + camera_config.link_masks = parse_link_masks(arg.substr(13)); + } + else if (arg.find("--nito-dir=") == 0) + { + camera_config.nito_dir = arg.substr(11); + } + else if (arg == "--list-sensors") + { + list_sensors = true; + } + else if (arg.find("--livekit-url=") == 0) + { + camera_config.livekit.url = arg.substr(14); + } + else if (arg.find("--livekit-token-file=") == 0) + { + livekit_token_file = arg.substr(21); + } + else if (arg.find("--livekit-fps=") == 0) + { + camera_config.livekit.fps = static_cast(std::stoul(arg.substr(14))); + } + else if (arg.find("--livekit-bitrate=") == 0) + { + camera_config.livekit.bitrate_bps = static_cast(std::stoul(arg.substr(18))); + } + else if (arg.find("--livekit-scale-div=") == 0) + { + camera_config.livekit.scale_div = static_cast(std::stoul(arg.substr(20))); + } + else if (arg.find("--gop=") == 0) + { + camera_config.gop = static_cast(std::stoul(arg.substr(6))); + } + else if (arg.find("--gpu-id=") == 0) + { + camera_config.gpu_id = std::stoi(arg.substr(9)); + } + else if (arg.find("--isp0-buffers=") == 0) + { + camera_config.isp0_buffers = static_cast(std::stoul(arg.substr(15))); + } + else if (arg == "--full-range") + { + camera_config.full_range = true; + } + else if (arg == "--swap-uv") + { + camera_config.swap_uv = true; + } + else if (arg.find("--plugin-root-id=") == 0) + { + // plugin-root-id is a default argument, so we don't need to store it + } + else + { + std::cerr << "Unknown option: " << arg << std::endl; + print_usage(argv[0]); + return 1; + } + } + + if (camera_config.platform_config_json.empty()) + { + camera_config.platform_config_json = default_platform_config(); + } + if (camera_config.platform_config_json.empty()) + { + std::cerr << "Error: no platform config. configs/shw5g.json was not found beside this\n" + "binary, so pass --platform-config=PATH. SIPL cannot enumerate sensors\n" + "without one." + << std::endl; + print_usage(argv[0]); + return 1; + } + + // Query is hardware-free -- it only parses the driver database and the JSON -- so --list-sensors + // works on a rig whose cameras are unplugged, and the poll interval below can be derived before + // anything is opened. + const auto sensors = SiplCamera::query( + camera_config.platform_config_json, camera_config.platform_config_name, camera_config.link_masks); + + if (list_sensors) + { + std::cout << camera_config.platform_config_name << ": " << sensors.size() << " sensor(s)\n"; + for (const auto& s : sensors) + { + std::cout << " sensor=" << s.id << " " << s.name << " " << s.width << "x" << s.height << " @ " << s.fps + << " fps\n"; + } + return 0; + } + + if (stream_map.empty()) + { + std::cerr << "Error: at least one --add-stream is required." << std::endl; + print_usage(argv[0]); + return 1; + } + std::vector stream_configs; + stream_configs.reserve(stream_map.size()); + for (auto& [_, cfg] : stream_map) + { + stream_configs.push_back(std::move(cfg)); + } + + // Check the settings that need no hardware or network before anything is opened. + if (camera_config.livekit.url.empty() || livekit_token_file.empty()) + throw std::runtime_error("--livekit-url and --livekit-token-file are required"); + if (camera_config.livekit.fps == 0) + throw std::runtime_error("--livekit-fps must be at least 1"); + if (camera_config.livekit.bitrate_bps < 100'000) + throw std::runtime_error("--livekit-bitrate must be at least 100000"); + + // The token is read from a file so it never appears in a process listing, and is not printed. + std::ifstream token_file(livekit_token_file); + if (!token_file) + throw std::runtime_error("cannot read --livekit-token-file " + livekit_token_file); + std::getline(token_file, camera_config.livekit.token); + const auto last = camera_config.livekit.token.find_last_not_of(" \t\r\n"); + camera_config.livekit.token.erase(last == std::string::npos ? 0 : last + 1); + if (camera_config.livekit.token.empty()) + throw std::runtime_error("--livekit-token-file " + livekit_token_file + " is empty"); + + // This process captures and never renders, but EGL is on its critical path: + // NvBufSurfaceMapEglImage() is how a SIPL NvSciBufObj reaches CUDA, and + // libnvbufsurface resolves its own display via + // eglGetDisplay(EGL_DEFAULT_DISPLAY). GLVND hands out Mesa's EGL rather than + // Tegra's whenever DISPLAY names an X server Tegra EGL cannot drive (Xvfb, + // X11 forwarding), and that choice cannot be made per-call -- DISPLAY has to + // be gone from the environment before the first EGL call. + ::unsetenv("DISPLAY"); + + std::signal(SIGINT, signal_handler); + std::signal(SIGTERM, signal_handler); + + std::cout << "============================================================" << std::endl; + std::cout << "SENSING Camera Plugin Starting (SIPL)" << std::endl; + std::cout << "============================================================" << std::endl; + + SensingCamera camera(camera_config, stream_configs); + + std::cout << "------------------------------------------------------------" << std::endl; + std::cout << "Running capture loop. Press Ctrl+C to stop." << std::endl; + + constexpr auto stats_interval = std::chrono::seconds(5); + auto last_stats_time = std::chrono::steady_clock::now(); + + // SiplCamera::latest() is a non-blocking mailbox read, so poll at roughly + // twice the fastest sensor's frame rate rather than spinning a core flat + // out. Capture itself runs on SIPL's own threads and is unaffected. + double fastest_fps = 30.0; + for (const auto& s : sensors) + { + fastest_fps = std::max(fastest_fps, s.fps); + } + const auto poll_interval = std::chrono::microseconds(static_cast(500'000.0 / fastest_fps)); + + while (!g_stop_requested.load(std::memory_order_relaxed)) + { + camera.update(); + + auto now = std::chrono::steady_clock::now(); + if (now - last_stats_time >= stats_interval) + { + camera.print_stats(); + last_stats_time = now; + } + + std::this_thread::sleep_for(poll_interval); + } + + std::cout << "------------------------------------------------------------" << std::endl; + std::cout << "Shutting down SENSING Camera Plugin..." << std::endl; + camera.flush(); + camera.print_stats(); + std::cout << "Plugin stopped" << std::endl; + std::cout << "============================================================" << std::endl; + + return 0; +} +catch (const std::exception& e) +{ + std::cerr << argv[0] << ": " << e.what() << std::endl; + return 1; +} +catch (...) +{ + std::cerr << argv[0] << ": Unknown error occurred" << std::endl; + return 1; +} diff --git a/src/plugins/sensing/core/CMakeLists.txt b/src/plugins/sensing/core/CMakeLists.txt new file mode 100644 index 0000000000..54365535e1 --- /dev/null +++ b/src/plugins/sensing/core/CMakeLists.txt @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# SIPL capture, colour conversion, the hardware encoder and the LiveKit track output. +add_library(sensing_core STATIC + sensing_camera.cpp + sipl_camera.cpp + jetson_encoder.cpp + livekit_video_output.cpp + yuv_to_rgba.cu + rgba_to_nv12.cu + rgba_box_downscale.cu + inc/sensing_core/sensing_camera.hpp + inc/sensing_core/sensing_types.hpp + inc/sensing_core/sipl_camera.hpp + ${JETSON_MMAPI_SOURCES} +) +add_library(sensing::core ALIAS sensing_core) + +# RESOLVE_DEVICE_SYMBOLS as well as SEPARABLE_COMPILATION: without it the device-link step only happens +# for consumers in a directory where CUDA is an enabled language. Resolving here makes the archive +# self-contained. +set_target_properties(sensing_core PROPERTIES + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON +) + +target_include_directories(sensing_core + INTERFACE + $ + PRIVATE + # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay private: they drag in + # NvMedia and X11-style macros. + ${SIPL_INCLUDE_DIRS} + ${EGL_INCLUDE_DIRS} + # The vendored encoder headers come first, so they win over the installed tree's copies of the same names. + ${SENSING_MMAPI_VENDORED_DIR}/include + ${JETSON_MMAPI_DIR}/include +) + +# The Multimedia API reference sources carry legacy warnings. +target_compile_options(sensing_core PRIVATE + $<$:-Wno-reorder -Wno-uninitialized -Wno-unused-variable + -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare> +) + +target_link_libraries(sensing_core + PUBLIC + sensing::livekit + PRIVATE + CUDA::cudart + CUDA::cuda_driver + ${SIPL_LIBRARIES} + ${EGL_LIBRARIES} + ${NVBUFSURFACE_LIBRARY} + ${V4L2_LIBRARY} +) diff --git a/src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp b/src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp new file mode 100644 index 0000000000..0822da3460 --- /dev/null +++ b/src/plugins/sensing/core/inc/sensing_core/sensing_camera.hpp @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "sensing_types.hpp" +#include "sipl_camera.hpp" + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +class LiveKitSession; +class LiveKitVideoOutput; + +/** + * @brief Multi-sensor SENSING camera publisher. + * + * One SiplCamera owns every sensor, because INvSIPLCamera::GetInstance() is process-wide -- there is no + * per-sensor capture object to hold. Each update() polls every sensor's latest-frame mailbox and hands a new + * frame to that sensor's LiveKit track. Geometry is per sensor and comes from the platform config. + */ +class SensingCamera +{ +public: + SensingCamera(const SensingConfig& config, const std::vector& streams); + ~SensingCamera(); + + SensingCamera(const SensingCamera&) = delete; + SensingCamera& operator=(const SensingCamera&) = delete; + SensingCamera(SensingCamera&&) = delete; + SensingCamera& operator=(SensingCamera&&) = delete; + + /** @brief Poll every sensor and submit newly captured frames. Throws once the room is lost. */ + void update(); + + /** @brief Flush each track's queued frames. */ + void flush(); + + /** @brief Print per-sensor frame counts to stdout. */ + void print_stats() const; + +private: + struct Stream + { + uint32_t sensor_id = 0; + std::unique_ptr livekit; + /// SIPL publish counter of the frame already handled; skips re-reads. + uint64_t last_sequence = 0; + bool have_last_sequence = false; + uint64_t frame_count = 0; + /// Captures the poll loop never saw, from gaps in the SIPL sequence. + uint64_t missed_captures = 0; + }; + + SensingConfig m_config; + std::unique_ptr m_camera; + /// Declared before the streams so it is destroyed after their tracks have been unpublished. + std::unique_ptr m_livekit; + std::vector m_streams; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/inc/sensing_core/sensing_types.hpp b/src/plugins/sensing/core/inc/sensing_core/sensing_types.hpp new file mode 100644 index 0000000000..9fb8fcba2a --- /dev/null +++ b/src/plugins/sensing/core/inc/sensing_core/sensing_types.hpp @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Plain configuration types, deliberately free of the SIPL and NvSci headers, which define X11-style +// macros (Success, Status, None) that collide with other libraries'. + +#pragma once + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/// One captured sensor and the LiveKit track that carries it. +struct StreamConfig +{ + /// SIPL pipeline index, validated against the platform config at startup. + uint32_t sensor_id = 0; + /// Name of the LiveKit camera track carrying this sensor. + std::string livekit_track; +}; + +/// Where and how the tracks are published. +struct LiveKitConfig +{ + std::string url; + /// The join token itself, not a path. It names the room and the participant. Never logged. + std::string token; + /// Publication rate cap, independent of the capture rate. + uint32_t fps = 30; + /// Output-only uniform reduction of every track; 1 publishes the sensor's native size. + uint32_t scale_div = 1; + /// Target bitrate of each track's hardware encoder; the VBR ceiling is half as much again. + uint32_t bitrate_bps = 8'000'000; +}; + +struct SensingConfig +{ + /// Vendor platform config JSON. Required -- SIPL cannot enumerate without one. + std::string platform_config_json; + /// Named config inside it. + std::string platform_config_name = "SHW5G_2"; + /// One mask per deserializer, in transport order. + std::vector link_masks{ 0x0000, 0x1100 }; + std::string nito_dir = "/var/nvidia/nvcam/settings/sipl"; + int gpu_id = 0; + + /// IDR period in frames; 0 defers to the encoder default of fps*5. + uint32_t gop = 0; + + bool full_range = false; + /// Defaults off: set_sipl_buf_attributes() requests the ISP0 surface order explicitly. + bool swap_uv = false; + /// ISP0 buffers per sensor, ~6 MB each at 2560x1984 NV12. + uint32_t isp0_buffers = 6; + + LiveKitConfig livekit; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/inc/sensing_core/sipl_camera.hpp b/src/plugins/sensing/core/inc/sensing_core/sipl_camera.hpp new file mode 100644 index 0000000000..52e645f52c --- /dev/null +++ b/src/plugins/sensing/core/inc/sensing_core/sipl_camera.hpp @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// SIPL capture for the SENSING SG8A carrier: every sensor in one platform +// config, ISP0 YUV out, converted to RGBA8 on the GPU. +// +// One instance owns every sensor. This is not a style choice -- +// INvSIPLCamera::GetInstance() hands out a single process-wide object, so a +// per-sensor camera class is not expressible. Do not reintroduce one. +// +// The SIPL and NvSci headers are pimpl'd away because they drag in NvMedia and +// X11-style macros (Success, Status, None) that collide with other libraries'; keeping them out of +// this header is what lets other files include it freely. + +#pragma once + +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct SiplConfig +{ + /// Vendor platform config, e.g. /query/sg8a_agth_g2a/shw5g.json. + std::string platform_config_json; + /// Named config inside it, e.g. "SHW5G_2". + std::string platform_config_name; + /// One mask per deserializer, in transport order. Required for GMSL. + std::vector link_masks; + /// Directory holding .nito; the ISP has no tuning without it. + std::string nito_dir = "/var/nvidia/nvcam/settings/sipl"; + int gpu_id = 0; + /// Overrides the range implied by the reconciled colour standard. + bool full_range = false; + /// Swap the chroma planes. Defaults off: the ISP0 surface order is + /// requested explicitly, so it does not need discovering from the frame. + bool swap_uv = false; + /// ISP0 buffers per sensor. Six is the SIPL default; more absorbs a slow + /// consumer at the cost of 6 MB each at 2560x1984 NV12. + uint32_t isp0_buffers = 6; + uint32_t frame_timeout_ms = 1000; +}; + +/// One sensor as the platform config describes it. `id` is the SIPL pipeline +/// index, which is NOT the GMSL link index, the CSI virtual channel, or the +/// JSON sensorInfo.id -- for SHW5G_2 those are 2 and 3 while this is 0 and 1. +struct SensorInfo +{ + uint32_t id = 0; + std::string name; + uint32_t width = 0; + uint32_t height = 0; + double fps = 0.0; +}; + +/** + * @brief Borrowed view of the most recently converted frame. + * + * `ptr` addresses one of three producer-owned slots and stays valid until the + * second subsequent latest() call for the same sensor, at which point the + * producer may overwrite it. Copy out before then; do not cache a FrameView + * across update() ticks. + */ +struct FrameView +{ + uintptr_t ptr = 0; + size_t pitch = 0; + uint32_t width = 0; + uint32_t height = 0; + /// CLOCK_MONOTONIC, stamped at YUV->RGBA conversion. Published as the frame's + /// transport timestamp; the encoder does not return it, so units pair with captures by order. + uint64_t timestamp_ns = 0; + /// CLOCK_REALTIME (Unix time) read at the same instant as timestamp_ns: processing time, after capture and + /// conversion, so later than the capture event. Published as the frame's application timestamp; it is + /// only as accurate as the two hosts' clock synchronisation and is not a capture time. + uint64_t unix_ns = 0; + /// SIPL's raw frame capture counter, shared by every sensor on the rig, so it pairs the two eyes. Its units + /// are unscaled ticks until the SIPL headers confirm otherwise; it is not nanoseconds or a host time. + uint64_t capture_tsc = 0; + uint64_t sequence = 0; +}; + +class SiplCamera +{ +public: + explicit SiplCamera(const SiplConfig& config); + ~SiplCamera(); + + SiplCamera(const SiplCamera&) = delete; + SiplCamera& operator=(const SiplCamera&) = delete; + + /// Enumerate a platform config without touching the hardware. Cheap: the + /// query API only parses the driver database and the JSON. + static std::vector query(const std::string& platform_config_json, + const std::string& platform_config_name, + const std::vector& link_masks); + + /// Sensors this instance configured, in pipeline-index order. + const std::vector& sensors() const; + + void start(); + void stop(); + + /// Non-blocking mailbox read. nullopt when no new frame has arrived. + std::optional latest(uint32_t sensor_id); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/jetson_encoder.cpp b/src/plugins/sensing/core/jetson_encoder.cpp new file mode 100644 index 0000000000..8dfff056cc --- /dev/null +++ b/src/plugins/sensing/core/jetson_encoder.cpp @@ -0,0 +1,325 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "jetson_encoder.hpp" + +#include "rgba_to_nv12.cuh" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +constexpr uint32_t kOutputBuffers = 6; +constexpr uint32_t kCaptureBuffers = 6; + +void check(int ret, const char* what) +{ + if (ret < 0) + throw std::runtime_error(std::string("JetsonEncoder: ") + what + " failed"); +} + +void check_cuda(cudaError_t err, const char* what) +{ + if (err != cudaSuccess) + throw std::runtime_error(std::string("JetsonEncoder: ") + what + ": " + cudaGetErrorString(err)); +} + +} // namespace + +struct JetsonEncoder::Impl +{ + EncoderConfig config; + NvVideoEncoder* encoder = nullptr; + + // NV12 staging in device memory; the V4L2 output plane is host-mapped, so + // each frame costs one device-to-host copy per plane. + uint8_t* d_y = nullptr; + uint8_t* d_uv = nullptr; + size_t d_y_pitch = 0; + size_t d_uv_pitch = 0; + + std::mutex mutex; + std::deque ready; + bool eos_reached = false; + bool eos_sent = false; // driving thread only + bool dq_thread_started = false; + + ~Impl() + { + // The capture thread dereferences this object, so it must be joined here, not only in ~JetsonEncoder: + // a constructor that throws after starting it never runs ~JetsonEncoder. + if (encoder && dq_thread_started) + { + encoder->capture_plane.stopDQThread(); + encoder->capture_plane.waitForDQThread(1000); + } + if (d_y) + cudaFree(d_y); + if (d_uv) + cudaFree(d_uv); + delete encoder; + } + + // Capture-plane dequeue callback; runs on the encoder's own thread. + static bool capture_dq(struct v4l2_buffer* v4l2_buf, NvBuffer* buffer, NvBuffer* /*shared*/, void* arg) + { + auto* self = static_cast(arg); + + if (!v4l2_buf) + return false; + + if (buffer && buffer->planes[0].bytesused > 0) + { + const auto* data = static_cast(buffer->planes[0].data); + // V4L2 M2M encoders may copy the OUTPUT buffer's timestamp onto the CAPTURE buffer, but the Orin driver + // observed here returns its own frame-rate numbering instead: treat this stamp as the unit's own. + EncodedUnit unit; + unit.data.assign(data, data + buffer->planes[0].bytesused); + unit.timestamp_ns = static_cast(v4l2_buf->timestamp.tv_sec) * 1000000000LL + + static_cast(v4l2_buf->timestamp.tv_usec) * 1000LL; + unit.keyframe = (v4l2_buf->flags & V4L2_BUF_FLAG_KEYFRAME) != 0; + std::lock_guard lock(self->mutex); + self->ready.push_back(std::move(unit)); + } + + // A zero-length unit marks end of stream; stop the thread rather than + // re-queueing, otherwise the dq thread spins on a finished encoder. + if (buffer && buffer->planes[0].bytesused == 0) + { + std::lock_guard lock(self->mutex); + self->eos_reached = true; + return false; + } + + if (self->encoder->capture_plane.qBuffer(*v4l2_buf, nullptr) < 0) + return false; + + return true; + } +}; + +JetsonEncoder::JetsonEncoder(const EncoderConfig& config) : m_impl(std::make_unique()) +{ + if (config.width == 0 || config.height == 0) + throw std::runtime_error("JetsonEncoder: width/height must be non-zero"); + if ((config.width % 2) != 0 || (config.height % 2) != 0) + throw std::runtime_error("JetsonEncoder: width/height must be even for NV12"); + + m_impl->config = config; + + m_impl->encoder = NvVideoEncoder::createVideoEncoder("enc0"); + if (!m_impl->encoder) + throw std::runtime_error("JetsonEncoder: createVideoEncoder failed (is /dev/v4l2-nvenc present?)"); + + auto* enc = m_impl->encoder; + + // Capture format must be set before the output format. + const uint32_t bitstream_size = config.width * config.height * 3 / 2; + check(enc->setCapturePlaneFormat(V4L2_PIX_FMT_H264, config.width, config.height, bitstream_size), + "setCapturePlaneFormat"); + check(enc->setOutputPlaneFormat(V4L2_PIX_FMT_NV12M, config.width, config.height), "setOutputPlaneFormat"); + + check(enc->setBitrate(config.bitrate_bps), "setBitrate"); + check(enc->setProfile(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH), "setProfile"); + + // Level is not inferred reliably, and 2560x1984@60 is 1,190,400 macroblocks + // per second against Level 5.1's ceiling of 983,040 -- 21% over. An unset + // or too-low level encodes fine here and then gets refused downstream. + check(enc->setLevel(V4L2_MPEG_VIDEO_H264_LEVEL_5_2), "setLevel"); + + if (config.peak_bitrate_bps > config.bitrate_bps) + { + check(enc->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR), "setRateControlMode"); + check(enc->setPeakBitrate(config.peak_bitrate_bps), "setPeakBitrate"); + } + else + { + check(enc->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR), "setRateControlMode"); + } + check(enc->setFrameRate(config.fps ? config.fps : 30, 1), "setFrameRate"); + + const uint32_t gop = config.gop ? config.gop : (config.fps ? config.fps * 5 : 150); + check(enc->setIDRInterval(gop), "setIDRInterval"); + check(enc->setIFrameInterval(gop), "setIFrameInterval"); + + // Low-latency shape: no B-frames, SPS/PPS on every IDR so a receiver can + // join mid-stream, and the max-performance clock preset. + check(enc->setNumBFrames(0), "setNumBFrames"); + check(enc->setInsertSpsPpsAtIdrEnabled(true), "setInsertSpsPpsAtIdrEnabled"); + check(enc->setMaxPerfMode(1), "setMaxPerfMode"); + // A separate control from the max-performance clocks: it trades quality for encoder throughput. + // FAST suits live streams; four native-size images at 30 fps may need ULTRAFAST, to be measured. + check(enc->setHWPresetType(V4L2_ENC_HW_PRESET_FAST), "setHWPresetType"); + + check(enc->output_plane.setupPlane(V4L2_MEMORY_MMAP, kOutputBuffers, true, false), "output setupPlane"); + check(enc->capture_plane.setupPlane(V4L2_MEMORY_MMAP, kCaptureBuffers, true, false), "capture setupPlane"); + + check(enc->output_plane.setStreamStatus(true), "output setStreamStatus"); + check(enc->capture_plane.setStreamStatus(true), "capture setStreamStatus"); + + enc->capture_plane.setDQThreadCallback(&Impl::capture_dq); + enc->capture_plane.startDQThread(m_impl.get()); + m_impl->dq_thread_started = true; + + // Prime the capture plane so the encoder always has somewhere to write. + for (uint32_t i = 0; i < enc->capture_plane.getNumBuffers(); ++i) + { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(planes, 0, sizeof(planes)); + v4l2_buf.index = i; + v4l2_buf.m.planes = planes; + check(enc->capture_plane.qBuffer(v4l2_buf, nullptr), "capture qBuffer"); + } + + check_cuda(cudaMallocPitch(reinterpret_cast(&m_impl->d_y), &m_impl->d_y_pitch, config.width, config.height), + "cudaMallocPitch(Y)"); + check_cuda( + cudaMallocPitch(reinterpret_cast(&m_impl->d_uv), &m_impl->d_uv_pitch, config.width, config.height / 2), + "cudaMallocPitch(UV)"); +} + +JetsonEncoder::~JetsonEncoder() = default; + +bool JetsonEncoder::submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns) +{ + auto* enc = m_impl->encoder; + const auto& config = m_impl->config; + + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(planes, 0, sizeof(planes)); + v4l2_buf.m.planes = planes; + + NvBuffer* buffer = nullptr; + // Until every output buffer has been queued once, index i is free by + // construction; after that a dequeue is what frees one. + if (m_queued < enc->output_plane.getNumBuffers()) + { + buffer = enc->output_plane.getNthBuffer(m_queued); + v4l2_buf.index = m_queued; + ++m_queued; + } + else if (enc->output_plane.dqBuffer(v4l2_buf, &buffer, nullptr, 0) < 0) + { + return false; // encoder still busy; drop this frame rather than block + } + + // An input buffer is ours now, so convert: a frame that would be dropped costs no conversion. A + // failure from here is terminal for the encoder, as a failed qBuffer is. + launch_rgba_to_nv12(reinterpret_cast(rgba_device_ptr), static_cast(row_pitch_bytes), m_impl->d_y, + static_cast(m_impl->d_y_pitch), m_impl->d_uv, static_cast(m_impl->d_uv_pitch), + static_cast(config.width), static_cast(config.height), config.full_range, nullptr); + check_cuda(cudaGetLastError(), "rgba_to_nv12 launch"); + check_cuda(cudaStreamSynchronize(nullptr), "rgba_to_nv12 sync"); + + check_cuda(cudaMemcpy2D(buffer->planes[0].data, buffer->planes[0].fmt.stride, m_impl->d_y, m_impl->d_y_pitch, + config.width, config.height, cudaMemcpyDeviceToHost), + "cudaMemcpy2D(Y)"); + check_cuda(cudaMemcpy2D(buffer->planes[1].data, buffer->planes[1].fmt.stride, m_impl->d_uv, m_impl->d_uv_pitch, + config.width, config.height / 2, cudaMemcpyDeviceToHost), + "cudaMemcpy2D(UV)"); + + buffer->planes[0].bytesused = buffer->planes[0].fmt.stride * config.height; + buffer->planes[1].bytesused = buffer->planes[1].fmt.stride * (config.height / 2); + v4l2_buf.m.planes[0].bytesused = buffer->planes[0].bytesused; + v4l2_buf.m.planes[1].bytesused = buffer->planes[1].bytesused; + + // Microsecond granularity is all a timeval carries, well below the frame interval. + v4l2_buf.timestamp.tv_sec = static_cast(timestamp_ns / 1000000000LL); + v4l2_buf.timestamp.tv_usec = static_cast((timestamp_ns % 1000000000LL) / 1000LL); + + check(enc->output_plane.qBuffer(v4l2_buf, nullptr), "output qBuffer"); + return true; +} + +EncodedUnit JetsonEncoder::poll() +{ + std::lock_guard lock(m_impl->mutex); + if (m_impl->ready.empty()) + return {}; + + auto out = std::move(m_impl->ready.front()); + m_impl->ready.pop_front(); + return out; +} + +std::vector JetsonEncoder::drain() +{ + auto* enc = m_impl->encoder; + + // A zero-length output buffer is the EOS marker for the V4L2 encoder. It is sent once: a second + // drain only returns what is left. + if (!m_impl->eos_sent) + { + m_impl->eos_sent = true; + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(planes, 0, sizeof(planes)); + v4l2_buf.m.planes = planes; + + NvBuffer* buffer = nullptr; + if (enc->output_plane.dqBuffer(v4l2_buf, &buffer, nullptr, 10) >= 0) + { + v4l2_buf.m.planes[0].bytesused = 0; + v4l2_buf.m.planes[1].bytesused = 0; + enc->output_plane.qBuffer(v4l2_buf, nullptr); + } + + enc->capture_plane.waitForDQThread(2000); + } + + std::lock_guard lock(m_impl->mutex); + std::vector units( + std::make_move_iterator(m_impl->ready.begin()), std::make_move_iterator(m_impl->ready.end())); + m_impl->ready.clear(); + return units; +} + +EncodedUnit JetsonEncoder::end_of_stream() +{ + // The tail is concatenated, so it carries the stamp of the last unit in it. + EncodedUnit out; + for (auto& chunk : drain()) + { + out.data.insert(out.data.end(), chunk.data.begin(), chunk.data.end()); + out.timestamp_ns = chunk.timestamp_ns; + } + return out; +} + +void JetsonEncoder::request_keyframe() +{ + check(m_impl->encoder->forceIDR(), "forceIDR"); +} + +bool JetsonEncoder::set_bitrate(uint32_t bitrate_bps) +{ + return m_impl->encoder->setBitrate(bitrate_bps) >= 0; +} + +const char* JetsonEncoder::backend_name() +{ + return "Jetson V4L2 NVENC"; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/jetson_encoder.hpp b/src/plugins/sensing/core/jetson_encoder.hpp new file mode 100644 index 0000000000..aabd3dcfcd --- /dev/null +++ b/src/plugins/sensing/core/jetson_encoder.hpp @@ -0,0 +1,111 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct EncoderConfig +{ + uint32_t width = 0; + uint32_t height = 0; + uint32_t bitrate_bps = 40'000'000; + /// VBR ceiling; 0 keeps CBR. The H.264 output is a recording, not a + /// fixed-rate transport, so CBR would pad static scenes and starve motion. + uint32_t peak_bitrate_bps = 60'000'000; + uint32_t fps = 30; + /// IDR period in frames; 0 defers to fps*5. + uint32_t gop = 0; + bool full_range = false; +}; + +/// One encoded Annex-B unit. +struct EncodedUnit +{ + std::vector data; + /// The encoder's own frame-rate numbering, not the stamp given to submit(). + int64_t timestamp_ns = 0; + /// The unit is an IDR (with its parameter sets), decodable on its own. Not set on the + /// concatenated tail from end_of_stream(). + bool keyframe = false; + + bool empty() const + { + return data.empty(); + } +}; + +/** + * @brief H.264 encoder on the Jetson V4L2 M2M engine (/dev/v4l2-nvenc). + * + * Jetson has no libnvidia-encode — the NVIDIA Video Codec SDK is dGPU-only — + * so this wraps NvVideoEncoder from the Jetson Multimedia API instead. + * + * One thread must drive an encoder: submit(), poll(), request_keyframe() and set_bitrate() are not + * safe to call concurrently. + * + * Submission is asynchronous: the encoder needs several input frames before it + * emits the first bitstream unit, so submit() never blocks waiting for output + * and poll() drains whatever a capture-plane thread has completed. + */ +class JetsonEncoder +{ +public: + explicit JetsonEncoder(const EncoderConfig& config); + ~JetsonEncoder(); + + JetsonEncoder(const JetsonEncoder&) = delete; + JetsonEncoder& operator=(const JetsonEncoder&) = delete; + JetsonEncoder(JetsonEncoder&&) = delete; + JetsonEncoder& operator=(JetsonEncoder&&) = delete; + + /** + * @brief Convert one GPU-resident RGBA8 frame to NV12 and queue it. + * @param rgba_device_ptr Device pointer to a HxWx4 RGBA8 buffer. + * @param row_pitch_bytes Byte stride between rows. + * @param timestamp_ns Stamp set on the V4L2 output buffer. On the verified Orin driver the unit + * poll() returns does not carry it back (units are numbered from the encoder's own frame rate), so + * callers pair units with submissions by order; that numbering is the only hint of a skipped frame. + * @return false when no input buffer is free (frame dropped). + */ + bool submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_bytes, int64_t timestamp_ns); + + /** @brief Take any completed unit; empty during encoder warmup. */ + EncodedUnit poll(); + + /** @brief Signal EOS and drain the remaining bitstream, concatenated into one unit. */ + EncodedUnit end_of_stream(); + + /** @brief Signal EOS and drain the remaining bitstream, one unit per frame. */ + std::vector drain(); + + /** @brief Make the next submitted frame an IDR, preceded by SPS/PPS. */ + void request_keyframe(); + + /** + * @brief Change the target bitrate of the running stream. + * @return false when this backend cannot change it while encoding. + */ + bool set_bitrate(uint32_t bitrate_bps); + + /** @brief Names the encoder behind this class, for diagnostics. */ + static const char* backend_name(); + +private: + struct Impl; + std::unique_ptr m_impl; + + /// Output buffers queued so far; below the buffer count they are free by + /// construction and need no dequeue first. + uint32_t m_queued = 0; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/livekit_video_output.cpp b/src/plugins/sensing/core/livekit_video_output.cpp new file mode 100644 index 0000000000..934ae3ecce --- /dev/null +++ b/src/plugins/sensing/core/livekit_video_output.cpp @@ -0,0 +1,519 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "livekit_video_output.hpp" + +#include "jetson_encoder.hpp" +#include "rgba_box_downscale.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +void check_cuda(cudaError_t err, const char* what) +{ + if (err != cudaSuccess) + throw std::runtime_error(std::string("LiveKit output: ") + what + ": " + cudaGetErrorString(err)); +} + +// Congestion control never drives the encoder below this. +constexpr uint32_t kMinBitrateBps = 100'000; +// A target is applied only when it differs from the current one by this fraction, and this long after +// the last change. +constexpr double kMinBitrateChange = 0.25; +constexpr std::chrono::seconds kMinBitrateInterval{ 3 }; + +} // namespace + +struct LiveKitVideoOutput::Impl +{ + struct Slot + { + uint8_t* ptr = nullptr; + size_t pitch = 0; + uint64_t timestamp_ns = 0; + uint64_t unix_ns = 0; + uint64_t capture_tsc = 0; + uint64_t sequence = 0; + }; + + Impl(std::unique_ptr track, const LiveKitOutputConfig& cfg) : sink(std::move(track)), config(cfg) + { + validate_geometry(config.source_width, config.source_height, config.scale_div); + if (config.fps == 0) + throw std::runtime_error("LiveKit output: the publication rate must be at least 1 fps"); + out_width = config.source_width / config.scale_div; + out_height = config.source_height / config.scale_div; + if (sink->width() != out_width || sink->height() != out_height) + throw std::runtime_error("LiveKit output: the track size does not match the output size"); + period_ns = 1'000'000'000ull / config.fps; + if (config.bitrate_bps < kMinBitrateBps) + { + throw std::runtime_error("LiveKit output: the bitrate must be at least " + std::to_string(kMinBitrateBps) + + " bps"); + } + // Congestion control moves the target between the floor and the ceiling; the configured target + // is never below the floor, so the range is never inverted. + max_bitrate = std::max(config.bitrate_bps, config.peak_bitrate_bps); + + // A constructor that throws never runs the destructor, so free what was allocated here. + try + { + check_cuda(cudaSetDevice(config.gpu_id), "cudaSetDevice"); + + // First, so a machine without a usable hardware encoder fails before anything is allocated. + EncoderConfig encoder_config; + encoder_config.width = out_width; + encoder_config.height = out_height; + encoder_config.bitrate_bps = config.bitrate_bps; + encoder_config.peak_bitrate_bps = config.peak_bitrate_bps; + encoder_config.fps = config.fps; + encoder_config.gop = config.gop; + encoder_config.full_range = config.full_range; + encoder = std::make_unique(encoder_config); + applied_bitrate = config.bitrate_bps; + + for (auto& slot : slots) + { + check_cuda(cudaMallocPitch(reinterpret_cast(&slot.ptr), &slot.pitch, + static_cast(config.source_width) * 4, config.source_height), + "cudaMallocPitch(snapshot)"); + } + if (config.scale_div > 1) + { + check_cuda(cudaMallocPitch(reinterpret_cast(&d_scaled), &scaled_pitch, + static_cast(out_width) * 4, out_height), + "cudaMallocPitch(reduced)"); + } + check_cuda(cudaStreamCreateWithFlags(&snapshot_stream, cudaStreamNonBlocking), "cudaStreamCreate"); + check_cuda(cudaStreamCreateWithFlags(&work_stream, cudaStreamNonBlocking), "cudaStreamCreate"); + + worker = std::thread([this] { run(); }); + } + catch (...) + { + release(); + throw; + } + } + + ~Impl() + { + { + std::lock_guard lock(mutex); + stopping = true; + } + wake.notify_all(); + if (worker.joinable()) + worker.join(); + release(); + } + + void release() + { + for (auto& slot : slots) + cudaFree(slot.ptr); + cudaFree(d_scaled); + if (snapshot_stream) + cudaStreamDestroy(snapshot_stream); + if (work_stream) + cudaStreamDestroy(work_stream); + } + + void fail(const std::string& message) + { + std::lock_guard lock(failure_mutex); + if (!failed_flag.load()) + { + failure_message = message; + failed_flag.store(true); + } + } + + // Polling thread. + void submit(const FrameView& view) + { + ++offered; + if (failed_flag.load()) + return; + + // Pace on the capture clock, so poll jitter does not drop a frame that is due. A tenth of a + // period of slack lets 60 fps capture feed a 30 fps track every second capture. + if (have_last && view.timestamp_ns < last_accepted_ns + period_ns * 9 / 10) + { + ++paced_out; + return; + } + + try + { + std::lock_guard lock(mutex); + int target = -1; + for (int i = 0; i < 2; ++i) + { + if (i != inflight && i != pending) + { + target = i; + break; + } + } + if (target < 0) + { + // The worker holds one slot and the other waits: the newer capture takes the waiting one. + target = pending; + } + // A capture still waiting is overwritten by this newer one. + if (pending >= 0) + ++replaced; + + // Complete before returning: the camera may rewrite its slot at the second following poll. + Slot& slot = slots[target]; + check_cuda(cudaMemcpy2DAsync(slot.ptr, slot.pitch, reinterpret_cast(view.ptr), view.pitch, + static_cast(config.source_width) * 4, config.source_height, + cudaMemcpyDeviceToDevice, snapshot_stream), + "snapshot copy"); + check_cuda(cudaStreamSynchronize(snapshot_stream), "snapshot sync"); + + slot.timestamp_ns = view.timestamp_ns; + slot.unix_ns = view.unix_ns; + slot.capture_tsc = view.capture_tsc; + slot.sequence = view.sequence; + pending = target; + } + catch (const std::exception& e) + { + fail(e.what()); + return; + } + ++snapshots; + have_last = true; + last_accepted_ns = view.timestamp_ns; + wake.notify_one(); + } + + // Worker thread. + void run() + { + try + { + check_cuda(cudaSetDevice(config.gpu_id), "cudaSetDevice(worker)"); + for (;;) + { + int index = -1; + bool flush_now = false; + { + std::unique_lock lock(mutex); + // Frames in the encoder come out a little later: poll for them soon, and otherwise + // wake now and then to pick up the receiving side's requests. + const auto wait = in_encoder.empty() ? std::chrono::milliseconds(100) : std::chrono::milliseconds(2); + wake.wait_for(lock, wait, [&] { return stopping || pending >= 0 || flush_requested; }); + if (stopping) + return; + index = pending; + pending = -1; + inflight = index; + flush_now = flush_requested; + } + + apply_feedback(); + if (index >= 0) + { + encode(slots[index]); + std::lock_guard lock(mutex); + inflight = -1; + } + if (flush_now) + { + for (auto& unit : encoder->drain()) + publish_unit(unit); + // Anything still unreturned once the encoder is drained is not coming back. + encoder_dropped += in_encoder.size(); + in_encoder.clear(); + std::lock_guard lock(mutex); + flush_requested = false; + flushed.notify_all(); + } + else + { + for (auto unit = encoder->poll(); !unit.empty(); unit = encoder->poll()) + publish_unit(unit); + } + } + } + catch (const std::exception& e) + { + fail(e.what()); + std::lock_guard lock(mutex); + flushed.notify_all(); + } + } + + // Worker thread: what the receiving side asked for since the last frame. + void apply_feedback() + { + const EncoderFeedback feedback = sink->take_feedback(); + if (feedback.keyframe_requested) + { + ++keyframes_requested; + keyframe_wanted = true; + } + if (feedback.target_bitrate_bps) + { + const uint32_t target = std::clamp(*feedback.target_bitrate_bps, kMinBitrateBps, max_bitrate); + // Congestion control re-estimates all the time. Follow it only for a real change, and not + // more than every few seconds: a change reopens the encoder session. + const auto now = std::chrono::steady_clock::now(); + const double change = std::abs(static_cast(target) - applied_bitrate) / applied_bitrate; + if (change >= kMinBitrateChange && now - last_bitrate_change >= kMinBitrateInterval && + encoder->set_bitrate(target)) + { + applied_bitrate = target; + last_bitrate_change = now; + } + } + } + + // Worker thread. + void encode(const Slot& slot) + { + const uint8_t* rgba = slot.ptr; + size_t pitch = slot.pitch; + if (config.scale_div > 1) + { + launch_rgba_box_downscale(rgba, static_cast(pitch), d_scaled, static_cast(scaled_pitch), + static_cast(out_width), static_cast(out_height), + static_cast(config.scale_div), work_stream); + check_cuda(cudaGetLastError(), "reduction kernel"); + // The encoder reads on the default stream, which a non-blocking stream does not order against. + check_cuda(cudaStreamSynchronize(work_stream), "reduction sync"); + rgba = d_scaled; + pitch = scaled_pitch; + } + + if (keyframe_wanted) + { + encoder->request_keyframe(); + keyframe_wanted = false; + } + const int64_t stamp_ns = static_cast(slot.timestamp_ns); + if (!encoder->submit(reinterpret_cast(rgba), pitch, stamp_ns)) + { + // No input buffer: dropped before encoding, so no reference is lost. A keyframe already + // requested of the encoder applies to the next frame it takes. + ++encoder_busy; + return; + } + in_encoder.push_back( + { stamp_ns / 1000, static_cast(slot.sequence), slot.capture_tsc, slot.unix_ns / 1000 }); + } + + // Worker thread: one encoded unit becomes one published frame, with the sequence and capture time + // of the capture it came from. + void publish_unit(const EncodedUnit& unit) + { + // The Jetson encoder does not return the stamp it was given: it numbers its units from its own frame + // rate (0, 1/fps, ...). Units pair with captures by order, and a jump in that numbering is the only + // evidence of frames the encoder consumed without returning a unit. Metadata is retired only by a + // matched unit or by such a jump; a jump that leaves no frame for this unit, or a numbering that goes + // backwards, is not trusted and just resets the baseline. + const int64_t index = std::llround(static_cast(unit.timestamp_ns) * config.fps / 1e9); + if (have_unit_index) + { + const int64_t gap = index - last_unit_index - 1; + if (gap > 0 && static_cast(gap) < in_encoder.size()) + { + in_encoder.erase(in_encoder.begin(), in_encoder.begin() + gap); + encoder_dropped += static_cast(gap); + } + } + last_unit_index = index; + have_unit_index = true; + + // A unit with no frame to pair with is skipped rather than published with someone else's metadata. + if (in_encoder.empty()) + { + ++unmatched_units; + return; + } + const FrameMeta meta = in_encoder.front(); + in_encoder.pop_front(); + + // After a refusal the stream is only usable from its next keyframe. + if (awaiting_keyframe && !unit.keyframe) + return; + + EncodedVideoFrame frame; + frame.data = unit.data.data(); + frame.size = unit.data.size(); + frame.keyframe = unit.keyframe; + frame.timestamp_us = static_cast(meta.stamp_us); + frame.frame_id = meta.sequence; + frame.capture_tsc = meta.capture_tsc; + frame.user_timestamp_us = meta.unix_us; + if (sink->submit(frame)) + { + awaiting_keyframe = false; + ++published; + } + else + { + ++rejected; + awaiting_keyframe = true; + keyframe_wanted = true; + } + } + + std::unique_ptr sink; + LiveKitOutputConfig config; + uint32_t out_width = 0; + uint32_t out_height = 0; + uint64_t period_ns = 0; + + struct FrameMeta + { + int64_t stamp_us = 0; + uint32_t sequence = 0; + uint64_t capture_tsc = 0; + /// Unix time of the same instant as stamp_us, for the frame's application timestamp. + uint64_t unix_us = 0; + }; + + std::unique_ptr encoder; + uint32_t max_bitrate = 0; + + // Worker thread only. + std::deque in_encoder; ///< Submitted to the encoder, no unit back yet. + int64_t last_unit_index = 0; ///< The encoder's own number of the last unit returned. + bool have_unit_index = false; + bool keyframe_wanted = false; + bool awaiting_keyframe = false; + uint32_t applied_bitrate = 0; + std::chrono::steady_clock::time_point last_bitrate_change = std::chrono::steady_clock::now(); + + Slot slots[2]; + uint8_t* d_scaled = nullptr; + size_t scaled_pitch = 0; + cudaStream_t snapshot_stream = nullptr; + cudaStream_t work_stream = nullptr; + + // Guards pending, inflight, stopping and the snapshot slots' contents. + std::mutex mutex; + std::condition_variable wake; + int pending = -1; + int inflight = -1; + bool stopping = false; + bool flush_requested = false; + std::condition_variable flushed; + std::thread worker; + + // Polling thread only. + bool have_last = false; + uint64_t last_accepted_ns = 0; + + std::atomic offered{ 0 }; + std::atomic paced_out{ 0 }; + std::atomic replaced{ 0 }; + std::atomic snapshots{ 0 }; + std::atomic published{ 0 }; + std::atomic encoder_busy{ 0 }; + std::atomic rejected{ 0 }; + std::atomic keyframes_requested{ 0 }; + std::atomic encoder_dropped{ 0 }; + std::atomic unmatched_units{ 0 }; + + std::mutex failure_mutex; + std::string failure_message; + std::atomic failed_flag{ false }; +}; + +void LiveKitVideoOutput::validate_geometry(uint32_t width, uint32_t height, uint32_t divisor) +{ + if (divisor == 0) + throw std::runtime_error("LiveKit output: the scale divisor must be at least 1"); + if (width == 0 || height == 0) + throw std::runtime_error("LiveKit output: the source size is empty"); + if (width % divisor != 0 || height % divisor != 0) + { + throw std::runtime_error("LiveKit output: divisor " + std::to_string(divisor) + " does not divide " + + std::to_string(width) + "x" + std::to_string(height) + " exactly"); + } + if ((width / divisor) % 2 != 0 || (height / divisor) % 2 != 0) + { + throw std::runtime_error("LiveKit output: " + std::to_string(width / divisor) + "x" + + std::to_string(height / divisor) + " is not even; NV12 needs even dimensions"); + } +} + +LiveKitVideoOutput::LiveKitVideoOutput(std::unique_ptr sink, const LiveKitOutputConfig& config) + : m_impl(std::make_unique(std::move(sink), config)) +{ +} + +LiveKitVideoOutput::~LiveKitVideoOutput() = default; + +void LiveKitVideoOutput::submit(const FrameView& view) +{ + m_impl->submit(view); +} + +LiveKitOutputStats LiveKitVideoOutput::stats() const +{ + LiveKitOutputStats s; + s.offered = m_impl->offered.load(); + s.paced_out = m_impl->paced_out.load(); + s.replaced = m_impl->replaced.load(); + s.snapshots = m_impl->snapshots.load(); + s.published = m_impl->published.load(); + s.encoder_busy = m_impl->encoder_busy.load(); + s.rejected = m_impl->rejected.load(); + s.keyframes_requested = m_impl->keyframes_requested.load(); + s.encoder_dropped = m_impl->encoder_dropped.load(); + s.unmatched_units = m_impl->unmatched_units.load(); + return s; +} + +bool LiveKitVideoOutput::failed() const +{ + return m_impl->failed_flag.load(); +} + +std::string LiveKitVideoOutput::failure() const +{ + std::lock_guard lock(m_impl->failure_mutex); + return m_impl->failure_message; +} + +std::string LiveKitVideoOutput::describe() +{ + std::string text = std::string(JetsonEncoder::backend_name()) + ", pre-encoded passthrough"; + const std::string transport = m_impl->sink->describe(); + if (!transport.empty()) + text += "; transport: " + transport; + return text; +} + +void LiveKitVideoOutput::flush() +{ + std::unique_lock lock(m_impl->mutex); + m_impl->flush_requested = true; + m_impl->wake.notify_one(); + m_impl->flushed.wait(lock, [&] { return !m_impl->flush_requested || m_impl->failed_flag.load(); }); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/livekit_video_output.hpp b/src/plugins/sensing/core/livekit_video_output.hpp new file mode 100644 index 0000000000..004c9e6044 --- /dev/null +++ b/src/plugins/sensing/core/livekit_video_output.hpp @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "inc/sensing_core/sipl_camera.hpp" + +#include + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct LiveKitOutputConfig +{ + uint32_t source_width = 0; + uint32_t source_height = 0; + /// Uniform output-only reduction; 1 publishes the sensor's native size. + uint32_t scale_div = 1; + /// Publication rate cap, independent of the capture rate. + uint32_t fps = 30; + int gpu_id = 0; + /// Hardware encoder settings for the published pictures; the peak is also the ceiling for the + /// bitrate that congestion control may ask for. + uint32_t bitrate_bps = 8'000'000; + uint32_t peak_bitrate_bps = 0; + /// IDR period in frames; 0 defers to the encoder's default. + uint32_t gop = 0; + bool full_range = false; +}; + +struct LiveKitOutputStats +{ + uint64_t offered = 0; ///< Captures handed to submit(). + uint64_t paced_out = 0; ///< Skipped to hold the publication rate. + uint64_t replaced = 0; ///< Snapshots overwritten by a newer capture before the worker took them. + uint64_t snapshots = 0; ///< GPU snapshots taken. + uint64_t published = 0; ///< Frames the SDK accepted. Not proof that a viewer received them. + uint64_t encoder_busy = 0; ///< Snapshots the encoder had no input buffer for, dropped before encoding. + uint64_t rejected = 0; ///< Encoded frames the SDK refused; a keyframe follows. + uint64_t keyframes_requested = 0; ///< Keyframes the receiving side asked for. + uint64_t encoder_dropped = 0; ///< Accepted frames the encoder never returned; their metadata is retired. + uint64_t unmatched_units = 0; ///< Encoded units with no capture to pair with, skipped. +}; + +/** + * @brief Encodes one sensor's frames on hardware and feeds them to a pre-encoded video track. + * + * submit() runs on the caller's polling loop. It takes a snapshot of the borrowed camera frame in + * GPU memory and finishes that copy before returning, because the camera may overwrite its slot at + * the second following poll. Everything else -- the optional reduction, the hardware encode and the + * SDK call -- happens on this output's own worker thread, so a slow network cannot hold up capture + * or the other sensors. + * + * Two snapshot slots bound the memory: one the worker is encoding, one waiting. A newer capture + * replaces the waiting one; the worker never waits for it. The encoder is the JetsonEncoder class: + * the Jetson V4L2 hardware engine. There is no software encoder: if the hardware encoder cannot + * start, construction fails. + * + * The receiving side's keyframe requests and bitrate targets are applied before the next frame is + * encoded. A frame the SDK refuses breaks the reference chain, so the output asks for a keyframe and + * publishes nothing else until one arrives. + */ +class LiveKitVideoOutput +{ +public: + /// @throws std::runtime_error if the geometry is unusable (see validate_geometry), the hardware + /// encoder cannot start, or CUDA fails. + LiveKitVideoOutput(std::unique_ptr sink, const LiveKitOutputConfig& config); + + /// Stops the worker; a snapshot not yet published is dropped. + ~LiveKitVideoOutput(); + + LiveKitVideoOutput(const LiveKitVideoOutput&) = delete; + LiveKitVideoOutput& operator=(const LiveKitVideoOutput&) = delete; + + /// Never blocks on the network. A view older than the publication rate allows is skipped. + void submit(const FrameView& view); + + LiveKitOutputStats stats() const; + + bool failed() const; + std::string failure() const; + + /// Names the hardware encoder and the transport's view of the stream, for diagnostics. + std::string describe(); + + /** + * Encodes what is waiting, drains the encoder and publishes every remaining unit with its own + * stamp. Call once, at the end of the stream: the encoder cannot take frames afterwards. + */ + void flush(); + + /** + * Output size after dividing: both source dimensions must divide exactly and leave positive, + * even results. Nothing is rounded or stretched. + * @throws std::runtime_error naming the problem. + */ + static void validate_geometry(uint32_t width, uint32_t height, uint32_t divisor); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/rgba_box_downscale.cu b/src/plugins/sensing/core/rgba_box_downscale.cu new file mode 100644 index 0000000000..cc169b3b92 --- /dev/null +++ b/src/plugins/sensing/core/rgba_box_downscale.cu @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "rgba_box_downscale.cuh" + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +__global__ void box_downscale( + const uint8_t* src, int src_pitch, uint8_t* dst, int dst_pitch, int width, int height, int divisor) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || y >= height) + return; + + int sum[4] = { 0, 0, 0, 0 }; + for (int j = 0; j < divisor; ++j) + { + const uint8_t* row = + src + static_cast(y * divisor + j) * src_pitch + static_cast(x * divisor) * 4; + for (int i = 0; i < divisor; ++i) + { + for (int c = 0; c < 4; ++c) + sum[c] += row[i * 4 + c]; + } + } + + const int count = divisor * divisor; + uint8_t* out = dst + static_cast(y) * dst_pitch + static_cast(x) * 4; + for (int c = 0; c < 4; ++c) + out[c] = static_cast((sum[c] + count / 2) / count); +} + +} // namespace + +void launch_rgba_box_downscale(const uint8_t* src, + int src_pitch, + uint8_t* dst, + int dst_pitch, + int dst_width, + int dst_height, + int divisor, + cudaStream_t stream) +{ + const dim3 block(16, 16); + const dim3 grid((dst_width + block.x - 1) / block.x, (dst_height + block.y - 1) / block.y); + box_downscale<<>>(src, src_pitch, dst, dst_pitch, dst_width, dst_height, divisor); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/rgba_box_downscale.cuh b/src/plugins/sensing/core/rgba_box_downscale.cuh new file mode 100644 index 0000000000..4a96e37a80 --- /dev/null +++ b/src/plugins/sensing/core/rgba_box_downscale.cuh @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/** + * @brief Reduce RGBA8 by an integer divisor, averaging each divisor x divisor block. + * + * The aspect ratio is kept exactly: the source size must divide evenly, so nothing is stretched, + * cropped or rounded. + * + * @param dst_width src_width / divisor. + * @param dst_height src_height / divisor. + */ +void launch_rgba_box_downscale(const uint8_t* src, + int src_pitch, + uint8_t* dst, + int dst_pitch, + int dst_width, + int dst_height, + int divisor, + cudaStream_t stream); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/rgba_to_nv12.cu b/src/plugins/sensing/core/rgba_to_nv12.cu new file mode 100644 index 0000000000..2c0ca5ec6c --- /dev/null +++ b/src/plugins/sensing/core/rgba_to_nv12.cu @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "rgba_to_nv12.cuh" + +namespace +{ + +__device__ __forceinline__ uint8_t clamp_u8(float v) +{ + return static_cast(fminf(fmaxf(v, 0.0f), 255.0f) + 0.5f); +} + +// BT.601 RGB->YCbCr, matching the coefficients yuv_to_rgba.cu inverts on the +// capture side so a round trip through RGBA is close to identity. +__device__ __forceinline__ float rgb_to_y(float r, float g, float b, bool full_range) +{ + return full_range ? (0.299f * r + 0.587f * g + 0.114f * b) : (16.0f + 0.257f * r + 0.504f * g + 0.098f * b); +} + +__device__ __forceinline__ float rgb_to_u(float r, float g, float b, bool full_range) +{ + return full_range ? (128.0f - 0.168736f * r - 0.331264f * g + 0.5f * b) : + (128.0f - 0.148f * r - 0.291f * g + 0.439f * b); +} + +__device__ __forceinline__ float rgb_to_v(float r, float g, float b, bool full_range) +{ + return full_range ? (128.0f + 0.5f * r - 0.418688f * g - 0.081312f * b) : + (128.0f + 0.439f * r - 0.368f * g - 0.071f * b); +} + +__global__ void rgba_to_nv12_kernel(const uint8_t* __restrict__ rgba, + int rgba_pitch, + uint8_t* __restrict__ y_plane, + int y_pitch, + uint8_t* __restrict__ uv_plane, + int uv_pitch, + int width, + int height, + bool full_range) +{ + // One thread per 2x2 block: writes four luma samples and one chroma pair. + const int bx = blockIdx.x * blockDim.x + threadIdx.x; + const int by = blockIdx.y * blockDim.y + threadIdx.y; + const int x = bx * 2; + const int y = by * 2; + if (x >= width || y >= height) + return; + + for (int dy = 0; dy < 2; ++dy) + { + for (int dx = 0; dx < 2; ++dx) + { + const int px = x + dx; + const int py = y + dy; + if (px >= width || py >= height) + continue; + + const uint8_t* pixel = rgba + static_cast(py) * rgba_pitch + static_cast(px) * 4; + const float r = pixel[0]; + const float g = pixel[1]; + const float b = pixel[2]; + y_plane[static_cast(py) * y_pitch + px] = clamp_u8(rgb_to_y(r, g, b, full_range)); + } + } + + const uint8_t* top_left = rgba + static_cast(y) * rgba_pitch + static_cast(x) * 4; + const float r = top_left[0]; + const float g = top_left[1]; + const float b = top_left[2]; + + uint8_t* uv = uv_plane + static_cast(by) * uv_pitch + static_cast(bx) * 2; + uv[0] = clamp_u8(rgb_to_u(r, g, b, full_range)); + uv[1] = clamp_u8(rgb_to_v(r, g, b, full_range)); +} + +} // namespace + +namespace plugins +{ +namespace sensing +{ + +void launch_rgba_to_nv12(const uint8_t* rgba, + int rgba_pitch, + uint8_t* y_plane, + int y_pitch, + uint8_t* uv_plane, + int uv_pitch, + int width, + int height, + bool full_range, + cudaStream_t stream) +{ + const dim3 block(16, 16); + const dim3 grid((width / 2 + block.x - 1) / block.x, (height / 2 + block.y - 1) / block.y); + rgba_to_nv12_kernel<<>>( + rgba, rgba_pitch, y_plane, y_pitch, uv_plane, uv_pitch, width, height, full_range); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/rgba_to_nv12.cuh b/src/plugins/sensing/core/rgba_to_nv12.cuh new file mode 100644 index 0000000000..a296be6181 --- /dev/null +++ b/src/plugins/sensing/core/rgba_to_nv12.cuh @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/** + * @brief Convert packed RGBA8 to NV12 (Y plane + interleaved UV plane). + * + * Chroma is point-sampled from the top-left pixel of each 2x2 block rather + * than averaged: the ISP already delivered 4:2:0, so the RGBA the + * capture path produced was upsampled from it and averaging would only blur + * chroma that was never independent. Width and height must be even. + * + * @param full_range true for [0,255] luma, false for broadcast [16,235]. + */ +void launch_rgba_to_nv12(const uint8_t* rgba, + int rgba_pitch, + uint8_t* y_plane, + int y_pitch, + uint8_t* uv_plane, + int uv_pitch, + int width, + int height, + bool full_range, + cudaStream_t stream); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/sensing_camera.cpp b/src/plugins/sensing/core/sensing_camera.cpp new file mode 100644 index 0000000000..b6ed078044 --- /dev/null +++ b/src/plugins/sensing/core/sensing_camera.cpp @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_core/sensing_camera.hpp" + +#include "livekit_video_output.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +SiplConfig make_sipl_config(const SensingConfig& config) +{ + SiplConfig sipl{}; + sipl.platform_config_json = config.platform_config_json; + sipl.platform_config_name = config.platform_config_name; + sipl.link_masks = config.link_masks; + sipl.nito_dir = config.nito_dir; + sipl.gpu_id = config.gpu_id; + sipl.full_range = config.full_range; + sipl.swap_uv = config.swap_uv; + sipl.isp0_buffers = config.isp0_buffers; + return sipl; +} + +LiveKitOutputConfig make_output_config(const SensingConfig& config, const SensorInfo& sensor) +{ + LiveKitOutputConfig output; + output.source_width = sensor.width; + output.source_height = sensor.height; + output.scale_div = config.livekit.scale_div; + output.fps = config.livekit.fps; + output.gpu_id = config.gpu_id; + output.bitrate_bps = config.livekit.bitrate_bps; + output.peak_bitrate_bps = config.livekit.bitrate_bps + config.livekit.bitrate_bps / 2; + output.gop = config.gop; + output.full_range = config.full_range; + return output; +} + +} // namespace + +SensingCamera::SensingCamera(const SensingConfig& config, const std::vector& streams) : m_config(config) +{ + if (streams.empty()) + throw std::runtime_error("SensingCamera: no streams requested"); + + m_camera = std::make_unique(make_sipl_config(config)); + const auto& sensors = m_camera->sensors(); + + // Everything that needs no network is checked first, so a bad name or divisor fails before any + // connection is made. + std::set tracks; + for (const auto& stream_config : streams) + { + if (!tracks.insert(stream_config.livekit_track).second) + throw std::runtime_error("LiveKit track name '" + stream_config.livekit_track + "' is used twice"); + const auto found = std::find_if( + sensors.begin(), sensors.end(), [&](const SensorInfo& s) { return s.id == stream_config.sensor_id; }); + if (found == sensors.end()) + { + std::ostringstream oss; + oss << "sensor=" << stream_config.sensor_id << " is not a pipeline in '" << config.platform_config_name + << "'. Available:"; + for (const auto& s : sensors) + oss << ' ' << s.id; + oss << ". Note this is the SIPL pipeline index, not the GMSL link index."; + throw std::runtime_error(oss.str()); + } + LiveKitVideoOutput::validate_geometry(found->width, found->height, config.livekit.scale_div); + } + + m_livekit = LiveKitSession::connect(config.livekit.url, config.livekit.token); + m_streams.reserve(streams.size()); + for (const auto& stream_config : streams) + { + const auto found = std::find_if( + sensors.begin(), sensors.end(), [&](const SensorInfo& s) { return s.id == stream_config.sensor_id; }); + const LiveKitOutputConfig output = make_output_config(config, *found); + + PublishSpec spec; + spec.name = stream_config.livekit_track; + spec.width = found->width / output.scale_div; + spec.height = found->height / output.scale_div; + spec.max_bitrate_bps = output.peak_bitrate_bps; + spec.max_fps = std::min(output.fps, found->fps); + + Stream stream; + stream.sensor_id = found->id; + stream.livekit = std::make_unique(m_livekit->publish_video(spec), output); + m_streams.push_back(std::move(stream)); + + std::cout << "Sensor " << found->id << " (" << found->name << ") -> LiveKit track '" << spec.name + << "': " << spec.width << "x" << spec.height << " @ up to " << spec.max_fps << " fps" << std::endl; + } + + m_camera->start(); +} + +SensingCamera::~SensingCamera() +{ + if (m_camera) + m_camera->stop(); +} + +void SensingCamera::update() +{ + // The room is gone for good: stop rather than keep capturing for nobody. + if (m_livekit->failed()) + throw std::runtime_error(m_livekit->failure()); + + for (auto& stream : m_streams) + { + auto view = m_camera->latest(stream.sensor_id); + if (view.has_value() && (!stream.have_last_sequence || view->sequence != stream.last_sequence)) + { + if (stream.have_last_sequence && view->sequence > stream.last_sequence + 1) + stream.missed_captures += view->sequence - stream.last_sequence - 1; + stream.last_sequence = view->sequence; + stream.have_last_sequence = true; + ++stream.frame_count; + + // A GPU snapshot only; the conversion and the network are on the output's own thread. + stream.livekit->submit(*view); + } + + if (stream.livekit->failed()) + throw std::runtime_error("sensor " + std::to_string(stream.sensor_id) + ": " + stream.livekit->failure()); + } +} + +void SensingCamera::flush() +{ + for (auto& stream : m_streams) + stream.livekit->flush(); +} + +void SensingCamera::print_stats() const +{ + for (const auto& stream : m_streams) + { + const auto s = stream.livekit->stats(); + std::cout << " sensor " << stream.sensor_id << ": " << stream.frame_count << " frames"; + if (stream.missed_captures) + std::cout << " | " << stream.missed_captures << " captures missed"; + std::cout << " | livekit " << s.published << " published, " << s.paced_out << " paced out, " << s.replaced + << " replaced"; + if (s.encoder_busy) + std::cout << ", " << s.encoder_busy << " encoder busy"; + if (s.rejected) + std::cout << ", " << s.rejected << " rejected"; + if (s.encoder_dropped) + std::cout << ", " << s.encoder_dropped << " dropped in encoder"; + if (s.unmatched_units) + std::cout << ", " << s.unmatched_units << " unmatched units"; + if (s.keyframes_requested) + std::cout << ", " << s.keyframes_requested << " keyframes requested"; + const std::string encoder = stream.livekit->describe(); + if (!encoder.empty()) + std::cout << " (" << encoder << ")"; + std::cout << std::endl; + } +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/sipl_camera.cpp b/src/plugins/sensing/core/sipl_camera.cpp new file mode 100644 index 0000000000..6a362ac25b --- /dev/null +++ b/src/plugins/sensing/core/sipl_camera.cpp @@ -0,0 +1,1241 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions adapted from NVIDIA's sipl_coe_unit_sample +// (/usr/src/jetson_multimedia_api/samples/unittest_samples/), BSD-3-Clause, +// Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. + +#include "inc/sensing_core/sipl_camera.hpp" + +#include "yuv_to_rgba.cuh" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ +namespace +{ + +using nvsipl::INvSIPLCamera; +using nvsipl::INvSIPLCameraQuery; +using nvsipl::INvSIPLClient; +using nvsipl::NVSIPL_STATUS_OK; +using nvsipl::SIPLStatus; + +constexpr auto kIsp0 = INvSIPLClient::ConsumerDesc::OutputType::ISP0; +constexpr auto kIcp = INvSIPLClient::ConsumerDesc::OutputType::ICP; + +/// SIPL allocates and registers ICP capture buffers unconditionally, even when +/// only an ISP output is consumed -- CNvSIPLMaster::AllocateBuffers hardcodes +/// the ICP slot to true. Skip them and RegisterImages fails with a bare +/// NVSIPL_STATUS_ERROR that names nothing. +constexpr uint32_t kIcpBuffers = 6; + +uint64_t monotonic_ns() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +uint64_t unix_ns() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count()); +} + +/// SIPL reports almost every failure as NVSIPL_STATUS_ERROR (10), which says +/// nothing on its own -- the reason only appears in its own log, above a +/// threshold that defaults to silent. SENSING_SIPL_TRACE=1..4 raises it +/// (1=error, 2=warn, 3=info, 4=debug). +void init_sipl_trace() +{ + const char* level = std::getenv("SENSING_SIPL_TRACE"); + if (level == nullptr) + { + return; + } + const int value = std::atoi(level); + if (value <= 0) + { + return; + } + auto* trace = nvsipl::INvSIPLTrace::GetInstance(); + if (trace != nullptr) + { + trace->SetLevel(static_cast(value)); + } +} + +void check_sipl(SIPLStatus status, const char* what) +{ + if (status != NVSIPL_STATUS_OK) + { + std::ostringstream oss; + oss << what << " failed with SIPLStatus " << static_cast(status); + throw std::runtime_error(oss.str()); + } +} + +void check_sci(NvSciError err, const char* what) +{ + if (err != NvSciError_Success) + { + std::ostringstream oss; + oss << what << " failed with NvSciError " << static_cast(err); + throw std::runtime_error(oss.str()); + } +} + +void check_cuda(CUresult result, const char* what) +{ + if (result == CUDA_SUCCESS) + { + return; + } + const char* name = nullptr; + const char* text = nullptr; + cuGetErrorName(result, &name); + cuGetErrorString(result, &text); + std::ostringstream oss; + oss << what << " failed"; + if (name) + { + oss << ": " << name; + } + if (text) + { + oss << " (" << text << ")"; + } + throw std::runtime_error(oss.str()); +} + +cudaTextureObject_t texture_for_array(CUarray array); + +void check_runtime(cudaError_t result, const char* what) +{ + if (result != cudaSuccess) + { + std::ostringstream oss; + oss << what << " failed: " << cudaGetErrorString(result); + throw std::runtime_error(oss.str()); + } +} + +cudaTextureObject_t texture_for_array(CUarray array) +{ + cudaResourceDesc resource_desc{}; + resource_desc.resType = cudaResourceTypeArray; + resource_desc.res.array.array = reinterpret_cast(array); + + cudaTextureDesc texture_desc{}; + texture_desc.addressMode[0] = cudaAddressModeClamp; + texture_desc.addressMode[1] = cudaAddressModeClamp; + texture_desc.filterMode = cudaFilterModePoint; + texture_desc.readMode = cudaReadModeElementType; + texture_desc.normalizedCoords = 0; + + cudaTextureObject_t texture = 0; + check_runtime(cudaCreateTextureObject(&texture, &resource_desc, &texture_desc, nullptr), "cudaCreateTextureObject"); + return texture; +} + +/** + * @brief Buffer attributes for the ISP0 output. + * + * Block-linear and REC709_ER are what the ISP produces, not what we would + * prefer. Neither is negotiable: GetImageAttributes rejects an attribute list + * asking for pitch-linear or BT.601 with NVSIPL_STATUS_BAD_ARGUMENT. So the + * conversion adapts to the ISP rather than the other way round -- convert() + * takes the CUDA-array path, and yuv_to_rgba.cu is called with bt709=true. + * `nvsipl_camera -v 3` prints the same thing: "YUV 420 SEMI-PLANAR UINT8 BL + * REC_709ER". + */ +NvSciError set_sipl_buf_attributes(NvSciBufAttrList attr_list) +{ + NvSciBufType buf_type = NvSciBufType_Image; + NvSciBufAttrValAccessPerm access_perm = NvSciBufAccessPerm_ReadWrite; + // Must be true, and it is not about us wanting CPU access. NvmmImageFill- + // NvSciBufAttrList sets NeedCpuAccess=true on the NvBufSurface list, so + // reconciliation ORs it in; if the producer list says false, SIPL's + // GetVerifiedBufferAttributes sees a producer/reconciled mismatch and + // RegisterImages fails with a bare NVSIPL_STATUS_ERROR. + bool cpu_access = true; + bool cpu_cache = true; + NvSciBufSurfMemLayout mem_layout = NvSciSurfMemLayout_SemiPlanar; + NvSciBufAttrValImageLayoutType layout_type = NvSciBufImage_BlockLinearType; + NvSciBufSurfType surf_type = NvSciSurfType_YUV; + NvSciBufSurfSampleType sample_type = NvSciSurfSampleType_420; + NvSciBufSurfBPC surf_bpc = NvSciSurfBPC_8; + NvSciBufSurfComponentOrder comp_order = NvSciSurfComponentOrder_YUV; + NvSciBufAttrValColorStd color_std[] = { NvSciColorStd_REC709_ER }; + + NvSciBufAttrKeyValuePair kvp[] = { + { NvSciBufGeneralAttrKey_Types, &buf_type, sizeof(buf_type) }, + { NvSciBufGeneralAttrKey_RequiredPerm, &access_perm, sizeof(access_perm) }, + { NvSciBufGeneralAttrKey_NeedCpuAccess, &cpu_access, sizeof(cpu_access) }, + { NvSciBufGeneralAttrKey_EnableCpuCache, &cpu_cache, sizeof(cpu_cache) }, + { NvSciBufImageAttrKey_Layout, &layout_type, sizeof(layout_type) }, + { NvSciBufImageAttrKey_SurfType, &surf_type, sizeof(surf_type) }, + { NvSciBufImageAttrKey_SurfMemLayout, &mem_layout, sizeof(mem_layout) }, + { NvSciBufImageAttrKey_SurfSampleType, &sample_type, sizeof(sample_type) }, + { NvSciBufImageAttrKey_SurfBPC, &surf_bpc, sizeof(surf_bpc) }, + { NvSciBufImageAttrKey_SurfComponentOrder, &comp_order, sizeof(comp_order) }, + { NvSciBufImageAttrKey_SurfColorStd, &color_std, sizeof(color_std) }, + }; + + return NvSciBufAttrListSetAttrs(attr_list, kvp, sizeof(kvp) / sizeof(kvp[0])); +} + + +/// Fill a second attribute list with the NvBufSurface attributes matching the +/// SIPL list, so the allocated buffer satisfies both consumers. Without it the +/// buffer is allocated for SIPL alone and RegisterImages rejects it with +/// NVSIPL_STATUS_INVALID_STATE -- a status whose only documented meaning is +/// "pipeline had seen an init error", which sends you looking in the wrong +/// place entirely. +/// +/// Adapted from SetNvBufNvSciBufAttributes in NVIDIA's NvNvSciBufHelper.cpp +/// (BSD-3-Clause). Its colour table has exactly two entries, both REC709_ER; +/// that is the whole set this ISP path can produce. +NvSciError set_nvbuf_attributes(NvSciBufAttrList nvbuf_attrs, NvSciBufAttrList sipl_attrs, int gpu_id) +{ + NvSciBufAttrKeyValuePair img[] = { + { NvSciBufImageAttrKey_SurfWidthBase, nullptr, 0 }, { NvSciBufImageAttrKey_SurfHeightBase, nullptr, 0 }, + { NvSciBufImageAttrKey_SurfSampleType, nullptr, 0 }, { NvSciBufImageAttrKey_SurfColorStd, nullptr, 0 }, + { NvSciBufImageAttrKey_SurfMemLayout, nullptr, 0 }, { NvSciBufImageAttrKey_SurfBPC, nullptr, 0 }, + { NvSciBufImageAttrKey_Layout, nullptr, 0 }, + }; + const NvSciError err = NvSciBufAttrListGetAttrs(sipl_attrs, img, sizeof(img) / sizeof(img[0])); + if (err != NvSciError_Success) + { + return err; + } + + const auto width = *static_cast(img[0].value); + const auto height = *static_cast(img[1].value); + const auto sample = *static_cast(img[2].value); + const auto color = *static_cast(img[3].value); + const auto mem = *static_cast(img[4].value); + const auto bpc = *static_cast(img[5].value); + const auto layout = *static_cast(img[6].value); + + NvBufSurfaceColorFormat pixfmt; + if (color == NvSciColorStd_REC709_ER && sample == NvSciSurfSampleType_420 && bpc == NvSciSurfBPC_8 && + mem == NvSciSurfMemLayout_SemiPlanar) + { + pixfmt = NVBUF_COLOR_FORMAT_NV12_709_ER; + } + else if (color == NvSciColorStd_REC709_ER && sample == NvSciSurfSampleType_420 && bpc == NvSciSurfBPC_8 && + mem == NvSciSurfMemLayout_Planar) + { + pixfmt = NVBUF_COLOR_FORMAT_YUV420_709_ER; + } + else + { + std::cerr << "[sipl] no NvBufSurface colour format for colourStd=" << static_cast(color) + << " sample=" << static_cast(sample) << " bpc=" << static_cast(bpc) + << " memLayout=" << static_cast(mem) << std::endl; + return NvSciError_BadParameter; + } + + NvmmImageParams params{}; + params.gpuId = static_cast(gpu_id); + params.width = width; + params.height = height; + params.colorFormat = pixfmt; + params.layout = (layout == NvSciBufImage_PitchLinearType) ? NVBUF_LAYOUT_PITCH : NVBUF_LAYOUT_BLOCK_LINEAR; + params.displayscanformat = NVBUF_DISPLAYSCANFORMAT_PROGRESSIVE; + params.isProtected = false; + + return (NvmmImageFillNvSciBufAttrList(¶ms, nvbuf_attrs) == 0) ? NvSciError_Success : NvSciError_BadParameter; +} + +/// Read back what reconciliation settled on, and report whether it is extended +/// range. _ER is full range (0-255), _SR is studio range (16-235); decoding one +/// as the other subtracts a black level that is not there and crushes shadows, +/// which on a dim scene loses most of the image. Returns true for full range. +bool check_color_std(NvSciBufAttrList reconciled) +{ + NvSciBufAttrKeyValuePair query{ NvSciBufImageAttrKey_SurfColorStd, nullptr, 0 }; + if (NvSciBufAttrListGetAttrs(reconciled, &query, 1) != NvSciError_Success || query.value == nullptr) + { + std::cerr << "[sipl] warning: could not read back SurfColorStd; assuming BT.709 extended range" << std::endl; + return true; + } + const auto* value = static_cast(query.value); + if (*value != NvSciColorStd_REC709_ER && *value != NvSciColorStd_REC709_SR) + { + std::ostringstream oss; + oss << "ISP0 reconciled to colour standard " << static_cast(*value) + << "; yuv_to_rgba.cu implements BT.601 and BT.709 only, and decoding anything else as" + " 709 would be a silent hue error"; + throw std::runtime_error(oss.str()); + } + return *value == NvSciColorStd_REC709_ER; +} + +std::vector load_nito(const std::string& dir, const std::string& module) +{ + const std::string path = dir + "/" + module + ".nito"; + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in) + { + throw std::runtime_error("cannot open NITO file " + path + + " (the ISP has no tuning; run the vendor install.sh on the host)"); + } + const auto size = static_cast(in.tellg()); + in.seekg(0); + std::vector blob(static_cast(size)); + if (!in.read(reinterpret_cast(blob.data()), size)) + { + throw std::runtime_error("short read on NITO file " + path); + } + return blob; +} + +/// Walk a SensorSystemConfig into flat SensorInfo records. The nesting is +/// module -> variant -> sensorConfigs -> variant, and `id` is the pipeline +/// index SetPipelineCfg wants. +std::vector flatten(const nvsipl::sensorconfig::SensorSystemConfig& cfg) +{ + std::vector out; + for (const auto& module : cfg.modules) + { + std::visit( + [&](const auto& mod) + { + for (const auto& sensor_variant : mod.sensorConfigs) + { + std::visit( + [&](const auto& sensor) + { + SensorInfo info; + info.id = sensor.id; + info.name = module.name; + if (!sensor.vcInfoList.empty()) + { + const auto& vc = sensor.vcInfoList.front(); + info.width = vc.resolution.width; + info.height = vc.resolution.height; + info.fps = static_cast(vc.fps); + } + out.push_back(std::move(info)); + }, + sensor_variant); + } + }, + module.moduleType); + } + return out; +} + +} // namespace + +// ============================================================================= +// Impl +// ============================================================================= + +struct SiplCamera::Impl +{ + /// One ISP0 buffer, imported into CUDA once at registration. Nothing here + /// is re-created per frame -- that is the whole point of registering. + struct Slot + { + NvSciBufObj buf = nullptr; + NvBufSurface* surface = nullptr; + CUgraphicsResource resource = nullptr; + CUeglFrame egl_frame{}; + }; + + struct DeviceBuffer + { + uint8_t* ptr = nullptr; + size_t pitch = 0; + }; + + struct Sensor + { + SensorInfo info; + nvsipl::NvSIPLPipelineConfiguration pipeline_config{}; + nvsipl::NvSIPLPipelineQueues queues{}; + + /// From the reconciled surface, not from config: the ISP decides. + bool full_range = true; + NvSciBufAttrList buf_attrs = nullptr; + std::vector buf_objects; + /// ICP pool. Never read -- allocated only because SIPL requires it. + NvSciBufAttrList icp_attrs = nullptr; + std::vector icp_buf_objects; + std::vector slots; + NvSciSyncObj sync_obj = nullptr; + + /// Triple-buffered RGBA8 mailbox. A reader leases the published slot + /// until its next latest() call, so the producer never overwrites it. + std::array buffers{}; + mutable std::mutex publish_mutex; + int publish_idx = -1; + int lease_idx = -1; + uint64_t published_sequence = 0; + uint64_t consumed_sequence = 0; + uint64_t published_timestamp_ns = 0; + uint64_t published_unix_ns = 0; + uint64_t published_capture_tsc = 0; + + std::thread frame_thread; + std::thread event_thread; + }; + + explicit Impl(const SiplConfig& cfg) : config(cfg) + { + } + + SiplConfig config; + + NvSciBufModule sci_buf_module = nullptr; + NvSciSyncModule sci_sync_module = nullptr; + NvSciSyncCpuWaitContext cpu_wait_context = nullptr; + + CUdevice cu_device = 0; + CUcontext cu_context = nullptr; + bool cu_context_retained = false; + CUstream convert_stream = nullptr; + + std::unique_ptr camera; + std::unique_ptr query_api; + std::unique_ptr system_config; + + std::vector sensor_infos; + std::vector> sensors; + + std::atomic running{ false }; + std::atomic failed{ false }; + mutable std::mutex error_mutex; + std::string failure_message; + + Sensor* find(uint32_t sensor_id) + { + for (auto& s : sensors) + { + if (s->info.id == sensor_id) + { + return s.get(); + } + } + return nullptr; + } + + void set_failure(const std::string& message) + { + std::lock_guard guard(error_mutex); + if (failure_message.empty()) + { + failure_message = message; + } + failed.store(true); + } + + void throw_if_failed() const + { + if (!failed.load()) + { + return; + } + std::lock_guard guard(error_mutex); + throw std::runtime_error(failure_message.empty() ? "SIPL capture failed" : failure_message); + } + + void init_cuda(); + void init_nvsci(); + void configure(); + void allocate_buffers(Sensor& sensor); + void allocate_icp_buffers(Sensor& sensor); + void register_buffers(Sensor& sensor); + void allocate_sync(Sensor& sensor); + void register_sync(Sensor& sensor); + void register_nito(Sensor& sensor); + void frame_loop(Sensor& sensor); + void event_loop(Sensor& sensor); + void convert(Sensor& sensor, const Slot& slot, uint32_t write_idx); + void publish(Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_stamp_ns, uint64_t capture_tsc); + uint32_t pick_write_index(const Sensor& sensor) const; + void cleanup(); +}; + +void SiplCamera::Impl::init_cuda() +{ + check_cuda(cuInit(0), "cuInit"); + check_cuda(cuDeviceGet(&cu_device, config.gpu_id), "cuDeviceGet"); + check_cuda(cuDevicePrimaryCtxRetain(&cu_context, cu_device), "cuDevicePrimaryCtxRetain"); + cu_context_retained = true; + check_cuda(cuCtxSetCurrent(cu_context), "cuCtxSetCurrent"); + check_cuda(cuStreamCreate(&convert_stream, CU_STREAM_NON_BLOCKING), "cuStreamCreate"); +} + +void SiplCamera::Impl::init_nvsci() +{ + check_sci(NvSciBufModuleOpen(&sci_buf_module), "NvSciBufModuleOpen"); + check_sci(NvSciSyncModuleOpen(&sci_sync_module), "NvSciSyncModuleOpen"); + check_sci(NvSciSyncCpuWaitContextAlloc(sci_sync_module, &cpu_wait_context), "NvSciSyncCpuWaitContextAlloc"); +} + +void SiplCamera::Impl::configure() +{ + init_sipl_trace(); + query_api = INvSIPLCameraQuery::GetInstance(); + if (!query_api) + { + throw std::runtime_error("INvSIPLCameraQuery::GetInstance returned null"); + } + // Loads the UDDF driver .so libs out of /usr/lib/nvsipl_drv. + check_sipl(query_api->ParseDatabase(), "ParseDatabase"); + check_sipl(query_api->ParseJsonFile(config.platform_config_json), "ParseJsonFile"); + + system_config = std::make_unique(); + check_sipl(query_api->GetSensorSystemConfig(config.platform_config_name, *system_config), "GetSensorSystemConfig"); + if (config.link_masks.empty()) + { + throw std::runtime_error("link masks are required for a GMSL platform config"); + } + check_sipl(query_api->ApplyMask(*system_config, config.link_masks), "ApplyMask"); + if (system_config->modules.empty()) + { + throw std::runtime_error("no modules left in '" + config.platform_config_name + + "' after applying the link masks -- check the mask against the config"); + } + + sensor_infos = flatten(*system_config); + if (sensor_infos.empty()) + { + throw std::runtime_error("platform config '" + config.platform_config_name + "' declares no sensors"); + } + + camera = INvSIPLCamera::GetInstance(); + if (!camera) + { + throw std::runtime_error("INvSIPLCamera::GetInstance returned null"); + } + check_sipl(camera->SetPlatformCfg(*system_config), "SetPlatformCfg"); + + for (const auto& info : sensor_infos) + { + auto sensor = std::make_unique(); + sensor->info = info; + // ISP0 only. ICP would double the capture bandwidth for a raw stream + // nothing here consumes, and ISP1/ISP2 are a downscale we do not want. + sensor->pipeline_config.captureOutputRequested = false; + sensor->pipeline_config.isp0OutputRequested = true; + sensor->pipeline_config.isp1OutputRequested = false; + sensor->pipeline_config.isp2OutputRequested = false; + // Defaults to false, i.e. subframe ENABLED, which needs a sliceCount we + // do not set. NVIDIA's own samples disable it unless asked for. + sensor->pipeline_config.disableSubframe = true; + sensor->pipeline_config.bufferCfg.maxIsp0BufferCount = config.isp0_buffers; + check_sipl(camera->SetPipelineCfg(info.id, sensor->pipeline_config, sensor->queues), "SetPipelineCfg"); + sensors.push_back(std::move(sensor)); + } + + // Order is not ours to choose. Everything that *describes* a pipeline -- + // GetImageAttributes and FillNvSciSyncAttrList -- must happen before Init(); + // everything that *binds* a resource to it -- RegisterImages and + // RegisterNvSciSyncObj -- must happen after. Get it wrong and Init() + // still returns OK, but the pipeline is left in an error state and the + // first Register* call returns NVSIPL_STATUS_INVALID_STATE with no clue + // why. Verified against a -v 4 trace of NVIDIA's own nvsipl_camera. + for (auto& sensor : sensors) + { + allocate_icp_buffers(*sensor); + allocate_buffers(*sensor); + allocate_sync(*sensor); + } + + check_sipl(camera->Init(), "INvSIPLCamera::Init"); + + for (auto& sensor : sensors) + { + register_buffers(*sensor); + register_sync(*sensor); + register_nito(*sensor); + } +} + +void SiplCamera::Impl::allocate_buffers(Sensor& sensor) +{ + NvSciBufAttrList sipl_attrs = nullptr; + NvSciBufAttrList nvbuf_attrs = nullptr; + NvSciBufAttrList conflict = nullptr; + check_sci(NvSciBufAttrListCreate(sci_buf_module, &sipl_attrs), "NvSciBufAttrListCreate sipl"); + check_sci(NvSciBufAttrListCreate(sci_buf_module, &nvbuf_attrs), "NvSciBufAttrListCreate nvbuf"); + check_sci(set_sipl_buf_attributes(sipl_attrs), "NvSciBufAttrListSetAttrs"); + check_sipl(camera->GetImageAttributes(sensor.info.id, kIsp0, sipl_attrs), "GetImageAttributes"); + + // Both consumers must be described before allocation: SIPL writes the + // surface, NvBufSurface reads it on the way to CUDA. Reconciling the SIPL + // list alone yields a buffer RegisterImages will not accept. + check_sci(set_nvbuf_attributes(nvbuf_attrs, sipl_attrs, config.gpu_id), "set_nvbuf_attributes"); + + NvSciBufAttrList unreconciled[] = { sipl_attrs, nvbuf_attrs }; + check_sci(NvSciBufAttrListReconcile(unreconciled, 2U, &sensor.buf_attrs, &conflict), "NvSciBufAttrListReconcile"); + NvSciBufAttrListFree(sipl_attrs); + NvSciBufAttrListFree(nvbuf_attrs); + if (conflict) + { + NvSciBufAttrListFree(conflict); + } + sensor.full_range = check_color_std(sensor.buf_attrs); + + sensor.slots.resize(config.isp0_buffers); + for (uint32_t i = 0; i < config.isp0_buffers; ++i) + { + Slot& slot = sensor.slots[i]; + check_sci(NvSciBufObjAlloc(sensor.buf_attrs, &slot.buf), "NvSciBufObjAlloc"); + sensor.buf_objects.push_back(slot.buf); + + if (NvmmNvSciBufToNvBufSurface(slot.buf, &slot.surface) != 0 || slot.surface == nullptr) + { + throw std::runtime_error("NvmmNvSciBufToNvBufSurface failed"); + } + slot.surface->numFilled = 1; + } + + // RGBA8 destination slots. Allocated from the reported geometry, never from + // a CLI flag -- see "Geometry comes from the query" in the plan. + for (auto& dest : sensor.buffers) + { + void* ptr = nullptr; + size_t pitch = 0; + check_runtime(cudaMallocPitch(&ptr, &pitch, static_cast(sensor.info.width) * 4, sensor.info.height), + "cudaMallocPitch"); + dest.ptr = static_cast(ptr); + dest.pitch = pitch; + } +} + +void SiplCamera::Impl::allocate_icp_buffers(Sensor& sensor) +{ + NvSciBufAttrList attrs = nullptr; + NvSciBufAttrList conflict = nullptr; + check_sci(NvSciBufAttrListCreate(sci_buf_module, &attrs), "NvSciBufAttrListCreate icp"); + + // Only the type and permissions: GetImageAttributes fills the raw format in. + NvSciBufType buf_type = NvSciBufType_Image; + NvSciBufAttrValAccessPerm perm = NvSciBufAccessPerm_ReadWrite; + NvSciBufAttrKeyValuePair kvp[] = { + { NvSciBufGeneralAttrKey_Types, &buf_type, sizeof(buf_type) }, + { NvSciBufGeneralAttrKey_RequiredPerm, &perm, sizeof(perm) }, + }; + check_sci(NvSciBufAttrListSetAttrs(attrs, kvp, 2), "NvSciBufAttrListSetAttrs icp"); + check_sipl(camera->GetImageAttributes(sensor.info.id, kIcp, attrs), "GetImageAttributes ICP"); + + NvSciBufAttrList unreconciled[] = { attrs }; + check_sci(NvSciBufAttrListReconcile(unreconciled, 1U, &sensor.icp_attrs, &conflict), "NvSciBufAttrListReconcile icp"); + NvSciBufAttrListFree(attrs); + if (conflict) + { + NvSciBufAttrListFree(conflict); + } + + for (uint32_t i = 0; i < kIcpBuffers; ++i) + { + NvSciBufObj obj = nullptr; + check_sci(NvSciBufObjAlloc(sensor.icp_attrs, &obj), "NvSciBufObjAlloc icp"); + sensor.icp_buf_objects.push_back(obj); + } +} + +void SiplCamera::Impl::register_buffers(Sensor& sensor) +{ + check_sipl(camera->RegisterImages(sensor.info.id, kIcp, sensor.icp_buf_objects), "RegisterImages ICP"); + check_sipl(camera->RegisterImages(sensor.info.id, kIsp0, sensor.buf_objects), "RegisterImages ISP0"); + + // Import into CUDA only once SIPL owns the buffers. Doing it at allocation + // time hands SIPL a surface that already has an EGL image bound to it. + // Registering per frame instead would put an EGL round trip on the 60 Hz + // path for no benefit. + for (auto& slot : sensor.slots) + { + if (NvBufSurfaceMapEglImage(slot.surface, 0) != 0) + { + throw std::runtime_error("NvBufSurfaceMapEglImage failed"); + } + check_cuda(cuGraphicsEGLRegisterImage(&slot.resource, slot.surface->surfaceList[0].mappedAddr.eglImage, + CU_GRAPHICS_REGISTER_FLAGS_READ_ONLY), + "cuGraphicsEGLRegisterImage"); + check_cuda(cuGraphicsResourceGetMappedEglFrame(&slot.egl_frame, slot.resource, 0, 0), + "cuGraphicsResourceGetMappedEglFrame"); + } +} + +void SiplCamera::Impl::allocate_sync(Sensor& sensor) +{ + NvSciSyncAttrList signaler = nullptr; + NvSciSyncAttrList waiter = nullptr; + NvSciSyncAttrList reconciled = nullptr; + NvSciSyncAttrList conflict = nullptr; + + check_sci(NvSciSyncAttrListCreate(sci_sync_module, &signaler), "NvSciSyncAttrListCreate signaler"); + check_sci(NvSciSyncAttrListCreate(sci_sync_module, &waiter), "NvSciSyncAttrListCreate waiter"); + + // We wait on the CPU before handing the surface to the conversion kernel. + NvSciSyncAccessPerm perm = NvSciSyncAccessPerm_WaitOnly; + bool cpu_waiter = true; + NvSciSyncAttrKeyValuePair kvp[] = { + { NvSciSyncAttrKey_NeedCpuAccess, &cpu_waiter, sizeof(cpu_waiter) }, + { NvSciSyncAttrKey_RequiredPerm, &perm, sizeof(perm) }, + }; + check_sci(NvSciSyncAttrListSetAttrs(waiter, kvp, 2), "NvSciSyncAttrListSetAttrs"); + check_sipl( + camera->FillNvSciSyncAttrList(sensor.info.id, kIsp0, signaler, nvsipl::SIPL_SIGNALER), "FillNvSciSyncAttrList"); + + NvSciSyncAttrList unreconciled[] = { signaler, waiter }; + check_sci(NvSciSyncAttrListReconcile(unreconciled, 2U, &reconciled, &conflict), "NvSciSyncAttrListReconcile"); + check_sci(NvSciSyncObjAlloc(reconciled, &sensor.sync_obj), "NvSciSyncObjAlloc"); + NvSciSyncAttrListFree(signaler); + NvSciSyncAttrListFree(waiter); + if (reconciled) + { + NvSciSyncAttrListFree(reconciled); + } + if (conflict) + { + NvSciSyncAttrListFree(conflict); + } +} + +void SiplCamera::Impl::register_sync(Sensor& sensor) +{ + check_sipl(camera->RegisterNvSciSyncObj(sensor.info.id, kIsp0, nvsipl::NVSIPL_EOFSYNCOBJ, sensor.sync_obj), + "RegisterNvSciSyncObj"); +} + +void SiplCamera::Impl::register_nito(Sensor& sensor) +{ + auto blob = load_nito(config.nito_dir, sensor.info.name); + nvsipl::ISiplControlAuto* auto_control = nullptr; + check_sipl(camera->RegisterAutoControlPlugin(sensor.info.id, nvsipl::NV_PLUGIN, auto_control, blob), + "RegisterAutoControlPlugin"); +} + +void SiplCamera::Impl::convert(Sensor& sensor, const Slot& slot, uint32_t write_idx) +{ + const CUeglFrame& frame = slot.egl_frame; + DeviceBuffer& dest = sensor.buffers[write_idx]; + + if (frame.planeCount < 2) + { + throw std::runtime_error("ISP0 surface has fewer than two planes"); + } + + // Requested SemiPlanar YUV, so this is NV12 unless swap_uv flips it, and + // REC709_ER, so the kernel gets bt709=true. + const YuvLayout layout = config.swap_uv ? YuvLayout::YVU420SemiPlanar : YuvLayout::YUV420SemiPlanar; + constexpr bool kBt709 = true; + + if (frame.frameType == CU_EGL_FRAME_TYPE_PITCH) + { + const auto* y_plane = static_cast(frame.frame.pPitch[0]); + const auto* uv_plane = static_cast(frame.frame.pPitch[1]); + launch_yuv420_pitch_to_rgba(y_plane, uv_plane, nullptr, static_cast(frame.pitch), + static_cast(frame.pitch), 0, sensor.info.width, sensor.info.height, dest.ptr, + static_cast(dest.pitch), layout, sensor.full_range, kBt709, + reinterpret_cast(convert_stream)); + } + else if (frame.frameType == CU_EGL_FRAME_TYPE_ARRAY) + { + // Block-linear: CUDA exposes the planes as arrays, so the kernel reads + // them through texture objects. Those are built per frame because the + // CUarray handles come from the per-frame CUeglFrame; the expensive + // part, cuGraphicsEGLRegisterImage, is done once per buffer. + cudaTextureObject_t y_tex = texture_for_array(frame.frame.pArray[0]); + cudaTextureObject_t uv_tex = texture_for_array(frame.frame.pArray[1]); + launch_yuv420_array_to_rgba(y_tex, uv_tex, 0, sensor.info.width, sensor.info.height, dest.ptr, + static_cast(dest.pitch), layout, sensor.full_range, kBt709, + reinterpret_cast(convert_stream)); + check_cuda(cuStreamSynchronize(convert_stream), "cuStreamSynchronize"); + check_runtime(cudaDestroyTextureObject(y_tex), "cudaDestroyTextureObject"); + check_runtime(cudaDestroyTextureObject(uv_tex), "cudaDestroyTextureObject"); + check_runtime(cudaGetLastError(), "SIPL YUV to RGBA kernel"); + return; + } + else + { + throw std::runtime_error("unsupported CUDA EGL frame type from ISP0"); + } + + check_cuda(cuStreamSynchronize(convert_stream), "cuStreamSynchronize"); + check_runtime(cudaGetLastError(), "SIPL YUV to RGBA kernel"); +} + +void SiplCamera::Impl::frame_loop(Sensor& sensor) +{ + try + { + check_cuda(cuCtxSetCurrent(cu_context), "cuCtxSetCurrent frame_loop"); + while (running.load()) + { + INvSIPLClient::INvSIPLBuffer* buffer = nullptr; + const SIPLStatus status = sensor.queues.isp0CompletionQueue->Get(buffer, config.frame_timeout_ms * 1000U); + if (status == nvsipl::NVSIPL_STATUS_TIMED_OUT) + { + continue; + } + if (status == nvsipl::NVSIPL_STATUS_EOF) + { + break; + } + check_sipl(status, "isp0CompletionQueue->Get"); + if (!buffer) + { + continue; + } + + auto* nvm = dynamic_cast(buffer); + if (!nvm) + { + buffer->Release(); + throw std::runtime_error("ISP0 buffer is not an INvSIPLNvMBuffer"); + } + + // The ISP is still writing until this fence clears. + NvSciSyncFence fence = NvSciSyncFenceInitializer; + check_sipl(nvm->GetEOFNvSciSyncFence(&fence), "GetEOFNvSciSyncFence"); + const NvSciError wait = NvSciSyncFenceWait(&fence, cpu_wait_context, config.frame_timeout_ms * 1000UL); + NvSciSyncFenceClear(&fence); + check_sci(wait, "NvSciSyncFenceWait"); + + const NvSciBufObj arrived = nvm->GetNvSciBufImage(); + const Slot* slot = nullptr; + for (const auto& candidate : sensor.slots) + { + if (candidate.buf == arrived) + { + slot = &candidate; + break; + } + } + if (!slot) + { + buffer->Release(); + throw std::runtime_error("ISP0 delivered a buffer this process never registered"); + } + + // The TSC is what makes the two eyes pairable; do not substitute a + // host clock, which carries this sensor's own queueing jitter. + const auto& meta = nvm->GetImageData(); + const uint64_t capture_tsc = meta.frameCaptureTSC; + // SENSING_TSC_PROBE=1 logs, twice a second, the capture TSC beside the host clocks read at dequeue. + // Uncalibrated samples: the counter's units are unconfirmed, so it is logged beside the host clocks and + // never subtracted from them. The environment is read once, so a disabled probe costs a branch per frame. + static const bool tsc_probe = std::getenv("SENSING_TSC_PROBE") != nullptr; + if (tsc_probe) + { + static thread_local uint64_t tsc_probe_n = 0; + if ((tsc_probe_n++ % 30) == 0) + { + const auto clock_ns = [](clockid_t id) + { + timespec ts{}; + clock_gettime(id, &ts); + return static_cast(ts.tv_sec) * 1'000'000'000LL + ts.tv_nsec; + }; + const int64_t raw_ns = clock_ns(CLOCK_MONOTONIC_RAW); + isaaccapture::Logger::get("isaaccapture.plugins.sensing.SiplCamera") + ->info("[tsc] sensor {} tsc={} monotonic_raw_ns={} monotonic_ns={} realtime_ns={}", + sensor.info.id, capture_tsc, raw_ns, clock_ns(CLOCK_MONOTONIC), clock_ns(CLOCK_REALTIME)); + } + } + // SENSING_AE_PROBE=1 logs the sensor's exposure and gain once a + // second. AE failures are invisible in the frame rate and hard to + // judge by eye, and this is the only place the numbers surface. + if (std::getenv("SENSING_AE_PROBE") != nullptr) + { + static thread_local uint64_t probe_n = 0; + if ((probe_n++ % 60) == 0) + { + std::cerr << "[ae] sensor " << sensor.info.id << " numExp=" << meta.numExposures + << " expValid=" << static_cast(meta.sensorExpInfo.expTimeValid) + << " exp[0]=" << meta.sensorExpInfo.exposureTime[0] << "s" + << " gainValid=" << static_cast(meta.sensorExpInfo.gainValid) + << " gain[0]=" << meta.sensorExpInfo.sensorGain[0] << std::endl; + } + } + + const uint32_t write_idx = pick_write_index(sensor); + convert(sensor, *slot, write_idx); + if (std::getenv("SENSING_LUMA_PROBE") != nullptr) + { + static thread_local uint64_t luma_n = 0; + if ((luma_n++ % 60) == 0) + { + // Mean over one row out of every 64, straight off the RGBA + // the consumer receives -- the only number that says what + // the operator actually sees. + const auto& dst = sensor.buffers[write_idx]; + const uint32_t step = 64; + std::vector row(dst.pitch); + double sum = 0.0; + uint64_t n = 0; + for (uint32_t y = 0; y < sensor.info.height; y += step) + { + if (cudaMemcpy(row.data(), dst.ptr + static_cast(y) * dst.pitch, dst.pitch, + cudaMemcpyDeviceToHost) != cudaSuccess) + break; + for (uint32_t x = 0; x < sensor.info.width; ++x) + { + sum += 0.2126 * row[x * 4 + 0] + 0.7152 * row[x * 4 + 1] + 0.0722 * row[x * 4 + 2]; + ++n; + } + } + if (n) + std::cerr << "[luma] sensor " << sensor.info.id << " mean=" << (sum / n) << "/255" << std::endl; + } + } + buffer->Release(); + // Both clocks are read back to back, so the pair names one instant. + const uint64_t stamp_ns = monotonic_ns(); + publish(sensor, write_idx, stamp_ns, unix_ns(), capture_tsc); + } + } + catch (const std::exception& e) + { + std::ostringstream oss; + oss << "SIPL sensor " << sensor.info.id << " capture error: " << e.what(); + std::cerr << "[sipl] " << oss.str() << std::endl; + set_failure(oss.str()); + running.store(false); + } +} + +void SiplCamera::Impl::event_loop(Sensor& sensor) +{ + // A GMSL link drop arrives here and nowhere else. Without this drain the + // symptom is a silent stall on the frame queue. + while (running.load()) + { + nvsipl::NvSIPLPipelineNotifier::NotificationData data{}; + const SIPLStatus status = sensor.queues.notificationQueue->Get(data, config.frame_timeout_ms * 1000U); + if (status != NVSIPL_STATUS_OK) + { + continue; + } + using Notif = nvsipl::NvSIPLPipelineNotifier; + switch (data.eNotifType) + { + case Notif::NOTIF_ERROR_DESERIALIZER_FAILURE: + case Notif::NOTIF_ERROR_SERIALIZER_FAILURE: + case Notif::NOTIF_ERROR_SENSOR_FAILURE: + case Notif::NOTIF_ERROR_INTERNAL_FAILURE: + { + std::ostringstream oss; + oss << "SIPL sensor " << sensor.info.id << " pipeline error, notification " + << static_cast(data.eNotifType); + std::cerr << "[sipl] " << oss.str() << std::endl; + set_failure(oss.str()); + running.store(false); + break; + } + default: + break; + } + } +} + +void SiplCamera::Impl::publish( + Sensor& sensor, uint32_t write_idx, uint64_t timestamp_ns, uint64_t unix_stamp_ns, uint64_t capture_tsc) +{ + std::lock_guard guard(sensor.publish_mutex); + sensor.publish_idx = static_cast(write_idx); + sensor.published_timestamp_ns = timestamp_ns; + sensor.published_unix_ns = unix_stamp_ns; + sensor.published_capture_tsc = capture_tsc; + ++sensor.published_sequence; +} + +uint32_t SiplCamera::Impl::pick_write_index(const Sensor& sensor) const +{ + std::lock_guard guard(sensor.publish_mutex); + if (sensor.publish_idx < 0) + { + return 0; + } + // Skip the published slot and any slot still leased to a reader. With three + // slots one is always free, so this cannot spin. + uint32_t idx = static_cast((sensor.publish_idx + 1) % 3); + if (static_cast(idx) == sensor.lease_idx) + { + idx = (idx + 1) % 3; + } + return idx; +} + +void SiplCamera::Impl::cleanup() +{ + running.store(false); + for (auto& sensor : sensors) + { + if (sensor->frame_thread.joinable()) + { + sensor->frame_thread.join(); + } + if (sensor->event_thread.joinable()) + { + sensor->event_thread.join(); + } + } + if (camera) + { + camera->Stop(); + camera->Deinit(); + } + + for (auto& sensor : sensors) + { + for (auto& slot : sensor->slots) + { + if (slot.resource) + { + cuGraphicsUnregisterResource(slot.resource); + } + if (slot.surface) + { + NvBufSurfaceUnMapEglImage(slot.surface, 0); + NvBufSurfaceDestroy(slot.surface); + } + if (slot.buf) + { + NvSciBufObjFree(slot.buf); + } + } + sensor->slots.clear(); + sensor->buf_objects.clear(); + for (auto& obj : sensor->icp_buf_objects) + { + NvSciBufObjFree(obj); + } + sensor->icp_buf_objects.clear(); + if (sensor->icp_attrs) + { + NvSciBufAttrListFree(sensor->icp_attrs); + sensor->icp_attrs = nullptr; + } + if (sensor->buf_attrs) + { + NvSciBufAttrListFree(sensor->buf_attrs); + sensor->buf_attrs = nullptr; + } + if (sensor->sync_obj) + { + NvSciSyncObjFree(sensor->sync_obj); + sensor->sync_obj = nullptr; + } + for (auto& dest : sensor->buffers) + { + if (dest.ptr) + { + cudaFree(dest.ptr); + dest.ptr = nullptr; + } + } + } + + camera.reset(); + query_api.reset(); + + if (cpu_wait_context) + { + NvSciSyncCpuWaitContextFree(cpu_wait_context); + cpu_wait_context = nullptr; + } + if (sci_sync_module) + { + NvSciSyncModuleClose(sci_sync_module); + sci_sync_module = nullptr; + } + if (sci_buf_module) + { + NvSciBufModuleClose(sci_buf_module); + sci_buf_module = nullptr; + } + if (convert_stream) + { + cuStreamDestroy(convert_stream); + convert_stream = nullptr; + } + if (cu_context_retained) + { + cuDevicePrimaryCtxRelease(cu_device); + cu_context_retained = false; + cu_context = nullptr; + } +} + +// ============================================================================= +// SiplCamera +// ============================================================================= + +SiplCamera::SiplCamera(const SiplConfig& config) : m_impl(std::make_unique(config)) +{ + if (config.platform_config_json.empty() || config.platform_config_name.empty()) + { + throw std::invalid_argument("SiplConfig needs both platform_config_json and platform_config_name"); + } + try + { + m_impl->init_cuda(); + m_impl->init_nvsci(); + m_impl->configure(); + } + catch (...) + { + m_impl->cleanup(); + throw; + } +} + +SiplCamera::~SiplCamera() +{ + m_impl->cleanup(); +} + +std::vector SiplCamera::query(const std::string& platform_config_json, + const std::string& platform_config_name, + const std::vector& link_masks) +{ + auto api = INvSIPLCameraQuery::GetInstance(); + if (!api) + { + throw std::runtime_error("INvSIPLCameraQuery::GetInstance returned null"); + } + check_sipl(api->ParseDatabase(), "ParseDatabase"); + check_sipl(api->ParseJsonFile(platform_config_json), "ParseJsonFile"); + + nvsipl::sensorconfig::SensorSystemConfig cfg; + check_sipl(api->GetSensorSystemConfig(platform_config_name, cfg), "GetSensorSystemConfig"); + if (!link_masks.empty()) + { + check_sipl(api->ApplyMask(cfg, link_masks), "ApplyMask"); + } + return flatten(cfg); +} + +const std::vector& SiplCamera::sensors() const +{ + return m_impl->sensor_infos; +} + +void SiplCamera::start() +{ + if (m_impl->running.load()) + { + return; + } + m_impl->running.store(true); + for (auto& sensor : m_impl->sensors) + { + sensor->frame_thread = std::thread([this, s = sensor.get()] { m_impl->frame_loop(*s); }); + sensor->event_thread = std::thread([this, s = sensor.get()] { m_impl->event_loop(*s); }); + } + try + { + check_sipl(m_impl->camera->Start(), "INvSIPLCamera::Start"); + } + catch (...) + { + m_impl->running.store(false); + for (auto& sensor : m_impl->sensors) + { + if (sensor->frame_thread.joinable()) + { + sensor->frame_thread.join(); + } + if (sensor->event_thread.joinable()) + { + sensor->event_thread.join(); + } + } + throw; + } +} + +void SiplCamera::stop() +{ + m_impl->running.store(false); + for (auto& sensor : m_impl->sensors) + { + if (sensor->frame_thread.joinable()) + { + sensor->frame_thread.join(); + } + if (sensor->event_thread.joinable()) + { + sensor->event_thread.join(); + } + } + if (m_impl->camera) + { + m_impl->camera->Stop(); + } +} + +std::optional SiplCamera::latest(uint32_t sensor_id) +{ + m_impl->throw_if_failed(); + Impl::Sensor* sensor = m_impl->find(sensor_id); + if (!sensor) + { + throw std::runtime_error("no SIPL pipeline with index " + std::to_string(sensor_id)); + } + + std::lock_guard guard(sensor->publish_mutex); + if (sensor->publish_idx < 0 || sensor->consumed_sequence == sensor->published_sequence) + { + return std::nullopt; + } + sensor->consumed_sequence = sensor->published_sequence; + + // The caller now holds this slot; pick_write_index() skips it until the + // next latest() moves the lease on. + sensor->lease_idx = sensor->publish_idx; + + FrameView view; + view.ptr = reinterpret_cast(sensor->buffers[sensor->publish_idx].ptr); + view.pitch = sensor->buffers[sensor->publish_idx].pitch; + view.width = sensor->info.width; + view.height = sensor->info.height; + view.timestamp_ns = sensor->published_timestamp_ns; + view.unix_ns = sensor->published_unix_ns; + view.capture_tsc = sensor->published_capture_tsc; + view.sequence = sensor->published_sequence; + return view; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/yuv_to_rgba.cu b/src/plugins/sensing/core/yuv_to_rgba.cu new file mode 100644 index 0000000000..7186e46ae8 --- /dev/null +++ b/src/plugins/sensing/core/yuv_to_rgba.cu @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// YUV420 -> RGBA8 conversion for the SIPL ISP0 output. +// +// BT.601 and BT.709, selected per call. SIPL's ISP0 output on this rig is +// REC709_ER and the request is not negotiable -- GetImageAttributes rejects a +// BT.601 attribute list with NVSIPL_STATUS_BAD_ARGUMENT -- so the 709 matrix is +// the one the SIPL path actually uses. rgba_to_nv12.cu on the encode side is +// still 601, so an encode round trip is not identity; that only matters if the +// H.264 output is ever compared against the RGBA it came from. + +#include "yuv_to_rgba.cuh" + +namespace +{ + +__device__ __forceinline__ unsigned char clamp_u8(float v) +{ + return static_cast(v < 0.f ? 0.f : (v > 255.f ? 255.f : v)); +} + +__device__ __forceinline__ void yuv_to_rgb( + int Y, int Cb, int Cr, bool full_range, bool bt709, unsigned char& r, unsigned char& g, unsigned char& b) +{ + // Full range uses Y as-is; limited range expands 16..235 first. + const float yf = full_range ? static_cast(Y) : (static_cast(Y) - 16.f) * 1.16438f; + const float u = static_cast(Cb) - 128.f; + const float v = static_cast(Cr) - 128.f; + + // Limited-range coefficients already fold in the 255/219 luma gain applied + // above, so they differ from the full-range pair by more than rounding. + float kr, kgu, kgv, kb; + if (bt709) + { + if (full_range) + { + kr = 1.5748f; + kgu = 0.18733f; + kgv = 0.46813f; + kb = 1.8556f; + } + else + { + kr = 1.79274f; + kgu = 0.21325f; + kgv = 0.53291f; + kb = 2.11240f; + } + } + else + { + if (full_range) + { + kr = 1.402f; + kgu = 0.344136f; + kgv = 0.714136f; + kb = 1.772f; + } + else + { + kr = 1.596f; + kgu = 0.391f; + kgv = 0.813f; + kb = 2.018f; + } + } + + r = clamp_u8(yf + kr * v); + g = clamp_u8(yf - kgu * u - kgv * v); + b = clamp_u8(yf + kb * u); +} + +__global__ void yuv420_pitch_to_rgba_kernel(const uint8_t* __restrict__ y_plane, + const uint8_t* __restrict__ uv_or_u_plane, + const uint8_t* __restrict__ v_plane, + int y_pitch, + int uv_pitch, + int v_pitch, + int width, + int height, + uint8_t* __restrict__ rgba_out, + int rgba_row_bytes, + int layout_value, + int full_range_value, + int bt709_value) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || y >= height) + { + return; + } + + const auto layout = static_cast(layout_value); + const int Y = y_plane[y * y_pitch + x]; + int Cb = 128; + int Cr = 128; + + if (layout == plugins::sensing::YuvLayout::YUV420SemiPlanar || layout == plugins::sensing::YuvLayout::YVU420SemiPlanar) + { + const int uv_x = x & ~1; + const int uv_y = y >> 1; + const uint8_t a = uv_or_u_plane[uv_y * uv_pitch + uv_x + 0]; + const uint8_t b = uv_or_u_plane[uv_y * uv_pitch + uv_x + 1]; + if (layout == plugins::sensing::YuvLayout::YUV420SemiPlanar) + { + Cb = a; + Cr = b; + } + else + { + Cr = a; + Cb = b; + } + } + else + { + const int uv_x = x >> 1; + const int uv_y = y >> 1; + const uint8_t a = uv_or_u_plane[uv_y * uv_pitch + uv_x]; + const uint8_t b = v_plane[uv_y * v_pitch + uv_x]; + if (layout == plugins::sensing::YuvLayout::YUV420Planar) + { + Cb = a; + Cr = b; + } + else + { + Cr = a; + Cb = b; + } + } + + const int idx = y * rgba_row_bytes + x * 4; + yuv_to_rgb( + Y, Cb, Cr, full_range_value != 0, bt709_value != 0, rgba_out[idx + 0], rgba_out[idx + 1], rgba_out[idx + 2]); + rgba_out[idx + 3] = 255; +} + +__global__ void yuv420_array_to_rgba_kernel(cudaTextureObject_t y_tex, + cudaTextureObject_t uv_or_u_tex, + cudaTextureObject_t v_tex, + int width, + int height, + uint8_t* __restrict__ rgba_out, + int rgba_row_bytes, + int layout_value, + int full_range_value, + int bt709_value) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || y >= height) + { + return; + } + + const auto layout = static_cast(layout_value); + const int Y = tex2D(y_tex, x, y); + int Cb = 128; + int Cr = 128; + + if (layout == plugins::sensing::YuvLayout::YUV420SemiPlanar || layout == plugins::sensing::YuvLayout::YVU420SemiPlanar) + { + const int uv_x = x >> 1; + const int uv_y = y >> 1; + const uchar2 uv = tex2D(uv_or_u_tex, uv_x, uv_y); + if (layout == plugins::sensing::YuvLayout::YUV420SemiPlanar) + { + Cb = uv.x; + Cr = uv.y; + } + else + { + Cr = uv.x; + Cb = uv.y; + } + } + else + { + const int uv_x = x >> 1; + const int uv_y = y >> 1; + const uint8_t a = tex2D(uv_or_u_tex, uv_x, uv_y); + const uint8_t b = tex2D(v_tex, uv_x, uv_y); + if (layout == plugins::sensing::YuvLayout::YUV420Planar) + { + Cb = a; + Cr = b; + } + else + { + Cr = a; + Cb = b; + } + } + + const int idx = y * rgba_row_bytes + x * 4; + yuv_to_rgb( + Y, Cb, Cr, full_range_value != 0, bt709_value != 0, rgba_out[idx + 0], rgba_out[idx + 1], rgba_out[idx + 2]); + rgba_out[idx + 3] = 255; +} + +} // namespace + +namespace plugins +{ +namespace sensing +{ + +void launch_yuv420_pitch_to_rgba(const uint8_t* y_plane, + const uint8_t* uv_or_u_plane, + const uint8_t* v_plane, + int y_pitch, + int uv_pitch, + int v_pitch, + int width, + int height, + uint8_t* rgba_out, + int rgba_row_bytes, + YuvLayout layout, + bool full_range, + bool bt709, + cudaStream_t stream) +{ + const dim3 block(16, 16, 1); + const dim3 grid((width + 15) / 16, (height + 15) / 16, 1); + yuv420_pitch_to_rgba_kernel<<>>(y_plane, uv_or_u_plane, v_plane, y_pitch, uv_pitch, v_pitch, + width, height, rgba_out, rgba_row_bytes, + static_cast(layout), full_range ? 1 : 0, bt709 ? 1 : 0); +} + +void launch_yuv420_array_to_rgba(cudaTextureObject_t y_tex, + cudaTextureObject_t uv_or_u_tex, + cudaTextureObject_t v_tex, + int width, + int height, + uint8_t* rgba_out, + int rgba_row_bytes, + YuvLayout layout, + bool full_range, + bool bt709, + cudaStream_t stream) +{ + const dim3 block(16, 16, 1); + const dim3 grid((width + 15) / 16, (height + 15) / 16, 1); + yuv420_array_to_rgba_kernel<<>>(y_tex, uv_or_u_tex, v_tex, width, height, rgba_out, + rgba_row_bytes, static_cast(layout), + full_range ? 1 : 0, bt709 ? 1 : 0); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/core/yuv_to_rgba.cuh b/src/plugins/sensing/core/yuv_to_rgba.cuh new file mode 100644 index 0000000000..3645202521 --- /dev/null +++ b/src/plugins/sensing/core/yuv_to_rgba.cuh @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +enum class YuvLayout +{ + YUV420Planar, + YVU420Planar, + YUV420SemiPlanar, + YVU420SemiPlanar, +}; + +void launch_yuv420_pitch_to_rgba(const uint8_t* y_plane, + const uint8_t* uv_or_u_plane, + const uint8_t* v_plane, + int y_pitch, + int uv_pitch, + int v_pitch, + int width, + int height, + uint8_t* rgba_out, + int rgba_row_bytes, + YuvLayout layout, + bool full_range, + /// BT.709 matrix instead of BT.601. + bool bt709, + cudaStream_t stream); + +void launch_yuv420_array_to_rgba(cudaTextureObject_t y_tex, + cudaTextureObject_t uv_or_u_tex, + cudaTextureObject_t v_tex, + int width, + int height, + uint8_t* rgba_out, + int rgba_row_bytes, + YuvLayout layout, + bool full_range, + /// BT.709 matrix instead of BT.601. + bool bt709, + cudaStream_t stream); + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/images/setup-camera-front.jpg b/src/plugins/sensing/images/setup-camera-front.jpg new file mode 100644 index 0000000000..2221a644aa --- /dev/null +++ b/src/plugins/sensing/images/setup-camera-front.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85edd8d44afd7e59075f6f23e7fa9942137f71de5b1537ac791a65b86622a927 +size 94547 diff --git a/src/plugins/sensing/images/setup-camera-front.jpg.license b/src/plugins/sensing/images/setup-camera-front.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-camera-front.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-camera-rear.jpg b/src/plugins/sensing/images/setup-camera-rear.jpg new file mode 100644 index 0000000000..3e1d73a2fc --- /dev/null +++ b/src/plugins/sensing/images/setup-camera-rear.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1e11a13cd8a0d691e7bfe36d65262b566e12ff72613b2c08bfd789f4a6f4fe8 +size 69846 diff --git a/src/plugins/sensing/images/setup-camera-rear.jpg.license b/src/plugins/sensing/images/setup-camera-rear.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-camera-rear.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-carrier-front-panel.jpg b/src/plugins/sensing/images/setup-carrier-front-panel.jpg new file mode 100644 index 0000000000..18fc6b8b78 --- /dev/null +++ b/src/plugins/sensing/images/setup-carrier-front-panel.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c98de586d8da78d945ad1f309655c58a1422bda4be985c444eedf25c1f0e7c9c +size 73457 diff --git a/src/plugins/sensing/images/setup-carrier-front-panel.jpg.license b/src/plugins/sensing/images/setup-carrier-front-panel.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-carrier-front-panel.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg b/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg new file mode 100644 index 0000000000..f544282a2f --- /dev/null +++ b/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeeabae35f8242468b5c6748edfd8cae4327286cf81d7fcb8abfd087a6ebfd47 +size 94271 diff --git a/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg.license b/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-carrier-inside-devkit.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-harness-connectors.jpg b/src/plugins/sensing/images/setup-harness-connectors.jpg new file mode 100644 index 0000000000..8788fa6f41 --- /dev/null +++ b/src/plugins/sensing/images/setup-harness-connectors.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8e93a67d2f1611e1aa1bb2b6df8798c0877d48fdf866481a60d7cba96fd9bca +size 76646 diff --git a/src/plugins/sensing/images/setup-harness-connectors.jpg.license b/src/plugins/sensing/images/setup-harness-connectors.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-harness-connectors.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-harness-in-cn1.jpg b/src/plugins/sensing/images/setup-harness-in-cn1.jpg new file mode 100644 index 0000000000..2d213040cf --- /dev/null +++ b/src/plugins/sensing/images/setup-harness-in-cn1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06899102d2d4d4cb76bdb525ca431e6c52239e5810e9634397453ebfb672ebbe +size 71799 diff --git a/src/plugins/sensing/images/setup-harness-in-cn1.jpg.license b/src/plugins/sensing/images/setup-harness-in-cn1.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-harness-in-cn1.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/images/setup-parts-overview.jpg b/src/plugins/sensing/images/setup-parts-overview.jpg new file mode 100644 index 0000000000..9302543e5d --- /dev/null +++ b/src/plugins/sensing/images/setup-parts-overview.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93776cd44ea1a86e67086120bd34b7c34860ee376c52bf84c71785503d536b49 +size 133909 diff --git a/src/plugins/sensing/images/setup-parts-overview.jpg.license b/src/plugins/sensing/images/setup-parts-overview.jpg.license new file mode 100644 index 0000000000..817df24424 --- /dev/null +++ b/src/plugins/sensing/images/setup-parts-overview.jpg.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 diff --git a/src/plugins/sensing/livekit/CMakeLists.txt b/src/plugins/sensing/livekit/CMakeLists.txt new file mode 100644 index 0000000000..6151bae977 --- /dev/null +++ b/src/plugins/sensing/livekit/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The LiveKit SDK behind a small interface: join a room, publish a video track, submit H.264 access units. +# No LiveKit type appears in an installed header, and nothing here touches CUDA. +add_library(sensing_livekit STATIC + livekit_session.cpp + inc/sensing_livekit/livekit_session.hpp + inc/sensing_livekit/video_track_sink.hpp +) + +target_include_directories(sensing_livekit + INTERFACE + $ +) + +target_link_libraries(sensing_livekit + PUBLIC + log_bridge::log_bridge_core + PRIVATE + LiveKit::livekit +) + +add_library(sensing::livekit ALIAS sensing_livekit) diff --git a/src/plugins/sensing/livekit/inc/sensing_livekit/livekit_session.hpp b/src/plugins/sensing/livekit/inc/sensing_livekit/livekit_session.hpp new file mode 100644 index 0000000000..7cf33bcfea --- /dev/null +++ b/src/plugins/sensing/livekit/inc/sensing_livekit/livekit_session.hpp @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "video_track_sink.hpp" + +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +struct LiveKitSessionImpl; + +/** + * @brief Keeps the LiveKit SDK initialised while it is held. + * + * The SDK is process-wide and must be shut down only when nothing in the process uses it, so every + * user holds one of these and the last release shuts it down. + */ +class LiveKitSdkReference +{ +public: + LiveKitSdkReference(); + ~LiveKitSdkReference(); + LiveKitSdkReference(const LiveKitSdkReference&) = delete; + LiveKitSdkReference& operator=(const LiveKitSdkReference&) = delete; +}; + +struct PublishSpec +{ + /// Name subscribers see for the track. + std::string name; + uint32_t width = 0; + uint32_t height = 0; + /// Upper bound the upstream encoder may use; WebRTC adapts below it. + uint64_t max_bitrate_bps = 0; + double max_fps = 30.0; +}; + +/** + * @brief One participant in one LiveKit room. + * + * The room and the participant's identity come from the join token. A terminal loss of the room + * (the server went away, the participant was removed) sets failed(); the SDK's own bounded + * reconnection runs first and is not wrapped in another supervisor. + */ +class LiveKitSession +{ +public: + /// @throws std::runtime_error if the room cannot be joined. The token is never logged. + static std::unique_ptr connect(const std::string& url, const std::string& token); + + /// Unpublish every track (destroy them first), then leave the room. + ~LiveKitSession(); + + LiveKitSession(const LiveKitSession&) = delete; + LiveKitSession& operator=(const LiveKitSession&) = delete; + + /// Publishes an H.264 camera track that carries pre-encoded access units; the SDK does not encode. + /// @throws std::runtime_error if publication fails. + std::unique_ptr publish_video(const PublishSpec& spec); + + /// True once the room has been lost for good. + bool failed() const; + std::string failure() const; + +private: + explicit LiveKitSession(std::shared_ptr impl); + std::shared_ptr m_impl; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp new file mode 100644 index 0000000000..85dac890df --- /dev/null +++ b/src/plugins/sensing/livekit/inc/sensing_livekit/video_track_sink.hpp @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +/// One complete encoded H.264 access unit, with what accompanies it. +struct EncodedVideoFrame +{ + /// Annex-B bytes: an IDR carries its SPS/PPS in front of it. + const uint8_t* data = nullptr; + size_t size = 0; + /// The unit is an IDR, decodable on its own. + bool keyframe = false; + /// CLOCK_MONOTONIC microseconds at capture: the host timeline, not sensor time. + uint64_t timestamp_us = 0; + /// Unix time in microseconds on the publishing host when the frame finished conversion: processing + /// time, not a capture time. Carried as the frame's application timestamp; the transport timestamp + /// above stays monotonic. + uint64_t user_timestamp_us = 0; + /// The camera's own frame sequence. + uint32_t frame_id = 0; + /// The sensor's capture time on its own clock, carried as user data so a consumer can pair + /// frames from different sensors. + uint64_t capture_tsc = 0; +}; + +/// What the receiving side asked of the encoder since the last call. +struct EncoderFeedback +{ + /// The next frame must be an IDR with its parameter sets. + bool keyframe_requested = false; + /// Latest bitrate target from congestion control, if one was requested. + std::optional target_bitrate_bps; +}; + +/** + * @brief One published video track that carries frames already encoded by hardware. + * + * The SDK forwards each access unit without encoding it again. A single thread must drive one sink. + */ +class VideoTrackSink +{ +public: + virtual ~VideoTrackSink() = default; + + /// Size of the encoded pictures this track was published for. + virtual uint32_t width() const = 0; + virtual uint32_t height() const = 0; + + /// Publishes one access unit; the bytes are copied before this returns. + /// @return false when the SDK did not accept the frame, which can break the reference chain: + /// the caller must follow with a keyframe. + /// @throws std::runtime_error when the frame cannot be published at all. + virtual bool submit(const EncodedVideoFrame& frame) = 0; + + /// Keyframe and rate requests accumulated since the previous call. + virtual EncoderFeedback take_feedback() = 0; + + /// One line naming the transport's view of the stream, or empty if unavailable. + virtual std::string describe() = 0; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/livekit/livekit_session.cpp b/src/plugins/sensing/livekit/livekit_session.cpp new file mode 100644 index 0000000000..b3ad56fe3e --- /dev/null +++ b/src/plugins/sensing/livekit/livekit_session.cpp @@ -0,0 +1,303 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/sensing_livekit/livekit_session.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +std::mutex g_sdk_mutex; +int g_sdk_users = 0; + +const char* reason_name(livekit::DisconnectReason reason) +{ + using R = livekit::DisconnectReason; + switch (reason) + { + case R::ClientInitiated: + return "client initiated"; + case R::DuplicateIdentity: + return "another participant joined with the same identity"; + case R::ServerShutdown: + return "the server is shutting down"; + case R::ParticipantRemoved: + return "the participant was removed from the room"; + case R::RoomDeleted: + return "the room was deleted"; + default: + return "connection lost"; + } +} + +} // namespace + +LiveKitSdkReference::LiveKitSdkReference() +{ + std::lock_guard lock(g_sdk_mutex); + if (g_sdk_users == 0) + livekit::initialize(livekit::LogLevel::Warn); + ++g_sdk_users; +} + +LiveKitSdkReference::~LiveKitSdkReference() +{ + std::lock_guard lock(g_sdk_mutex); + if (--g_sdk_users == 0) + livekit::shutdown(); +} + +struct LiveKitSessionImpl : livekit::RoomDelegate +{ + ~LiveKitSessionImpl() override + { + // The members below are destroyed before the room: stop callbacks reaching them, then leave the + // room while the SDK is still up (its reference is released after the room). + room.setDelegate(nullptr); + room.disconnect(); + } + + void onDisconnected(livekit::Room&, const livekit::DisconnectedEvent& event) override + { + if (event.reason == livekit::DisconnectReason::ClientInitiated) + return; + fail(std::string("LiveKit room lost: ") + reason_name(event.reason)); + } + + void onReconnecting(livekit::Room&, const livekit::ReconnectingEvent&) override + { + logger->warn("LiveKit connection interrupted; the SDK is reconnecting"); + } + + void onReconnected(livekit::Room&, const livekit::ReconnectedEvent&) override + { + logger->info("LiveKit connection restored"); + } + + void fail(const std::string& message) + { + std::lock_guard lock(mutex); + if (!failed.load()) + { + failure = message; + failed.store(true); + logger->error("{}", message); + } + } + + // Declared before the room, so destroyed after it. + LiveKitSdkReference sdk; + livekit::Room room; + std::atomic failed{ false }; + mutable std::mutex mutex; + std::string failure; + std::shared_ptr logger = isaaccapture::Logger::get("isaaccapture.plugins.sensing.LiveKitSession"); +}; + +namespace +{ + +/// A track fed with H.264 access units that were encoded upstream. +class LiveKitVideoTrack : public VideoTrackSink +{ +public: + LiveKitVideoTrack(std::shared_ptr session, const PublishSpec& spec) + : m_session(std::move(session)), m_width(spec.width), m_height(spec.height) + { + m_source = std::make_shared( + livekit::VideoCodec::H264, static_cast(spec.width), static_cast(spec.height)); + m_track = livekit::LocalVideoTrack::createLocalVideoTrack(spec.name, m_source); + + livekit::TrackPublishOptions options; + options.source = livekit::TrackSource::SOURCE_CAMERA; + options.video_codec = livekit::VideoCodec::H264; + // Pre-encoded access units go through as they are: no software or SDK-selected encoder. + options.video_encoder = livekit::VideoEncoderBackend::PreEncoded; + options.simulcast = false; + options.video_encoding = livekit::VideoEncodingOptions{ spec.max_bitrate_bps, spec.max_fps }; + options.frame_metadata_features = livekit::FrameMetadataFeatures{ true, true, true }; + + auto participant = m_session->room.localParticipant().lock(); + if (!participant) + throw std::runtime_error("LiveKit: no local participant to publish '" + spec.name + "'"); + try + { + participant->publishTrack(m_track, options); + } + catch (const std::exception& e) + { + throw std::runtime_error("LiveKit: cannot publish track '" + spec.name + "': " + e.what()); + } + m_participant = participant; + } + + ~LiveKitVideoTrack() override + { + try + { + if (auto participant = m_participant.lock()) + participant->unpublishTrack(m_track->sid()); + } + catch (const std::exception& e) + { + m_session->logger->warn("LiveKit: unpublish failed: {}", e.what()); + } + } + + uint32_t width() const override + { + return m_width; + } + uint32_t height() const override + { + return m_height; + } + + bool submit(const EncodedVideoFrame& frame) override + { + if (m_session->failed.load()) + throw std::runtime_error(m_session->failure); + + livekit::EncodedVideoSource::Frame out; + out.is_keyframe = frame.keyframe; + out.width = m_width; + out.height = m_height; + out.timestamp_us = static_cast(frame.timestamp_us); + out.data.assign(frame.data, frame.data + frame.size); + livekit::VideoFrameMetadata metadata; + metadata.user_timestamp_us = frame.user_timestamp_us; + metadata.frame_id = frame.frame_id; + std::vector tsc(8); + for (int i = 0; i < 8; ++i) + tsc[i] = static_cast((frame.capture_tsc >> (8 * i)) & 0xFF); + metadata.user_data = std::move(tsc); + out.metadata = std::move(metadata); + + try + { + return m_source->captureFrame(out); + } + catch (const std::exception& e) + { + throw std::runtime_error(std::string("LiveKit: cannot submit a frame: ") + e.what()); + } + } + + EncoderFeedback take_feedback() override + { + EncoderFeedback feedback; + try + { + const auto pending = m_source->takeFeedback(); + feedback.keyframe_requested = pending.keyframe_requested; + if (pending.rate_control && pending.rate_control->target_bitrate_bps > 0) + feedback.target_bitrate_bps = static_cast(pending.rate_control->target_bitrate_bps); + } + catch (const std::exception& e) + { + throw std::runtime_error(std::string("LiveKit: cannot read encoder feedback: ") + e.what()); + } + return feedback; + } + + // Called from the polling thread, which must not wait on the network: this returns the last answer + // and starts, or collects, a stats request that was asked for earlier. + std::string describe() override + { + try + { + if (m_stats.valid() && m_stats.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + { + for (const auto& entry : m_stats.get()) + { + const auto* out = std::get_if(&entry.stats); + if (!out || out->outbound.frame_width == 0) + continue; + const auto& o = out->outbound; + m_description = "encoder " + o.encoder_implementation + ", " + std::to_string(o.frame_width) + "x" + + std::to_string(o.frame_height) + " @ " + + std::to_string(static_cast(o.frames_per_second)) + " fps, " + + std::to_string(static_cast(o.target_bitrate / 1000)) + " kbps target"; + break; + } + } + if (!m_stats.valid()) + m_stats = m_track->getStats(); + } + catch (const std::exception&) + { + } + return m_description; + } + +private: + std::shared_ptr m_session; + uint32_t m_width; + uint32_t m_height; + std::shared_ptr m_source; + std::shared_ptr m_track; + std::weak_ptr m_participant; + decltype(std::declval().getStats()) m_stats; + std::string m_description; +}; + +} // namespace + +LiveKitSession::LiveKitSession(std::shared_ptr impl) : m_impl(std::move(impl)) +{ +} + +LiveKitSession::~LiveKitSession() = default; + +std::unique_ptr LiveKitSession::connect(const std::string& url, const std::string& token) +{ + auto impl = std::make_shared(); + impl->room.setDelegate(impl.get()); + + livekit::RoomOptions options; + options.auto_subscribe = false; // this process only publishes + options.dynacast = false; + if (!impl->room.connect(url, token, options)) + { + throw std::runtime_error("LiveKit: cannot join the room at " + url + + " (check the URL, that the server is reachable and that the token is valid)"); + } + impl->logger->info("Joined LiveKit room at {}", url); + return std::unique_ptr(new LiveKitSession(std::move(impl))); +} + +std::unique_ptr LiveKitSession::publish_video(const PublishSpec& spec) +{ + return std::make_unique(m_impl, spec); +} + +bool LiveKitSession::failed() const +{ + return m_impl->failed.load(); +} + +std::string LiveKitSession::failure() const +{ + std::lock_guard lock(m_impl->mutex); + return m_impl->failure; +} + +} // namespace sensing +} // namespace plugins diff --git a/src/plugins/sensing/plugin.yaml b/src/plugins/sensing/plugin.yaml new file mode 100644 index 0000000000..2f7cb80087 --- /dev/null +++ b/src/plugins/sensing/plugin.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: sensing_camera +description: "SENSING GMSL camera plugin (SIPL capture, hardware H.264, LiveKit publication)" +command: "./camera_plugin_sensing" +version: "1.0.0" +devices: + - path: "/camera" + type: "camera" + description: "Camera video stream" diff --git a/src/plugins/sensing/scripts/setup_orin_crosscompile.py b/src/plugins/sensing/scripts/setup_orin_crosscompile.py new file mode 100755 index 0000000000..e3e2c1f796 --- /dev/null +++ b/src/plugins/sensing/scripts/setup_orin_crosscompile.py @@ -0,0 +1,724 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Set up compiling the SENSING camera producer for a Jetson Orin on an x86 PC, without root. + + setup_orin_crosscompile.py doctor what is present and what is missing + setup_orin_crosscompile.py fetch build the Orin sysroot and download the aarch64 compiler + setup_orin_crosscompile.py syntax type-check every producer source against the Orin headers (fast) + setup_orin_crosscompile.py toolchain write the CMake toolchain file + setup_orin_crosscompile.py build configure and build camera_plugin_sensing for aarch64 + +Everything lands under one directory (``--root``, default ``~/orin-cross-``); nothing outside it +changes and no step needs sudo. The sysroot is unpacked from NVIDIA's Jetson Linux package and the Multimedia and +SIPL API packages, which ``fetch`` takes from the SDK Manager download directory (``--sdk-downloads``) when they +are there and otherwise from NVIDIA's apt repository, checking the repository's SHA-256 for each. The compiler is +Arm's GNU toolchain, checked against the SHA-256 Arm publishes beside it. + +``syntax`` needs only clang and the fetched headers, and catches the compile errors a build on the Orin would. +``build`` also needs the CUDA cross toolchain for aarch64 (``cuda-cross-sbsa``), CMake and network access for +the dependencies CMake fetches. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import shutil +import subprocess +import sys +import tarfile +import urllib.request +from pathlib import Path + +RELEASE = "39.2.1" +L4T_REPO = "https://repo.download.nvidia.com/jetson/common" +L4T_SUITE = "r39.2" +ARM_GCC_VERSION = "13.3.rel1" +ARM_GCC_NAME = f"arm-gnu-toolchain-{ARM_GCC_VERSION}-x86_64-aarch64-none-linux-gnu" +ARM_GCC_URL = f"https://developer.arm.com/-/media/Files/downloads/gnu/{ARM_GCC_VERSION}/binrel/{ARM_GCC_NAME}.tar.xz" +JETSON_LINUX = f"Jetson_Linux_R{RELEASE}_aarch64.tbz2" +SIPL_PACKAGE = f"Jetson_SIPL_API_R{RELEASE}_aarch64.tbz2" +MMAPI_PACKAGE = "nvidia-l4t-jetson-multimedia-api" +# The L4T packages inside the Jetson Linux archive that hold the libraries and headers the producer links against. +L4T_DEBS = ( + "nvidia-l4t-core", + "nvidia-l4t-camera", + "nvidia-l4t-multimedia-utils", + "nvidia-l4t-multimedia-nvgpu", + "nvidia-l4t-nvsci", + "nvidia-l4t-3d-core", + "nvidia-l4t-cuda-nvgpu", + "nvidia-l4t-cuda-utils", + "nvidia-l4t-wayland", +) +# Ubuntu's own arm64 packages for the C library and the libraries the build finds by name; the Orin runs Ubuntu 24.04, +# so these match its glibc. Indexes are read over https and each file is checked against its index's SHA-256. +UBUNTU_PORTS = "https://ports.ubuntu.com/ubuntu-ports" +UBUNTU_SUITES = ("noble-updates", "noble-security", "noble") +UBUNTU_DEBS = ( + "libc6", + "libc6-dev", + "linux-libc-dev", + "libgcc-s1", + "libstdc++6", + "libegl1", + "libegl-dev", + "libglvnd0", + "libv4l-0t64", + "libv4l-dev", + "libv4lconvert0t64", +) +# What the build needs from the sysroot: (name shown, glob under the sysroot). +SYSROOT_LIBS = ( + ("libnvsipl", "usr/lib/aarch64-linux-gnu/nvidia/libnvsipl.so"), + ("libnvsipl_query", "usr/lib/aarch64-linux-gnu/nvidia/libnvsipl_query.so"), + ("libnvscibuf", "usr/lib/aarch64-linux-gnu/nvidia/libnvscibuf.so*"), + ("libnvscisync", "usr/lib/aarch64-linux-gnu/nvidia/libnvscisync.so*"), + ("libnvbufsurface", "usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurface.so"), + ( + "libnvbufsurface_nvsci", + "usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurface_nvsci.so", + ), + ( + "libnvargus_socketclient", + "usr/lib/aarch64-linux-gnu/nvidia/libnvargus_socketclient.so", + ), + ("libcuda", "opt/nvidia/l4t-gpu-libs/nvgpu/libcuda.so"), + ("libc (crt1.o)", "usr/lib/aarch64-linux-gnu/crt1.o"), + ("libEGL", "usr/lib/aarch64-linux-gnu/libEGL.so"), + ("libv4l2", "usr/lib/aarch64-linux-gnu/libv4l2.so*"), +) +SYSROOT_HEADERS = ( + ( + "NvSIPLCamera.hpp (SIPL API)", + "usr/src/jetson_sipl_api/sipl/include/NvSIPLCamera.hpp", + ), + ( + "nvbufsurface.h (Multimedia API)", + "usr/src/jetson_multimedia_api/include/nvbufsurface.h", + ), + ( + "Argus/Argus.h (Multimedia API)", + "usr/src/jetson_multimedia_api/argus/include/Argus/Argus.h", + ), +) +CUDA_CROSS_ROOT = Path("/usr/local/cuda/targets/sbsa-linux") + + +class SetupError(Exception): + """A step that cannot continue; the message says what to do.""" + + +def say(message: str = "") -> None: + """Progress and diagnostics go to stderr, like the other scripts here.""" + print(message, file=sys.stderr, flush=True) + + +def repo_root() -> Path: + """The checkout this script lives in.""" + return Path(__file__).resolve().parents[4] + + +def sensing_dir() -> Path: + return repo_root() / "src" / "plugins" / "sensing" + + +class Layout: + """Where everything lives under the chosen root.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.sysroot = root / "sysroot" + self.toolchain_dir = root / ARM_GCC_NAME + self.cache = root / "downloads" + self.cmake_file = root / "orin-aarch64.cmake" + self.build_dir = root / "build" + + @property + def gxx(self) -> Path: + return self.toolchain_dir / "bin" / "aarch64-none-linux-gnu-g++" + + @property + def gcc(self) -> Path: + return self.toolchain_dir / "bin" / "aarch64-none-linux-gnu-gcc" + + @property + def spdlog(self) -> Path: + return self.root / "spdlog" + + +def sha256_of(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def download(url: str, target: Path, expected_sha256: str | None) -> Path: + """Fetch ``url`` to ``target`` (kept between runs) and verify its checksum; a bad file is never left in place.""" + if target.exists() and expected_sha256 and sha256_of(target) == expected_sha256: + return target + target.parent.mkdir(parents=True, exist_ok=True) + partial = target.with_suffix(target.suffix + ".part") + say(f" downloading {url}") + with urllib.request.urlopen(url, timeout=60) as response, partial.open("wb") as out: # noqa: S310 - fixed https URLs + shutil.copyfileobj(response, out, 1 << 20) + if expected_sha256 and sha256_of(partial) != expected_sha256: + partial.unlink() + raise SetupError( + f"{url}: the SHA-256 does not match the published {expected_sha256}" + ) + partial.replace(target) + return target + + +def fetch_text(url: str) -> str: + with urllib.request.urlopen(url, timeout=60) as response: # noqa: S310 - fixed https URLs + return response.read().decode() + + +def apt_index() -> dict[str, dict[str, str]]: + """NVIDIA's package index for this release: name -> {Version, Filename, SHA256} (the newest entry wins).""" + import gzip + + raw = urllib.request.urlopen( + f"{L4T_REPO}/dists/{L4T_SUITE}/main/binary-arm64/Packages.gz", timeout=60 + ).read() # noqa: S310 + packages: dict[str, dict[str, str]] = {} + for stanza in gzip.decompress(raw).decode().split("\n\n"): + fields = dict( + line.split(": ", 1) + for line in stanza.splitlines() + if ": " in line and not line.startswith(" ") + ) + if "Package" in fields: + packages.setdefault(fields["Package"], fields) + if fields.get("Version", "") > packages[fields["Package"]].get( + "Version", "" + ): + packages[fields["Package"]] = fields + return packages + + +def ubuntu_index() -> dict[str, dict[str, str]]: + """Ubuntu 24.04's arm64 package index: name -> fields, taking the entry from the first suite that lists it.""" + import gzip + + packages: dict[str, dict[str, str]] = {} + for suite in UBUNTU_SUITES: + for component in ("main", "universe"): + url = f"{UBUNTU_PORTS}/dists/{suite}/{component}/binary-arm64/Packages.gz" + raw = gzip.decompress( + urllib.request.urlopen(url, timeout=120).read() + ).decode() # noqa: S310 + for stanza in raw.split("\n\n"): + fields = dict( + line.split(": ", 1) + for line in stanza.splitlines() + if ": " in line and not line.startswith(" ") + ) + if ( + fields.get("Package") in UBUNTU_DEBS + and fields["Package"] not in packages + ): + packages[fields["Package"]] = fields + return packages + + +def extract_deb(deb: Path, destination: Path) -> None: + """Unpack a .deb's files under ``destination`` without root (dpkg-deb reads it; nothing is installed).""" + if not shutil.which("dpkg-deb"): + raise SetupError( + "dpkg-deb is needed to unpack the packages (it ships with every Debian or Ubuntu host)" + ) + destination.mkdir(parents=True, exist_ok=True) + subprocess.run(["dpkg-deb", "-x", str(deb), str(destination)], check=True) + + +def find_sdk_file(args: argparse.Namespace, pattern: str) -> Path | None: + """The first file in the SDK Manager download directory whose name matches ``pattern``.""" + directory = Path(args.sdk_downloads).expanduser() + matches = sorted(directory.glob(pattern)) if directory.is_dir() else [] + return matches[-1] if matches else None + + +def fetch_sysroot(args: argparse.Namespace, layout: Layout) -> None: + """Unpack the L4T libraries, the Multimedia API and the SIPL API into the sysroot.""" + layout.sysroot.mkdir(parents=True, exist_ok=True) + marker = layout.sysroot / ".fetched" + if marker.exists() and not args.force: + say( + f"sysroot already built ({marker.read_text().strip()}); --force rebuilds it" + ) + return + # 1. The L4T libraries, from the Jetson Linux archive. + archive = find_sdk_file(args, JETSON_LINUX) + if archive is None: + raise SetupError( + f"{JETSON_LINUX} is not in {args.sdk_downloads}. Download it with NVIDIA SDK Manager (or from the Jetson " + "Linux release page) and pass its folder as --sdk-downloads." + ) + say( + f"unpacking L4T libraries from {archive.name} (the archive is large; this takes a minute)" + ) + staging = layout.cache / "l4t_debs" + staging.mkdir(parents=True, exist_ok=True) + wanted = {f"{name}_{RELEASE}-": name for name in L4T_DEBS} + with tarfile.open(archive, "r:bz2") as tar: + for member in tar: + base = os.path.basename(member.name) + if "l4t_deb_packages" in member.name and any( + base.startswith(prefix) for prefix in wanted + ): + member.name = base + tar.extract(member, staging) + found = sorted(staging.glob("*.deb")) + missing = [ + name + for prefix, name in wanted.items() + if not any(d.name.startswith(prefix) for d in found) + ] + if missing: + raise SetupError(f"not in {archive.name}: {', '.join(missing)}") + for deb in found: + extract_deb(deb, layout.sysroot) + # 1b. The C library and the libraries found by name, from Ubuntu 24.04 (what the Orin runs). + index = ubuntu_index() + absent = [name for name in UBUNTU_DEBS if name not in index] + if absent: + raise SetupError(f"not in Ubuntu's arm64 index: {', '.join(absent)}") + for name in UBUNTU_DEBS: + entry = index[name] + deb = download( + f"{UBUNTU_PORTS}/{entry['Filename']}", + layout.cache / Path(entry["Filename"]).name, + entry["SHA256"], + ) + extract_deb(deb, layout.sysroot) + # 2. The Multimedia API: the SDK Manager copy when present, else NVIDIA's apt repository, checksum verified. + mmapi = find_sdk_file(args, f"{MMAPI_PACKAGE}_{RELEASE}-*_arm64.deb") + if mmapi is None: + entry = apt_index().get(MMAPI_PACKAGE) + if not entry or not entry["Version"].startswith(RELEASE): + raise SetupError( + f"{MMAPI_PACKAGE} {RELEASE} is not in the {L4T_SUITE} repository" + ) + mmapi = download( + f"{L4T_REPO}/{entry['Filename']}", + layout.cache / Path(entry["Filename"]).name, + entry["SHA256"], + ) + say(f"unpacking the Multimedia API from {mmapi.name}") + extract_deb(mmapi, layout.sysroot) + # 3. The SIPL API headers. + sipl = find_sdk_file(args, SIPL_PACKAGE) + if sipl is None: + raise SetupError( + f"{SIPL_PACKAGE} is not in {args.sdk_downloads}. Download it with NVIDIA SDK Manager or from the Jetson " + "Linux release page." + ) + say(f"unpacking the SIPL API from {sipl.name}") + with tarfile.open(sipl, "r:bz2") as tar: + tar.extractall(layout.sysroot, filter="data") + fix_sysroot_layout(layout.sysroot) + marker.write_text(f"L4T {RELEASE}\n") + + +def fix_sysroot_layout(sysroot: Path) -> None: + """Ubuntu's libraries sit under usr/ with /lib a link to usr/lib; unpacking debs alone does not make that link.""" + link = sysroot / "lib" + if not link.exists() and not link.is_symlink(): + link.symlink_to("usr/lib") + + +def fetch_compiler(layout: Layout, force: bool) -> None: + """Download Arm's aarch64 GNU toolchain next to the sysroot and check it against Arm's published SHA-256.""" + if layout.gxx.exists() and not force: + say(f"compiler already present: {layout.gxx}") + return + checksum_text = fetch_text(ARM_GCC_URL + ".sha256asc") + match = re.search(r"\b([0-9a-f]{64})\b", checksum_text) + if not match: + raise SetupError(f"could not read the SHA-256 published for {ARM_GCC_NAME}") + archive = download( + ARM_GCC_URL, layout.cache / f"{ARM_GCC_NAME}.tar.xz", match.group(1) + ) + say(f"unpacking {archive.name}") + with tarfile.open(archive, "r:xz") as tar: + tar.extractall(layout.root, filter="data") + + +def pinned_spdlog_commit() -> str: + """The spdlog revision the repository's own build fetches, so the type check sees the same headers.""" + text = (repo_root() / "deps" / "third_party" / "CMakeLists.txt").read_text() + match = re.search(r"spdlog\.git\s+GIT_TAG\s+([0-9a-f]{40})", text) + if not match: + raise SetupError( + "could not read the pinned spdlog revision from deps/third_party/CMakeLists.txt" + ) + return match.group(1) + + +def fetch_spdlog(layout: Layout, force: bool) -> None: + """Fetch spdlog at the pinned commit with git, which checks the object hashes, and confirm the checkout.""" + commit = pinned_spdlog_commit() + if (layout.spdlog / "include" / "spdlog" / "spdlog.h").exists() and not force: + say(f"spdlog already present: {layout.spdlog}") + return + if not shutil.which("git"): + raise SetupError("git is needed to fetch spdlog") + say(f"fetching spdlog {commit[:12]}") + shutil.rmtree(layout.spdlog, ignore_errors=True) + layout.spdlog.mkdir(parents=True) + for step in ( + ["git", "init", "-q"], + [ + "git", + "fetch", + "-q", + "--depth", + "1", + "https://github.com/gabime/spdlog.git", + commit, + ], + ["git", "checkout", "-q", "FETCH_HEAD"], + ): + subprocess.run(step, cwd=layout.spdlog, check=True) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=layout.spdlog, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if head != commit: + raise SetupError(f"spdlog checked out {head}, not the pinned {commit}") + + +def cmd_fetch(args: argparse.Namespace) -> int: + layout = Layout(Path(args.root).expanduser()) + layout.root.mkdir(parents=True, exist_ok=True) + fetch_sysroot(args, layout) + fetch_spdlog(layout, args.force) + if not args.no_compiler: + fetch_compiler(layout, args.force) + say("done; run `doctor` to see what is still missing") + return 0 + + +def check_rows(layout: Layout) -> list[tuple[str, bool, str]]: + """Every prerequisite as (name, present, hint).""" + rows: list[tuple[str, bool, str]] = [] + for tool, hint in ( + ("clang++", "needed by `syntax`: sudo apt install clang"), + ("cmake", "needed by `build`: sudo apt install cmake"), + ("nvcc", "needed by `build`: install the CUDA toolkit"), + ("dpkg-deb", "needed by `fetch`"), + ): + rows.append((tool, shutil.which(tool) is not None, hint)) + rows.append( + ( + "aarch64 g++ (Arm toolchain)", + layout.gxx.exists(), + "run `fetch` (downloads Arm's toolchain)", + ) + ) + rows.append( + ( + "spdlog headers", + (layout.spdlog / "include" / "spdlog" / "spdlog.h").exists(), + "run `fetch`", + ) + ) + rows.append( + ( + "CUDA aarch64 cross target", + (CUDA_CROSS_ROOT / "include" / "cuda_runtime.h").exists(), + "needed by `build`: install cuda-cross-sbsa for your CUDA version", + ) + ) + for label, pattern in SYSROOT_HEADERS + SYSROOT_LIBS: + present = bool(list(layout.sysroot.glob(pattern))) + rows.append((label, present, "run `fetch`" if not present else "")) + return rows + + +def cmd_doctor(args: argparse.Namespace) -> int: + layout = Layout(Path(args.root).expanduser()) + say(f"root: {layout.root}") + rows = check_rows(layout) + for name, present, hint in rows: + say( + f" [{'ok' if present else '--'}] {name}" + + ("" if present or not hint else f" ({hint})") + ) + return 0 if all(present for _, present, _ in rows) else 1 + + +def include_dirs(layout: Layout, args: argparse.Namespace) -> list[Path]: + """The include path the producer's sources need, in the order the build uses.""" + sensing = sensing_dir() + sysroot = layout.sysroot + dirs = [ + sensing / "core" / "inc", + sensing / "livekit" / "inc", + sensing / "third_party" / "jetson_multimedia_api" / "include", + sysroot / "usr/src/jetson_multimedia_api/include", + sysroot / "usr/src/jetson_multimedia_api/argus/include", + sysroot / "usr/src/jetson_sipl_api/sipl/include", + sysroot / "usr/src/jetson_sipl_api/sipl/include/query/include", + sysroot / "usr/src/jetson_sipl_api/sipl/include/nvsci", + repo_root() / "src/core/log_bridge/cpp/inc", + Path(args.cuda).expanduser() / "include", + ] + if args.spdlog_include: + dirs.append(Path(args.spdlog_include).expanduser()) + return dirs + + +def find_spdlog(explicit: str | None, layout: Layout) -> str | None: + """A spdlog include directory: the one given, else the copy `fetch` made.""" + if explicit: + return explicit + fetched = layout.spdlog / "include" + return str(fetched) if (fetched / "spdlog" / "spdlog.h").exists() else None + + +def cmd_syntax(args: argparse.Namespace) -> int: + """Type-check every producer C++ source with clang against the sysroot's headers; no linking, no CUDA.""" + layout = Layout(Path(args.root).expanduser()) + clang = shutil.which("clang++") + if not clang: + raise SetupError("clang++ is not installed (sudo apt install clang)") + if not all(list(layout.sysroot.glob(pattern)) for _, pattern in SYSROOT_HEADERS): + raise SetupError("the Orin headers are missing; run `fetch` first") + args.spdlog_include = find_spdlog(args.spdlog_include, layout) + if not args.spdlog_include: + raise SetupError( + "no spdlog headers found; run `fetch`, or pass --spdlog-include DIR" + ) + sensing = sensing_dir() + sources = sorted( + [ + *sensing.glob("core/*.cpp"), + *sensing.glob("app/*.cpp"), + *sensing.glob("third_party/**/classes/*.cpp"), + ] + ) + flags = ["-std=c++17", "-fsyntax-only", "-Wno-everything", "-DSENSING_HAS_SIPL"] + if (sensing / "core" / "argus_capture.cpp").exists(): + flags.append("-DSENSING_HAS_ARGUS") + for directory in include_dirs(layout, args): + flags += ["-I", str(directory)] + failed = 0 + for source in sources: + result = subprocess.run( + [clang, *flags, str(source)], capture_output=True, text=True, check=False + ) + errors = [ + line + for line in result.stderr.splitlines() + if "error" in line or "fatal" in line + ] + if result.returncode != 0: + failed += 1 + say(f"FAIL {source.relative_to(repo_root())}") + for line in errors[:8]: + say(f" {line}") + elif args.verbose: + say(f"ok {source.relative_to(repo_root())}") + say(f"{len(sources) - failed} of {len(sources)} sources type-check") + return 1 if failed else 0 + + +def cmd_toolchain(args: argparse.Namespace) -> int: + layout = Layout(Path(args.root).expanduser()) + if not layout.gxx.exists(): + raise SetupError("the aarch64 compiler is missing; run `fetch` first") + cuda_cross = Path(args.cuda).expanduser() + # CMake finds a cross target's device library through /usr/lib/cuda (Debian's cross layout). + cuda_link = layout.sysroot / "usr" / "lib" / "cuda" + if not cuda_link.exists() and not cuda_link.is_symlink(): + cuda_link.symlink_to(cuda_cross.resolve()) + # The nvcc must come from the same toolkit as the cross target; a distribution's nvcc on PATH is another version. + toolkit_nvcc = cuda_cross.resolve().parents[1] / "bin" / "nvcc" + cuda_nvcc = ( + str(toolkit_nvcc) if toolkit_nvcc.exists() else (shutil.which("nvcc") or "nvcc") + ) + extra_roots = "" + layout.cmake_file.write_text( + f"""# Generated by setup_orin_crosscompile.py; regenerate rather than edit. +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(CMAKE_C_COMPILER "{layout.gcc}") +set(CMAKE_CXX_COMPILER "{layout.gxx}") +set(CMAKE_SYSROOT "{layout.sysroot}") +# The Tegra libraries sit beside their own dependencies; shared libraries' own needs are not re-resolved on the host. +set(_orin_libs "{layout.sysroot}/usr/lib/aarch64-linux-gnu/nvidia" "{layout.sysroot}/usr/lib/aarch64-linux-gnu" "{layout.sysroot}/opt/nvidia/l4t-gpu-libs/nvgpu") +foreach(_dir IN LISTS _orin_libs) + string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " -Wl,-rpath-link,${{_dir}} -L${{_dir}}") +endforeach() +string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " -Wl,--allow-shlib-undefined") +# Arm's compiler looks for the C runtime and glibc's architecture headers in the plain usr/lib and usr/include; Ubuntu +# keeps them in the aarch64-linux-gnu subdirectories. +foreach(_lang C CXX) + string(APPEND CMAKE_${{_lang}}_FLAGS_INIT " -B{layout.sysroot}/usr/lib/aarch64-linux-gnu -isystem {layout.sysroot}/usr/include/aarch64-linux-gnu") +endforeach() +string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " -B{layout.sysroot}/usr/lib/aarch64-linux-gnu") +# The CUDA link rule does not add --sysroot on its own, and glibc's linker scripts name /lib/... absolutely. +string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " --sysroot={layout.sysroot}") +# Libraries and headers come from the Orin sysroot; build tools (cmake, python) from this host. +set(CMAKE_FIND_ROOT_PATH "{layout.sysroot}" "{cuda_cross}"{extra_roots}) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +# The fetched LiveKit SDK sits in the build tree, outside the sysroot; its config files are searched by explicit path. +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH) +# nvcc compiles the device code for the Orin's GPU and hands the host code to the aarch64 compiler. +set(CMAKE_CUDA_COMPILER "{cuda_nvcc}") +set(CMAKE_CUDA_HOST_COMPILER "{layout.gxx}") +set(CMAKE_CUDA_FLAGS_INIT "--target-directory sbsa-linux -isystem {layout.sysroot}/usr/include/aarch64-linux-gnu") +set(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES "{cuda_cross}/include") +set(CMAKE_CUDA_ARCHITECTURES 87) +set(CUDAToolkit_ROOT "{cuda_cross}") +""" + ) + say(f"wrote {layout.cmake_file}") + return 0 + + +def write_project(layout: Layout) -> Path: + """A small top-level project that builds only the producer. + + The repository's own top-level CMake pulls in the OpenXR SDK, Python and the viewer, which need an aarch64 + desktop stack that has no place here; the producer needs only spdlog and the logging bridge from outside its + own directory. + """ + project = layout.root / "project" + project.mkdir(parents=True, exist_ok=True) + (project / "CMakeLists.txt").write_text( + f"""# Generated by setup_orin_crosscompile.py; regenerate rather than edit. +cmake_minimum_required(VERSION 3.24) +project(sensing_cross LANGUAGES C CXX) +include(FetchContent) +FetchContent_Declare(spdlog SOURCE_DIR "{layout.spdlog}") +set(SPDLOG_BUILD_EXAMPLE OFF CACHE BOOL "" FORCE) +set(SPDLOG_BUILD_TESTS OFF CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(spdlog) +add_subdirectory("{repo_root() / "src/core/log_bridge/cpp"}" log_bridge) +add_subdirectory("{sensing_dir()}" sensing) +""" + ) + return project + + +def cmd_build(args: argparse.Namespace) -> int: + """Configure and build ``camera_plugin_sensing`` for aarch64 into ``/build``.""" + layout = Layout(Path(args.root).expanduser()) + for tool in ("cmake", "nvcc"): + if not shutil.which(tool): + raise SetupError(f"{tool} is not installed") + if not (CUDA_CROSS_ROOT / "include" / "cuda_runtime.h").exists(): + raise SetupError( + "the CUDA aarch64 cross target is missing (install cuda-cross-sbsa for your CUDA version)" + ) + cmd_toolchain(args) + configure = [ + "cmake", + "-S", + str(write_project(layout)), + "-B", + str(layout.build_dir), + f"-DCMAKE_TOOLCHAIN_FILE={layout.cmake_file}", + "-DCMAKE_BUILD_TYPE=Release", + f"-DSIPL_API_DIR={layout.sysroot}/usr/src/jetson_sipl_api", + f"-DJETSON_MMAPI_DIR={layout.sysroot}/usr/src/jetson_multimedia_api", + *args.cmake_arg, + ] + say("configuring: " + " ".join(configure)) + if subprocess.run(configure, check=False).returncode != 0: + return 1 + build = [ + "cmake", + "--build", + str(layout.build_dir), + "--target", + args.target, + "-j", + str(args.jobs), + ] + say("building: " + " ".join(build)) + return subprocess.run(build, check=False).returncode + + +def parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument( + "--root", + default=f"~/orin-cross-{RELEASE}", + help="directory holding the sysroot and compiler", + ) + ap.add_argument( + "--sdk-downloads", + default="~/Downloads/nvidia/sdkm_downloads", + help="folder with the Jetson Linux and SIPL API archives from SDK Manager", + ) + ap.add_argument( + "--cuda", default=str(CUDA_CROSS_ROOT), help="the CUDA aarch64 target directory" + ) + ap.add_argument( + "--spdlog-include", + help="spdlog include directory (default: the copy `fetch` makes)", + ) + sub = ap.add_subparsers(dest="command", required=True) + sub.add_parser( + "doctor", help="show what is present and what is missing" + ).set_defaults(run=cmd_doctor) + fetch = sub.add_parser( + "fetch", help="build the sysroot and download the aarch64 compiler" + ) + fetch.add_argument( + "--force", action="store_true", help="redo steps that look finished" + ) + fetch.add_argument( + "--no-compiler", action="store_true", help="sysroot only (enough for `syntax`)" + ) + fetch.set_defaults(run=cmd_fetch) + syntax = sub.add_parser( + "syntax", help="type-check the producer's sources against the Orin headers" + ) + syntax.add_argument( + "-v", "--verbose", action="store_true", help="also list the files that pass" + ) + syntax.set_defaults(run=cmd_syntax) + sub.add_parser("toolchain", help="write the CMake toolchain file").set_defaults( + run=cmd_toolchain + ) + build = sub.add_parser("build", help="cross-build camera_plugin_sensing") + build.add_argument("--target", default="camera_plugin_sensing") + build.add_argument("--jobs", type=int, default=os.cpu_count() or 4) + build.add_argument( + "--cmake-arg", + action="append", + default=[], + help="extra CMake argument, repeatable", + ) + build.set_defaults(run=cmd_build) + return ap + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + return args.run(args) + except SetupError as exc: + say(f"error: {exc}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE b/src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE new file mode 100644 index 0000000000..d6ab34509d --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/README.md b/src/plugins/sensing/third_party/jetson_multimedia_api/README.md new file mode 100644 index 0000000000..d326aed359 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/README.md @@ -0,0 +1,43 @@ + + +# Jetson Multimedia API encoder classes + +The SENSING producer drives the Jetson hardware H.264 encoder through NVIDIA's `NvVideoEncoder` class and the +V4L2 element classes it is built on. The Jetson Multimedia API documents these classes and ships their headers +in `include/`, but ships their implementation only as source, in the package's `samples/common/classes/` +directory: the package contains no prebuilt library for them, and NVIDIA's own samples compile them in. So they +are copied here and compiled into `sensing_core`, instead of being compiled from the installed `/usr/src` tree. + +- **Source:** the `nvidia-l4t-jetson-multimedia-api` package, version 39.2.1-20260806224157 (L4T R39.2.1), + from `https://repo.download.nvidia.com/jetson/common/pool/main/n/nvidia-l4t-jetson-multimedia-api/`. The files + were taken unchanged from `/usr/src/jetson_multimedia_api/samples/common/classes/` (`classes/`) and + `/usr/src/jetson_multimedia_api/include/` (`include/`), except for the change below. +- **Files:** `classes/` holds `NvVideoEncoder`, `NvV4l2Element`, `NvV4l2ElementPlane`, `NvElement`, + `NvElementProfiler`, `NvBuffer` and `NvLogging` (`.cpp`); `include/` holds the matching headers. This is the + complete set the encoder needs; the producer calls only the encoder and its planes. +- **License:** BSD-3-Clause. Every file carries its own copy of the license text in its header and keeps it; + [`LICENSE`](LICENSE) repeats the text, and the build installs it beside the plugin binary under + `licenses/jetson-multimedia-api`. The package's own copyright file names the NVIDIA Driver License Agreement as + the default for files that carry no license of their own; none of the files here is in that category. +- **Local change:** `classes/NvVideoEncoder.cpp` has the four `cout` lines in `setBitrate()` and `setPeakBitrate()` + removed (they printed the plane pixel-format codes to stdout on every encoder start). The two places are marked + with a comment. + +## Not copied + +These headers stay in the installed tree (`JETSON_MMAPI_DIR`, default `/usr/src/jetson_multimedia_api`), because +their license does not allow copying them here or they must match the installed driver: + +- `nvbufsurface.h`: NVIDIA proprietary license. +- `libv4l2.h`: LGPL-2.1. +- `v4l2_nv_extensions.h`: BSD-3-Clause, but the driver's own interface, which has to match the installed L4T. + +The Argus headers and the `libnvbufsurface` and V4L2 libraries also come from the installed JetPack. + +## Updating + +Extract the package for the target release (`dpkg-deb -x`), copy the same fourteen files over these, compare the +result against the previous copy, and reapply the local change above. Check that the license headers are unchanged. diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvBuffer.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvBuffer.cpp new file mode 100644 index 0000000000..cf5a19050f --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvBuffer.cpp @@ -0,0 +1,514 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvBuffer.h" +#include "NvLogging.h" + +#include +#include +#include +#include + +#define CAT_NAME "Buffer" + +#define MAX(a,b) (a > b ? a : b) + +NvBuffer::NvBuffer(enum v4l2_buf_type buf_type, enum v4l2_memory memory_type, + uint32_t n_planes, NvBufferPlaneFormat * fmt, uint32_t index) + :buf_type(buf_type), + memory_type(memory_type), + index(index), + n_planes(n_planes) +{ + uint32_t i; + + mapped = false; + allocated = false; + + memset(planes, 0, sizeof(planes)); + for (i = 0; i < n_planes; i++) + { + this->planes[i].fd = -1; + this->planes[i].fmt = fmt[i]; + } + + ref_count = 0; + pthread_mutex_init(&ref_lock, NULL); + shared_buffer = NULL; +} + +NvBuffer::NvBuffer(uint32_t pixfmt, uint32_t width, uint32_t height, + uint32_t index) + :buf_type(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE), + memory_type(V4L2_MEMORY_USERPTR), + index(index) +{ + uint32_t i = 0; + NvBuffer::NvBufferPlaneFormat fmt[MAX_PLANES] = {{0}}; + + mapped = false; + allocated = false; + + fill_buffer_plane_format(&n_planes, fmt, width, height, pixfmt); + + for (i = 0; i < MAX_PLANES; i++) + { + this->planes[i].fd = -1; + this->planes[i].data = NULL; + this->planes[i].bytesused = 0; + this->planes[i].mem_offset = 0; + this->planes[i].length = 0; + this->planes[i].fmt = fmt[i]; + this->planes[i].fmt.sizeimage = fmt[i].width * fmt[i].height * + fmt[i].bytesperpixel; + this->planes[i].fmt.stride = fmt[i].width * fmt[i].bytesperpixel; + } + + ref_count = 0; + pthread_mutex_init(&ref_lock, NULL); + shared_buffer = NULL; +} + +NvBuffer::NvBuffer(uint32_t size, uint32_t index) + :buf_type(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE), + memory_type(V4L2_MEMORY_USERPTR), + index(index) +{ + uint32_t i; + + mapped = false; + allocated = false; + + n_planes = 1; + for (i = 0; i < n_planes; i++) + { + this->planes[i].fd = -1; + this->planes[i].data = NULL; + this->planes[i].bytesused = 0; + this->planes[i].mem_offset = 0; + this->planes[i].length = 0; + this->planes[i].fmt.sizeimage = size; + } + + ref_count = 0; + pthread_mutex_init(&ref_lock, NULL); + shared_buffer = NULL; +} + +NvBuffer::~NvBuffer() +{ + if (mapped) + { + unmap(); + } + if (allocated) + { + deallocateMemory(); + } + + pthread_mutex_destroy(&ref_lock); +} + +int +NvBuffer::map() +{ + uint32_t j; + NvBufSurface *nvbuf_surf = 0; + int ret = 0; + + if (memory_type != V4L2_MEMORY_MMAP) + { + CAT_WARN_MSG("Buffer " << index << "already mapped"); + return -1; + } + + if (mapped) + { + CAT_WARN_MSG("Buffer " << index << "already mapped"); + return 0; + } + + ret = NvBufSurfaceFromFd(planes[0].fd, (void **)(&nvbuf_surf)); + if (ret < 0) + { + CAT_ERROR_MSG("Could not get NvBufSurface" << planes[0].fd); + return -1; + } + + if (nvbuf_surf->memType != NVBUF_MEM_CUDA_DEVICE && nvbuf_surf->memType != NVBUF_MEM_CUDA_ARRAY) + { + for (j = 0; j < n_planes; j++) + { + if (planes[j].fd == -1) + { + return -1; + } + + ret = NvBufSurfaceMap(nvbuf_surf, 0, j, NVBUF_MAP_READ_WRITE); + + planes[j].data = (unsigned char *)nvbuf_surf->surfaceList[0].mappedAddr.addr[j]; + if (planes[j].data == NULL) + { + CAT_ERROR_MSG("Could not map buffer " << index << ", plane " << j); + return -1; + } + else + { + CAT_DEBUG_MSG("Mapped buffer " << index << ", plane " << j << " to " + << planes[j].data); + } + } + } + mapped = true; + return 0; +} + +void +NvBuffer::unmap() +{ + NvBufSurface *nvbuf_surf = 0; + int ret = 0; + + if (memory_type != V4L2_MEMORY_MMAP || !mapped) + { + CAT_WARN_MSG("Cannot Unmap Buffer " << index << + ". Only mapped MMAP buffer can be unmapped"); + return; + } + + ret = NvBufSurfaceFromFd(planes[0].fd, (void **)(&nvbuf_surf)); + if (ret < 0) + { + CAT_ERROR_MSG("Could not get NvBufSurface" << planes[0].fd); + return; + } + + if (nvbuf_surf->memType != NVBUF_MEM_CUDA_DEVICE && nvbuf_surf->memType != NVBUF_MEM_CUDA_ARRAY) + { + for (uint32_t j = 0; j < n_planes; j++) + { + if (planes[j].data) + { + ret = NvBufSurfaceUnMap(nvbuf_surf, 0, j); + if (ret < 0) + { + CAT_WARN_MSG("Error while Unmapping buffer"); + return; + } + } + planes[j].data = NULL; + } + } + mapped = false; + CAT_DEBUG_MSG("Buffer " << index << " unmapped "); +} + +int +NvBuffer::allocateMemory() +{ + uint32_t j; + + if (memory_type != V4L2_MEMORY_USERPTR) + { + CAT_ERROR_MSG("Only USERPTR buffers can be allocated"); + return -1; + } + + if (allocated) + { + CAT_WARN_MSG("Buffer " << index << "already allocated memory"); + return 0; + } + + for (j = 0; j < n_planes; j++) + { + if (planes[j].data) + { + ERROR_MSG("Buffer " << index << ", Plane " << j << + " already allocated"); + return -1; + } + + planes[j].length = MAX(planes[j].fmt.sizeimage, + planes[j].fmt.width * + planes[j].fmt.bytesperpixel * + planes[j].fmt.height); + planes[j].data = new unsigned char [planes[j].length]; + + if (planes[j].data == MAP_FAILED) + { + SYS_ERROR_MSG("Error while allocating buffer " << index << + " plane " << j); + return -1; + } + else + { + DEBUG_MSG("Buffer " << index << ", Plane " << j << + " allocated to " << (void *) planes[j].data); + } + } + allocated = true; + return 0; +} + +void +NvBuffer::deallocateMemory() +{ + uint32_t j; + + if (memory_type != V4L2_MEMORY_USERPTR || !allocated) + { + ERROR_MSG("Only allocated USERPTR buffers can be deallocated"); + return; + } + + for (j = 0; j < n_planes; j++) + { + if (!planes[j].data) + { + DEBUG_MSG("Buffer " << index << ", Plane " << j << + " not allocated"); + continue; + } + delete[] planes[j].data; + planes[j].data = NULL; + } + allocated = false; + DEBUG_MSG("Buffer " << index << " deallocated"); +} + +int +NvBuffer::ref() +{ + int ref_count; + + pthread_mutex_lock(&ref_lock); + ref_count = ++this->ref_count; + pthread_mutex_unlock(&ref_lock); + return ref_count; +} + +int +NvBuffer::unref() +{ + int ref_count; + + pthread_mutex_lock(&ref_lock); + if (this->ref_count > 0) + { + --this->ref_count; + } + ref_count = this->ref_count; + pthread_mutex_unlock(&ref_lock); + return ref_count; +} + +int +NvBuffer::fill_buffer_plane_format(uint32_t *num_planes, + NvBuffer::NvBufferPlaneFormat *planefmts, + uint32_t width, uint32_t height, uint32_t raw_pixfmt) +{ + switch (raw_pixfmt) + { + case V4L2_PIX_FMT_YUV444M: + case V4L2_PIX_FMT_YUV444: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width; + planefmts[2].width = width; + + planefmts[0].height = height; + planefmts[1].height = height; + planefmts[2].height = height; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_YUV422M: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + planefmts[2].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height; + planefmts[2].height = height; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_YUV422RM: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width; + planefmts[2].width = width; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + planefmts[2].height = height / 2; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_YUV420M: + case V4L2_PIX_FMT_YVU420M: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + planefmts[2].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + planefmts[2].height = height / 2; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_NV12M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 2; + break; + case V4L2_PIX_FMT_NV16M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 2; + break; + case V4L2_PIX_FMT_GREY: + *num_planes = 1; + + planefmts[0].width = width; + + planefmts[0].height = height; + + planefmts[0].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_YUYV: + case V4L2_PIX_FMT_YVYU: + case V4L2_PIX_FMT_UYVY: + case V4L2_PIX_FMT_VYUY: + *num_planes = 1; + + planefmts[0].width = width; + + planefmts[0].height = height; + + planefmts[0].bytesperpixel = 2; + break; + case V4L2_PIX_FMT_ABGR32: + case V4L2_PIX_FMT_XRGB32: + *num_planes = 1; + + planefmts[0].width = width; + + planefmts[0].height = height; + + planefmts[0].bytesperpixel = 4; + break; + case V4L2_PIX_FMT_P010M: + case V4L2_PIX_FMT_P012M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + + planefmts[0].bytesperpixel = 2; + planefmts[1].bytesperpixel = 4; + break; + case V4L2_PIX_FMT_NV24M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width; + + planefmts[0].height = height; + planefmts[1].height = height; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 2; + break; + case V4L2_PIX_FMT_NV24_10LE: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width; + + planefmts[0].height = height; + planefmts[1].height = height; + + planefmts[0].bytesperpixel = 2; + planefmts[1].bytesperpixel = 4; + break; + case V4L2_PIX_FMT_YUV444_10LE: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width; + planefmts[2].width = width; + + planefmts[0].height = height; + planefmts[1].height = height; + planefmts[2].height = height; + + planefmts[0].bytesperpixel = 2; + planefmts[1].bytesperpixel = 2; + planefmts[2].bytesperpixel = 2; + break; + default: + ERROR_MSG("Unsupported pixel format " << raw_pixfmt); + return -1; + } + return 0; +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElement.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElement.cpp new file mode 100644 index 0000000000..7d32974d8f --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElement.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvElement.h" + +void NvElement::getProfilingData(NvElementProfiler::NvElementProfilerData &data) +{ + profiler.getProfilerData(data); +} + +void NvElement::printProfilingStats(std::ostream &out_stream) +{ + out_stream << "----------- Element = " << comp_name << " -----------" << std::endl; + profiler.printProfilerData(out_stream); + out_stream << "-------------------------------------" << std::endl; +} + +void NvElement::enableProfiling() +{ + profiler.enableProfiling(true); +} + +bool NvElement::isProfilingEnabled() +{ + return profiler.enabled; +} + +NvElement::NvElement(const char *name, NvElementProfiler::ProfilerField fields) + :profiler(fields) +{ + is_in_error = 0; + if (!name) + is_in_error = 1; + this->comp_name = name; +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElementProfiler.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElementProfiler.cpp new file mode 100644 index 0000000000..e5855f5736 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvElementProfiler.cpp @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include "NvElementProfiler.h" + +#define LOCK() pthread_mutex_lock(&profiler_lock) +#define UNLOCK() pthread_mutex_unlock(&profiler_lock) + +#define RETURN_IF_DISABLED() \ + if (!enabled) { \ + UNLOCK(); \ + return; \ + } + +#define GET_TIME(timeval) gettimeofday(timeval, NULL); + +#define TIMESPEC_DIFF_USEC(timespec1, timespec2) \ + (((timespec1)->tv_sec - (timespec2)->tv_sec) * 1000000L + \ + (timespec1)->tv_usec - (timespec2)->tv_usec) + +using namespace std; + +NvElementProfiler::NvElementProfiler(ProfilerField fields) + :valid_fields(fields) +{ + enabled = false; + unit_id_counter = 0; + + reset(); + + pthread_mutex_init(&profiler_lock, NULL); +} + +NvElementProfiler::~NvElementProfiler() +{ + LOCK(); + reset(); + UNLOCK(); + pthread_mutex_destroy(&profiler_lock); +} +void +NvElementProfiler::enableProfiling(bool reset_data) +{ + LOCK(); + if (enabled) + { + UNLOCK(); + return; + } + + if(reset_data) + { + reset(); + } + + enabled = true; + UNLOCK(); +} + +void +NvElementProfiler::disableProfiling() +{ + LOCK(); + RETURN_IF_DISABLED(); + + data_int.accumulated_time.tv_sec += + (data_int.stop_time.tv_sec - data_int.start_time.tv_sec); + data_int.accumulated_time.tv_usec += + (data_int.stop_time.tv_usec - data_int.start_time.tv_usec); + data_int.start_time.tv_sec = 0; + data_int.start_time.tv_usec = 0; + data_int.stop_time.tv_sec = 0; + data_int.stop_time.tv_usec = 0; + enabled = false; + UNLOCK(); +} + +void NvElementProfiler::getProfilerData(NvElementProfiler::NvElementProfilerData &data) +{ + uint64_t total_time; + + LOCK(); + + total_time = data_int.accumulated_time.tv_sec * 1000000L + + data_int.accumulated_time.tv_usec + + TIMESPEC_DIFF_USEC(&data_int.stop_time, &data_int.start_time); + + if (data_int.total_processed_units == 0 || total_time == 0) + { + data.average_fps = 0; + } + else + { + data.average_fps = ((float) (data_int.total_processed_units - 1)) * + 1000000 / total_time; + } + + if (data_int.total_processed_units == 0) + { + data.max_latency_usec = 0; + data.min_latency_usec = 0; + data.average_latency_usec = 0; + } + else + { + data.max_latency_usec = data_int.max_latency_usec; + data.min_latency_usec = data_int.min_latency_usec; + data.average_latency_usec = + data_int.total_latency / data_int.total_processed_units; + } + + data.profiling_time.tv_sec = + data_int.accumulated_time.tv_sec + data_int.stop_time.tv_sec - + data_int.start_time.tv_sec; + + data.profiling_time.tv_usec = + data_int.accumulated_time.tv_usec + data_int.stop_time.tv_usec - + data_int.start_time.tv_usec; + + if (data.profiling_time.tv_usec < 0) + { + data.profiling_time.tv_usec += 1000000; + data.profiling_time.tv_sec--; + } + + if (data.profiling_time.tv_usec > 1000000) + { + data.profiling_time.tv_usec -= 1000000; + data.profiling_time.tv_sec++; + } + + data.total_processed_units = data_int.total_processed_units; + data.num_late_units = data_int.num_late_units; + data.valid_fields = valid_fields; + UNLOCK(); +} + +void NvElementProfiler::printProfilerData(ostream &out_stream) +{ + NvElementProfilerData data; + getProfilerData(data); + + if (data.valid_fields & PROFILER_FIELD_FPS) + { + out_stream << "Total Profiling time = " << + (data.profiling_time.tv_sec + + (data.profiling_time.tv_usec / 1000000.0)) << endl; + out_stream << "Average FPS = " << data.average_fps << endl; + } + if (data.valid_fields & PROFILER_FIELD_TOTAL_UNITS) + { + out_stream << "Total units processed = " << + data.total_processed_units << endl; + } + if (data.valid_fields & PROFILER_FIELD_LATE_UNITS) + { + out_stream << "Num. of late units = " << + data.num_late_units << endl; + } + if (data.valid_fields & PROFILER_FIELD_LATENCIES) + { + out_stream << "Average latency(usec) = " << + data.average_latency_usec << endl; + out_stream << "Minimum latency(usec) = " << + data.min_latency_usec << endl; + out_stream << "Maximum latency(usec) = " << + data.max_latency_usec << endl; + } +} + +void +NvElementProfiler::reset() +{ + memset(&data_int, 0, sizeof(data_int)); + data_int.min_latency_usec = (uint64_t) -1; + + unit_start_time_queue.clear(); +} + +uint64_t +NvElementProfiler::startProcessing() +{ + struct timeval time; + uint64_t ret = 0; + LOCK(); + if (enabled) + { + std::map::iterator it = + unit_start_time_queue.end(); + + unit_id_counter++; + GET_TIME(&time); + + unit_start_time_queue.insert(it, + std::pair(unit_id_counter, time)); + + ret = unit_id_counter; + } + UNLOCK(); + return ret; +} + +void +NvElementProfiler::finishProcessing(uint64_t id, bool is_late) +{ + struct timeval unit_start_time; + struct timeval stop_time; + uint64_t latency; + + LOCK(); + RETURN_IF_DISABLED(); + + if ((valid_fields & PROFILER_FIELD_LATENCIES) && + unit_start_time_queue.empty()) + { + UNLOCK(); + return; + } + + GET_TIME(&stop_time); + + if (valid_fields & PROFILER_FIELD_LATENCIES) + { + std::map::iterator it; + if (id) + { + it = unit_start_time_queue.find(id); + } + else + { + it = unit_start_time_queue.begin(); + } + + if (it == unit_start_time_queue.end()) + { + UNLOCK(); + return; + } + unit_start_time = it->second; + unit_start_time_queue.erase(it); + + latency = TIMESPEC_DIFF_USEC(&stop_time, &unit_start_time); + data_int.total_latency += latency; + + if (latency < data_int.min_latency_usec) + { + data_int.min_latency_usec = latency; + } + if(latency > data_int.max_latency_usec) + { + data_int.max_latency_usec = latency; + } + } + + data_int.stop_time = stop_time; + + if (!data_int.start_time.tv_sec && !data_int.start_time.tv_usec) + { + data_int.start_time = data_int.stop_time; + } + + if (is_late) + { + data_int.num_late_units++; + } + data_int.total_processed_units++; + + UNLOCK(); +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvLogging.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvLogging.cpp new file mode 100644 index 0000000000..a8069ac032 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvLogging.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvLogging.h" + +int log_level = DEFAULT_LOG_LEVEL; + +const char *log_level_name[] = {"INFO", "ERROR", "WARN", "DEBUG"}; \ No newline at end of file diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2Element.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2Element.cpp new file mode 100644 index 0000000000..0ce51c63d2 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2Element.cpp @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvV4l2Element.h" +#include "NvLogging.h" + +#include +#include +#include +#include + +#define CAT_NAME "V4l2Element" + +using namespace std; + +/*initialization of mutex for NvV4l2Element*/ +pthread_mutex_t initializer_mutex = PTHREAD_MUTEX_INITIALIZER; + +NvV4l2Element::NvV4l2Element(const char *comp_name, const char *dev_node, int flags, NvElementProfiler::ProfilerField fields) + :NvElement(comp_name, fields), + output_plane(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, comp_name, + fd, !(flags & O_NONBLOCK), profiler), + capture_plane(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, comp_name, + fd, !(flags & O_NONBLOCK), profiler) +{ + struct v4l2_capability caps; + int ret; + + app_data = NULL; + output_plane_pixfmt = 0; + capture_plane_pixfmt = 0; + + /*Synchronization issue of libv4l2 open source library fixing here,adding lock for that*/ + pthread_mutex_lock(&initializer_mutex); + fd = v4l2_open(dev_node, flags | O_RDWR); + if (fd == -1) + { + COMP_SYS_ERROR_MSG("Could not open device '" << dev_node << "'"); + is_in_error = 1; + pthread_mutex_unlock(&initializer_mutex); + return; + } + pthread_mutex_unlock(&initializer_mutex); + + COMP_DEBUG_MSG("Opened, fd = " << fd); + + ret = v4l2_ioctl(fd, VIDIOC_QUERYCAP, &caps); + if (ret != 0) + { + COMP_SYS_ERROR_MSG("Error in VIDIOC_QUERYCAP"); + is_in_error = 1; + return; + } + if (!(caps.capabilities & V4L2_CAP_VIDEO_M2M_MPLANE)) + { + COMP_ERROR_MSG("Device does not support V4L2_CAP_VIDEO_M2M_MPLANE"); + is_in_error = 1; + return; + } +} + +NvV4l2Element::~NvV4l2Element() +{ + output_plane.deinitPlane(); + capture_plane.deinitPlane(); + + if (fd != -1) + { + v4l2_close(fd); + CAT_DEBUG_MSG("Device closed, fd = " << fd); + } +} + +int +NvV4l2Element::dqEvent(struct v4l2_event &ev, uint32_t max_wait_ms) +{ + int ret; + + do + { + ret = v4l2_ioctl(fd, VIDIOC_DQEVENT, &ev); + + if (ret == 0) + { + COMP_DEBUG_MSG("DQed event " << hex << ev.type << dec); + } + else if (errno != EAGAIN) + { + COMP_SYS_ERROR_MSG("Error while DQing event"); + break; + } + else if (max_wait_ms-- == 0) + { + COMP_WARN_MSG("Error while DQing event: Resource temporarily unavailable"); + break; + } + else + { + usleep(1000); + } + } + while (ret && (output_plane.getStreamStatus() || capture_plane.getStreamStatus())); + + return ret; +} + +int +NvV4l2Element::setControl(uint32_t id, int32_t value) +{ + struct v4l2_control ctl; + int ret; + + ctl.id = id; + ctl.value = value; + + ret = v4l2_ioctl(fd, VIDIOC_S_CTRL, &ctl); + + if (ret < 0) + { + COMP_SYS_ERROR_MSG("Error setting value " << value << " on control " << + id); + } + else + { + COMP_DEBUG_MSG("Set value " << value << " on control " << id); + } + return ret; +} + +int +NvV4l2Element::getControl(uint32_t id, int32_t &value) +{ + struct v4l2_control ctl; + int ret; + + ctl.id = id; + + ret = v4l2_ioctl(fd, VIDIOC_G_CTRL, &ctl); + + if (ret < 0) + { + COMP_SYS_ERROR_MSG("Error getting value of control " << id); + } + else + { + COMP_DEBUG_MSG("Got value " << ctl.value << " for control " << id); + value = ctl.value; + } + return ret; +} + +int +NvV4l2Element::setExtControls(v4l2_ext_controls &ctl) +{ + int ret; + + ret = v4l2_ioctl(fd, VIDIOC_S_EXT_CTRLS, &ctl); + + if (ret < 0) + { + COMP_SYS_ERROR_MSG("Error setting controls"); + } + else + { + COMP_DEBUG_MSG("Set controls"); + } + return ret; +} + +int +NvV4l2Element::getExtControls(v4l2_ext_controls &ctl) +{ + int ret; + + ret = v4l2_ioctl(fd, VIDIOC_G_EXT_CTRLS, &ctl); + + if (ret < 0) + { + COMP_SYS_ERROR_MSG("Error getting value of controls"); + } + else + { + COMP_DEBUG_MSG("Got controls"); + } + return ret; +} + +int +NvV4l2Element::subscribeEvent(uint32_t type, uint32_t id, uint32_t flags) +{ + struct v4l2_event_subscription sub; + int ret; + + memset(&sub, 0, sizeof(struct v4l2_event_subscription)); + + sub.type = type; + sub.id = id; + sub.flags = flags; + + ret = v4l2_ioctl(fd, VIDIOC_SUBSCRIBE_EVENT, &sub); + if (ret == 0) + { + COMP_DEBUG_MSG("Successfully subscribed to event " << type); + } + else + { + COMP_SYS_ERROR_MSG + ("Error while subscribing to event " << type); + } + + return ret; +} + +int +NvV4l2Element::abort() +{ + int ret = 0; + + ret |= output_plane.setStreamStatus(false); + ret |= capture_plane.setStreamStatus(false); + + return ret; +} + +int +NvV4l2Element::waitForIdle(uint32_t max_wait_ms) +{ + COMP_ERROR_MSG("wait_for_idle not implemented"); + return 0; +} + +int +NvV4l2Element::isInError() +{ + int error = is_in_error; + error |= capture_plane.isInError(); + error |= output_plane.isInError(); + return error; +} + +void +NvV4l2Element::enableProfiling() +{ + if (output_plane_pixfmt || capture_plane_pixfmt) + { + COMP_ERROR_MSG("Profiling must be enabled before setting formats on either plane"); + return; + } + profiler.enableProfiling(true); +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2ElementPlane.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2ElementPlane.cpp new file mode 100644 index 0000000000..335382ed6d --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvV4l2ElementPlane.cpp @@ -0,0 +1,943 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvV4l2ElementPlane.h" +#include "NvLogging.h" + +#include +#include +#include +#include +#include +#include "nvbufsurface.h" + +#define CHECK_V4L2_RETURN(ret, str) \ + if (ret < 0) { \ + PLANE_SYS_ERROR_MSG(str << ": failed"); \ + return -1; \ + } else { \ + PLANE_DEBUG_MSG(str << ": success"); \ + return 0; \ + } + +using namespace std; + +NvV4l2ElementPlane::NvV4l2ElementPlane(enum v4l2_buf_type buf_type, + const char *device_name, int &fd, bool blocking, + NvElementProfiler &profiler) + :fd(fd), + v4l2elem_profiler(profiler), + comp_name(device_name) +{ + this->buf_type = buf_type; + this->blocking = blocking; + is_in_error = 0; + switch (buf_type) + { + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + plane_name = "Output Plane"; + break; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + plane_name = "Capture Plane"; + break; + default: + ERROR_MSG("Unsupported v4l2_buf_type " << buf_type); + plane_name = "Unknown"; + is_in_error = 1; + } + + num_buffers = 0; + buffers = NULL; + + n_planes = 0; + memset(&planefmts, 0, sizeof(planefmts)); + + num_queued_buffers = 0; + total_queued_buffers = 0; + total_dequeued_buffers = 0; + + streamon = false; + pthread_mutex_init(&plane_lock, NULL); + pthread_cond_init(&plane_cond, NULL); + + dqthread_running = false; + stop_dqthread = false; + dq_thread = 0; + callback = NULL; + + memory_type = V4L2_MEMORY_MMAP; + + dqThread_data = NULL; +} + +NvV4l2ElementPlane::~NvV4l2ElementPlane() +{ + pthread_mutex_destroy(&plane_lock); + pthread_cond_destroy(&plane_cond); +} + +NvBuffer * +NvV4l2ElementPlane::getNthBuffer(uint32_t n) +{ + if (n >= num_buffers) + { + PLANE_DEBUG_MSG("WARNING:Requested " << n << "th buffer out of " << + num_buffers << "buffers. Returning NULL"); + return NULL; + } + return buffers[n]; +} + +int +NvV4l2ElementPlane::dqBuffer(struct v4l2_buffer &v4l2_buf, NvBuffer ** buffer, + NvBuffer ** shared_buffer, uint32_t num_retries) +{ + int ret; + + v4l2_buf.type = buf_type; + v4l2_buf.memory = memory_type; + do + { + ret = v4l2_ioctl(fd, VIDIOC_DQBUF, &v4l2_buf); + + if (ret == 0) + { + pthread_mutex_lock(&plane_lock); + if (buffer) + *buffer = buffers[v4l2_buf.index]; + if (shared_buffer && memory_type == V4L2_MEMORY_DMABUF) + { + *shared_buffer = + (NvBuffer *) buffers[v4l2_buf.index]->shared_buffer; + } + for (uint32_t i = 0; i < buffers[v4l2_buf.index]->n_planes; i++) + { + buffers[v4l2_buf.index]->planes[i].bytesused = + v4l2_buf.m.planes[i].bytesused; + } + if (buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) + { + v4l2elem_profiler.finishProcessing(0, false); + } + + total_dequeued_buffers++; + num_queued_buffers--; + pthread_cond_broadcast(&plane_cond); + PLANE_DEBUG_MSG("DQed buffer " << v4l2_buf.index); + pthread_mutex_unlock(&plane_lock); + } + else if (errno == EAGAIN) + { + pthread_mutex_lock(&plane_lock); + if (v4l2_buf.flags & V4L2_BUF_FLAG_LAST) + { + pthread_mutex_unlock(&plane_lock); + break; + } + pthread_mutex_unlock(&plane_lock); + + if (num_retries-- == 0) + { + PLANE_WARN_MSG("Error while DQing buffer: Resource temporarily unavailable"); + break; + } + if (!blocking) + { + break; + } + } + else + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while DQing buffer"); + break; + } + } + while (ret && !is_in_error); + + return ret; +} + +int +NvV4l2ElementPlane::qBuffer(struct v4l2_buffer &v4l2_buf, NvBuffer * shared_buffer) +{ + int ret; + uint32_t i; + NvBuffer *buffer; + + pthread_mutex_lock(&plane_lock); + buffer = buffers[v4l2_buf.index]; + + v4l2_buf.type = buf_type; + v4l2_buf.memory = memory_type; + v4l2_buf.length = n_planes; + + switch (memory_type) + { + case V4L2_MEMORY_USERPTR: + buffer->shared_buffer = shared_buffer; + for (i = 0; i < buffer->n_planes; i++) + { + if (shared_buffer) + { + v4l2_buf.m.planes[i].m.userptr = + (unsigned long) shared_buffer->planes[i].data; + v4l2_buf.m.planes[i].bytesused = + shared_buffer->planes[i].bytesused; + } + else + { + v4l2_buf.m.planes[i].m.userptr = + (unsigned long) buffer->planes[i].data; + v4l2_buf.m.planes[i].bytesused = + buffer->planes[i].bytesused; + } + } + break; + case V4L2_MEMORY_MMAP: + for (i = 0; i < buffer->n_planes; i++) + { + v4l2_buf.m.planes[i].bytesused = buffer->planes[i].bytesused; + } + break; + case V4L2_MEMORY_DMABUF: + buffer->shared_buffer = shared_buffer; + if (shared_buffer) + { + for (i = 0; i < buffer->n_planes; i++) + { + v4l2_buf.m.planes[i].m.fd = shared_buffer->planes[i].fd; + v4l2_buf.m.planes[i].bytesused = + shared_buffer->planes[i].bytesused; + } + } + break; + default: + pthread_cond_broadcast(&plane_cond); + pthread_mutex_unlock(&plane_lock); + return -1; + } + + if(buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + { + v4l2elem_profiler.startProcessing(); + } + + ret = v4l2_ioctl(fd, VIDIOC_QBUF, &v4l2_buf); + if (ret) + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while Qing buffer"); + } + else + { + PLANE_DEBUG_MSG("Qed buffer " << v4l2_buf.index); + pthread_cond_broadcast(&plane_cond); + total_queued_buffers++; + num_queued_buffers++; + } + pthread_mutex_unlock(&plane_lock); + + return ret; +} + +int +NvV4l2ElementPlane::mapOutputBuffers(struct v4l2_buffer &v4l2_buf, int dmabuff_fd) +{ + int ret; + uint32_t i; + pthread_mutex_lock(&plane_lock); + unsigned char *data; + NvBufSurface *nvbuf_surf = 0; + + switch (memory_type) + { + case V4L2_MEMORY_DMABUF: + ret = NvBufSurfaceFromFd (dmabuff_fd, (void**)(&nvbuf_surf)); + if(ret < 0) + { + PLANE_SYS_ERROR_MSG("Error: NvBufSurfaceFromFd Failed\n"); + pthread_mutex_unlock(&plane_lock); + return ret; + } + for (i = 0; i < n_planes; i++) + { + buffers[v4l2_buf.index]->planes[i].fd = dmabuff_fd; + v4l2_buf.m.planes[i].m.fd = buffers[v4l2_buf.index]->planes[i].fd; + buffers[v4l2_buf.index]->planes[i].mem_offset = nvbuf_surf->surfaceList[0].planeParams.offset[i]; + + if (nvbuf_surf->memType != NVBUF_MEM_CUDA_DEVICE && nvbuf_surf->memType != NVBUF_MEM_CUDA_ARRAY) + { + ret = NvBufSurfaceMap(nvbuf_surf, 0, i, NVBUF_MAP_READ_WRITE); + if (ret < 0) + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while Mapping buffer"); + pthread_mutex_unlock(&plane_lock); + return ret; + } + data = (unsigned char *)nvbuf_surf->surfaceList[0].mappedAddr.addr[i]; + buffers[v4l2_buf.index]->planes[i].data = data; + } + } + break; + default: + pthread_mutex_unlock(&plane_lock); + return -1; + } + if(ret == 0) + { + PLANE_DEBUG_MSG("Mapped Nvbuffer to buffers " << v4l2_buf.index); + } + pthread_mutex_unlock(&plane_lock); + + return ret; +} + +int +NvV4l2ElementPlane::unmapOutputBuffers(int index, int dmabuff_fd) +{ + int ret = 0; + uint32_t i; + NvBufSurface *nvbuf_surf = 0; + pthread_mutex_lock(&plane_lock); + + switch (memory_type) + { + case V4L2_MEMORY_DMABUF: + for (i = 0; i < n_planes; i++) + { + ret = NvBufSurfaceFromFd (dmabuff_fd, (void**)(&nvbuf_surf)); + if (ret < 0) + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while NvBufSurfaceFromFd"); + pthread_mutex_unlock(&plane_lock); + return ret; + } + + if (nvbuf_surf->memType != NVBUF_MEM_CUDA_DEVICE && nvbuf_surf->memType != NVBUF_MEM_CUDA_ARRAY) + { + ret = NvBufSurfaceUnMap(nvbuf_surf, 0, i); + if (ret < 0) + { + is_in_error = 1; + PLANE_SYS_ERROR_MSG("Error while Unmapping buffer"); + pthread_mutex_unlock(&plane_lock); + return ret; + } + } + } + break; + default: + pthread_mutex_unlock(&plane_lock); + return -1; + } + if(ret == 0) + { + PLANE_DEBUG_MSG("Unmapped Nvbuffer to buffers " << index); + } + pthread_mutex_unlock(&plane_lock); + + return ret; +} + +int +NvV4l2ElementPlane::getFormat(struct v4l2_format &format) +{ + format.type = buf_type; + CHECK_V4L2_RETURN(v4l2_ioctl(fd, VIDIOC_G_FMT, &format), + "Getting format"); +} + +int +NvV4l2ElementPlane::setFormat(struct v4l2_format &format) +{ + int ret; + int j; + + format.type = buf_type; + ret = v4l2_ioctl(fd, VIDIOC_S_FMT, &format); + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in VIDIOC_S_FMT"); + is_in_error = 1; + } + else + { + PLANE_DEBUG_MSG("VIDIOC_S_FMT at capture plane successful"); + n_planes = format.fmt.pix_mp.num_planes; + for (j = 0; j < n_planes; j++) + { + planefmts[j].stride = format.fmt.pix_mp.plane_fmt[j].bytesperline; + planefmts[j].sizeimage = format.fmt.pix_mp.plane_fmt[j].sizeimage; + } + } + + return ret; +} + +int +NvV4l2ElementPlane::getCrop(struct v4l2_crop & crop) +{ + crop.type = buf_type; + + CHECK_V4L2_RETURN(v4l2_ioctl(fd, VIDIOC_G_CROP, &crop), + "Getting crop params"); +} + +int +NvV4l2ElementPlane::setSelection(uint32_t target, uint32_t flags, struct v4l2_rect & rect) +{ + struct v4l2_selection select; + + switch (buf_type) + { + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + select.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + break; + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + select.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + break; + default: + PLANE_ERROR_MSG("Unsupported v4l2_buf_type " << buf_type); + return -1; + } + + select.target = target; + select.flags = flags; + select.r = rect; + + CHECK_V4L2_RETURN(v4l2_ioctl(fd, VIDIOC_S_SELECTION, &select), + "Setting selection"); +} + +void +NvV4l2ElementPlane::setBufferPlaneFormat(int n_planes, + NvBuffer::NvBufferPlaneFormat * planefmts) +{ + int i; + + this->n_planes = n_planes; + for (i = 0; i < n_planes; i++) + { + this->planefmts[i] = planefmts[i]; + } +} + +void +NvV4l2ElementPlane::deinitPlane() +{ + setStreamStatus(false); + waitForDQThread(-1); + for (uint32_t i = 0; i < num_buffers; i++) + { + switch (memory_type) + { + case V4L2_MEMORY_USERPTR: + buffers[i]->deallocateMemory(); + break; + case V4L2_MEMORY_MMAP: + buffers[i]->unmap(); + break; + case V4L2_MEMORY_DMABUF: + break; + default: + return; + } + } + reqbufs(memory_type, 0); + PLANE_DEBUG_MSG("deinit successful"); +} + +int +NvV4l2ElementPlane::reqbufs(enum v4l2_memory mem_type, uint32_t num) +{ + struct v4l2_requestbuffers reqbufs; + int ret; + + memset(&reqbufs, 0, sizeof(struct v4l2_requestbuffers)); + reqbufs.count = num; + reqbufs.type = buf_type; + switch (mem_type) + { + case V4L2_MEMORY_USERPTR: + for (uint32_t i = 0; i < n_planes; i++) + { + planefmts[i].stride = + planefmts[i].width * planefmts[i].bytesperpixel; + if(!planefmts[i].sizeimage) + { + planefmts[i].sizeimage = + planefmts[i].width * planefmts[i].height; + } + } + break; + case V4L2_MEMORY_MMAP: + case V4L2_MEMORY_DMABUF: + break; + default: + PLANE_ERROR_MSG("Error in VIDIOC_REQBUFS:Unknown memory type " << + mem_type); + return -1; + } + memory_type = mem_type; + + reqbufs.memory = mem_type; + ret = v4l2_ioctl(fd, VIDIOC_REQBUFS, &reqbufs); + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in VIDIOC_REQBUFS at output plane"); + is_in_error = 1; + } + else + { + if (reqbufs.count) + { + buffers = new NvBuffer *[reqbufs.count]; + for (uint32_t i = 0; i < reqbufs.count; i++) + { + buffers[i] = + new NvBuffer(buf_type, mem_type, n_planes, planefmts, i); + } + } + else + { + for (uint32_t i = 0; i < num_buffers; i++) + { + delete buffers[i]; + } + delete[] buffers; + buffers = NULL; + } + num_buffers = reqbufs.count; + PLANE_DEBUG_MSG("Reqbuf returned " << reqbufs.count << " buffers"); + } + + return ret; +} + +int +NvV4l2ElementPlane::setStreamStatus(bool status) +{ + int ret; + + pthread_mutex_lock(&plane_lock); + if (status == streamon) + { + PLANE_DEBUG_MSG("Already in " << ((status) ? "STREAMON" : "STREAMOFF")); + pthread_mutex_unlock(&plane_lock); + return 0; + } + + if (status) + { + ret = v4l2_ioctl(fd, VIDIOC_STREAMON, &buf_type); + } + else + { + ret = v4l2_ioctl(fd, VIDIOC_STREAMOFF, &buf_type); + } + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in " << + ((status) ? "STREAMON" : "STREAMOFF")); + is_in_error = 1; + } + else + { + PLANE_DEBUG_MSG(((status) ? "STREAMON" : "STREAMOFF") << " successful"); + streamon = status; + if (!streamon) + { + num_queued_buffers = 0; + pthread_cond_broadcast(&plane_cond); + } + + if (buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) + { + if (status == false) + { + v4l2elem_profiler.disableProfiling(); + } + } + + } + + pthread_mutex_unlock(&plane_lock); + return ret; +} + +bool +NvV4l2ElementPlane::getStreamStatus() +{ + return streamon; +} + +int +NvV4l2ElementPlane::setStreamParms(struct v4l2_streamparm &parm) +{ + int ret; + + parm.type = buf_type; + ret = v4l2_ioctl(fd, VIDIOC_S_PARM, &parm); + + if(ret == 0) + { + PLANE_DEBUG_MSG("Successfully set stream parameters"); + } + else + { + PLANE_SYS_ERROR_MSG("Error while setting stream parameters"); + } + return ret; +} + +int +NvV4l2ElementPlane::queryBuffer(uint32_t i) +{ + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + int ret; + uint32_t j; + + memset(&v4l2_buf, 0, sizeof(struct v4l2_buffer)); + memset(planes, 0, sizeof(planes)); + v4l2_buf.index = i; + v4l2_buf.type = buf_type; + v4l2_buf.memory = memory_type; + v4l2_buf.m.planes = planes; + v4l2_buf.length = n_planes; + + ret = v4l2_ioctl(fd, VIDIOC_QUERYBUF, &v4l2_buf); + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in QueryBuf for " << i << "th buffer"); + is_in_error = 1; + } + else + { + PLANE_DEBUG_MSG("QueryBuf for " << i << "th buffer successful"); + + for (j = 0; j < v4l2_buf.length; j++) + { + buffers[i]->planes[j].length = v4l2_buf.m.planes[j].length; + buffers[i]->planes[j].mem_offset = + v4l2_buf.m.planes[j].m.mem_offset; + } + } + + return ret; +} + +int +NvV4l2ElementPlane::exportBuffer(uint32_t i) +{ + struct v4l2_exportbuffer expbuf; + int ret; + int j; + + memset(&expbuf, 0, sizeof(expbuf)); + expbuf.type = buf_type; + expbuf.index = i; + + for (j = 0; j < n_planes; j++) + { + expbuf.plane = j; + ret = v4l2_ioctl(fd, VIDIOC_EXPBUF, &expbuf); + if (ret) + { + PLANE_SYS_ERROR_MSG("Error in ExportBuf for Buffer " << i << + ", Plane " << j); + is_in_error = 1; + return -1; + } + else + { + PLANE_DEBUG_MSG("ExportBuf successful for Buffer " << i << + ", Plane " << j << ", fd = " << expbuf.fd); + buffers[i]->planes[j].fd = expbuf.fd; + } + } + return 0; +} + +int +NvV4l2ElementPlane::setupPlane(enum v4l2_memory mem_type, uint32_t num_buffers, + bool map, bool allocate) +{ + uint32_t i; + + if (reqbufs(mem_type, num_buffers)) + { + goto error; + } + + for (i = 0; i < this->num_buffers; i++) + { + switch (mem_type) + { + case V4L2_MEMORY_USERPTR: + if (allocate) + { + if (buffers[i]->allocateMemory()) + { + goto error; + } + } + break; + case V4L2_MEMORY_MMAP: + if (queryBuffer(i)) + { + goto error; + } + if (exportBuffer(i)) + { + goto error; + } + if (map) + { + if (buffers[i]->map()) + { + goto error; + } + } + break; + default: + continue; + } + } + return 0; + +error: + PLANE_ERROR_MSG("Error during setup"); + is_in_error = 1; + deinitPlane(); + return -1; +} + +int +NvV4l2ElementPlane::waitAllBuffersQueued(uint32_t max_wait_ms) +{ + struct timespec timeToWait; + struct timeval now; + int return_val = 0; + int ret; + + gettimeofday(&now, NULL); + + timeToWait.tv_nsec = (now.tv_usec + (max_wait_ms % 1000) * 1000L) * 1000L; + timeToWait.tv_sec = now.tv_sec + max_wait_ms / 1000 + + timeToWait.tv_nsec / 1000000000L; + timeToWait.tv_nsec = timeToWait.tv_nsec % 1000000000L; + + pthread_mutex_lock(&plane_lock); + while (num_queued_buffers < num_buffers) + { + ret = pthread_cond_timedwait(&plane_cond, &plane_lock, &timeToWait); + if (ret == ETIMEDOUT) + { + return_val = -1; + break; + } + } + pthread_mutex_unlock(&plane_lock); + + CHECK_V4L2_RETURN(return_val, "Waiting for all buffers to get queued"); +} + +int +NvV4l2ElementPlane::waitAllBuffersDequeued(uint32_t max_wait_ms) +{ + struct timespec timeToWait; + struct timeval now; + int return_val = 0; + int ret; + + gettimeofday(&now, NULL); + + timeToWait.tv_nsec = (now.tv_usec + (max_wait_ms % 1000) * 1000L) * 1000L; + timeToWait.tv_sec = now.tv_sec + max_wait_ms / 1000 + + timeToWait.tv_nsec / 1000000000L; + timeToWait.tv_nsec = timeToWait.tv_nsec % 1000000000L; + + pthread_mutex_lock(&plane_lock); + while (num_queued_buffers) + { + ret = pthread_cond_timedwait(&plane_cond, &plane_lock, &timeToWait); + if (ret == ETIMEDOUT) + { + return_val = -1; + break; + } + } + pthread_mutex_unlock(&plane_lock); + + CHECK_V4L2_RETURN(return_val, "Waiting for all buffers to get dequeued"); +} + +bool NvV4l2ElementPlane::setDQThreadCallback(dqThreadCallback callback) +{ + if (dqthread_running) + return false; + this->callback = callback; + return true; +} + +void * +NvV4l2ElementPlane::dqThread(void *data) +{ + NvV4l2ElementPlane *plane = (NvV4l2ElementPlane *) data; + const char *comp_name = plane->comp_name; + const char *plane_name = plane->plane_name; + + PLANE_DEBUG_MSG("Starting DQthread"); + prctl (PR_SET_NAME, plane_name, 0, 0, 0); + plane->stop_dqthread = false; + while (!plane->stop_dqthread) + { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + NvBuffer *buffer; + NvBuffer *shared_buffer; + bool ret = 0; + + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, sizeof(planes)); + v4l2_buf.m.planes = planes; + v4l2_buf.length = plane->n_planes; + + if (plane->dqBuffer(v4l2_buf, &buffer, &shared_buffer, -1) < 0) + { + if (errno != EAGAIN) + { + plane->is_in_error = 1; + } + if (errno != EAGAIN || plane->streamon) + { + ret = plane->callback(NULL, NULL, NULL, plane->dqThread_data); + } + if (!plane->streamon) + { + break; + } + } + else + { + ret = plane->callback(&v4l2_buf, buffer, shared_buffer, + plane->dqThread_data); + } + if (!ret) + { + break; + } + } + plane->stop_dqthread = false; + + pthread_mutex_lock(&plane->plane_lock); + plane->dqthread_running = false; + pthread_cond_broadcast(&plane->plane_cond); + pthread_mutex_unlock(&plane->plane_lock); + PLANE_DEBUG_MSG("Exiting DQthread"); + return NULL; +} + +int +NvV4l2ElementPlane::startDQThread(void *data) +{ + pthread_mutex_lock(&plane_lock); + if (dqthread_running) + { + PLANE_DEBUG_MSG("DQ Thread already started"); + pthread_mutex_unlock(&plane_lock); + return 0; + } + dqThread_data = data; + pthread_create(&dq_thread, NULL, dqThread, this); + dqthread_running = true; + pthread_mutex_unlock(&plane_lock); + PLANE_DEBUG_MSG("Started DQ Thread"); + return 0; +} + +int +NvV4l2ElementPlane::stopDQThread() +{ + if (blocking) + { + PLANE_WARN_MSG("Should not be called in blocking mode"); + return 0; + } + stop_dqthread = true; + pthread_join(dq_thread, NULL); + dq_thread = 0; + PLANE_DEBUG_MSG("Stopped DQ Thread"); + return 0; +} + +int +NvV4l2ElementPlane::waitForDQThread(uint32_t max_wait_ms) +{ + struct timespec timeToWait; + struct timeval now; + int return_val = 0; + int ret = 0; + + gettimeofday(&now, NULL); + + timeToWait.tv_nsec = (now.tv_usec + (max_wait_ms % 1000) * 1000L) * 1000L; + timeToWait.tv_sec = now.tv_sec + max_wait_ms / 1000 + + timeToWait.tv_nsec / 1000000000L; + timeToWait.tv_nsec = timeToWait.tv_nsec % 1000000000L; + + pthread_mutex_lock(&plane_lock); + while (dqthread_running) + { + ret = pthread_cond_timedwait(&plane_cond, &plane_lock, &timeToWait); + if (ret == ETIMEDOUT) + { + return_val = -1; + break; + } + } + pthread_mutex_unlock(&plane_lock); + + if (ret == 0) + { + pthread_join(dq_thread, NULL); + dq_thread = 0; + PLANE_DEBUG_MSG("Stopped DQ Thread"); + } + else + { + PLANE_ERROR_MSG("Timed out waiting for dqthread"); + } + return return_val; +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvVideoEncoder.cpp b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvVideoEncoder.cpp new file mode 100644 index 0000000000..ddd3d2ea0f --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/classes/NvVideoEncoder.cpp @@ -0,0 +1,1701 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NvVideoEncoder.h" +#include "NvLogging.h" + +#include +#include +#include + +#include +#include +#include "nvbufsurface.h" + +#define ENCODER_DEV "/dev/nvhost-msenc" +#define ENCODER_DEV_ALT "/dev/v4l2-nvenc" +#define ENCODER_GPU_DEV "/dev/nvidia0" + +#define ENCODER_COMP_NAME "NVENC" + +#define CHECK_V4L2_RETURN(ret, str) \ + if (ret < 0) { \ + COMP_SYS_ERROR_MSG(str << ": failed"); \ + return -1; \ + } else { \ + COMP_DEBUG_MSG(str << ": success"); \ + return 0; \ + } + +#define RETURN_ERROR_IF_FORMATS_SET() \ + if (output_plane_pixfmt != 0 || capture_plane_pixfmt != 0) { \ + COMP_ERROR_MSG("Should be called before setting plane formats") \ + return -1; \ + } + +#define RETURN_ERROR_IF_OUTPUT_PLANE_FORMAT_NOT_SET() \ +if (output_plane_pixfmt != 0) { \ + COMP_ERROR_MSG("Should be called before setting output plane format") \ + return -1; \ +} + +#define RETURN_ERROR_IF_BUFFERS_REQUESTED() \ + if (output_plane.getNumBuffers() != 0 && capture_plane.getNumBuffers() != 0) { \ + COMP_ERROR_MSG("Should be called before requesting buffers on either plane") \ + return -1; \ + } + +#define RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED() \ + if (output_plane.getNumBuffers() == 0 || capture_plane.getNumBuffers() == 0) { \ + COMP_ERROR_MSG("Should be called before requesting buffers on either plane") \ + return -1; \ + } + +#define RETURN_ERROR_IF_FORMATS_NOT_SET() \ + if (output_plane_pixfmt == 0 || capture_plane_pixfmt == 0) { \ + COMP_ERROR_MSG("Should be called after setting plane formats") \ + return -1; \ + } + +using namespace std; + +NvVideoEncoder::NvVideoEncoder(const char *name, const char *dev_node, int flags) + : NvV4l2Element(name, dev_node, flags, valid_fields) +{ + gpu_enabled = false; +} + +NvVideoEncoder * +NvVideoEncoder::createVideoEncoder(const char *name, int flags) +{ + NvVideoEncoder *enc; + NvBufSurfaceDeviceInfo deviceInfo; + memset(&deviceInfo, 0, sizeof(deviceInfo)); + NvBufSurfaceGetDeviceInfo(&deviceInfo); + if (deviceInfo.driverType == NVBUF_DRIVER_TYPE_NVGPU && deviceInfo.isIntegratedGpu) + { + if (access (ENCODER_DEV, F_OK) == 0) + { + enc = new NvVideoEncoder(name, ENCODER_DEV, flags); + } + else if (access(ENCODER_DEV_ALT, F_OK) == 0) + { + enc = new NvVideoEncoder(name, ENCODER_DEV_ALT, flags); + } + else + { + return NULL; + } + } + else + { + if (access(ENCODER_GPU_DEV, F_OK) == 0) + { + enc = new NvVideoEncoder(name, ENCODER_GPU_DEV, flags); + enc->gpu_enabled = true; + } + else + return NULL; + } + + if (enc->isInError()) + { + delete enc; + return NULL; + } + return enc; +} + +NvVideoEncoder::~NvVideoEncoder() +{ +} + +bool +NvVideoEncoder::isGPUEnabled() +{ + return this->gpu_enabled; +} + +int NvVideoEncoder::setOutputPlaneFormat(uint32_t pixfmt, uint32_t width, + uint32_t height, v4l2_colorspace cs) +{ + struct v4l2_format format; + uint32_t num_bufferplanes; + NvBuffer::NvBufferPlaneFormat planefmts[MAX_PLANES] = {{0}}; + + if (pixfmt != V4L2_PIX_FMT_YUV420M && + pixfmt != V4L2_PIX_FMT_P010M && pixfmt != V4L2_PIX_FMT_NV12M && + pixfmt != V4L2_PIX_FMT_NV24M && pixfmt != V4L2_PIX_FMT_NV24_10LE && + pixfmt != V4L2_PIX_FMT_YUV444M && pixfmt != V4L2_PIX_FMT_YUV444_10LE) + { + COMP_ERROR_MSG("Only YUV420M, NV24, NV24_10LE, YUV444, P010M and NV12M are supported"); + return -1; + } + + output_plane_pixfmt = pixfmt; + NvBuffer::fill_buffer_plane_format(&num_bufferplanes, planefmts, width, + height, pixfmt); + output_plane.setBufferPlaneFormat(num_bufferplanes, planefmts); + + memset(&format, 0, sizeof(struct v4l2_format)); + format.type = output_plane.getBufType(); + format.fmt.pix_mp.width = width; + format.fmt.pix_mp.height = height; + format.fmt.pix_mp.pixelformat = pixfmt; + format.fmt.pix_mp.num_planes = num_bufferplanes; + format.fmt.pix_mp.colorspace = cs; + + return output_plane.setFormat(format); +} + +int NvVideoEncoder::setOutputPlaneFormat(uint32_t pixfmt, uint32_t width, + uint32_t height, v4l2_colorspace cs, + bool extended) +{ + struct v4l2_format format; + uint32_t num_bufferplanes; + NvBuffer::NvBufferPlaneFormat planefmts[MAX_PLANES]; + + if (pixfmt != V4L2_PIX_FMT_YUV420M && + pixfmt != V4L2_PIX_FMT_P010M && pixfmt != V4L2_PIX_FMT_NV12M && + pixfmt != V4L2_PIX_FMT_NV24M && pixfmt != V4L2_PIX_FMT_NV24_10LE && + pixfmt != V4L2_PIX_FMT_YUV444M && pixfmt != V4L2_PIX_FMT_YUV444_10LE) + { + COMP_ERROR_MSG("Only YUV420M, NV24, NV24_10LE, YUV444, P010M and NV12M are supported"); + return -1; + } + + output_plane_pixfmt = pixfmt; + NvBuffer::fill_buffer_plane_format(&num_bufferplanes, planefmts, width, + height, pixfmt); + output_plane.setBufferPlaneFormat(num_bufferplanes, planefmts); + + memset(&format, 0, sizeof(struct v4l2_format)); + format.type = output_plane.getBufType(); + format.fmt.pix_mp.width = width; + format.fmt.pix_mp.height = height; + format.fmt.pix_mp.pixelformat = pixfmt; + format.fmt.pix_mp.num_planes = num_bufferplanes; + + /* Extended color format maps to full color range; otherwise signal the + * limited (studio) range explicitly. */ + format.fmt.pix_mp.quantization = + extended ? V4L2_QUANTIZATION_FULL_RANGE : V4L2_QUANTIZATION_LIM_RANGE; + + format.fmt.pix_mp.colorspace = cs; + + /* Program the matrix coefficients (ycbcr_enc) and transfer function + * (xfer_func) to match the colorspace. V4L2 has no dedicated constant for + * the BT.2020 10-/12-bit transfer function, and BT.601 shares the BT.709 + * OETF, so for those colorspaces the transfer characteristic is left as + * DEFAULT and derived from the colorspace by the encoder backend (this + * lets BT.601 be signalled as SMPTE 170M rather than BT.709). + * https://www.kernel.org/doc/html/v6.8/userspace-api/media/v4l/colorspaces-defs.html */ + switch (format.fmt.pix_mp.colorspace) + { + case V4L2_COLORSPACE_REC709: + format.fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_709; + format.fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_709; + break; + case V4L2_COLORSPACE_BT2020: + format.fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_BT2020; + format.fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT; + break; + case V4L2_COLORSPACE_SMPTE170M: + format.fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_601; + format.fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT; + break; + default: + format.fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + format.fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT; + break; + } + + return output_plane.setFormat(format); +} + +int +NvVideoEncoder::setCapturePlaneFormat(uint32_t pixfmt, uint32_t width, + uint32_t height, uint32_t sizeimage) +{ + struct v4l2_format format; + + memset(&format, 0, sizeof(struct v4l2_format)); + format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + switch (pixfmt) + { + case V4L2_PIX_FMT_H264: + case V4L2_PIX_FMT_H265: + case V4L2_PIX_FMT_VP8: + case V4L2_PIX_FMT_VP9: + case V4L2_PIX_FMT_AV1: + capture_plane_pixfmt = pixfmt; + break; + default: + ERROR_MSG("Unknown supported pixel format for encoder " << pixfmt); + return -1; + } + + format.fmt.pix_mp.pixelformat = pixfmt; + format.fmt.pix_mp.width = width; + format.fmt.pix_mp.height = height; + format.fmt.pix_mp.num_planes = 1; + format.fmt.pix_mp.plane_fmt[0].sizeimage = sizeimage; + + return capture_plane.setFormat(format); +} + +int +NvVideoEncoder::setFrameRate(uint32_t framerate_num, uint32_t framerate_den) +{ + struct v4l2_streamparm parms; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&parms, 0, sizeof(parms)); + parms.parm.output.timeperframe.numerator = framerate_den; + parms.parm.output.timeperframe.denominator = framerate_num; + parms.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + + CHECK_V4L2_RETURN(output_plane.setStreamParms(parms), + "Setting framerate to " << framerate_num << "/" << framerate_den); +} + +int +NvVideoEncoder::setBitrate(uint32_t bitrate) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + // Local change: the plane pixel-format debug prints are removed (see ../README.md). + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_BITRATE; + control.value = bitrate; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder bitrate to " << bitrate); +} + +int +NvVideoEncoder::setEncoderCommand(int cmd, int flags) +{ + int ret=0; + struct v4l2_encoder_cmd v4l2_enc_cmd; + v4l2_enc_cmd.cmd = cmd; + v4l2_enc_cmd.flags = flags; + + ret = v4l2_ioctl(fd, VIDIOC_ENCODER_CMD, v4l2_enc_cmd); + if (ret < 0) + printf(" Error in encoder command \n"); + + return ret; +} + +int +NvVideoEncoder::setPeakBitrate(uint32_t peak_bitrate) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + // Local change: the plane pixel-format debug prints are removed (see ../README.md). + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_BITRATE_PEAK; + control.value = peak_bitrate; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder peak bitrate to " << peak_bitrate); +} + +int +NvVideoEncoder::setProfile(uint32_t profile) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + switch (capture_plane_pixfmt) + { + case V4L2_PIX_FMT_H264: + control.id = V4L2_CID_MPEG_VIDEO_H264_PROFILE; + break; + case V4L2_PIX_FMT_H265: + control.id = V4L2_CID_MPEG_VIDEO_H265_PROFILE; + break; + default: + COMP_ERROR_MSG("Unsupported encoder type"); + return -1; + } + control.value = profile; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder profile to " << profile); +} + +int +NvVideoEncoder::setLevel(uint32_t level) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + switch (capture_plane_pixfmt) + { + case V4L2_PIX_FMT_H264: + control.id = V4L2_CID_MPEG_VIDEO_H264_LEVEL; + control.value = (enum v4l2_mpeg_video_h264_level)level; + break; + case V4L2_PIX_FMT_H265: + control.id = V4L2_CID_MPEG_VIDEOENC_H265_LEVEL; + control.value = (enum v4l2_mpeg_video_h265_level)level; + break; + default: + COMP_WARN_MSG("Currently only supported for H.264 and H.265"); + return 0; + } + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder level to " << level); +} + +int +NvVideoEncoder::setConstantQp(bool enabled_rc) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE; + control.value = enabled_rc; // if false: disable rate control + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder constant qp to " << enabled_rc); +} + +int +NvVideoEncoder::setRateControlMode(enum v4l2_mpeg_video_bitrate_mode mode) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_BITRATE_MODE; + control.value = mode; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder rate control mode to " << mode); +} + +int +NvVideoEncoder::setMaxPerfMode(int flag) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_MAX_PERFORMANCE; + control.value = flag; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling Maximum Performance "); +} + +int +NvVideoEncoder::setIFrameInterval(uint32_t interval) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_GOP_SIZE; + control.value = interval; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder I-frame interval to " << interval); +} + +int +NvVideoEncoder::setIDRInterval(uint32_t interval) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_IDR_INTERVAL; + control.value = interval; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder IDR interval to " << interval); +} + +int +NvVideoEncoder::forceIDR() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Forcing IDR"); +} + +int +NvVideoEncoder::setTemporalTradeoff(v4l2_enc_temporal_tradeoff_level_type level) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_TEMPORAL_TRADEOFF_LEVEL; + control.value = level; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder temporal tradeoff level to " << level); +} + +int +NvVideoEncoder::setSliceLength(v4l2_enc_slice_length_type type, uint32_t length) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_enc_slice_length_param param = {type, length}; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_SLICE_LENGTH_PARAM; + control.string = (char *) ¶m; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder packet type to " << type << ", length to " << + length); +} + +int +NvVideoEncoder::setHWPresetType(v4l2_enc_hw_preset_type type) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_HW_PRESET_TYPE_PARAM; + control.value = type; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder HW Preset type to " << type); +} + +int +NvVideoEncoder::setROIParams(uint32_t buffer_index, + v4l2_enc_frame_ROI_params ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ROI_PARAMS; + control.string = (char *) ¶ms; + params.config_store = buffer_index; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder ROI params for buffer " << buffer_index); +} + +int +NvVideoEncoder::setPPEInitParams(v4l2_enc_ppe_init_params ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_PPE_INIT_PARAMS; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder PPE initialization params"); +} + +int +NvVideoEncoder::SetInputMetaParams(uint32_t buffer_index, + v4l2_ctrl_videoenc_input_metadata ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INPUT_METADATA; + control.string = (char *) ¶ms; + params.config_store = buffer_index; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder input metadata Params"); +} + +int +NvVideoEncoder::enableROI(v4l2_enc_enable_roi_param ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_ROI_PARAM; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder ROI"); +} + +int +NvVideoEncoder::enableReconCRC(v4l2_enc_enable_reconcrc_param ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_RECONCRC_PARAM; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder ReconCRC"); +} + +int +NvVideoEncoder::enableExternalRPS(v4l2_enc_enable_ext_rps_ctr ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_EXTERNAL_RPS_CONTROL; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder External RPS"); +} + +int +NvVideoEncoder::enableExternalRC(v4l2_enc_enable_ext_rate_ctr ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_EXTERNAL_RATE_CONTROL; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder External RC"); +} + +int NvVideoEncoder::setH265DisableSAO(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_H265_DISABLE_SAO; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder SAO filter to " << enabled); +} + +int +NvVideoEncoder::enableAV1Tile(v4l2_enc_av1_tile_config ¶ms) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_TILE_CONFIGURATION; + control.string = (char *) ¶ms; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder AV1 tile"); +} + +int +NvVideoEncoder::setAV1SsimRdo(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_SSIMRDO; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder Ssim Rdo to " << enabled); +} + +int +NvVideoEncoder::setAV1DisableCDFUpdate(bool disabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_DISABLE_CDF_UPDATE; + control.value = disabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder CDF update to " << !disabled); +} + +int NvVideoEncoder::enableAV1TileGroups(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_TILE_GROUPS; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder tile groups present flag to " << enabled); +} + +int NvVideoEncoder::setAV1FrameIDPresentFlag(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_FRAMEID_NUMBERS; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder frameID present flag to " << enabled); +} + +int +NvVideoEncoder::setAV1ErrResilienceMode(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_AV1_ERR_RESILIENT_MODE; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder Error Resilient Mode to " << enabled); +} + +int +NvVideoEncoder::setVirtualBufferSize(uint32_t size) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_enc_virtual_buffer_size buffer_size = {size}; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_VIRTUALBUFFER_SIZE; + control.string = (char *) &buffer_size; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder virtual buffer size to " << size); +} + +int +NvVideoEncoder::setNumReferenceFrames(uint32_t num_frames) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_enc_num_ref_frames frames = { num_frames }; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_NUM_REFERENCE_FRAMES; + control.string = (char *) &frames; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder number of reference frames to " << num_frames); +} + +int +NvVideoEncoder::setSliceIntrarefresh(uint32_t interval) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_enc_slice_intrarefresh_param param = {interval}; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_SLICE_INTRAREFRESH_PARAM; + control.string = (char *) ¶m; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder slice intrarefresh interval to " << interval); +} + +int NvVideoEncoder::setCUDASliceIntrarefresh(uint32_t interval, uint32_t count) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + v4l2_ctrl_intra_refresh param = {0}; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + param.enableIntraRefresh = 1; + param.intraRefreshPeriod = interval; + param.intraRefreshCnt = count; + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_SLICE_INTRAREFRESH_PARAM; + control.string = (char *)¶m; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder slice intrarefresh interval to " << interval); +} + +int +NvVideoEncoder::setNumBFrames(uint32_t num) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_NUM_BFRAMES; + control.value = num; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder number of B frames to " << num); +} + +int +NvVideoEncoder::setInsertSpsPpsAtIdrEnabled(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INSERT_SPS_PPS_AT_IDR; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder SPSPPSatIDR to " << enabled); +} + +int +NvVideoEncoder::setCABAC(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder CABAC encoding to " << enabled); +} + +int +NvVideoEncoder::setSliceLevelEncode(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_SLICE_LEVEL_ENCODE; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encode slice level encoding to " << enabled); +} + +int +NvVideoEncoder::enableMotionVectorReporting() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_METADATA_MV; + control.value = 1; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Enabling encoder motion vector reporting"); +} + +int +NvVideoEncoder::getMetadata(uint32_t buffer_index, + v4l2_ctrl_videoenc_outputbuf_metadata &enc_metadata) +{ + v4l2_ctrl_video_metadata metadata; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + metadata.buffer_index = buffer_index; + metadata.VideoEncMetadata = &enc_metadata; + + control.id = V4L2_CID_MPEG_VIDEOENC_METADATA; + control.string = (char *)&metadata; + + CHECK_V4L2_RETURN(getExtControls(ctrls), + "Getting encoder output metadata for buffer " << buffer_index); +} + +int +NvVideoEncoder::getMotionVectors(uint32_t buffer_index, + v4l2_ctrl_videoenc_outputbuf_metadata_MV &enc_mv_metadata) +{ + v4l2_ctrl_video_metadata metadata; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_BUFFERS_NOT_REQUESTED(); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + metadata.buffer_index = buffer_index; + metadata.VideoEncMetadataMV = &enc_mv_metadata; + + control.id = V4L2_CID_MPEG_VIDEOENC_METADATA_MV; + control.string = (char *)&metadata; + + CHECK_V4L2_RETURN(getExtControls(ctrls), + "Getting encoder output MV metadata for buffer " << buffer_index); +} + +int +NvVideoEncoder::setQpRange(uint32_t MinQpI, uint32_t MaxQpI, uint32_t MinQpP, + uint32_t MaxQpP, uint32_t MinQpB, uint32_t MaxQpB) +{ + v4l2_ctrl_video_qp_range qprange; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + qprange.MinQpI = MinQpI; + qprange.MaxQpI = MaxQpI; + qprange.MinQpP = MinQpP; + qprange.MaxQpP = MaxQpP; + qprange.MinQpB = MinQpB; + qprange.MaxQpB = MaxQpB; + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_QP_RANGE; + control.string = (char *)&qprange; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder Qp range " << ctrls.count); +} + +int NvVideoEncoder::setCudaConstantQp(uint32_t qpI, uint32_t qpP, + uint32_t qpB) +{ + v4l2_ctrl_video_constqp qp_param; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + qp_param.constQpI = qpI; + qp_param.constQpP = qpP; + qp_param.constQpB = qpB; + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_CUDA_CONSTQP; + control.string = (char *)&qp_param; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder const Qp " << ctrls.count); +} + +int +NvVideoEncoder::setSampleAspectRatioWidth(uint32_t sar_width) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + switch (capture_plane_pixfmt) + { + case V4L2_PIX_FMT_H264: + control.id = V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_WIDTH; + break; + case V4L2_PIX_FMT_H265: + control.id = V4L2_CID_MPEG_VIDEOENC_H265_VUI_EXT_SAR_WIDTH; + break; + default: + COMP_ERROR_MSG("Unsupported encoder type"); + return -1; + } + control.value = sar_width; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder SAR width to " << sar_width); +} + +int +NvVideoEncoder::setSampleAspectRatioHeight(uint32_t sar_height) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + switch (capture_plane_pixfmt) + { + case V4L2_PIX_FMT_H264: + control.id = V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_HEIGHT; + break; + case V4L2_PIX_FMT_H265: + control.id = V4L2_CID_MPEG_VIDEOENC_H265_VUI_EXT_SAR_HEIGHT; + break; + default: + COMP_ERROR_MSG("Unsupported encoder type"); + return -1; + } + control.value = sar_height; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder SAR width to " << sar_height); +} + +int +NvVideoEncoder::setInsertVuiEnabled(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INSERT_VUI; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder InsertVUI to " << enabled); +} + +int +NvVideoEncoder::setExtendedColorFormat(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_EXTEDED_COLORFORMAT; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder extended colorformat to " << enabled); +} + +int +NvVideoEncoder::setInsertAudEnabled(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INSERT_AUD; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder InsertAUD to " << enabled); +} + +int +NvVideoEncoder::setAlliFramesEncode(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_ALLIFRAME_ENCODE; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder InsertAUD to " << enabled); +} + +int +NvVideoEncoder::DevicePoll(v4l2_ctrl_video_device_poll *devicepoll) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_DEVICE_POLL; + control.string = (char *)devicepoll; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Done calling video device poll "); +} + +int +NvVideoEncoder::SetPollInterrupt() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_SET_POLL_INTERRUPT; + control.value = 1; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder poll interrupt to 1 "); +} + +int +NvVideoEncoder::ClearPollInterrupt() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_SET_POLL_INTERRUPT; + control.value = 0; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder poll interrupt to 0 "); +} + +int +NvVideoEncoder::setPocType(uint32_t pocType) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_POC_TYPE; + control.value = pocType; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting Pic_order_cnt_type to " << pocType); +} + +int NvVideoEncoder::setInitQP(uint32_t IinitQP, + uint32_t PinitQP, uint32_t BinitQP) +{ + v4l2_ctrl_video_init_qp initqp; + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + initqp.IInitQP = IinitQP; + initqp.PInitQP = PinitQP; + initqp.BInitQP = BinitQP; + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_INIT_FRAME_QP; + control.string = (char *)&initqp; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting encoder Init QP " << ctrls.count); +} + +int +NvVideoEncoder::setFramesToEncode(uint32_t framesToEncode) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_FRAMES_TO_ENCODE; + control.value = framesToEncode; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting num_frames_to_encode to " << framesToEncode); +} + +int +NvVideoEncoder::setChromaFactorIDC(uint8_t value) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_H265_CHROMA_FACTOR_IDC; + control.value = value; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting chroma_format_idc to " << value); +} + +int +NvVideoEncoder::setLossless(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_LOSSLESS; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting lossless encoding to " << enabled); +} + +int NvVideoEncoder::setDisableAMP(bool enabled) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_H265_DISABLE_AMP; + control.value = enabled; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting disable AMP types to " << enabled); +} + +int NvVideoEncoder::enableTwoPassCBR() +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEOENC_TWO_PASS_CBR; + control.value = 1; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting two pass CBR to 1"); +} + +int NvVideoEncoder::setGPUId(uint32_t gpuId) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_FORMATS_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_CUDA_GPU_ID; + control.value = gpuId; + + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting GPU Id to " << gpuId); +} + +int NvVideoEncoder::setCudaMemType(uint32_t cumemType) +{ + struct v4l2_ext_control control; + struct v4l2_ext_controls ctrls; + + RETURN_ERROR_IF_OUTPUT_PLANE_FORMAT_NOT_SET(); + RETURN_ERROR_IF_BUFFERS_REQUESTED(); + + memset(&control, 0, sizeof(control)); + memset(&ctrls, 0, sizeof(ctrls)); + + ctrls.count = 1; + ctrls.controls = &control; + ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG; + + control.id = V4L2_CID_MPEG_VIDEO_CUDA_MEM_TYPE; + control.value = cumemType; + CHECK_V4L2_RETURN(setExtControls(ctrls), + "Setting cumemType to " << cumemType); +} diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvBuffer.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvBuffer.h new file mode 100644 index 0000000000..1c1be52cac --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvBuffer.h @@ -0,0 +1,296 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: Buffer API + * + * @b Description: This file declares the NvBuffer APIs. + */ + +#ifndef __NV_BUFFER_H__ +#define __NV_BUFFER_H__ + +#include +#include +#include + +#include "v4l2_nv_extensions.h" +#include "nvbufsurface.h" + +/** + * + * @defgroup l4t_mm_nvbuffer_group Buffer API + * + * The @c %NvBuffer API provides buffer functionality, including reference + * count functionality and convenience methods. + * @ingroup aa_framework_api_group + * @{ + */ + +/** + * Specifies the maximum number of planes a buffer can contain. + */ +#define MAX_PLANES 3 + +/** + * @brief Class representing a buffer. + * + * The NvBuffer class is modeled on the basis of the @c v4l2_buffer + * structure. The buffer has @c buf_type @c v4l2_buf_type, @c + * memory_type @c v4l2_memory, and an index. It contains an + * NvBufferPlane array similar to the array of @c v4l2_plane + * structures in @c v4l2_buffer.m.planes. It also contains a + * corresponding NvBufferPlaneFormat array that describes the + * format of each of the planes. + * + * Even though @c %NvBuffer closely resembles v4l2 structures, it can + * be easily used with other non-v4l2 components. @c %NvBuffer + * contains data pointers, buffer length, file descriptor (FD) of + * buffer planes, buffer format (height, width, stride, etc.), and + * other members that are required by such components. + * + * This class also provides buffer reference count functionality. This + * is useful when the same buffer is being used by multiple elements. + * + * In the case of a V4L2 MMAP, this class provides convenience methods + * for mapping or unmapping the contents of the buffer to or from + * memory, allocating or deallocating software memory depending on its + * format. + */ +class NvBuffer +{ +public: + /** + * Holds the buffer plane format. + */ + typedef struct + { + uint32_t width; /**< Holds the width of the plane in pixels. */ + uint32_t height; /**< Holds the height of the plane in pixels. */ + + uint32_t bytesperpixel; /**< Holds the bytes used to represent one + pixel in the plane. */ + uint32_t stride; /**< Holds the stride of the plane in bytes. */ + uint32_t sizeimage; /**< Holds the size of the plane in bytes. */ + } NvBufferPlaneFormat; + + /** + * Holds the buffer plane parameters. + */ + typedef struct + { + NvBufferPlaneFormat fmt; /**< Holds the format of the plane. */ + + unsigned char *data; /**< Holds a pointer to the plane memory. */ + uint32_t bytesused; /**< Holds the number of valid bytes in the plane. */ + + int fd; /**< Holds the file descriptor (FD) of the plane of the + exported buffer, in the case of V4L2 MMAP buffers. */ + uint32_t mem_offset; /**< Holds the offset of the first valid byte + from the data pointer. */ + uint32_t length; /**< Holds the size of the buffer in bytes. */ + } NvBufferPlane; + + /** + * Creates a new NvBuffer object. + * + * This convenience method for V4L2 elements creates a new buffer + * with the planes array memset to zero and the refcount + * initialized to zero. + * + + * @param[in] buf_type Type of buffer, enumerated as @c + * v4l2_buf_type. + * @param[in] memory_type @c %NvBuffer memory, enumerated as an + * @c v4l2_memory enum. + * @param[in] n_planes Number of planes in the buffer. + * @param[in] fmt Specifies a pointer to the array of buffer plane formats. + * Should contain at least @a n_planes elements. + * @param[in] index Index of the buffer in the plane. + */ + NvBuffer(enum v4l2_buf_type buf_type, enum v4l2_memory memory_type, + uint32_t n_planes, NvBufferPlaneFormat *fmt, uint32_t index); + + /** + * Creates a new NvBuffer for raw pixel formats. + * + * This convenience method for V4L2 elements is an @c %NvBuffer + * constructor for raw pixel formats only. It requires width, + * height, and pixel format to be specified. + * + * The planes array is memset to zero and the refcount is + * initialized to zero. + * + * @attention The memory must be allocated by the application + * by calling NvBuffer::allocateMemory. + * + * @param[in] pixfmt Pixel format of the buffer. + * @param[in] width Width of the buffer in pixels. + * @param[in] height Height of the buffer in pixels. + * @param[in] index Index/ID of the buffer. + */ + NvBuffer(uint32_t pixfmt, uint32_t width, uint32_t height, uint32_t index); + + /** + * Creates a new NvBuffer object for non-raw pixel formats. + * + * This convenience method for V4L2 elements is an @c %NvBuffer + * constructor for non raw pixel formats. It requires size of the + * buffer to be supplied. + * + * The planes array is memset to zero and refcount initialized to + * zero. + * + * @attention The memory needs to be allocated by the application + * by calling NvBuffer::allocateMemory. + * + * @param[in] size Size of the buffer in bytes. + * @param[in] index Index/ID of the buffer. + */ + NvBuffer(uint32_t size, uint32_t index); + + /** + * Destroys an NvBuffer object. + * + * This method cleans up class instances, unmapping any mapped + * planes. + */ + ~NvBuffer(); + + /** + * Maps the contents of the buffer to memory. + * + * This method maps the file descriptor (FD) of the planes to + * a data pointer of @c planes. (MMAP buffers only.) + * + * @return 0 on success, -1 otherwise. + */ + int map(); + /** + * Unmaps the contents of the buffer from memory. (MMAP buffers only.) + * + */ + void unmap(); + + /** + * Allocates software memory for the buffer. + * + * @warning This method works only for @c V4L2_MEMORY_USERPTR memory. + * + * This method allocates memory on the basis of the buffer format: + * @a height, @a width, @a bytesperpixel, and @a sizeimage. + * + * @return 0 for success, -1 otherwise. + */ + int allocateMemory(); + /** + * Deallocates buffer memory. + * + * @warning This method works only for @c V4L2_MEMORY_USERPTR memory and if + * the memory was previously allocated using NvBuffer::allocateMemory. + */ + void deallocateMemory(); + + /** + * Increases the reference count of the buffer. + * + * This method is thread safe. + * + * @return Reference count of the buffer after the operation. + */ + int ref(); + /** + * Decreases the reference count of the buffer. + * + * This thread-safe method decreases the buffer reference count if the + * buffer reference count is above 0. + * + * @return Reference count of the buffer after the operation. + */ + int unref(); + + const enum v4l2_buf_type buf_type; /**< Type of the buffer. */ + const enum v4l2_memory memory_type; /**< Type of memory associated + with the buffer. */ + + const uint32_t index; /**< Holds the buffer index. */ + + uint32_t n_planes; /**< Holds the number of planes in the buffer. */ + NvBufferPlane planes[MAX_PLANES]; /**< Holds the data pointer, plane file + descriptor (FD), plane format, etc. */ + + /** + * Fills the NvBuffer::NvBufferPlaneFormat array. + * + * This convenience method populates the + * @c %NvBuffer::NvBufferPlaneFormat array on the basis of @a width, + * @a height and pixel format (@a raw_pixfmt). It also returns the number of planes + * required for the pixel format in @a num_planes. + * + * + * @param[out] num_planes The number of planes. Must not be NULL. + * @param[in,out] planefmts Array of %NvBuffer::NvBufferPlaneFormat to + * fill. Must be at least \a num_planes in length. For best + * results, pass an array of length #MAX_PLANES. + * @param[in] width Width of the buffer in pixels. + * @param[in] height Height of the buffer in pixels. + * @param[in] raw_pixfmt Raw V4L2 pixel formats. + * @return 0 for success, -1 for an unsupported pixel format. + */ + static int fill_buffer_plane_format(uint32_t *num_planes, + NvBuffer::NvBufferPlaneFormat *planefmts, + uint32_t width, uint32_t height, uint32_t raw_pixfmt); +private: + uint32_t ref_count; /**< Holds the reference count of the buffer. */ + pthread_mutex_t ref_lock; /**< Mutex to synchronize increment/ + decrement operations of @c ref_count. */ + + bool mapped; /**< Indicates if the buffer is mapped to + memory. */ + bool allocated; /**< Indicates if the buffer is allocated + memory. */ + NvBuffer *shared_buffer; /**< If this is a DMABUF buffer, @c shared_buffer + points to the MMAP @c NvBuffer whose FD was + sent when this buffer was queued. */ + + /** + * Disallows copy constructor. + */ + NvBuffer(const NvBuffer& that); + /** + * Disallows assignment. + */ + void operator=(NvBuffer const&); + + friend class NvV4l2ElementPlane; +}; +/** @} */ +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElement.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElement.h new file mode 100644 index 0000000000..543f47b324 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElement.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: %NvElement Base Class + * + * @b This file declares the NvElement base class. + */ + +#ifndef __NV_ELEMENT_H__ +#define __NV_ELEMENT_H__ + +#include +#include +#include +#include + +#include "NvElementProfiler.h" + +/** + * + * @defgroup l4t_mm_nvelement_base_group NvElement Base Class + * @ingroup l4t_mm_nvelement_group + * + * This class is the class from which both V4L2 and non-V4L2 + * components are derived. + * + * @{ + */ +/** + * Every element has a unique name that can be used for identifying + * the element in debug logs. + * + * @c %NvElement also provides other common functionality, such as keeping + * track of errors. + */ +class NvElement +{ +public: + /** + * Indicates whether the element encountered an error during its operation. + * + * @return 0 if no error was encountered, a non-zero value if an + * error was encountered. + */ + virtual int isInError() + { + return is_in_error; + } + virtual ~NvElement() + { + } + + /** + * Gets profiling data for the element. + * + * @return A constant reference to the element's profiling data. + */ + void getProfilingData(NvElementProfiler::NvElementProfilerData &data); + + /** + * Prints profiling data for the element to an output stream. + * + * @param[in] out_stream Output stream of type std::ostream to print the + * data to. It takes the default value std::cout if not specified. + */ + void printProfilingStats(std::ostream &out_stream = std::cout); + + /** + * Enables profiling for the element. + */ + virtual void enableProfiling(); + + /** + * Checks whether profiling is enabled for the element. + * + * @return Boolean value indicating if profiling is enabled. + */ + bool isProfilingEnabled(); + +protected: + + /** + * Creates a new NvElement object with name @a name. + * + * If the @a name parameter is NULL, this method sets the internal + * error variable. + * + * @param[in] name If non-NULL, a pointer to the name of the + * element. + */ + NvElement(const char *name, NvElementProfiler::ProfilerField = NvElementProfiler::PROFILER_FIELD_NONE); + + int is_in_error; /**< Indicates if an error was encountered during + the operation of the element. */ + const char *comp_name; /**< Specifies the name of the component, + for debugging. */ + NvElementProfiler profiler; /**< Profiler for the element. */ + + /** + * Disallows copy constructor. + */ + NvElement(const NvElement& that); + /** + * Disallows assignment. + */ + void operator=(NvElement const&); + +}; +/** @} */ +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElementProfiler.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElementProfiler.h new file mode 100644 index 0000000000..945bb93765 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvElementProfiler.h @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NV_ELEMENT_PROFILER_H__ +#define __NV_ELEMENT_PROFILER_H__ + +#include +#include +#include +#include +#include + +/** + * @file + * NVIDIA Multimedia API: Element Profiler API + * + * @b Description: This file profiles the performance of individual elements. + */ + + +/** + * + * Helper class for profiling the performance of individual elements. + * + * NvElementProfiler currently measures processing latencies, average processing rate, and + * the number of units that arrived late at the element. Components should use this + * information internally. + * + * If you require latency measurements, + * you must call startProcessing() to indicate that a unit has been submitted + * for processing and finishProcessing() to indicate that a unit has finished processing. + * If you require only averaging processing rate or the number of units that + * arrived late need not call startProcessing(). + * + * You can get data from NvElementProfiler using getProfilerData(). This function + * fills the [NvElementProfilerData](@ref NvElementProfiler::NvElementProfilerData) + * structure. Components that do not support all + * the fields available in the structure must use the variable + * [valid_fields](@ref NvElementProfiler::NvElementProfilerData::valid_fields) of + * type [ProfilerField](@ref NvElementProfiler::ProfilerField), which is also + * included in the structure. + * + * @defgroup l4t_mm_nvelementprofiler_group Element Profiler API + * @ingroup aa_framework_api_group + * @{ + */ +class NvElementProfiler { +public: + /** + * @defgroup Defines @c valid_field values for the #NvElementProfilerData structure. + * @ingroup l4t_mm_nvelementprofiler_group + * @{ + */ + typedef int ProfilerField; + static const ProfilerField PROFILER_FIELD_NONE = 0; + static const ProfilerField PROFILER_FIELD_TOTAL_UNITS = 1; + static const ProfilerField PROFILER_FIELD_LATE_UNITS = 2; + static const ProfilerField PROFILER_FIELD_LATENCIES = 4; + static const ProfilerField PROFILER_FIELD_FPS = 8; + static const ProfilerField PROFILER_FIELD_ALL = (PROFILER_FIELD_FPS << 1) - 1; + /** @} */ + + /** + * Holds profiling data for the element. + * + * Some elements may not support all the fields in the structure. User must check + * the @a valid_fields flag to determine which fields are valid. + */ + typedef struct { + /** Valid Fields which are supported by the element. */ + ProfilerField valid_fields; + + /** Average latency of all processed units, in microseconds. */ + uint64_t average_latency_usec; + /** Minimum of latencies for each processed units, in microseconds. */ + uint64_t min_latency_usec; + /** Maximum of latencies for each processed units, in microseconds. */ + uint64_t max_latency_usec; + + /** Total units processed. */ + uint64_t total_processed_units; + /** Number of units which arrived late at the element. */ + uint64_t num_late_units; + + /** Average rate at which the units were processed. */ + float average_fps; + + /** Total profiling time. */ + struct timeval profiling_time; + } NvElementProfilerData; + + /** + * Gets the profiling data for the element. + * + * @param[out] data Reference to the NvElementProfilerData structure which should be filled. + */ + void getProfilerData(NvElementProfilerData &data); + + /** + * Prints the element's profiling data to an output stream. + * + * @param[in] out_stream Reference to a std::ostream. + */ + void printProfilerData(std::ostream &out_stream = std::cout); + + /** + * Informs the profiler that processing has started. + * + * Has no effect if profiler is disabled. + * + * @return ID of the unit, to be supplied with finishProcessing();. + */ + uint64_t startProcessing(); + + /** + * Informs the profiler that processing has finished. + * + * Has no effect if profiler is disabled. + * + * @param[in] id ID of the unit whose processing is finished, + * 0 if the first unit in the profiler's queue should be picked. + * @param[in] is_late Should be true if the frame arrived late at the element. + */ + void finishProcessing(uint64_t id, bool is_late); + + /** + * Enables the profiler. + * + * startProcessing() and finishProcessing() are ineffective until the profiler is enabled. + * + * @param[in] reset_data Reset the profiled data. + */ + void enableProfiling(bool reset_data); + + /** + * Disables the profiler. + */ + void disableProfiling(); +private: + /** + * Resets the profiler data. + */ + void reset(); + + pthread_mutex_t profiler_lock; /**< Mutex to synchronize multithreaded access to profiler data. */ + + bool enabled; /**< Flag indicating if profiler is enabled. */ + + const ProfilerField valid_fields; /**< Valid fields for the element. */ + + struct NvElementProfilerDataInternal : NvElementProfilerData { + /** Wall-clock time at which the first unit was processed. */ + struct timeval start_time; + + /** Wall-clock time at which the latest unit was processed. */ + struct timeval stop_time; + + /** Total accumulated time. + * When performance measurement is restarted @a start_time and @a stop_time + * are reset. This field is used to accumulate time before + * resetting. */ + struct timeval accumulated_time; + + /** Total accumulated latency for all units, in microseconds. */ + uint64_t total_latency; + } data_int; + + /** Queue used to maintain the timestamps of when the unit + * processing started. Required to calculate latency. */ + std::map unit_start_time_queue; + + uint64_t unit_id_counter; /**< Unique ID of the last unit. */ + + /** + * Constructor for NvElementProfiler. + * + * Initializes internal data structures. The profiler is disabled by default. + * @param fields + */ + NvElementProfiler(ProfilerField fields); + + /** + * Disallow copy constructor. + */ + NvElementProfiler(const NvElementProfiler& that); + /** + * Disallow assignment. + */ + void operator=(NvElementProfiler const&); + + ~NvElementProfiler(); + + friend class NvElement; +}; + +/** @} */ + +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvLogging.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvLogging.h new file mode 100644 index 0000000000..e4e9ca9469 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvLogging.h @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: Logging API + * + * @brief Description: This file defines macros for logging messages. + */ + +#ifndef __NV_LOGGING_H_ +#define __NV_LOGGING_H_ + +#include +#include + +/** + * + * @defgroup l4t_mm_nvlogging_group Logging API + * + * This file defines macros that provide message logging + * functionality. + * + * @ingroup aa_framework_api_group + * @{ + */ + + +/** + * Specifies the log level for Information messages. + */ +#define LOG_LEVEL_INFO 0 +/** + * Specifies the log level for Error messages. + */ +#define LOG_LEVEL_ERROR 1 +/** + * Specifies the log level for Warning messages. + */ +#define LOG_LEVEL_WARN 2 +/** + * Specifies the log level for Debug messages. + */ +#define LOG_LEVEL_DEBUG 3 + +/** + * Holds the current log level at runtime by assignment of one of the + * @c LOG_LEVEL_* values. + */ +extern int log_level; + +/** + * Specifies the default log level. + */ +#define DEFAULT_LOG_LEVEL LOG_LEVEL_ERROR + +/** + * @cond + */ +#define NV_STRINGIFY(s) #s +#define NV_XSTRINGIFY(s) NV_STRINGIFY(s) +#define __LINE_NUM_STR__ NV_XSTRINGIFY(__LINE__) + +extern const char *log_level_name[]; +/** + * @endcond + */ + +/** + * + * Prints log messages. + * + * Prints a log message only if the current log_level is greater + * than or equal to the level of the message. + * + * Messages are in the following form: + * [LEVEL] (FILE: LINE_NUM) Message + * + * @param[in] level The Log level of the message. + * @param[in] str1 The NULL-terminated char array to print. + */ +#define PRINT_MSG(level, str1) if(level <= log_level) { \ + std::ostringstream ostr; \ + ostr << "[" << log_level_name[level] << "] (" << \ + __FILE__ << ":" __LINE_NUM_STR__ ") " << \ + str1 << std::endl; \ + std::cerr << ostr.str(); \ + } + +/** + * Prints a log message of level LOG_LEVEL_INFO. + */ +#define INFO_MSG(str) PRINT_MSG(LOG_LEVEL_INFO, str) +/** + * Prints a component-specific log message of level LOG_LEVEL_INFO. + * This is used by the components internally and should not be used by + * the application. + * + * Messages are in the following form: + * [LEVEL] (FILE: LINE_NUM) + */ +#define COMP_INFO_MSG(str) INFO_MSG("<" << comp_name << "> " << str) +/** + * Prints a category-specific (Component type) system error log + * message of level LOG_LEVEL_INFO. This is used by the components + * internally and should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE: LINE_NUM) + */ +#define CAT_INFO_MSG(str) INFO_MSG("<" CAT_NAME "> " << str) + +/** + * Prints a log message of level LOG_LEVEL_ERROR. + */ +#define ERROR_MSG(str) PRINT_MSG(LOG_LEVEL_ERROR, str) +/** + * Prints a component-specific log message of level + * LOG_LEVEL_ERROR. This is used by the components internally + * and should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define COMP_ERROR_MSG(str) ERROR_MSG("<" << comp_name << "> " << str) +/** + * Prints a category-specific (Component type) log message of level + * LOG_LEVEL_ERROR. This is used by the components internally and + * should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define CAT_ERROR_MSG(str) ERROR_MSG("<" CAT_NAME "> " << str) + +/** + * Prints a system error log message of level LOG_LEVEL_ERROR with + * the string description of the errno value appended. + */ +#define SYS_ERROR_MSG(str) ERROR_MSG(str << ": " << strerror(errno)) +/** + * Prints a component-specific system error log message of level + * LOG_LEVEL_ERROR. This is used by the components internally and + * should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define COMP_SYS_ERROR_MSG(str) SYS_ERROR_MSG("<" << comp_name << "> " << str) +/** + * Prints a category-specific (Component type) system error log + * message of level LOG_LEVEL_ERROR. This is used by the components + * internally and should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define CAT_SYS_ERROR_MSG(str) SYS_ERROR_MSG("<" CAT_NAME "> " << str) + +/** + * Prints a log message of level LOG_LEVEL_WARN. + */ +#define WARN_MSG(str) PRINT_MSG(LOG_LEVEL_WARN, str) +/** + * Prints a component-specific log message of level LOG_LEVEL_WARN. + * This is used by the components internally and should not be used by + * the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define COMP_WARN_MSG(str) WARN_MSG("<" << comp_name << "> :" << str) +/** + * Print a category-specific (Component type) log message of level + * LOG_LEVEL_WARN. + * This is used by the components internally and should not be used by the + * application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define CAT_WARN_MSG(str) WARN_MSG("<" CAT_NAME "> " << str) + +/** + * Prints a log message of level LOG_LEVEL_DEBUG. + */ +#define DEBUG_MSG(str) PRINT_MSG(LOG_LEVEL_DEBUG, str) +/** + * Prints a component-specific log message of level LOG_LEVEL_DEBUG. + * This is used by the components internally and should not be used by the + * application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define COMP_DEBUG_MSG(str) DEBUG_MSG("<" << comp_name << "> :" << str) +/** + * Prints a category-specific (Component type) log message of level + * LOG_LEVEL_DEBUG. This is used by the components internally and + * should not be used by the application. + * + * Messages are in the following form: + * [LEVEL] (FILE:LINE_NUM) + */ +#define CAT_DEBUG_MSG(str) DEBUG_MSG("<" CAT_NAME "> " << str) + +#endif +/** @} */ diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2Element.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2Element.h new file mode 100644 index 0000000000..2f7a72d7d4 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2Element.h @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: V4L2 Helper Class + * + * @b Description: This file declares a helper class for V4L2-based components. + */ + +/** + * @defgroup l4t_mm_nvv4l2element_group V4L2 Element Class + * @ingroup l4t_mm_nvelement_group + * + * Helper class that provides common functionality for V4L2-based components, + * such as encoder and decoder. Objects instantiated from this class create + * new V4L2 elements, subscribes to V4L2 events, dequeues an event from + * an element, and sets/gets control values. + * @{ + */ +#ifndef __NV_V4L2_ELEMENT_H__ +#define __NV_V4L2_ELEMENT_H__ + +#include "NvElement.h" +#include "NvV4l2ElementPlane.h" + +#include "v4l2_nv_extensions.h" + +/** + * Defines a helper class for V4L2 based components. + * + * This derived class provides common functionality for V4L2 components. V4L2-based + * components such as encoder/decoder extend from this class. + * + * This class is modeled on V4L2 M2M devices. It includes the file descriptor (FD) of the device + * opened using %v4l2_open, two planes (NvV4l2ElementPlane), output plane, + * capture plane, and other helper methods, such as setting/getting controls, + * subscribing/dequeueing events, etc. + */ +class NvV4l2Element:public NvElement +{ +public: + virtual ~NvV4l2Element(); + + /** + * Subscribes to an V4L2 event. + * + * Calls \c VIDIOC_SUBSCRIBE_EVENT IOCTL internally. + * + * @param[in] type Type of the event. + * @param[in] id ID of the event source. + * @param[in] flags Event flags. + * @return 0 for success, -1 otherwise. + */ + int subscribeEvent(uint32_t type, uint32_t id, uint32_t flags); + /** + * Dequeues an event from the element. + * + * Calls \c VIDIOC_DQEVENT IOCTL internally. The caller can specify the maximum time + * to wait for dequeuing the event. The call blocks until an event is + * dequeued successfully or timeout is reached. + * + * @param[in,out] event A reference to the \c v4l2_event structure to fill. + * @param[in] max_wait_ms Specifies the max wait time for dequeuing an event, + * in milliseconds. + * @return 0 for success, -1 otherwise. + */ + int dqEvent(struct v4l2_event &event, uint32_t max_wait_ms); + + /** + * Sets the value of a control. + * + * Calls \c VIDIOC_S_CTRL IOCTL internally. + * + * @param[in] id ID of the control to be set. + * @param[in] value Value to be set on the control. + * @return 0 for success, -1 otherwise. + */ + int setControl(uint32_t id, int32_t value); + /** + * Gets the value of a control. + * + * Calls \c VIDIOC_G_CTRL IOCTL internally. + * + * @param[in] id ID of the control to get. + * @param[out] value A reference to the variable into which the control value + is read. + * @return 0 for success, -1 otherwise. + */ + int getControl(uint32_t id, int32_t &value); + + /** + * Sets the value of several controls. + * + * Calls \c VIDIOC_S_EXT_CTRLS IOCTL internally. + * + * @param[in] ctl A pointer to the controls to set. + * @return 0 for success, -1 otherwise. + */ + int setExtControls(struct v4l2_ext_controls &ctl); + /** + * Gets the value of several controls. + * + * Calls \c VIDIOC_G_EXT_CTRLS IOCTL internally. + * + * @param[in,out] ctl A pointer to the controls to get. + * @return 0 for success, -1 otherwise. + */ + int getExtControls(struct v4l2_ext_controls &ctl); + + virtual int isInError(); + + /** + * Sets the output plane. + */ + NvV4l2ElementPlane output_plane; /**< Output plane of the element */ + /** + * Sets the capture plane. + */ + NvV4l2ElementPlane capture_plane; /**< Capture plane of the element */ + + /** + * + * Terminates processing of queued buffers immediately. All the buffers are + * returned to the application. + * + * Calls VIDIOC_STREAMOFF IOCTL on both of the planes internally. + * + * @return 0 for success, -1 otherwise. + */ + int abort(); + + /** + * Waits until the element processes all the output plane buffers. + * + * Objects extending @c V4l2Element must implement this because the idle + * condition is component-specific. + * + * @param[in] max_wait_ms Max time to wait in milliseconds. + * @return 0 for success, -1 otherwise. + */ + virtual int waitForIdle(uint32_t max_wait_ms); + + void *app_data; /**< A pointer to the application-specific data. */ + + /** + * Enables profiling for the V4l2Element. + * + * Must be called before setting either plane formats. + */ + void enableProfiling(); + +protected: + int fd; /**< Specifies the FD of the device opened using \c v4l2_open. */ + + uint32_t output_plane_pixfmt; /**< Pixel format of output plane buffers */ + uint32_t capture_plane_pixfmt; /**< Pixel format of capture plane buffers */ + + /** + * Creates a new V4l2Element named \a name. + * + * This constructor calls v4l2_open on the \a dev_node. It sets an error if + * v4l2_open fails. + * + * This function also checks if the device supports V4L2_CAP_VIDEO_M2M_MPLANE + * capability. + * + * @param[in] comp_name A pointer to the unique name to identity the element instance. + * @param[in] dev_node A pointer to /dev/ * node of the device. + * @param[in] flags Flags with which to open the device. + * @param[in] fields Profiler fields that are valid for the element. + */ + NvV4l2Element(const char *comp_name, const char *dev_node, int flags, NvElementProfiler::ProfilerField fields); +}; +/** @} */ +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2ElementPlane.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2ElementPlane.h new file mode 100644 index 0000000000..64378f1842 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvV4l2ElementPlane.h @@ -0,0 +1,586 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: V4L2 Element Plane + * + * @b Description: This file declares a helper class for operations + * performed on a V4L2 Element plane. + */ + +/** + * @defgroup l4t_mm_nvv4lelementplane_group NvV4l2ElementPlane Class + * @ingroup l4t_mm_nvelement_group + * + * Helper class for operations performed on a V4L2 element plane. + * This includes getting/setting plane formats, plane buffers, + * and cropping. + * @{ + */ + +#ifndef __NV_V4L2_ELELMENT_PLANE_H__ +#define __NV_V4L2_ELELMENT_PLANE_H__ + +#include +#include "NvElement.h" +#include "NvLogging.h" +#include "NvBuffer.h" + +/** + * Prints a plane-specific message of level LOG_LEVEL_DEBUG. + * Must not be used by applications. + */ +#define PLANE_DEBUG_MSG(str) COMP_DEBUG_MSG(plane_name << ":" << str); +/** + * Prints a plane-specific message of level LOG_LEVEL_INFO. + * Must not be used by applications. + */ +#define PLANE_INFO_MSG(str) COMP_INFO_MSG(plane_name << ":" << str); +/** + * Prints a plane-specific message of level LOG_LEVEL_WARN. + * Must not be used by applications. + */ +#define PLANE_WARN_MSG(str) COMP_WARN_MSG(plane_name << ":" << str); +/** + * Prints a plane-specific message of level LOG_LEVEL_ERROR. + * Must not be used by applications. + */ +#define PLANE_ERROR_MSG(str) COMP_ERROR_MSG(plane_name << ":" << str); +/** + * Prints a plane-specific system error message of level LOG_LEVEL_ERROR. + * Must not be used by applications. + */ +#define PLANE_SYS_ERROR_MSG(str) COMP_SYS_ERROR_MSG(plane_name << ":" << str); + +/** + * @brief Defines a helper class for operations performed on a V4L2 Element plane. + * + * This derived class is modeled on the planes of a V4L2 Element. It provides + * convenient wrapper methods around V4L2 IOCTLs associated with plane + * operations such as VIDIOC_G_FMT/VIDIOC_S_FMT, \c VIDIOC_REQBUFS, + * VIDIOC_STREAMON/VIDIOC_STREAMOFF, etc. + * + * The plane buffer type can be either \c V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE (for + * the output plane) or \c V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE (for the capture + * plane). + * + * The plane has an array of NvBuffer object pointers that is allocated and + * initialized during reqbuf call. These \c NvBuffer objects are similar to the + * \c v4l2_buffer structures that are queued/dequeued. + * + * This class provides another feature useful for multi-threading. On calling + * #startDQThread, it internally spawns a thread that runs infinitely until + * signaled to stop. This thread keeps trying to dequeue a buffer from the + * plane and calls a #dqThreadCallback method specified by the user on + * successful dequeue. + * + */ +class NvV4l2ElementPlane +{ + +public: + /** + * Gets the plane format. + * + * Calls @b VIDIOC_G_FMT \c IOCTL internally. + * + * @param[in,out] format A reference to the \c v4l2_format structure to be filled. + * @return 0 for success, -1 otherwise. + */ + int getFormat(struct v4l2_format & format); + /** + * Sets the plane format. + * + * Calls @b VIDIOC_S_FMT \c IOCTL internally. + * + * @param[in] format A reference to the \c v4l2_format structure to be set on the plane. + * @return 0 for success, -1 otherwise. + */ + int setFormat(struct v4l2_format & format); + + /** + * Maps the NvMMBuffer to NvBuffer for V4L2_MEMORY_DMABUF. + * + * @param[in] v4l2_buf Address of the NvBuffer to which the NvMMBuffer is mapped. + * @param[in] dmabuff_fd Index to the field that holds NvMMBuffer attributes. + * @return 0 for success, -1 otherwise. + */ + + int mapOutputBuffers(struct v4l2_buffer &v4l2_buf, int dmabuff_fd); + + /** + * Unmaps the NvMMBuffer for V4L2_MEMORY_DMABUF. + * + * @param[in] index for the current buffer index. + * @param[in] dmabuff_fd Index to the field that holds NvMMBuffer attributes. + * @return 0 for success, -1 otherwise. + */ + + int unmapOutputBuffers(int index, int dmabuff_fd); + + /** + * Gets the cropping rectangle for the plane. + * + * Calls @b VIDIOC_G_CROP \c IOCTL internally. + * + * @param[in] crop A reference to the \c v4l2_crop structure to be filled. + * @return 0 for success, -1 otherwise. + */ + int getCrop(struct v4l2_crop & crop); + + /** + * Sets the selection rectangle for the plane. + * + * Calls @b VIDIOC_S_SELECTION IOCTL internally. + * + * @param[in] target Specifies the rectangle selection type. + * @param[in] flags Specifies the flags to control selection adjustments. + * @param[in] rect A reference to the selection rectangle. + * @return 0 for success, -1 otherwise. + */ + int setSelection(uint32_t target, uint32_t flags, struct v4l2_rect & rect); + + /** + * Requests for buffers on the plane. + * + * Calls \c VIDIOC_REQBUFS IOCTL internally. Creates an array of NvBuffer of + * length equal to the count returned by the IOCTL. + * + * @param[in] mem_type Specifies the type of V4L2 memory to be requested. + * @param[in] num Specifies the number of buffers to request on the plane. + * @return 0 for success, -1 otherwise. + */ + int reqbufs(enum v4l2_memory mem_type, uint32_t num); + /** + * Queries the status of the buffer at the index. + * + * @warning This method works only for \c V4L2_MEMORY_MMAP memory. + * + * Calls \c VIDIOC_QUERYBUF IOCTL internally. Populates the \a length and \a + * mem_offset members of all the NvBuffer::NvBufferPlane members of the + * \c %NvBuffer object at index \a buf_index. + * + * @param[in] buf_index Specifies the index of the buffer to query. + * @return 0 for success, -1 otherwise. + */ + int queryBuffer(uint32_t buf_index); + /** + * Exports the buffer as DMABUF FD. + * + * @warning This method works only for \c V4L2_MEMORY_MMAP memory. + * + * Calls \c VIDIOC_EXPBUF IOCTL internally. Populates the \a fd member of all + * the \c NvBuffer::NvBufferPlane members of \c NvBuffer object at index \a buf_in. + * + * @param[in] buf_index Specifies the index of the buffer to export. + * @return 0 for success, -1 otherwise. + */ + int exportBuffer(uint32_t buf_index); + + /** + * Starts or stops streaming on the plane. + * + * Calls \c VIDIOC_STREAMON/VIDIOC_STREAMOFF IOCTLs internally. + * + * @param[in] status Must be TRUE to start the stream, FALSE to stop the stream. + * @return 0 for success, -1 otherwise. + */ + int setStreamStatus(bool status); + + /** + * Checks whether the plane is streaming. + * + * @returns true if the plane is streaming, false otherwise. + */ + bool getStreamStatus(); + + /** + * Sets streaming parameters. + * + * Calls \c VIDIOC_S_PARM IOCTL internally. + * + * @param[in] parm A reference to the \c v4l2_streamparm structure to be set on the + * plane. + * @return 0 for success, -1 otherwise. + */ + int setStreamParms(struct v4l2_streamparm & parm); + + /** + * Helper method that encapsulates all the method calls required to + * set up the plane for streaming. + * + * Calls reqbuf internally. Then, for each of the buffers, calls #queryBuffer, + * #exportBuffer and maps the buffer/allocates the buffer memory depending + * on the memory type. + * + * @sa deinitPlane + * + * @param[in] mem_type V4L2 Memory to use on the buffer. + * @param[in] num_buffers Number of buffer to request on the plane. + * @param[in] map boolean value indicating if the buffers should be mapped to + memory (Only for V4L2_MEMORY_MMAP). + * @param[in] allocate boolean valued indicating whether the buffers should be + allocated memory (Only for V4L2_MEMORY_USERPTR). + * @return 0 for success, -1 otherwise. + */ + int setupPlane(enum v4l2_memory mem_type, uint32_t num_buffers, bool map, bool allocate); + /** + * Helper method that encapsulates all the method calls required to + * deinitialize the plane for streaming. + * + * For each of the buffers, unmaps/deallocates memory depending on the + * memory type. Then, calls reqbuf with count zero. + * + * @sa setupPlane + */ + void deinitPlane(); + + /** + * Gets the streaming/buffer type of this plane. + * + * @returns Type of the buffer belonging to enum \c v4l2_buf_type. + */ + inline enum v4l2_buf_type getBufType() + { + return buf_type; + } + + /** + * Gets the \c NvBuffer object at index n. + * + * @returns \c %NvBuffer object at index n, NULL if n >= number of buffers. + */ + NvBuffer *getNthBuffer(uint32_t n); + + /** + * Dequeues a buffer from the plane. + * + * This is a blocking call. This call returns when a buffer is successfully + * dequeued or timeout is reached. If \a buffer is not NULL, returns the + * \c NvBuffer object at the index returned by the \c VIDIOC_DQBUF IOCTL. If this + * plane shares a buffer with other elements and \a shared_buffer is not + * NULL, returns the shared \c %NvBuffer object in \a shared_buffer. + * + * @param[in] v4l2_buf A reference to the \c v4l2_buffer structure to use for dequeueing. + * @param[out] buffer Returns a pointer to a pointer to the \c %NvBuffer object associated with the dequeued + * buffer. Can be NULL. + * @param[out] shared_buffer Returns a pointer to a pointer to the shared \c %NvBuffer object if the queued + buffer is shared with other elements. Can be NULL. + * @param[in] num_retries Number of times to try dequeuing a buffer before + * a failure is returned. In case of non-blocking + * mode, this is equivalent to the number of + * milliseconds to try to dequeue a buffer. + * @return 0 for success, -1 otherwise. + */ + int dqBuffer(struct v4l2_buffer &v4l2_buf, NvBuffer ** buffer, + NvBuffer ** shared_buffer, uint32_t num_retries); + /** + * Queues a buffer on the plane. + * + * This method calls \c VIDIOC_QBUF internally. If this plane is sharing a + * buffer with other elements, the application can pass the pointer to the + * shared NvBuffer object in \a shared_buffer. + * + * @param[in] v4l2_buf A reference to the \c v4l2_buffer structure to use for queueing. + * @param[in] shared_buffer A pointer to the shared \c %NvBuffer object. + * @return 0 for success, -1 otherwise. + */ + int qBuffer(struct v4l2_buffer &v4l2_buf, NvBuffer * shared_buffer); + + /** + * Gets the number of buffers allocated/requested on the plane. + * + * @returns Number of buffers. + */ + inline uint32_t getNumBuffers() + { + return num_buffers; + } + + /** + * Gets the number of planes buffers on this plane for the currently + * set format. + * + * @returns Number of planes. + */ + inline uint32_t getNumPlanes() + { + return n_planes; + } + + /** + * Sets the format of the planes of the buffer that is used with this + * plane. + * + * The buffer plane format must be set before calling reqbuf since + * these are needed by the NvBuffer constructor. + * + * @sa reqbufs + * + * @param[in] n_planes Number of planes in the buffer. + * @param[in] planefmts A pointer to the array of \c NvBufferPlaneFormat that describes the + * format of each of the plane. The array length must be at + * least @a n_planes. + */ + void setBufferPlaneFormat(int n_planes, NvBuffer::NvBufferPlaneFormat * planefmts); + + /** + * Gets the number of buffers currently queued on the plane. + * + * @returns Number of buffers currently queued on the plane. + */ + inline uint32_t getNumQueuedBuffers() + { + return num_queued_buffers; + } + + /** + * Gets the total number of buffers dequeued from the plane. + * + * @returns Total number of buffers dequeued from the plane. + */ + inline uint32_t getTotalDequeuedBuffers() + { + return total_dequeued_buffers; + } + + /** + * Gets the total number of buffers queued on the plane. + * + * @returns Total number of buffers queued on the plane. + */ + inline uint32_t getTotalQueuedBuffers() + { + return total_queued_buffers; + } + + /** + * Waits until all buffers of the plane are queued. + * + * This is a blocking call that returns when all the buffers are queued + * or timeout is reached. + + * @param[in] max_wait_ms Maximum time to wait, in milliseconds. + * @return 0 for success, -1 otherwise. + */ + int waitAllBuffersQueued(uint32_t max_wait_ms); + /** + * Waits until all buffers of the plane are dequeued. + * + * This is a blocking call that returns when all the buffers are dequeued + * or timeout is reached. + + * @param[in] max_wait_ms Maximum time to wait, in milliseconds + * @return 0 for success, -1 otherwise. + */ + int waitAllBuffersDequeued(uint32_t max_wait_ms); + + /** + * This is a callback function type method that is called by the DQ Thread when + * it successfully dequeues a buffer from the plane. Applications must implement + * this and set the callback using #setDQThreadCallback. + * + * Setting the stream to off automatically stops this thread. + * + * @sa setDQThreadCallback, #startDQThread + * + * @param v4l2_buf A pointer to the \c v4l2_buffer structure that is used for dequeueing. + * @param buffer A pointer to the NvBuffer object at the \c index contained in \c v4l2_buf. + * @param shared_buffer A pointer to the NvBuffer object if the plane shares a buffer with other elements, + * else NULL. + * @param data A pointer to application specific data that is set with + * #startDQThread. + * @returns If the application implementing this call returns FALSE, + * the DQThread is stopped; else, the DQ Thread continues running. + */ + typedef bool(*dqThreadCallback) (struct v4l2_buffer * v4l2_buf, + NvBuffer * buffer, NvBuffer * shared_buffer, + void *data); + + /** + * Sets the DQ Thread callback method. + * + * The callback method is called from the DQ Thread once a buffer is + * successfully dequeued. + + * @param[in] callback Method to be called upon succesful dequeue. + * @returns TRUE for success, FALSE for failure. + */ + bool setDQThreadCallback(dqThreadCallback callback); + /** + * Starts DQ Thread. + * + * This method starts a thread internally. On successful dequeue of a + * buffer from the plane, the #dqThreadCallback method set using + * #setDQThreadCallback is called. + * + * Setting the stream to off automatically stops the thread. + * + * @sa stopDQThread, waitForDQThread + * + * @param[in] data A pointer to the application data. This is provided as an + * argument in the \c dqThreadCallback method. + * @return 0 for success, -1 otherwise. + */ + int startDQThread(void *data); + /** + * Force stops the DQ Thread if it is running. + * + * Does not work when the device is opened in blocking mode. + * + * @sa startDQThread, waitForDQThread + * + * @return 0 for success, -1 otherwise. + */ + int stopDQThread(); + /** + * Waits for the DQ Thread to stop. + * + * This method waits until the DQ Thread stops or timeout is reached. + * + * @sa startDQThread, stopDQThread + * + * @param[in] max_wait_ms Maximum wait time, in milliseconds. + * @return 0 for success, -1 otherwise. + */ + int waitForDQThread(uint32_t max_wait_ms); + + pthread_mutex_t plane_lock; /**< Mutex lock used along with #plane_cond. */ + pthread_cond_t plane_cond; /**< Plane condition that application can wait on + to receive notifications from + #qBuffer/#dqBuffer. */ + +private: + int &fd; /**< A reference to the FD of the V4l2 Element the plane is associated with. */ + + const char *plane_name; /**< A pointer to the name of the plane. Could be "Output Plane" or + "Capture Plane". Used only for debug logs. */ + enum v4l2_buf_type buf_type; /**< Speciifes the type of the stream. */ + + bool blocking; /**< Specifies whether the V4l2 element is opened with + blocking mode. */ + + uint32_t num_buffers; /**< Holds the number of buffers returned by \c VIDIOC_REQBUFS + IOCTL. */ + NvBuffer **buffers; /**< A pointer to an array of NvBuffer object pointers. This array is + allocated and initialized in #reqbufs. */ + + uint8_t n_planes; /**< Specifies the number of planes in the buffers. */ + NvBuffer::NvBufferPlaneFormat planefmts[MAX_PLANES]; + /**< Format of the buffer planes. This must be + initialized before calling #reqbufs since this + is required by the \c %NvBuffer constructor. */ + + enum v4l2_memory memory_type; /**< Specifies the V4l2 memory type of the buffers. */ + + uint32_t num_queued_buffers; /**< Holds the number of buffers currently queued on the + plane. */ + uint32_t total_queued_buffers; /**< Holds the total number of buffers queued on the + plane. */ + uint32_t total_dequeued_buffers; /**< Holds the total number of buffers dequeued from + the plane. */ + + bool streamon; /**< Specifies whether the plane is streaming. */ + + bool dqthread_running; /**< Specifies whether DQ Thread is running. + Its value is toggled by the DQ Thread. */ + bool stop_dqthread; /**< Specifies the value used to signal the DQ Thread to stop. */ + + pthread_t dq_thread; /**< Speciifes the pthread ID of the DQ Thread. */ + + dqThreadCallback callback; /**< Specifies the callback method used by the DQ Thread. */ + + void *dqThread_data; /**< Application supplied pointer provided as an + argument in #dqThreadCallback. */ + + /** + * The DQ thread method. + * + * This method runs indefinitely until it is signaled to stop + * by #stopDQThread or the #dqThreadCallback method returns FALSE. + * It keeps on trying to dequeue a buffer from the plane and calls the + * #dqThreadCallback method on successful dequeue. + * + * @param[in] v4l2_element_plane A pointer to the NvV4l2ElementPlane object + * for which the thread started. + */ + static void *dqThread(void *v4l2_element_plane); + + NvElementProfiler &v4l2elem_profiler; /**< A reference to the profiler belonging + to the plane's parent element. */ + + /** + * Indicates whether the plane encountered an error during its operation. + * + * @return 0 if no error was encountered, a non-zero value if an + * error was encountered. + */ + inline int isInError() + { + return is_in_error; + } + + /** + * Creates a new V4l2Element plane. + * + * + * @param[in] buf_type Type of the stream. + * @param[in] device_name A pointer to the name of the element the plane belongs to. + * @param[in] fd A reference to the FD of the device opened using v4l2_open. + * @param[in] blocking A flag that indicates whether the device has been opened with blocking mode. + * @param[in] profiler The profiler. + */ + NvV4l2ElementPlane(enum v4l2_buf_type buf_type, const char *device_name, + int &fd, bool blocking, NvElementProfiler &profiler); + + /** + * Disallows copy constructor. + */ + NvV4l2ElementPlane(const NvV4l2ElementPlane& that); + /** + * Disallows assignment. + */ + void operator=(NvV4l2ElementPlane const&); + + /** + * NvV4l2ElementPlane destructor. + * + * Calls #deinitPlane internally. + */ + ~NvV4l2ElementPlane(); + + int is_in_error; /**< Indicates if an error was encountered during + the operation of the element. */ + const char *comp_name; /**< Specifies the name of the component, + for debugging. */ + + friend class NvV4l2Element; +}; +/** @} */ +#endif diff --git a/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvVideoEncoder.h b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvVideoEncoder.h new file mode 100644 index 0000000000..ca8f088868 --- /dev/null +++ b/src/plugins/sensing/third_party/jetson_multimedia_api/include/NvVideoEncoder.h @@ -0,0 +1,930 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVIDIA Multimedia API: Video Encode API + * + */ + +/** + * @defgroup l4t_mm_nvvideoencoder_group Video Encoder + * @ingroup l4t_mm_nvvideo_group + * + * Helper class that creates new V4L2 + * video encoders, and it sets encoder capture and output plane + * formats. + * @{ + */ + +#ifndef __NV_VIDEO_ENCODER_H__ +#define __NV_VIDEO_ENCODER_H__ + +#include "NvV4l2Element.h" + +/** + * @brief Defines a helper class for V4L2 Video Encoder. + * + * The video encoder device node is \c "/dev/nvhost-msenc". The category name + * for the encoder is \c "NVENC". + * + * Refer to [V4L2 Video Encoder](group__V4L2Enc.html) for more information on the encoder. + */ + +class NvVideoEncoder:public NvV4l2Element +{ +public: + /** + * Creates a new V4L2 Video Encoder object named \a name. + * + * This method internally calls \c v4l2_open on the encoder dev node + * \c "/dev/nvhost-msenc" and checks for \c V4L2_CAP_VIDEO_M2M_MPLANE + * capability on the device. This method allows the caller to specify + * additional flags with which the device must be opened. + * + * The device is opened in blocking mode, which can be modified by passing + * the @a O_NONBLOCK flag to this method. + * + * @returns Reference to the newly created encoder object, else NULL in + * case of failure during initialization. + */ + static NvVideoEncoder *createVideoEncoder(const char *name, int flags = 0); + + ~NvVideoEncoder(); + + /** + * Sets the format on the encoder output plane. + * + * Calls \c VIDIOC_S_FMT IOCTL internally on the output plane. + * + * @pre Applications must set the capture plane format using #setCapturePlaneFormat before calling this method. + * + * @param[in] pixfmt One of the raw V4L2 pixel formats. + * @param[in] width Width of the input buffers in pixels. + * @param[in] height Height of the input buffers in pixels. + * @return 0 for success, -1 otherwise. + */ + int setOutputPlaneFormat(uint32_t pixfmt, uint32_t width, uint32_t height, + enum v4l2_colorspace cs = V4L2_COLORSPACE_SMPTE170M); + + /** + * CUDA Enc: Sets the format on the encoder output plane. + * + * Calls \c VIDIOC_S_FMT IOCTL internally on the output plane. + * + * @pre Applications must set the capture plane format using #setCapturePlaneFormat before calling this method. + * + * @param[in] pixfmt One of the raw V4L2 pixel formats. + * @param[in] width Width of the input buffers in pixels. + * @param[in] height Height of the input buffers in pixels. + * @param[in] cs Colorspace of the input buffers. + * @param[in] extended Color Range the input buffers. + * @return 0 for success, -1 otherwise. + */ + int setOutputPlaneFormat(uint32_t pixfmt, uint32_t width, uint32_t height, + v4l2_colorspace cs, bool extended); + + /** + * Sets the format on the converter capture plane. + * + * Calls \c VIDIOC_S_FMT IOCTL internally on the capture plane. + * + * @param[in] pixfmt One of the coded V4L2 pixel formats. + * @param[in] width Width of the input buffers in pixels. + * @param[in] height Height of the input buffers in pixels. + * @param[in] sizeimage Maximum size of the encoded buffers on the capture. + * plane in bytes + * @return 0 for success, -1 otherwise. + */ + int setCapturePlaneFormat(uint32_t pixfmt, uint32_t width, + uint32_t height, uint32_t sizeimage); + + /** + * Sets the encode framerate. + * + * Calls the VIDIOC_S_PARM IOCTL on the encoder capture plane. Can be + * called any time after setFormat on both the planes. + * + * @param[in] framerate_num Numerator part of the framerate fraction. + * @param[in] framerate_den Denominator part of the framerate fraction. + * + * @return 0 for success, -1 otherwise. + */ + int setFrameRate(uint32_t framerate_num, uint32_t framerate_den); + + /** + * Sets the encoder bitrate. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_BITRATE. Can be called any time after setFormat on + * both the planes. + * + * @param[in] bitrate Bitrate of the encoded stream, in bits per second. + * + * @return 0 for success, -1 otherwise. + */ + int setBitrate(uint32_t bitrate); + + /** + * Sets the encoder peak bitrate. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_BITRATE_PEAK. Can be called any time after setFormat on + * both the planes. Takes effect in VBR mode + * + * @param[in] peak_bitrate Peak Bitrate of the encoded stream, in bits per second. + * + * @return 0 for success, -1 otherwise. + */ + int setPeakBitrate(uint32_t peak_bitrate); + + /** + * Sets the encoder profile. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_H264_PROFILE or #V4L2_CID_MPEG_VIDEO_H265_PROFILE, + * depending on the encoder type. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] profile Profile to be used for encoding. + * + * @return 0 for success, -1 otherwise. + */ + int setProfile(uint32_t profile); + + /** + * Sets the encoder command. + * + * Calls the VIDIOC_ENCODER_CMD internally with encoder commands. + * + * @return 0 for succes, -1 otherwise. + */ + int setEncoderCommand(int cmd, int flags); + + /** + * Set the encoder level. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_H264_LEVEL or #V4L2_CID_MPEG_VIDEOENC_H265_LEVEL, + * depending on encoder type. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] level Level to be used for encoding + * + * @return 0 for success, -1 otherwise. + */ + int setLevel(uint32_t level); + + /** + * Sets the encoder for maximum performance. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * V4L2_CID_MPEG_VIDEO_MAX_PERFORMANCE. Must be called after setFormat on both + * the planes and before requestBuffers on any of the planes. + * + * @param[in] flag Integer variable to indicate whether max performance is to be + * enabled/disabled. + * + * @return 0 for success, -1 otherwise. + */ + int setMaxPerfMode(int flag); + /** + * Sets constant QP for encoder. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE to enable/disable rate control. + * If the value false is given, it disables RC and set constant QP parameters. + * Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] enabled_rc Boolean value indicating whether to enable/disable + * the control. + * + * @return 0 for success, -1 otherwise. + */ + int setConstantQp(bool enabled_rc); + + /** + * Sets the encoder rate control mode. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_BITRATE_MODE. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] mode Type of rate control, one of enum + * v4l2_mpeg_video_bitrate_mode. + * + * @return 0 for success, -1 otherwise. + */ + int setRateControlMode(enum v4l2_mpeg_video_bitrate_mode mode); + + /** + * Sets the encoder I-frame interval. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_VIDEO_GOP_SIZE. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] interval Interval between two I frames, in number of frames. + * + * @return 0 for success, -1 otherwise. + */ + int setIFrameInterval(uint32_t interval); + + /** + * Sets the encoder IDR interval. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEO_IDR_INTERVAL. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] interval Interval between two IDR frames, in number of frames. + * + * @return 0 for success, -1 otherwise. + */ + int setIDRInterval(uint32_t interval); + + /** + * Forces an IDR frame. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * \c V4L2_CID_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE. Must be called after + * setFormat on both the planes. + * + * @return 0 for success, -1 otherwise. + */ + int forceIDR(); + + /** + * Sets the encoder Temporal Tradeoff. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_TEMPORAL_TRADEOFF_LEVEL. Must be called after + * setFormat on both the planes and before \c requestBuffers on any of the + * planes. + * + * @param[in] level Temporal tradeoff level, one of + * v4l2_enc_temporal_tradeoff_level_type. + * @return 0 for success, -1 otherwise. + */ + int setTemporalTradeoff(v4l2_enc_temporal_tradeoff_level_type level); + + /** + * Sets the encoder output slice length. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_SLICE_LENGTH_PARAM. Must be called after setFormat on + * both the planes and before \c requestBuffers on any of the planes. + * + * @param[in] type Slice length type, one of enum v4l2_enc_slice_length_type. + * @param[in] length Length of the slice, in bytes if the type is + * #V4L2_ENC_SLICE_LENGTH_TYPE_BITS, else in number of MBs if the type is + * #V4L2_ENC_SLICE_LENGTH_TYPE_MBLK. + * @return 0 for success, -1 otherwise. + */ + int setSliceLength(v4l2_enc_slice_length_type type, uint32_t length); + + /** + * Sets the encoder HW Preset Type. + * + * Calls the VIDIOC_S_EXT_CTRLS ioctl internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_HW_PRESET_TYPE_PARAM. Must be called after setFormat() on + * both the planes and before \c requestBuffers on any of the planes. + * + * @param[in] type HW Preset Type, one of + * enum v4l2_enc_hw_preset_type_param. + * @return 0 for success, -1 otherwise. + */ + int setHWPresetType(v4l2_enc_hw_preset_type type); + + /** + * Sets the Region of Interest (ROI) parameters for the next buffer, which will + * be queued on the output plane with index \a buffer_index. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ROI_PARAMS. Must be called after + * requesting buffer on both the planes. + * + * @param[in] buffer_index Index of the output plane buffer to apply the ROI + * params. + * @param[in] params A reference to the parameters to be applied on the frame, structure of + * type v4l2_enc_frame_ROI_params. + * @return 0 for success, -1 otherwise. + */ + int setROIParams(uint32_t buffer_index, v4l2_enc_frame_ROI_params & params); + + /** + * Sets the preprocessing enhancement (PPE) initialization parameters at the start of the session + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_PPE_INIT_PARAMS. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params A reference to the parameters to be applied on the frame, structure of + * type v4l2_enc_ppe_init_params. + * @return 0 for success, -1 otherwise. + */ + int setPPEInitParams(v4l2_enc_ppe_init_params & params); + + /** + * Enables External ROI. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_ROI_PARAM. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params Parameters to be applied on the frame, structure of + * type #v4l2_enc_enable_roi_param. + * @return 0 for success, -1 otherwise. + */ + int enableROI(v4l2_enc_enable_roi_param ¶ms); + + /** + * Enables Recon CRC. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_RECONCRC_PARAM. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params Parameters to be applied on the frame, structure of + * type #v4l2_enc_enable_reconcrc_param. + * @return 0 for success, -1 otherwise. + */ + int enableReconCRC(v4l2_enc_enable_reconcrc_param ¶ms); + + /** + * Enable External RPS + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_EXTERNAL_RPS_CONTROL. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params Parameters to be applied on the frame, structure of + * type #v4l2_enc_enable_ext_rps_ctr + * @return 0 for success, -1 otherwise. + */ + int enableExternalRPS(v4l2_enc_enable_ext_rps_ctr ¶ms); + + /** + * Enable External Picture RC + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_EXTERNAL_RATE_CONTROL. Must be called after + * requesting buffer on both the planes. + * + * @param[in] params Parameters to be applied on the frame, structure of + * type ##v4l2_enc_enable_ext_rate_ctr + * @return 0 for success, -1 otherwise. + */ + int enableExternalRC(v4l2_enc_enable_ext_rate_ctr ¶ms); + + /** + * Disable SAO filter for HEVC encoding. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_H265_DISABLE_SAO. Must be called after + * setFormat on both the planes. Disabled by default. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setH265DisableSAO(bool enabled); + + /** + * Enable AV1 Multi-tile configuration + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_TILE_CONFIGURATION. Must be called after + * setFormat on both the planes and before \c requestBuffers on any of the + * planes. + * + * @param[in] params Parameters to be applied, structure of + * type ##v4l2_enc_av1_tile_config + * @return 0 for success, -1 otherwise. + */ + int enableAV1Tile(v4l2_enc_av1_tile_config ¶ms); + + /** + * Enable AV1 variance based SSIM RDO. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_SSIMRDO. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAV1SsimRdo(bool enabled); + + /** + * Disable AV1 CDF update in the symbol decoding process. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_DISABLE_CDF_UPDATE. The value true + * disables the CDF update. Must be called after setFormat on both + * the planes. + * + * @param[in] disabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAV1DisableCDFUpdate(bool disabled); + + /** + * Enable Error Resilient Mode for AV1. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_ERR_RESILIENT_MODE. The value true + * disables the control. Must be called after setFormat on both + * the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAV1ErrResilienceMode(bool enabled); + + /** + * Enable AV1 frame_id_numbers_present_flag. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_FRAMEID_NUMBERS. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAV1FrameIDPresentFlag(bool enabled); + + /** + * Enable AV1 frame_id_numbers_present_flag. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_AV1_ENABLE_TILE_GROUPS. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int enableAV1TileGroups(bool enabled); + + /** + * Set input Metadata parameters for the next buffer which will + * be queued on output plane with index \a buffer_index + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_INPUT_METADATA. Must be called after + * requesting buffer on both the planes. + * + * @param[in] buffer_index Index of output plane buffer on which the external + * RC params should be applied. + * @param[in] params Parameters to be applied on the frame, structure of + * type #v4l2_ctrl_videoenc_input_metadata + * @return 0 for success, -1 otherwise. + */ + int SetInputMetaParams(uint32_t buffer_index, v4l2_ctrl_videoenc_input_metadata ¶ms); + + /** + * Sets the virtual buffer size of the encoder. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_VIRTUALBUFFER_SIZE. Must be called after + * setFormat on both the planes. + * + * @param[in] size Virtual buffer size, in bytes. + * @return 0 for success, -1 otherwise. + */ + int setVirtualBufferSize(uint32_t size); + + /** + * Sets the number of reference frames of the encoder. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_NUM_REFERENCE_FRAMES. Must be called after + * setFormat on both the planes. + * + * @param[in] num_frames Number of reference frames. + * @return 0 for success, -1 otherwise. + */ + int setNumReferenceFrames(uint32_t num_frames); + + /** + * Sets slice intra-refresh interval params. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_SLICE_INTRAREFRESH_PARAM. Must be called after + * setFormat on both the planes. + * + * @param[in] interval Slice intra-refresh interval, in number of slices. + * @return 0 for success, -1 otherwise. + */ + int setSliceIntrarefresh(uint32_t interval); + + /** + * Sets CUDA slice intra-refresh interval params. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_SLICE_INTRAREFRESH_PARAM. Must be called after + * setFormat on both the planes. + * + * @param[in] interval Slice intra-refresh interval. + * @param[in] count Slice intra-refresh count, in number of slices. + * @return 0 for success, -1 otherwise. + */ + int setCUDASliceIntrarefresh(uint32_t interval, uint32_t count); + + /** + * Sets the number of B frames to P frames. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_NUM_BFRAMES. Must be called after + * setFormat on both the planes. + * + * @param[in] num Number of B frames. + * @return 0 for success, -1 otherwise. + */ + int setNumBFrames(uint32_t num); + + /** + * Enables/disables insert SPS PPS at every IDR. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_INSERT_SPS_PPS_AT_IDR. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setInsertSpsPpsAtIdrEnabled(bool enabled); + + /** + * Enables/disables CABAC entropy encoding for H264. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setCABAC(bool enabled); + + /** + * Enables/disables Slice level encoding for H264 / HEVC. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_SLICE_LEVEL_ENCODE. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setSliceLevelEncode(bool enabled); + + /** + * Enables video encoder output motion vector metadata reporting. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_METADATA_MV. Must be called after setFormat on + * both the planes and before \c requestBuffers on any of the planes. + * + * @return 0 for success, -1 otherwise. + */ + int enableMotionVectorReporting(); + + /** + * Gets metadata for the encoded capture plane buffer. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_METADATA. Must be called for a buffer that has + * been dequeued from the capture plane. The returned metadata corresponds + * to the last dequeued buffer with index @a buffer_index. + * + * @param[in] buffer_index Index of the capture plane buffer whose metadata + * is required. + * @param[in,out] enc_metadata Reference to the metadata structure + * v4l2_ctrl_videoenc_outputbuf_metadata to be filled. + * + * @return 0 for success, -1 otherwise. + */ + int getMetadata(uint32_t buffer_index, + v4l2_ctrl_videoenc_outputbuf_metadata &enc_metadata); + + /** + * Gets motion vector metadata for the encoded capture plane buffer. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_METADATA_MV. Must be called for a buffer that has + * been dequeued from the capture plane. The returned metadata corresponds + * to the last dequeued buffer with index @a buffer_index. + * + * @param[in] buffer_index Index of the capture plane buffer whose metadata + * is required. + * @param[in,out] enc_mv_metadata Reference to the metadata structure + * v4l2_ctrl_videoenc_outputbuf_metadata_MV to be filled. + * + * @return 0 for success, -1 otherwise. + */ + int getMotionVectors(uint32_t buffer_index, + v4l2_ctrl_videoenc_outputbuf_metadata_MV &enc_mv_metadata); + + /** + * Sets QP values for I/P/B frames. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * %V4L2_CID_MPEG_VIDEOENC_QP_RANGE. Must be called after + * setFormat on both the planes. + * + * @param[in] MinQpI Minimum Qp Value for I frame. + * @param[in] MaxQpI Minimum Qp Value for I frame. + * @param[in] MinQpP Minimum Qp Value for P frame. + * @param[in] MaxQpP Minimum Qp Value for P frame. + * @param[in] MinQpB Minimum Qp Value for B frame. + * @param[in] MaxQpB Minimum Qp Value for B frame. + * @returns 0 for success, -1 otherwise. + */ + int setQpRange(uint32_t MinQpI, uint32_t MaxQpI, uint32_t MinQpP, + uint32_t MaxQpP, uint32_t MinQpB, uint32_t MaxQpB); + + /** + * Sets Const QP values for I/P/B frames. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * %V4L2_CID_MPEG_VIDEOENC_CUDA_CONSTQP. Must be called after + * setFormat on both the planes. + * + * @param[in] qpI Qp Value for I frame. + * @param[in] qpP Qp Value for P frame. + * @param[in] qpB Qp Value for B frame. + * @returns 0 for success, -1 otherwise. + */ + int setCudaConstantQp(uint32_t qpI, uint32_t qpP, + uint32_t qpB); + + /** + * Sets Sample Aspect Ratio width for VUI encoding. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * \c V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_WIDTH or + * #V4L2_CID_MPEG_VIDEOENC_H265_VUI_EXT_SAR_WIDTH, depending on the + * encoder type. Must be called after setFormat on both the planes. + * + * @param[in] sar_width SAR for width. + * @returns 0 for success, -1 otherwise. + */ + int setSampleAspectRatioWidth(uint32_t sar_width); + + /** + * Sets Sample Aspect Ratio height for VUI encoding. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * \c V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_HEIGHT or + * #V4L2_CID_MPEG_VIDEOENC_H265_VUI_EXT_SAR_HEIGHT, depending on the + * encoder type. Must be called after setFormat on both the planes. + * + * @param[in] sar_height SAR for height. + * @returns 0 for success, -1 otherwise. + */ + int setSampleAspectRatioHeight(uint32_t sar_height); + + /** + * Enables/disables insert VUI. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * @c V4L2_CID_MPEG_VIDEOENC_INSERT_VUI. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setInsertVuiEnabled(bool enabled); + + /** + * Enables/disables extended color format. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * @c V4L2_CID_MPEG_VIDEOENC_EXTEDED_COLORFORMAT. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setExtendedColorFormat(bool enabled); + + /** + * Enables/disables insert AUD. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * @c V4L2_CID_MPEG_VIDEO_H264_AUD_SAR_ENABLE. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setInsertAudEnabled(bool enabled); + + /** + * Enables/disables all i-frame encode. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * @c V4L2_CID_MPEG_VIDEOENC_ENABLE_ALLIFRAME_ENCODE. Must be called after + * setFormat on both the planes. + * + * @param[in] enabled Boolean value indicating whether to enable/disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setAlliFramesEncode(bool enabled); + + /** + * Sets the encoder Picture Order Control type. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_POC_TYPE. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] pocType Set the Picture Order Count for the encoder. + * + * @return 0 for success, -1 otherwise. + */ + int setPocType(uint32_t pocType); + + /** + * Sets the initial QP for I/P/B frames. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with control Id + * %V4L2_CID_MPEG_VIDEOENC_INIT_FRAME_QP. Must be called after + * setFormat on both the planes. + * + * + * @param[in] IinitQP Qp Value for I frame. + * @param[in] PinitQP Qp Value for P frame. + * @param[in] BinitQP Qp Value for B frame. + * + * @return 0 for success, -1 otherwise. + */ + int setInitQP(uint32_t IinitQP, uint32_t PinitQP, uint32_t BinitQP); + + /** + * Sets the number of frames to encode. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_FRAMES_TO_ENCODE. Must be called after + * setFormat on both the planes. + * + * @param[in] framesToEncode Set the number of frames to encode. + * + * @return 0 for success, -1 otherwise. + */ + int setFramesToEncode(uint32_t framesToEncode); + + /** + * Sets the H.265 encoder Chroma Format IDC. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_H265_CHROMA_FACTOR_IDC. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] crfactor Set @c chroma_factor_idc for the encoder. + * + * @return 0 for success, -1 otherwise. + */ + int setChromaFactorIDC(uint8_t crfactor); + + /** + * Sets the lossless encoding for H.264/H.265. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_ENABLE_LOSSLESS. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] enabled Indicates whether to enable or disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setLossless(bool enabled); + + /** + * Sets the Disable AMP types to true for H.265. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_H265_DISABLE_AMP. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @param[in] enabled Indicates whether to enable or disable + * the control. + * @return 0 for success, -1 otherwise. + */ + int setDisableAMP(bool enabled); + + /** + * Sets whether to enable two pass CBR for H.264/H.265. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEOENC_TWO_PASS_CBR. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @return 0 for success, -1 otherwise. + */ + int enableTwoPassCBR(); + + /** + * Issues Poll on the device which blocks until : + * a) Either there is something to dequeue from capture or output plane or any events. + * b) Poll was interrupted by a call to the device using V4L2_CID_SET_POLL_INTERRUPT + * c) Application has already interrupted polling by V4L2_CID_SET_POLL_INTERRUPT + */ + int DevicePoll(v4l2_ctrl_video_device_poll *devicepoll); + + /** + * Sets the polling interrupt, now if the application calls Poll, the device should + * not block, in other words polling is disabled. + */ + int SetPollInterrupt(); + + /** + * Clears the polling interrupt, now if the application calls Poll, the device should + * block until the event is triggered, in other words polling is enabled. + */ + int ClearPollInterrupt(); + + /** + * Checks if GPU configuration is enabled to switch to CUDA processing + */ + bool isGPUEnabled(); + + /** + * Sets GPU Id when GPU is enabled. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEO_CUDA_GPU_ID. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @return 0 for success, -1 otherwise. + */ + int setGPUId(uint32_t gpuId); + + /** + * Sets CUDA Memory type when GPU is enabled. + * + * Calls the VIDIOC_S_EXT_CTRLS IOCTL internally with Control ID + * #V4L2_CID_MPEG_VIDEO_CUDA_MEM_TYPE. Must be called after setFormat on both + * the planes and before \c requestBuffers on any of the planes. + * + * @return 0 for success, -1 otherwise. + */ + int setCudaMemType(uint32_t cumemType); + +private: + /** + * Constructor used by #createVideoEncoder. + */ + NvVideoEncoder(const char *name, const char* dev_node, int flags); + + static const NvElementProfiler::ProfilerField valid_fields = + NvElementProfiler::PROFILER_FIELD_TOTAL_UNITS | + NvElementProfiler::PROFILER_FIELD_LATENCIES | + NvElementProfiler::PROFILER_FIELD_FPS; + + bool gpu_enabled; +}; +/** @} */ + +#endif