From 701733cc7fe2d1a2aec1950e2f01d7ead54a6f14 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 9 Nov 2026 22:10:49 -0500 Subject: [PATCH] video_stage: add the Orin-to-PC video transport AVS1 records, clock exchange, H.264 access-unit helpers and the frame correlator in video_stage_wire, and the Orin-side direct sender in video_stage_sender, with their tests. Plain C++ and POSIX sockets, built by BUILD_VIDEO_STAGE; no CUDA and no LiveKit. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- CMakeLists.txt | 1 + src/CMakeLists.txt | 4 + src/video_stage/CMakeLists.txt | 21 + src/video_stage/sender/CMakeLists.txt | 27 + src/video_stage/sender/direct_sender.cpp | 486 ++++++++++++++++ .../sender/inc/video_stage/direct_sender.hpp | 134 +++++ src/video_stage/sender/tests/CMakeLists.txt | 11 + .../sender/tests/test_direct_sender.cpp | 168 ++++++ .../tests/test_direct_sender_target.cpp | 548 ++++++++++++++++++ src/video_stage/wire/CMakeLists.txt | 17 + src/video_stage/wire/avs1.cpp | 205 +++++++ src/video_stage/wire/h264_access_units.cpp | 92 +++ src/video_stage/wire/h264_sps.cpp | 401 +++++++++++++ src/video_stage/wire/inc/video_stage/avs1.hpp | 75 +++ .../wire/inc/video_stage/clock_exchange.hpp | 152 +++++ .../wire/inc/video_stage/clock_sync.hpp | 118 ++++ .../inc/video_stage/encoded_video_frame.hpp | 32 + .../wire/inc/video_stage/frame_correlator.hpp | 103 ++++ .../inc/video_stage/h264_access_units.hpp | 31 + .../wire/inc/video_stage/h264_sps.hpp | 56 ++ src/video_stage/wire/tests/CMakeLists.txt | 16 + src/video_stage/wire/tests/test_avs1.cpp | 316 ++++++++++ .../wire/tests/test_clock_exchange.cpp | 229 ++++++++ .../wire/tests/test_clock_sync.cpp | 134 +++++ .../wire/tests/test_encoded_video_frame.cpp | 17 + .../wire/tests/test_frame_correlator.cpp | 176 ++++++ .../wire/tests/test_h264_access_units.cpp | 231 ++++++++ src/video_stage/wire/tests/test_h264_sps.cpp | 353 +++++++++++ 28 files changed, 4154 insertions(+) create mode 100644 src/video_stage/CMakeLists.txt create mode 100644 src/video_stage/sender/CMakeLists.txt create mode 100644 src/video_stage/sender/direct_sender.cpp create mode 100644 src/video_stage/sender/inc/video_stage/direct_sender.hpp create mode 100644 src/video_stage/sender/tests/CMakeLists.txt create mode 100644 src/video_stage/sender/tests/test_direct_sender.cpp create mode 100644 src/video_stage/sender/tests/test_direct_sender_target.cpp create mode 100644 src/video_stage/wire/CMakeLists.txt create mode 100644 src/video_stage/wire/avs1.cpp create mode 100644 src/video_stage/wire/h264_access_units.cpp create mode 100644 src/video_stage/wire/h264_sps.cpp create mode 100644 src/video_stage/wire/inc/video_stage/avs1.hpp create mode 100644 src/video_stage/wire/inc/video_stage/clock_exchange.hpp create mode 100644 src/video_stage/wire/inc/video_stage/clock_sync.hpp create mode 100644 src/video_stage/wire/inc/video_stage/encoded_video_frame.hpp create mode 100644 src/video_stage/wire/inc/video_stage/frame_correlator.hpp create mode 100644 src/video_stage/wire/inc/video_stage/h264_access_units.hpp create mode 100644 src/video_stage/wire/inc/video_stage/h264_sps.hpp create mode 100644 src/video_stage/wire/tests/CMakeLists.txt create mode 100644 src/video_stage/wire/tests/test_avs1.cpp create mode 100644 src/video_stage/wire/tests/test_clock_exchange.cpp create mode 100644 src/video_stage/wire/tests/test_clock_sync.cpp create mode 100644 src/video_stage/wire/tests/test_encoded_video_frame.cpp create mode 100644 src/video_stage/wire/tests/test_frame_correlator.cpp create mode 100644 src/video_stage/wire/tests/test_h264_access_units.cpp create mode 100644 src/video_stage/wire/tests/test_h264_sps.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b02232bd7..4a90a6ce9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetc option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF) option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF) option(BUILD_PLUGIN_SENSING "Build SENSING GMSL camera plugin (Jetson only; requires SIPL, NVENC and the LiveKit SDK)" OFF) +option(BUILD_VIDEO_STAGE "Build the video stage: the Orin-to-PC video transport and the PC relay" OFF) option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) option(BUILD_TESTING "Build unit tests" ON) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cdef8fe380..95680ca540 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,6 +25,10 @@ if(BUILD_VIZ) add_subdirectory(viz) endif() +if(BUILD_VIDEO_STAGE) + add_subdirectory(video_stage) +endif() + if(BUILD_PLUGINS) add_subdirectory(plugins) endif() diff --git a/src/video_stage/CMakeLists.txt b/src/video_stage/CMakeLists.txt new file mode 100644 index 0000000000..6f1465ba6a --- /dev/null +++ b/src/video_stage/CMakeLists.txt @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ============================================================================== +# Video stage: the Orin-to-PC video transport +# ============================================================================== +# wire/ video_stage_wire -- AVS1 records, clock exchange, H.264 access-unit helpers, frame correlator +# sender/ video_stage_sender -- the Orin-side sender of AVS1 records over a direct link +# +# Plain C++ and POSIX sockets: no CUDA, no LiveKit. +# +# Configure with: +# cmake -B build -DBUILD_VIDEO_STAGE=ON +# ============================================================================== + +add_subdirectory(wire) +add_subdirectory(sender) + +if(BUILD_TESTING) + add_subdirectory(wire/tests) +endif() diff --git a/src/video_stage/sender/CMakeLists.txt b/src/video_stage/sender/CMakeLists.txt new file mode 100644 index 0000000000..71332d739e --- /dev/null +++ b/src/video_stage/sender/CMakeLists.txt @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The Orin-side sender of AVS1 records to the PC video stage: plain POSIX sockets and threads, no CUDA, so the camera +# producer, the mock producer and the tests all link the same code. +add_library(video_stage_sender STATIC + direct_sender.cpp + inc/video_stage/direct_sender.hpp +) +add_library(video_stage::sender ALIAS video_stage_sender) + +set_target_properties(video_stage_sender PROPERTIES POSITION_INDEPENDENT_CODE ON) + +target_include_directories(video_stage_sender + INTERFACE + $ +) + +target_link_libraries(video_stage_sender + PUBLIC + video_stage::wire + log_bridge::log_bridge_core +) + +if(BUILD_TESTING) + add_subdirectory(tests) +endif() diff --git a/src/video_stage/sender/direct_sender.cpp b/src/video_stage/sender/direct_sender.cpp new file mode 100644 index 0000000000..1797215edd --- /dev/null +++ b/src/video_stage/sender/direct_sender.cpp @@ -0,0 +1,486 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/video_stage/direct_sender.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +namespace +{ + +std::shared_ptr logger() +{ + return isaaccapture::Logger::get("isaaccapture.plugins.sensing.DirectSender"); +} + +bool recv_exact(int fd, uint8_t* out, std::size_t size) +{ + std::size_t done = 0; + while (done < size) + { + const ssize_t n = ::recv(fd, out + done, size - done, 0); + if (n == 0) + return false; + if (n < 0) + { + if (errno == EINTR) + continue; + return false; + } + done += static_cast(n); + } + return true; +} + +bool send_all(int fd, const uint8_t* data, std::size_t size) +{ + while (size > 0) + { + const ssize_t n = ::send(fd, data, size, MSG_NOSIGNAL); + if (n < 0) + { + if (errno == EINTR) + continue; + return false; + } + data += n; + size -= static_cast(n); + } + return true; +} + +} // namespace + +DirectSender::DirectSender(const DirectSenderConfig& config) + : m_config(config), m_host(config.host), m_port(config.port), m_has_target(!config.host.empty()) +{ + m_thread = std::thread(&DirectSender::run, this); +} + +DirectSender::~DirectSender() +{ + { + std::lock_guard lock(m_mutex); + m_stop = true; + } + m_stopping = true; + m_wake.notify_all(); + // Unblock a connect, send or recv that the worker thread is sitting in; it then sees m_stopping and exits. + { + std::lock_guard lock(m_fd_mutex); + for (const int fd : m_fds) + if (fd >= 0) + ::shutdown(fd, SHUT_RDWR); + } + if (m_thread.joinable()) + m_thread.join(); +} + +void DirectSender::flush_stale_locked() +{ + m_dropped += m_queue.size(); + m_queue.clear(); + m_queued_bytes = 0; + m_need_keyframe = true; + m_keyframe_requested = true; +} + +void DirectSender::set_target(const std::string& host, uint16_t port) +{ + { + std::lock_guard lock(m_target_mutex); + m_host = host; + m_port = host.empty() ? 0 : port; + m_has_target = !host.empty(); + ++m_target_generation; + } + set_error({}); + { + // Whatever waits was meant for the old destination, and the new one needs a keyframe first. + std::lock_guard lock(m_mutex); + m_queue.clear(); + m_queued_bytes = 0; + m_need_keyframe = true; + m_keyframe_requested = true; + } + { + // A connect, send or receive in progress is for the old destination: cut it so the worker moves on. + std::lock_guard lock(m_fd_mutex); + for (const int fd : m_fds) + if (fd >= 0) + ::shutdown(fd, SHUT_RDWR); + } + m_wake.notify_all(); +} + +std::string DirectSender::last_error() const +{ + std::lock_guard lock(m_error_mutex); + return m_last_error; +} + +void DirectSender::set_error(const std::string& error) +{ + std::lock_guard lock(m_error_mutex); + m_last_error = error; +} + +std::string DirectSender::host() const +{ + std::lock_guard lock(m_target_mutex); + return m_host; +} + +uint16_t DirectSender::port() const +{ + std::lock_guard lock(m_target_mutex); + return m_port; +} + +void DirectSender::current_target(std::string& host, uint16_t& port, uint64_t& generation) const +{ + std::lock_guard lock(m_target_mutex); + host = m_host; + port = m_port; + generation = m_target_generation; +} + +void DirectSender::submit(const EncodedVideoFrame& frame) +{ + // With no destination there is nothing to queue, and nothing to ask the encoder for. + if (!m_has_target.load()) + return; + std::lock_guard lock(m_mutex); + if (!m_queue.empty() && std::chrono::steady_clock::now() - m_queue.front().enqueued > m_config.max_queue_age) + flush_stale_locked(); + if (m_need_keyframe) + { + if (!frame.keyframe) + return; // a delta frame cannot start a stream; the keyframe request is already pending + m_need_keyframe = false; + } + if (m_queued_bytes + frame.size > m_config.max_queued_bytes) + { + // Dropping any frame breaks the chain after it: discard up to the next keyframe, and ask for one. + ++m_dropped; + m_need_keyframe = true; + m_keyframe_requested = true; + return; + } + video_stage::avs1::Sample sample; + sample.width = m_config.width; + sample.height = m_config.height; + sample.fps_numerator = m_config.fps; + sample.bundle_sequence = ++m_sequence; + sample.session_id = 1; + sample.frame_number = frame.frame_id; + sample.frame_timestamp_ns = frame.user_timestamp_us * 1000; + std::vector record; + video_stage::avs1::encode(sample, frame.data, frame.size, record); + m_queued_bytes += record.size(); + m_queue.push_back({ std::move(record), std::chrono::steady_clock::now() }); + m_wake.notify_one(); +} + +bool DirectSender::take_keyframe_request() +{ + return m_keyframe_requested.exchange(false); +} + +bool DirectSender::connect_once( + int& fd, int slot, const std::string& host, uint16_t port, uint64_t generation, std::string& why) +{ + addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + addrinfo* found = nullptr; + if (::getaddrinfo(host.c_str(), std::to_string(port).c_str(), &hints, &found) != 0) + { + why = "cannot resolve the address"; + return false; + } + fd = -1; + for (addrinfo* a = found; a != nullptr && fd < 0 && !m_stopping; a = a->ai_next) + { + const int s = ::socket(a->ai_family, a->ai_socktype, a->ai_protocol); + if (s < 0) + continue; + ::fcntl(s, F_SETFL, ::fcntl(s, F_GETFL, 0) | O_NONBLOCK); + // Set before connect so the handshake itself is bounded, and publish so the destructor can cancel it. + const unsigned user_timeout_ms = 5000; + ::setsockopt(s, IPPROTO_TCP, TCP_USER_TIMEOUT, &user_timeout_ms, sizeof(user_timeout_ms)); + { + std::lock_guard lock(m_fd_mutex); + m_fds[slot] = s; + } + bool ok = false; + int rc = ::connect(s, a->ai_addr, a->ai_addrlen); + if (rc == 0) + { + ok = true; + } + else if (errno == EINPROGRESS) + { + // Poll in short slices so a stop request is noticed within 100 ms. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!m_stopping && m_target_generation.load() == generation) + { + pollfd pfd{ s, POLLOUT, 0 }; + const int n = ::poll(&pfd, 1, 100); + if (n < 0 && errno != EINTR) + { + why = std::strerror(errno); + break; + } + if (n > 0) + { + int err = 0; + socklen_t len = sizeof(err); + ::getsockopt(s, SOL_SOCKET, SO_ERROR, &err, &len); + if (err == 0) + ok = true; + else + why = std::strerror(err); + break; + } + if (std::chrono::steady_clock::now() >= deadline) + { + why = "connection timed out"; + break; + } + } + } + else + { + why = std::strerror(errno); + } + if (ok && !m_stopping && m_target_generation.load() == generation) + { + ::fcntl(s, F_SETFL, ::fcntl(s, F_GETFL, 0) & ~O_NONBLOCK); + fd = s; + break; + } + { + std::lock_guard lock(m_fd_mutex); + m_fds[slot] = -1; + ::close(s); + } + } + ::freeaddrinfo(found); + if (fd < 0) + { + if (m_stopping) + why = "stopping"; + return false; + } + // The link may be wired or Wi-Fi, so make it behave under both: send small packets at once, keep the kernel's send + // buffer small so a backlog stays in our queue (where its age is bounded) instead of hiding in the kernel, and + // notice a dead link in seconds, not minutes. + const int one = 1; + ::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + const int send_buffer = 256 * 1024; + ::setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &send_buffer, sizeof(send_buffer)); + ::setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(one)); + const int idle = 5, interval = 2, count = 3; + ::setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle)); + ::setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &interval, sizeof(interval)); + ::setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &count, sizeof(count)); + return true; +} + +// The control connection carries only small messages from the PC: a keyframe request ('K') and clock-exchange +// requests ('Q', answered with 'R'). This thread blocks in recv so t2 is stamped when a request actually arrives, not +// when the record sender next looks at the socket; it ends when the connection does. +void DirectSender::control_loop(int fd) +{ + m_control_ended = false; + run_control(fd); + m_control_ended = true; +} + +void DirectSender::run_control(int fd) +{ + while (true) + { + uint8_t type = 0; + if (!recv_exact(fd, &type, 1)) + return; + const int64_t t2 = monotonic_ns(); + if (type == 'K') + { + m_keyframe_requested = true; + } + else if (type == 'Q') + { + std::vector request(kClockSyncRequestSize); + request[0] = type; + if (!recv_exact(fd, request.data() + 1, request.size() - 1)) + return; + const auto parsed = parse_clock_sync_request(request); + if (!parsed) + continue; + std::vector reply = build_clock_sync_reply(*parsed, t2, 0); + set_clock_sync_reply_t3(reply, monotonic_ns()); + if (!send_all(fd, reply.data(), reply.size())) + return; + } + // Anything else is ignored: the PC sends nothing more. + } +} + +void DirectSender::run() +{ + int data_fd = -1; + int control_fd = -1; + bool logged_failure = false; + std::chrono::steady_clock::time_point last_failure_log; + std::thread control; + uint64_t connected_generation = 0; + const auto close_connection = [&] + { + if (control_fd >= 0) + ::shutdown(control_fd, SHUT_RDWR); // unblocks the control thread + if (control.joinable()) + control.join(); + // Close under the lock the destructor shuts down under, so it never touches a closed or reused descriptor. + std::lock_guard lock(m_fd_mutex); + if (data_fd >= 0) + ::close(data_fd); + if (control_fd >= 0) + ::close(control_fd); + data_fd = control_fd = -1; + m_fds[0] = m_fds[1] = -1; + m_connected = false; + }; + while (true) + { + { + std::unique_lock lock(m_mutex); + if (m_stop) + break; + } + std::string host; + uint16_t port = 0; + uint64_t generation = 0; + current_target(host, port, generation); + if (data_fd >= 0 && generation != connected_generation) + close_connection(); // retargeted: leave the old destination + if (host.empty()) + { + // No destination: idle until one is set. + std::unique_lock lock(m_mutex); + m_wake.wait_for(lock, std::chrono::milliseconds(200), [this] { return m_stop; }); + continue; + } + if (data_fd < 0) + { + std::string why = "the hello could not be sent"; + const bool up = connect_once(data_fd, 0, host, port, generation, why) && + send_all(data_fd, reinterpret_cast("DATA"), 4) && + connect_once(control_fd, 1, host, port, generation, why) && + send_all(control_fd, reinterpret_cast("CTRL"), 4); + if (!up) + { + if (m_stopping) + { + close_connection(); + break; + } + if (m_target_generation.load() != generation) + { + close_connection(); // retargeted while connecting: the failure belongs to the old destination + continue; + } + // Say so on the first failure and then every 30 s: a producer that cannot reach the PC is otherwise + // indistinguishable from one that never tried. + const auto now = std::chrono::steady_clock::now(); + if (!logged_failure || now - last_failure_log >= std::chrono::seconds(30)) + { + logger()->warn("cannot connect to {}:{}: {}; retrying every second", host, port, why); + logged_failure = true; + last_failure_log = now; + } + set_error("cannot connect to " + host + ":" + std::to_string(port) + ": " + why); + close_connection(); + std::unique_lock lock(m_mutex); + m_wake.wait_for(lock, std::chrono::seconds(1), [this] { return m_stop; }); + continue; + } + logged_failure = false; + connected_generation = generation; + m_connected = true; + set_error({}); + logger()->info("connected to {}:{}", host, port); + control = std::thread(&DirectSender::control_loop, this, control_fd); + // The PC needs a keyframe first: drop anything queued before the connection and ask for one. + std::lock_guard lock(m_mutex); + m_queue.clear(); + m_queued_bytes = 0; + m_need_keyframe = true; + m_keyframe_requested = true; + } + std::vector record; + { + std::unique_lock lock(m_mutex); + // Wake periodically so a control connection that ended is noticed even when no frames are arriving. + m_wake.wait_for(lock, std::chrono::milliseconds(200), [this] { return m_stop || !m_queue.empty(); }); + if (m_stop) + break; + if (m_queue.empty()) + { + if (!control_alive()) + { + // Only if this is still the destination: a retarget cuts the old connection on purpose. + if (m_target_generation.load() == generation) + set_error(host + ":" + std::to_string(port) + " closed the control connection"); + logger()->warn("{}:{} closed the control connection; reconnecting", host, port); + lock.unlock(); + close_connection(); + } + continue; + } + if (std::chrono::steady_clock::now() - m_queue.front().enqueued > m_config.max_queue_age) + { + // The link could not keep up: what waits is already late, so drop it rather than send it. + flush_stale_locked(); + continue; + } + record = std::move(m_queue.front().bytes); + m_queue.pop_front(); + m_queued_bytes -= record.size(); + } + if (send_all(data_fd, record.data(), record.size())) + { + ++m_sent; + continue; + } + if (m_target_generation.load() == generation) + { + logger()->warn("send to {}:{} failed; reconnecting", host, port); + set_error("sending to " + host + ":" + std::to_string(port) + " failed"); + } + close_connection(); + } + close_connection(); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/video_stage/sender/inc/video_stage/direct_sender.hpp b/src/video_stage/sender/inc/video_stage/direct_sender.hpp new file mode 100644 index 0000000000..339083c7f7 --- /dev/null +++ b/src/video_stage/sender/inc/video_stage/direct_sender.hpp @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +using video_stage::EncodedVideoFrame; + +struct DirectSenderConfig +{ + std::string host; + uint16_t port = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t fps = 30; + /// Encoded bytes allowed to wait for the network. A frame that does not fit is dropped, and so is everything after + /// it up to the next keyframe, because those frames reference it. + std::size_t max_queued_bytes = 16U << 20; + /// A unit that has waited this long for the network is stale. On a slow or lossy link (Wi-Fi, a congested switch) a + /// backlog is worse than a gap: when the oldest queued unit is older than this, the whole queue is dropped and the + /// stream resumes at the next keyframe, so the PC never receives video that is already late. + std::chrono::milliseconds max_queue_age{ 250 }; +}; + +/** + * @brief Sends each encoded access unit to the PC over TCP as an AVS1 record, beside the LiveKit publication. + * + * submit() never blocks the encoder thread: it copies the unit into a bounded queue that a sender thread drains. The + * connection starts at a keyframe. When the link is down or the queue overflowed, units are discarded until the next + * keyframe and a keyframe is requested; take_keyframe_request() hands that request to the encoder's owner. The PC + * connects nothing back: this side connects and reconnects, so the worker can be restarted freely. + */ +class DirectSender +{ +public: + explicit DirectSender(const DirectSenderConfig& config); + ~DirectSender(); + + DirectSender(const DirectSender&) = delete; + DirectSender& operator=(const DirectSender&) = delete; + + /// Offer one unit. `frame.user_timestamp_us` and `frame.frame_id` travel as the record's stamp and frame number. + void submit(const EncodedVideoFrame& frame); + + /// True once after a (re)connection or a drop that needs a fresh keyframe. + bool take_keyframe_request(); + + /// Send to host:port from now on (empty host stops sending). Whatever is queued is discarded, the current + /// connection is cut, and the new destination starts at a keyframe. Safe to call from any thread. + void set_target(const std::string& host, std::uint16_t port); + std::string host() const; + /// Why the link is not up (the last connect or send failure), empty while it is up or not yet tried. + std::string last_error() const; + std::uint16_t port() const; + /// Both connections to the current destination are up. + bool connected() const + { + return m_connected.load(); + } + + std::uint64_t sent() const + { + return m_sent.load(); + } + std::uint64_t dropped() const + { + return m_dropped.load(); + } + +private: + void run(); + void control_loop(int fd); + void run_control(int fd); + /// Connects without blocking past a stop request; \p slot (0 data, 1 control) publishes the fd to the destructor. + bool connect_once( + int& fd, int slot, const std::string& host, std::uint16_t port, std::uint64_t generation, std::string& why); + void set_error(const std::string& error); + void current_target(std::string& host, std::uint16_t& port, std::uint64_t& generation) const; + /// Drops every queued unit and waits for a keyframe, asking the encoder for one. Caller holds m_mutex. + void flush_stale_locked(); + bool control_alive() const + { + return !m_control_ended.load(); + } + + DirectSenderConfig m_config; + mutable std::mutex m_target_mutex; + std::string m_host; ///< guarded by m_target_mutex + std::uint16_t m_port = 0; ///< guarded by m_target_mutex + std::atomic m_has_target{ false }; + std::atomic m_target_generation{ 0 }; + std::atomic m_connected{ false }; + mutable std::mutex m_error_mutex; + std::string m_last_error; ///< guarded by m_error_mutex + std::thread m_thread; + std::mutex m_mutex; + std::condition_variable m_wake; + struct Queued + { + std::vector bytes; + std::chrono::steady_clock::time_point enqueued; + }; + std::deque m_queue; + std::size_t m_queued_bytes = 0; + bool m_need_keyframe = true; ///< guarded by m_mutex + bool m_stop = false; ///< guarded by m_mutex + std::uint64_t m_sequence = 0; + std::atomic m_stopping{ false }; + std::mutex m_fd_mutex; + int m_fds[2] = { -1, -1 }; ///< live sockets, guarded by m_fd_mutex + std::atomic m_keyframe_requested{ false }; + std::atomic m_control_ended{ false }; + std::atomic m_sent{ 0 }; + std::atomic m_dropped{ 0 }; +}; + +} // namespace sensing +} // namespace plugins diff --git a/src/video_stage/sender/tests/CMakeLists.txt b/src/video_stage/sender/tests/CMakeLists.txt new file mode 100644 index 0000000000..4435988773 --- /dev/null +++ b/src/video_stage/sender/tests/CMakeLists.txt @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(video_stage_sender_tests + test_direct_sender.cpp + test_direct_sender_target.cpp +) + +target_link_libraries(video_stage_sender_tests PRIVATE video_stage::sender Catch2::Catch2WithMain) + +catch_discover_tests(video_stage_sender_tests) diff --git a/src/video_stage/sender/tests/test_direct_sender.cpp b/src/video_stage/sender/tests/test_direct_sender.cpp new file mode 100644 index 0000000000..6fd07ced39 --- /dev/null +++ b/src/video_stage/sender/tests/test_direct_sender.cpp @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Destroying a DirectSender must be bounded whatever its worker thread is blocked in: a connect that never +// completes, a send to a peer that does not read, or an idle connection. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using plugins::sensing::DirectSender; +using plugins::sensing::DirectSenderConfig; +using plugins::sensing::EncodedVideoFrame; + +namespace +{ + +using Clock = std::chrono::steady_clock; +constexpr auto kBound = std::chrono::seconds(2); + +class Listener +{ +public: + explicit Listener(int backlog) + { + m_fd = ::socket(AF_INET, SOCK_STREAM, 0); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + REQUIRE(::bind(m_fd, reinterpret_cast(&addr), sizeof(addr)) == 0); + REQUIRE(::listen(m_fd, backlog) == 0); + socklen_t len = sizeof(addr); + ::getsockname(m_fd, reinterpret_cast(&addr), &len); + m_port = ntohs(addr.sin_port); + } + + ~Listener() + { + ::close(m_fd); + for (const int fd : m_accepted) + ::close(fd); + } + + uint16_t port() const + { + return m_port; + } + + /// Accepts \p count connections and reads their 4-byte hellos, then never reads again. + bool accept_and_stall(int count) + { + for (int i = 0; i < count; ++i) + { + pollfd pfd{ m_fd, POLLIN, 0 }; + if (::poll(&pfd, 1, 5000) <= 0) + return false; + const int c = ::accept(m_fd, nullptr, nullptr); + if (c < 0) + return false; + m_accepted.push_back(c); + char hello[4]; + if (::recv(c, hello, sizeof(hello), MSG_WAITALL) != 4) + return false; + } + return true; + } + + /// Fills the accept queue with half-open clients so later SYNs are dropped and a connect hangs. + void saturate(std::vector& clients) + { + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = htons(m_port); + for (int i = 0; i < 8; ++i) + { + const int c = ::socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); + ::connect(c, reinterpret_cast(&addr), sizeof(addr)); + clients.push_back(c); + } + } + +private: + int m_fd = -1; + uint16_t m_port = 0; + std::vector m_accepted; +}; + +DirectSenderConfig config_for(uint16_t port) +{ + DirectSenderConfig config; + config.host = "127.0.0.1"; + config.port = port; + config.width = 64; + config.height = 48; + config.max_queued_bytes = 512U << 20; + config.max_queue_age = std::chrono::hours(1); + return config; +} + +std::chrono::milliseconds time_destruction(std::unique_ptr sender) +{ + const auto start = Clock::now(); + sender.reset(); + return std::chrono::duration_cast(Clock::now() - start); +} + +} // namespace + +TEST_CASE("destroying a sender whose connect never completes is bounded", "[direct_sender]") +{ + Listener listener(0); + std::vector clients; + listener.saturate(clients); + + auto sender = std::make_unique(config_for(listener.port())); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); // let the worker reach its connect + + CHECK(time_destruction(std::move(sender)) < kBound); + for (const int fd : clients) + ::close(fd); +} + +TEST_CASE("destroying a sender blocked in send to a peer that does not read is bounded", "[direct_sender]") +{ + Listener listener(8); + auto sender = std::make_unique(config_for(listener.port())); + REQUIRE(listener.accept_and_stall(2)); + + // Far more than the socket buffers hold, so the worker ends up blocked in send(). + const std::vector unit(1U << 20, 0x65); + constexpr int kUnits = 100; + for (int i = 0; i < kUnits; ++i) + { + EncodedVideoFrame frame; + frame.data = unit.data(); + frame.size = unit.size(); + frame.keyframe = (i == 0); + frame.frame_id = static_cast(i); + sender->submit(frame); + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + REQUIRE(sender->sent() < kUnits); + + CHECK(time_destruction(std::move(sender)) < kBound); +} + +TEST_CASE("destroying a connected, idle sender is bounded", "[direct_sender]") +{ + Listener listener(8); + auto sender = std::make_unique(config_for(listener.port())); + REQUIRE(listener.accept_and_stall(2)); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + CHECK(time_destruction(std::move(sender)) < kBound); +} diff --git a/src/video_stage/sender/tests/test_direct_sender_target.cpp b/src/video_stage/sender/tests/test_direct_sender_target.cpp new file mode 100644 index 0000000000..876584bac6 --- /dev/null +++ b/src/video_stage/sender/tests/test_direct_sender_target.cpp @@ -0,0 +1,548 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Retargeting a DirectSender at run time: the connection moves, the new destination starts at a keyframe, an empty +// host idles, and none of it can hang the caller or the destructor. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using plugins::sensing::DirectSender; +using plugins::sensing::DirectSenderConfig; +using plugins::sensing::EncodedVideoFrame; +using namespace std::chrono_literals; + +namespace +{ + +using Clock = std::chrono::steady_clock; + +/// A loopback receiver that accepts the sender's DATA and CTRL connections and reads AVS1 records from DATA. +class Peer +{ +public: + explicit Peer(int backlog = 8) + { + m_listen = ::socket(AF_INET, SOCK_STREAM, 0); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + REQUIRE(::bind(m_listen, reinterpret_cast(&addr), sizeof(addr)) == 0); + REQUIRE(::listen(m_listen, backlog) == 0); + socklen_t len = sizeof(addr); + ::getsockname(m_listen, reinterpret_cast(&addr), &len); + m_port = ntohs(addr.sin_port); + } + + ~Peer() + { + ::close(m_listen); + close_connections(); + } + + std::uint16_t port() const + { + return m_port; + } + + /// Accepts the DATA and CTRL connections; true when both hellos arrived. + bool accept_pair(int timeout_ms = 5000) + { + for (int i = 0; i < 2; ++i) + { + pollfd pfd{ m_listen, POLLIN, 0 }; + if (::poll(&pfd, 1, timeout_ms) <= 0) + return false; + const int fd = ::accept(m_listen, nullptr, nullptr); + if (fd < 0) + return false; + char hello[4]; + if (!read_exact(fd, reinterpret_cast(hello), 4, timeout_ms)) + { + ::close(fd); + return false; + } + const std::string tag(hello, 4); + if (tag == "DATA") + m_data = fd; + else if (tag == "CTRL") + m_control = fd; + else + { + ::close(fd); + return false; + } + } + return m_data >= 0 && m_control >= 0; + } + + /// One AVS1 record's bytes after the length prefix; nullopt on EOF or timeout. + std::optional> read_record(int timeout_ms = 5000) + { + std::uint8_t prefix[4]; + if (m_data < 0 || !read_exact(m_data, prefix, 4, timeout_ms)) + return std::nullopt; + std::vector body(video_stage::avs1::read_length(prefix)); + if (!read_exact(m_data, body.data(), body.size(), timeout_ms)) + return std::nullopt; + return body; + } + + /// True if the DATA connection was closed by the sender within the timeout (records still in flight are drained). + bool data_closed(int timeout_ms = 5000) + { + const auto until = Clock::now() + std::chrono::milliseconds(timeout_ms); + while (Clock::now() < until) + { + pollfd pfd{ m_data, POLLIN, 0 }; + if (::poll(&pfd, 1, 100) > 0) + { + std::uint8_t buffer[65536]; + const ssize_t n = ::recv(m_data, buffer, sizeof(buffer), 0); + if (n <= 0) + return true; + } + } + return false; + } + + /// Whether any new connection arrives within the timeout. + bool any_connection(int timeout_ms) + { + pollfd pfd{ m_listen, POLLIN, 0 }; + return ::poll(&pfd, 1, timeout_ms) > 0; + } + + /// The first record of the DATA stream must carry an IDR: the PC cannot start at a delta frame. + bool first_record_is_keyframe() + { + const auto body = read_record(); + if (!body) + return false; + std::string error; + const auto record = video_stage::avs1::parse(body->data(), body->size(), error); + return record && video_stage::avs1::contains_idr(record->payload, record->payload_size); + } + + void close_connections() + { + for (int* fd : { &m_data, &m_control }) + { + if (*fd >= 0) + ::close(*fd); + *fd = -1; + } + } + + /// Fills the accept queue with half-open clients so later SYNs are dropped and a connect hangs. + void saturate(std::vector& clients) + { + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = htons(m_port); + for (int i = 0; i < 8; ++i) + { + const int c = ::socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); + ::connect(c, reinterpret_cast(&addr), sizeof(addr)); + clients.push_back(c); + } + } + +private: + static bool read_exact(int fd, std::uint8_t* out, std::size_t size, int timeout_ms) + { + std::size_t done = 0; + while (done < size) + { + pollfd pfd{ fd, POLLIN, 0 }; + if (::poll(&pfd, 1, timeout_ms) <= 0) + return false; + const ssize_t n = ::recv(fd, out + done, size - done, 0); + if (n <= 0) + return false; + done += static_cast(n); + } + return true; + } + + int m_listen = -1; + int m_data = -1; + int m_control = -1; + std::uint16_t m_port = 0; +}; + +/// Plays the encoder: a unit every few milliseconds, a keyframe every tenth and whenever the sender asks. +class Feeder +{ +public: + Feeder(DirectSender& sender, std::size_t unit_bytes = 2000) : m_sender(sender), m_unit_bytes(unit_bytes) + { + m_thread = std::thread( + [this] + { + std::uint32_t id = 0; + bool force_key = true; + while (!m_stop) + { + force_key |= m_sender.take_keyframe_request(); + const bool key = force_key || (id % 10 == 0); + force_key = false; + std::vector unit = { 0, 0, 0, 1, static_cast(key ? 0x65 : 0x41), 0x88 }; + unit.resize(m_unit_bytes, 0x5a); + EncodedVideoFrame frame; + frame.data = unit.data(); + frame.size = unit.size(); + frame.keyframe = key; + frame.frame_id = ++id; + frame.user_timestamp_us = id * 1000ULL; + m_sender.submit(frame); + std::this_thread::sleep_for(5ms); + } + }); + } + ~Feeder() + { + m_stop = true; + m_thread.join(); + } + +private: + DirectSender& m_sender; + std::size_t m_unit_bytes; + std::atomic m_stop{ false }; + std::thread m_thread; +}; + +DirectSenderConfig config_for(const std::string& host, std::uint16_t port) +{ + DirectSenderConfig config; + config.host = host; + config.port = port; + config.width = 64; + config.height = 48; + config.max_queued_bytes = 512U << 20; + config.max_queue_age = std::chrono::hours(1); + return config; +} + +template +bool wait_for(Pred pred, std::chrono::milliseconds limit = 5s) +{ + const auto until = Clock::now() + limit; + while (Clock::now() < until) + { + if (pred()) + return true; + std::this_thread::sleep_for(5ms); + } + return pred(); +} + +} // namespace + +TEST_CASE("the configured target is reported by host() and port()", "[direct_sender_target]") +{ + Peer a; + DirectSender sender(config_for("127.0.0.1", a.port())); + CHECK(sender.host() == "127.0.0.1"); + CHECK(sender.port() == a.port()); + REQUIRE(a.accept_pair()); + CHECK(wait_for([&] { return sender.connected(); })); +} + +TEST_CASE("retargeting moves the connection and the new peer starts at a keyframe", "[direct_sender_target]") +{ + Peer a; + Peer b; + DirectSender sender(config_for("127.0.0.1", a.port())); + Feeder feeder(sender); + + REQUIRE(a.accept_pair()); + CHECK(a.first_record_is_keyframe()); + REQUIRE(wait_for([&] { return sender.connected(); })); + + sender.set_target("127.0.0.1", b.port()); + CHECK(sender.host() == "127.0.0.1"); + CHECK(sender.port() == b.port()); + + REQUIRE(b.accept_pair()); + CHECK(b.first_record_is_keyframe()); + CHECK(a.data_closed()); // the old destination was left + REQUIRE(wait_for([&] { return sender.connected(); })); + + // The new peer keeps receiving, and the old one gets nothing more and no reconnect. + for (int i = 0; i < 5; ++i) + CHECK(b.read_record()); + CHECK_FALSE(a.any_connection(300)); +} + +TEST_CASE("setting the same peer again reconnects it from a keyframe", "[direct_sender_target]") +{ + Peer a; + DirectSender sender(config_for("127.0.0.1", a.port())); + Feeder feeder(sender); + REQUIRE(a.accept_pair()); + REQUIRE(a.first_record_is_keyframe()); + + sender.set_target("127.0.0.1", a.port()); + REQUIRE(a.data_closed()); + a.close_connections(); + REQUIRE(a.accept_pair()); + CHECK(a.first_record_is_keyframe()); +} + +TEST_CASE("an empty host idles the sender", "[direct_sender_target]") +{ + Peer a; + DirectSender sender(config_for("127.0.0.1", a.port())); + Feeder feeder(sender); + REQUIRE(a.accept_pair()); + REQUIRE(wait_for([&] { return sender.connected(); })); + + sender.set_target("", 1234); + CHECK(sender.host().empty()); + CHECK(sender.port() == 0); + CHECK(wait_for([&] { return !sender.connected(); })); + CHECK(a.data_closed()); + + // Idle: no reconnect, and submitted units are not queued or counted as sent. + CHECK_FALSE(a.any_connection(500)); + const auto sent = sender.sent(); + std::this_thread::sleep_for(200ms); + CHECK(sender.sent() == sent); + CHECK_FALSE(sender.connected()); + + // A target set later wakes it again. + Peer b; + sender.set_target("127.0.0.1", b.port()); + REQUIRE(b.accept_pair()); + CHECK(b.first_record_is_keyframe()); + CHECK(wait_for([&] { return sender.connected(); })); +} + +TEST_CASE("a sender created without a target idles until one is set", "[direct_sender_target]") +{ + DirectSender sender(config_for("", 0)); + Feeder feeder(sender); + std::this_thread::sleep_for(300ms); + CHECK_FALSE(sender.connected()); + CHECK(sender.sent() == 0); + CHECK(sender.host().empty()); + + Peer a; + sender.set_target("127.0.0.1", a.port()); + REQUIRE(a.accept_pair()); + CHECK(a.first_record_is_keyframe()); +} + +TEST_CASE("set_target while a send is blocked does not hang and the new peer gets video", "[direct_sender_target]") +{ + Peer stalled; + Peer b; + DirectSender sender(config_for("127.0.0.1", stalled.port())); + REQUIRE(stalled.accept_pair()); + + // Units far larger than the socket buffers, to a peer that never reads: the worker ends up blocked in send(). + const std::vector unit = [] + { + std::vector bytes(1U << 20, 0x5a); + bytes[0] = 0; + bytes[1] = 0; + bytes[2] = 0; + bytes[3] = 1; + bytes[4] = 0x65; + return bytes; + }(); + for (int i = 0; i < 100; ++i) + { + EncodedVideoFrame frame; + frame.data = unit.data(); + frame.size = unit.size(); + frame.keyframe = (i == 0); + frame.frame_id = static_cast(i); + sender.submit(frame); + } + std::this_thread::sleep_for(500ms); + REQUIRE(sender.sent() < 100); + + const auto start = Clock::now(); + sender.set_target("127.0.0.1", b.port()); + CHECK(Clock::now() - start < 1s); + + Feeder feeder(sender); + REQUIRE(b.accept_pair()); + CHECK(b.first_record_is_keyframe()); +} + +TEST_CASE("destruction is bounded after a retarget", "[direct_sender_target]") +{ + const auto time_destruction = [](std::unique_ptr sender) + { + const auto start = Clock::now(); + sender.reset(); + return Clock::now() - start; + }; + + SECTION("to a peer that is connected") + { + Peer a; + Peer b; + auto sender = std::make_unique(config_for("127.0.0.1", a.port())); + REQUIRE(a.accept_pair()); + sender->set_target("127.0.0.1", b.port()); + REQUIRE(b.accept_pair()); + std::this_thread::sleep_for(200ms); + CHECK(time_destruction(std::move(sender)) < 2s); + } + SECTION("to a peer whose connect never completes") + { + Peer a; + Peer hung(0); + std::vector clients; + hung.saturate(clients); + auto sender = std::make_unique(config_for("127.0.0.1", a.port())); + REQUIRE(a.accept_pair()); + sender->set_target("127.0.0.1", hung.port()); + std::this_thread::sleep_for(500ms); // let the worker reach its connect + CHECK(time_destruction(std::move(sender)) < 2s); + for (const int fd : clients) + ::close(fd); + } + SECTION("while idle after an empty host") + { + Peer a; + auto sender = std::make_unique(config_for("127.0.0.1", a.port())); + REQUIRE(a.accept_pair()); + sender->set_target("", 0); + std::this_thread::sleep_for(300ms); + CHECK(time_destruction(std::move(sender)) < 2s); + } +} + +TEST_CASE("retargeting away from a connect that never completes is prompt", "[direct_sender_target]") +{ + Peer hung(0); + std::vector clients; + hung.saturate(clients); + Peer b; + DirectSender sender(config_for("127.0.0.1", hung.port())); + Feeder feeder(sender); + std::this_thread::sleep_for(500ms); + + sender.set_target("127.0.0.1", b.port()); + REQUIRE(b.accept_pair(3000)); // well inside the 5 s connect timeout + CHECK(b.first_record_is_keyframe()); + for (const int fd : clients) + ::close(fd); +} + +namespace +{ + +/// A loopback port nothing listens on: bound once to learn a free number, then released. +std::uint16_t closed_port() +{ + const int s = ::socket(AF_INET, SOCK_STREAM, 0); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + REQUIRE(::bind(s, reinterpret_cast(&addr), sizeof(addr)) == 0); + socklen_t len = sizeof(addr); + ::getsockname(s, reinterpret_cast(&addr), &len); + ::close(s); + return ntohs(addr.sin_port); +} + +bool starts_with(const std::string& text, const std::string& prefix) +{ + return text.compare(0, prefix.size(), prefix) == 0; +} + +} // namespace + +TEST_CASE("a connect to a closed port is reported by last_error", "[direct_sender_target][last_error]") +{ + const auto port = closed_port(); + DirectSender sender(config_for("127.0.0.1", port)); + REQUIRE(wait_for([&] { return !sender.last_error().empty(); })); + const auto error = sender.last_error(); + CHECK(starts_with(error, "cannot connect to 127.0.0.1:" + std::to_string(port) + ": ")); + CHECK_FALSE(sender.connected()); +} + +TEST_CASE("a successful connection leaves last_error empty", "[direct_sender_target][last_error]") +{ + Peer a; + DirectSender sender(config_for("127.0.0.1", a.port())); + REQUIRE(a.accept_pair()); + REQUIRE(wait_for([&] { return sender.connected(); })); + CHECK(sender.last_error().empty()); +} + +TEST_CASE("set_target to a reachable peer clears a connect error", "[direct_sender_target][last_error]") +{ + Peer a; + DirectSender sender(config_for("127.0.0.1", closed_port())); + REQUIRE(wait_for([&] { return !sender.last_error().empty(); })); + + sender.set_target("127.0.0.1", a.port()); + REQUIRE(a.accept_pair()); + REQUIRE(wait_for([&] { return sender.connected(); })); + CHECK(sender.last_error().empty()); +} + +TEST_CASE("set_target to an empty host clears last_error and it stays clear", "[direct_sender_target][last_error]") +{ + DirectSender sender(config_for("127.0.0.1", closed_port())); + REQUIRE(wait_for([&] { return !sender.last_error().empty(); })); + + sender.set_target("", 0); + CHECK(sender.last_error().empty()); + std::this_thread::sleep_for(300ms); + CHECK(sender.last_error().empty()); +} + +TEST_CASE("set_target to another closed port reports the new destination", "[direct_sender_target][last_error]") +{ + DirectSender sender(config_for("127.0.0.1", closed_port())); + REQUIRE(wait_for([&] { return !sender.last_error().empty(); })); + + const auto second = closed_port(); + sender.set_target("127.0.0.1", second); + REQUIRE(wait_for( + [&] { return starts_with(sender.last_error(), "cannot connect to 127.0.0.1:" + std::to_string(second)); })); +} + +TEST_CASE("a retarget away from a peer that then closes leaves no stale control error", + "[direct_sender_target][last_error]") +{ + Peer a; + Peer b; + DirectSender sender(config_for("127.0.0.1", a.port())); + REQUIRE(a.accept_pair()); + REQUIRE(wait_for([&] { return sender.connected(); })); + + sender.set_target("127.0.0.1", b.port()); + a.close_connections(); + REQUIRE(b.accept_pair()); + REQUIRE(wait_for([&] { return sender.connected(); })); + std::this_thread::sleep_for(500ms); + CHECK(sender.last_error().find("closed the control connection") == std::string::npos); + CHECK(sender.last_error().empty()); +} diff --git a/src/video_stage/wire/CMakeLists.txt b/src/video_stage/wire/CMakeLists.txt new file mode 100644 index 0000000000..4925e1aad6 --- /dev/null +++ b/src/video_stage/wire/CMakeLists.txt @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The AVS1 record codec: plain C++, no CUDA, so the Orin-side sender can link it too. +add_library(video_stage_wire STATIC + avs1.cpp + h264_sps.cpp + h264_access_units.cpp +) +add_library(video_stage::wire ALIAS video_stage_wire) + +set_target_properties(video_stage_wire PROPERTIES POSITION_INDEPENDENT_CODE ON) + +target_include_directories(video_stage_wire + INTERFACE + $ +) diff --git a/src/video_stage/wire/avs1.cpp b/src/video_stage/wire/avs1.cpp new file mode 100644 index 0000000000..90e899c8a0 --- /dev/null +++ b/src/video_stage/wire/avs1.cpp @@ -0,0 +1,205 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/video_stage/avs1.hpp" + +#include + +namespace video_stage +{ +namespace avs1 +{ + +namespace +{ + +constexpr uint16_t kStreamPresent = 0x0001; +constexpr uint16_t kSamplePresent = 0x0001; +constexpr uint16_t kSampleFrameTimestampValid = 0x0004; +constexpr uint16_t kSampleKnownFlags = kSamplePresent | 0x0002 | kSampleFrameTimestampValid | 0x0008; + +void put16(std::vector& out, uint16_t v) +{ + out.push_back(static_cast(v >> 8)); + out.push_back(static_cast(v)); +} + +void put32(std::vector& out, uint32_t v) +{ + for (int shift = 24; shift >= 0; shift -= 8) + out.push_back(static_cast(v >> shift)); +} + +void put64(std::vector& out, uint64_t v) +{ + for (int shift = 56; shift >= 0; shift -= 8) + out.push_back(static_cast(v >> shift)); +} + +uint16_t get16(const uint8_t* p) +{ + return static_cast((p[0] << 8) | p[1]); +} + +uint32_t get32(const uint8_t* p) +{ + return (static_cast(p[0]) << 24) | (static_cast(p[1]) << 16) | + (static_cast(p[2]) << 8) | p[3]; +} + +uint64_t get64(const uint8_t* p) +{ + return (static_cast(get32(p)) << 32) | get32(p + 4); +} + +std::optional fail(std::string& error, const char* why) +{ + error = why; + return std::nullopt; +} + +} // namespace + +void encode(const Sample& s, const uint8_t* payload, size_t payload_size, std::vector& out) +{ + const uint32_t descriptor_bytes = kStreamBytes + kSampleBytes; + const uint32_t record_length = kHeaderBytes + descriptor_bytes + static_cast(payload_size); + out.reserve(out.size() + 4 + record_length); + put32(out, record_length); + + // Bundle header (64 bytes). + out.insert(out.end(), { 'A', 'V', 'S', '1' }); + put16(out, kVersion); + put16(out, 0); // bundle flags: no sender wall-clock fields + put32(out, kHeaderBytes); + put16(out, 1); // stream_count + put16(out, 1); // sample_count + put64(out, s.bundle_sequence); + put64(out, s.session_id); + put64(out, 0); // sender_capture_ready_ns + put64(out, 0); // sender_tcp_enqueue_ns + put32(out, descriptor_bytes); + put32(out, static_cast(payload_size)); + put64(out, 0); // sender_capture_ready_tsc_ns + + // Stream descriptor (40 bytes). + put16(out, 0); // stream_index + put16(out, kStreamPresent); + out.push_back(static_cast(s.codec)); + out.push_back(1); // pixel_format NV12 + put16(out, 0); // rotation + put32(out, s.width); + put32(out, s.height); + put32(out, s.fps_numerator); + put32(out, 1); // fps_denominator + put64(out, s.bundle_sequence); // stream_sequence + put16(out, 0); // first_sample + put16(out, 1); // sample_count + put32(out, 0); // reserved + + // Sample descriptor (72 bytes). + put16(out, 0); // stream_index + out.push_back(0); // camera_index + out.push_back(0); // role: mono + put16(out, kSamplePresent | kSampleFrameTimestampValid); + put16(out, 0); // reserved + put32(out, 0); // device_index + put32(out, 0); // sensor_mode_index + put64(out, s.frame_number); + put64(out, 0); // sof_tsc_ns + put64(out, s.frame_timestamp_ns); + put64(out, 0); // sender_capture_ready_ns + put64(out, 0); // sender_process_submit_ns + put64(out, 0); // sender_process_done_ns + put32(out, 0); // payload_offset + put32(out, static_cast(payload_size)); + + out.insert(out.end(), payload, payload + payload_size); +} + +uint32_t read_length(const uint8_t prefix[4]) +{ + return get32(prefix); +} + +std::optional parse(const uint8_t* data, size_t size, std::string& error) +{ + constexpr size_t kFixed = kHeaderBytes + kStreamBytes + kSampleBytes; + if (size < kFixed) + return fail(error, "record shorter than its descriptors"); + if (std::memcmp(data, "AVS1", 4) != 0) + return fail(error, "bad magic"); + if (get16(data + 4) != kVersion) + return fail(error, "unsupported version"); + if (get16(data + 6) != 0) + return fail(error, "unknown bundle flags"); + if (get32(data + 8) != kHeaderBytes) + return fail(error, "bad header size"); + if (get16(data + 12) != 1 || get16(data + 14) != 1) + return fail(error, "this stage takes one stream with one sample"); + const uint32_t descriptor_bytes = get32(data + 48); + const uint32_t payload_bytes = get32(data + 52); + if (descriptor_bytes != kStreamBytes + kSampleBytes) + return fail(error, "bad descriptor size"); + if (size != kFixed + payload_bytes) + return fail(error, "payload size does not match the record length"); + if (get64(data + 32) != 0 || get64(data + 40) != 0 || get64(data + 56) != 0) + return fail(error, "sender wall-clock fields must be zero without their flags"); + + const uint8_t* stream = data + kHeaderBytes; + if (get16(stream + 2) != kStreamPresent) + return fail(error, "stream not present or unknown stream flags"); + if (stream[4] != static_cast(Codec::H264)) + return fail(error, "codec is not H.264"); + if (stream[5] != 1) + return fail(error, "pixel format is not NV12"); + if (get16(stream + 6) != 0) + return fail(error, "rotation not supported"); + if (get32(stream + 20) == 0) + return fail(error, "zero fps denominator"); + if (get16(stream + 32) != 0 || get16(stream + 34) != 1 || get32(stream + 36) != 0) + return fail(error, "bad sample indexing"); + + const uint8_t* sample = stream + kStreamBytes; + const uint16_t flags = get16(sample + 4); + if ((flags & ~kSampleKnownFlags) != 0) + return fail(error, "unknown sample flags"); + if ((flags & kSamplePresent) == 0 || (flags & kSampleFrameTimestampValid) == 0) + return fail(error, "sample must be present with a frame timestamp"); + if (get16(sample + 6) != 0) + return fail(error, "reserved sample field is nonzero"); + const uint32_t payload_offset = get32(sample + 64); + const uint32_t payload_length = get32(sample + 68); + if (payload_length == 0 || payload_offset != 0 || payload_length != payload_bytes) + return fail(error, "payload range is empty, offset, or does not cover the payload"); + + Record record; + record.sample.codec = Codec::H264; + record.sample.width = get32(stream + 8); + record.sample.height = get32(stream + 12); + record.sample.fps_numerator = get32(stream + 16); + record.sample.bundle_sequence = get64(data + 16); + record.sample.session_id = get64(data + 24); + record.sample.frame_number = get64(sample + 16); + record.sample.frame_timestamp_ns = get64(sample + 32); + record.payload = data + kFixed; + record.payload_size = payload_length; + return record; +} + +bool contains_idr(const uint8_t* au, size_t size) +{ + for (size_t i = 0; i + 3 < size; ++i) + { + if (au[i] == 0 && au[i + 1] == 0 && au[i + 2] == 1) + { + if ((au[i + 3] & 0x1F) == 5) + return true; + i += 2; + } + } + return false; +} + +} // namespace avs1 +} // namespace video_stage diff --git a/src/video_stage/wire/h264_access_units.cpp b/src/video_stage/wire/h264_access_units.cpp new file mode 100644 index 0000000000..425ac64f45 --- /dev/null +++ b/src/video_stage/wire/h264_access_units.cpp @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/video_stage/h264_access_units.hpp" + +namespace video_stage +{ + +namespace +{ + +struct Nal +{ + std::size_t start; ///< offset of the start code (including a leading zero byte when it is a 4-byte code) + int type; + std::size_t header; ///< offset of the NAL header byte +}; + +std::vector find_nals(const std::uint8_t* data, std::size_t size) +{ + std::vector nals; + std::size_t i = 0; + while (i + 3 < size) + { + if (data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1) + { + const std::size_t start = (i > 0 && data[i - 1] == 0) ? i - 1 : i; + nals.push_back({ start, data[i + 3] & 0x1F, i + 3 }); + i += 3; + } + else + { + ++i; + } + } + return nals; +} + +/// first_mb_in_slice, the Exp-Golomb code right after the NAL header byte; large values read as nonzero. +std::size_t first_mb_in_slice(const std::uint8_t* data, std::size_t size, std::size_t header) +{ + std::uint32_t bits = 0; + for (std::size_t k = 0; k < 4; ++k) + bits = (bits << 8) | (header + 1 + k < size ? data[header + 1 + k] : 0); + int zeros = 0; + while (zeros < 15 && !((bits >> (31 - zeros)) & 1)) + ++zeros; + const std::uint32_t code = (bits >> (32 - (2 * zeros + 1))) & ((1u << (2 * zeros + 1)) - 1); + return code - 1; +} + +bool is_trailing(int type) +{ + return type == 10 || type == 11 || type == 12; +} + +} // namespace + +std::vector split_access_units(const std::uint8_t* data, std::size_t size) +{ + const std::vector nals = find_nals(data, size); + std::vector starts; + bool previous_vcl = false; + for (std::size_t n = 0; n < nals.size(); ++n) + { + const bool vcl = nals[n].type == 1 || nals[n].type == 5; + bool begin = false; + if (n == 0) + begin = true; + else if (vcl) + begin = previous_vcl && first_mb_in_slice(data, size, nals[n].header) == 0; + else + begin = previous_vcl && !is_trailing(nals[n].type); + if (begin) + starts.push_back(nals[n].start); + previous_vcl = vcl || (previous_vcl && is_trailing(nals[n].type)); + } + starts.push_back(size); + + std::vector units; + for (std::size_t u = 0; u + 1 < starts.size(); ++u) + { + AccessUnitRange unit{ starts[u], starts[u + 1] - starts[u], false }; + for (const auto& nal : nals) + if (nal.start >= unit.offset && nal.start < unit.offset + unit.size && nal.type == 5) + unit.idr = true; + units.push_back(unit); + } + return units; +} + +} // namespace video_stage diff --git a/src/video_stage/wire/h264_sps.cpp b/src/video_stage/wire/h264_sps.cpp new file mode 100644 index 0000000000..a917eff86e --- /dev/null +++ b/src/video_stage/wire/h264_sps.cpp @@ -0,0 +1,401 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/video_stage/h264_sps.hpp" + +#include +#include + +namespace video_stage +{ +namespace h264 +{ + +namespace +{ + +// RBSP bytes of a NAL: the payload with emulation-prevention bytes (00 00 03) removed. +std::vector unescape(const uint8_t* p, size_t n) +{ + std::vector out; + out.reserve(n); + int zeros = 0; + for (size_t i = 0; i < n; ++i) + { + if (zeros >= 2 && p[i] == 3) + { + zeros = 0; + continue; + } + out.push_back(p[i]); + zeros = p[i] == 0 ? zeros + 1 : 0; + } + return out; +} + +std::vector escape(const std::vector& rbsp) +{ + std::vector out; + out.reserve(rbsp.size() + 8); + int zeros = 0; + for (uint8_t b : rbsp) + { + if (zeros >= 2 && b <= 3) + { + out.push_back(3); + zeros = 0; + } + out.push_back(b); + zeros = b == 0 ? zeros + 1 : 0; + } + return out; +} + +class BitReader +{ +public: + explicit BitReader(const std::vector& d) : m_data(d) + { + } + bool ok() const + { + return !m_error; + } + size_t position() const + { + return m_pos; + } + uint32_t bits(int n) + { + uint32_t v = 0; + for (int i = 0; i < n; ++i) + v = (v << 1) | bit(); + return v; + } + uint32_t bit() + { + if (m_pos >= m_data.size() * 8) + { + m_error = true; + return 0; + } + const uint32_t b = (m_data[m_pos >> 3] >> (7 - (m_pos & 7))) & 1; + ++m_pos; + return b; + } + uint32_t ue() + { + int zeros = 0; + while (!bit() && ok() && zeros < 32) + ++zeros; + if (zeros >= 32) + { + m_error = true; + return 0; + } + return (1u << zeros) - 1 + bits(zeros); + } + int32_t se() + { + const uint32_t k = ue(); + return (k & 1) ? static_cast((k + 1) / 2) : -static_cast(k / 2); + } + +private: + const std::vector& m_data; + size_t m_pos = 0; + bool m_error = false; +}; + +class BitWriter +{ +public: + void bit(uint32_t b) + { + if ((m_bits & 7) == 0) + m_data.push_back(0); + if (b) + m_data.back() |= static_cast(1u << (7 - (m_bits & 7))); + ++m_bits; + } + void bits(uint32_t v, int n) + { + for (int i = n - 1; i >= 0; --i) + bit((v >> i) & 1); + } + void ue(uint32_t v) + { + const uint32_t x = v + 1; + int len = 0; + while ((x >> len) > 1) + ++len; + bits(0, len); + bits(x, len + 1); + } + void copy(const std::vector& src, size_t from, size_t to) + { + for (size_t i = from; i < to; ++i) + bit((src[i >> 3] >> (7 - (i & 7))) & 1); + } + std::vector& finish() + { + bit(1); // rbsp_stop_one_bit, then zero bits to a byte boundary + return m_data; + } + +private: + std::vector m_data; + size_t m_bits = 0; +}; + +void skip_scaling_list(BitReader& r, int size) +{ + int last = 8, next = 8; + for (int j = 0; j < size; ++j) + { + if (next != 0) + { + next = (last + r.se() + 256) % 256; + } + last = next == 0 ? last : next; + } +} + +void skip_hrd(BitReader& r) +{ + const uint32_t cpb_cnt = r.ue() + 1; + r.bits(4); + r.bits(4); + for (uint32_t i = 0; i < cpb_cnt && r.ok(); ++i) + { + r.ue(); + r.ue(); + r.bit(); + } + r.bits(5); + r.bits(5); + r.bits(5); + r.bits(5); +} + +struct Parsed +{ + SpsInfo info; + std::vector rbsp; // the SPS RBSP, header byte first + size_t vui_flag_pos = 0; // bit position of vui_parameters_present_flag + size_t restriction_flag_pos = 0; // bit position of bitstream_restriction_flag (valid when vui_present) +}; + +std::optional parse_rbsp(std::vector rbsp) +{ + if (rbsp.size() < 4) + return std::nullopt; + Parsed out; + out.rbsp = std::move(rbsp); + BitReader r(out.rbsp); + r.bits(8); // NAL header + SpsInfo& s = out.info; + s.profile_idc = r.bits(8); + r.bits(8); + s.level_idc = r.bits(8); + r.ue(); + const uint32_t high[] = { 100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135 }; + if (std::find(std::begin(high), std::end(high), s.profile_idc) != std::end(high)) + { + s.chroma_format_idc = r.ue(); + if (s.chroma_format_idc == 3) + r.bit(); + r.ue(); + r.ue(); + r.bit(); + if (r.bit()) + { + for (int i = 0; i < (s.chroma_format_idc != 3 ? 8 : 12) && r.ok(); ++i) + { + if (r.bit()) + skip_scaling_list(r, i < 6 ? 16 : 64); + } + } + } + r.ue(); + s.pic_order_cnt_type = r.ue(); + if (s.pic_order_cnt_type == 0) + { + r.ue(); + } + else if (s.pic_order_cnt_type == 1) + { + r.bit(); + r.se(); + r.se(); + const uint32_t cycle = r.ue(); + for (uint32_t i = 0; i < cycle && r.ok(); ++i) + r.se(); + } + s.max_num_ref_frames = r.ue(); + r.bit(); + const uint32_t w_mbs = r.ue() + 1; + const uint32_t h_map_units = r.ue() + 1; + s.frame_mbs_only = r.bit() != 0; + if (!s.frame_mbs_only) + r.bit(); + r.bit(); + uint32_t crop[4] = { 0, 0, 0, 0 }; + if (r.bit()) + { + for (uint32_t& c : crop) + c = r.ue(); + } + s.width = w_mbs * 16; + s.height = h_map_units * (s.frame_mbs_only ? 1 : 2) * 16; + const uint32_t cx = (s.chroma_format_idc == 1 || s.chroma_format_idc == 2) ? 2 : 1; + const uint32_t cy = (s.chroma_format_idc == 1 ? 2 : 1) * (s.frame_mbs_only ? 1 : 2); + s.width -= cx * (crop[0] + crop[1]); + s.height -= cy * (crop[2] + crop[3]); + out.vui_flag_pos = r.position(); + s.vui_present = r.bit() != 0; + if (s.vui_present) + { + if (r.bit()) + { + if (r.bits(8) == 255) + r.bits(32); + } + if (r.bit()) + r.bit(); + if (r.bit()) + { + r.bits(3); + s.full_range = r.bit() != 0; + if (r.bit()) + r.bits(24); + } + if (r.bit()) + { + // chroma_loc_info_present_flag: the vertical sample positions of the chroma samples (H.264 Table E-1) + s.chroma_loc_present = true; + s.chroma_loc_top = r.ue(); + s.chroma_loc_bottom = r.ue(); + } + if (r.bit()) + { + r.bits(32); + r.bits(32); + r.bit(); + } + const bool nal_hrd = r.bit() != 0; + if (nal_hrd) + skip_hrd(r); + const bool vcl_hrd = r.bit() != 0; + if (vcl_hrd) + skip_hrd(r); + if (nal_hrd || vcl_hrd) + r.bit(); + r.bit(); + out.restriction_flag_pos = r.position(); + s.bitstream_restriction = r.bit() != 0; + if (s.bitstream_restriction) + { + r.bit(); + r.ue(); + r.ue(); + r.ue(); + r.ue(); + s.max_num_reorder_frames = r.ue(); + s.max_dec_frame_buffering = r.ue(); + } + } + if (!r.ok()) + return std::nullopt; + return out; +} + +// Finds the first NAL of `type`; returns the offsets of its header byte and its end (start of the next start code). +bool find_nal(const uint8_t* d, size_t n, int type, size_t& begin, size_t& end, size_t& start_code) +{ + for (size_t i = 0; i + 3 < n; ++i) + { + if (d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 1 && (d[i + 3] & 0x1F) == type) + { + begin = i + 3; + start_code = (i > 0 && d[i - 1] == 0) ? i - 1 : i; + size_t j = begin; + while (j + 2 < n && !(d[j] == 0 && d[j + 1] == 0 && (d[j + 2] == 1 || d[j + 2] == 0))) + ++j; + end = (j + 2 < n) ? j : n; + return true; + } + } + return false; +} + +} // namespace + +std::string SpsInfo::describe() const +{ + std::string s = "H.264 profile " + std::to_string(profile_idc) + " level " + std::to_string(level_idc) + ", " + + std::to_string(width) + "x" + std::to_string(height) + ", refs " + + std::to_string(max_num_ref_frames) + ", poc type " + std::to_string(pic_order_cnt_type); + if (bitstream_restriction) + s += ", bitstream_restriction: reorder " + std::to_string(max_num_reorder_frames) + ", dpb " + + std::to_string(max_dec_frame_buffering); + else + s += vui_present ? ", VUI without bitstream_restriction" : ", no VUI"; + return s; +} + +std::optional parse_sps(const uint8_t* annexb, size_t size) +{ + size_t begin = 0, end = 0, sc = 0; + if (!find_nal(annexb, size, 7, begin, end, sc)) + return std::nullopt; + auto parsed = parse_rbsp(unescape(annexb + begin, end - begin)); + if (!parsed) + return std::nullopt; + return parsed->info; +} + +std::optional> declare_no_reordering(const uint8_t* annexb, size_t size) +{ + size_t begin = 0, end = 0, sc = 0; + if (!find_nal(annexb, size, 7, begin, end, sc)) + return std::nullopt; + auto parsed = parse_rbsp(unescape(annexb + begin, end - begin)); + if (!parsed) + return std::nullopt; + const SpsInfo& info = parsed->info; + const std::vector& rbsp = parsed->rbsp; + // Drop the trailing stop bit and padding: copy the original bits up to the point the VUI tail is replaced. + BitWriter w; + if (!info.vui_present) + { + w.copy(rbsp, 0, parsed->vui_flag_pos); + w.bit(1); // vui_parameters_present_flag + // Minimal VUI: nothing present except bitstream_restriction. + for (int i = 0; i < 8; ++i) + w.bit(0); // aspect ratio, overscan, video signal, chroma loc, timing, nal hrd, vcl hrd, pic_struct + w.bit(1); // bitstream_restriction_flag + } + else + { + w.copy(rbsp, 0, parsed->restriction_flag_pos); + w.bit(1); // bitstream_restriction_flag + } + w.bit(1); // motion_vectors_over_pic_boundaries_flag + w.ue(0); // max_bytes_per_pic_denom + w.ue(1); // max_bits_per_mb_denom + w.ue(15); // log2_max_mv_length_horizontal + w.ue(15); // log2_max_mv_length_vertical + w.ue(0); // max_num_reorder_frames + w.ue(std::max(info.max_num_ref_frames, 1)); // max_dec_frame_buffering + const std::vector new_rbsp = w.finish(); + const std::vector payload = escape(new_rbsp); + std::vector out(annexb, annexb + sc); + out.insert(out.end(), { 0, 0, 0, 1 }); + out.insert(out.end(), payload.begin(), payload.end()); + out.insert(out.end(), annexb + end, annexb + size); + return out; +} + +} // namespace h264 +} // namespace video_stage diff --git a/src/video_stage/wire/inc/video_stage/avs1.hpp b/src/video_stage/wire/inc/video_stage/avs1.hpp new file mode 100644 index 0000000000..d38f16873b --- /dev/null +++ b/src/video_stage/wire/inc/video_stage/avs1.hpp @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +namespace video_stage +{ +namespace avs1 +{ + +/// AVS1 version 1, one mono stream with one sample per record. The layout is the repository's AVS1 wire format +/// (examples/camera_streamer/argus_sender/WIRE_FORMAT.md): a 4-byte big-endian record length, a 64-byte bundle +/// header, one 40-byte stream descriptor, one 72-byte sample descriptor, then the payload. Two things are specific +/// to this use: +/// - codec 2 is H.264: one Annex-B access unit per record, with SPS and PPS before every IDR; +/// - `frame_number` is the source frame id and `frame_timestamp_ns` carries the source stamp on the sender's own +/// monotonic clock, the same value the LiveKit track publishes. Neither is a wall-clock time. A zero stamp and a +/// zero frame number mean the sender could not tie the unit to a capture: the receiver publishes it without a +/// capture time. +constexpr uint16_t kVersion = 1; +constexpr uint32_t kHeaderBytes = 64; +constexpr uint32_t kStreamBytes = 40; +constexpr uint32_t kSampleBytes = 72; +constexpr uint32_t kMaxRecordBytes = 64U * 1024U * 1024U; + +enum class Codec : uint8_t +{ + RawNv12 = 0, + Hevc = 1, + H264 = 2, +}; + +/// What one record carries: the picture description and its identity. +struct Sample +{ + Codec codec = Codec::H264; + uint32_t width = 0; + uint32_t height = 0; + uint32_t fps_numerator = 30; + uint64_t bundle_sequence = 0; + uint64_t session_id = 0; + uint64_t frame_number = 0; + uint64_t frame_timestamp_ns = 0; +}; + +/// A parsed record. `payload` points into the buffer given to parse(); it is valid while that buffer is. +struct Record +{ + Sample sample; + const uint8_t* payload = nullptr; + size_t payload_size = 0; +}; + +/// Appends one complete record (length prefix included) to `out`. `payload` is one access unit. +void encode(const Sample& sample, const uint8_t* payload, size_t payload_size, std::vector& out); + +/// Parses the bytes after the 4-byte length prefix. Returns nullopt and sets `error` when the record is not a valid +/// version-1 mono H.264 record: bad magic or version, nonzero reserved fields, unknown flags, inconsistent presence, +/// a payload range outside the record, or an empty access unit. +std::optional parse(const uint8_t* data, size_t size, std::string& error); + +/// The record length from its 4-byte big-endian prefix. +uint32_t read_length(const uint8_t prefix[4]); + +/// True when an Annex-B access unit contains an IDR slice (NAL type 5). +bool contains_idr(const uint8_t* au, size_t size); + +} // namespace avs1 +} // namespace video_stage diff --git a/src/video_stage/wire/inc/video_stage/clock_exchange.hpp b/src/video_stage/wire/inc/video_stage/clock_exchange.hpp new file mode 100644 index 0000000000..6d0e48dad2 --- /dev/null +++ b/src/video_stage/wire/inc/video_stage/clock_exchange.hpp @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +namespace video_stage +{ + +/// One completed four-timestamp exchange: t1/t4 on the local clock, t2/t3 on the remote's. Nanoseconds. +struct Exchange +{ + int64_t t1 = 0; + int64_t t2 = 0; + int64_t t3 = 0; + int64_t t4 = 0; + + int64_t rtt_ns() const + { + return (t4 - t1) - (t3 - t2); + } + /// Remote minus local, assuming symmetric transit. + int64_t offset_ns() const + { + return ((t2 - t1) + (t3 - t4)) / 2; + } + bool well_formed() const + { + return t4 >= t1 && t3 >= t2 && rtt_ns() >= 0; + } +}; + +struct ClockMapping +{ + int64_t offset_ns = 0; ///< remote minus local + int64_t rtt_ns = 0; + /// Half the round trip plus the drift the exchange's age allows: what unknown asymmetry and rate can cause. + int64_t uncertainty_ns = 0; + int64_t accepted_at_ns = 0; + + int64_t to_local_ns(int64_t remote_ns) const + { + return remote_ns - offset_ns; + } +}; + +/** + * @brief Maps a remote monotonic clock onto this one from four-timestamp exchanges. + * + * Same rules as the Python estimator the recorder uses (egocap_web/clock_exchange.py): the best recent exchange by + * uncertainty, a mapping withheld when no exchange arrived within the stale limit, a new exchange that disagrees with + * the current mapping by more than both uncertainties plus a drift allowance flagged as a discontinuity, and the + * uncertainty aged to the time of the read. Not thread-safe; the owner serialises access. + */ +class ClockEstimator +{ +public: + static constexpr int64_t kMaxRttNs = 250'000'000; + static constexpr int64_t kStaleAfterNs = 5'000'000'000; + static constexpr size_t kWindow = 8; + static constexpr double kDriftAllowance = 200e-6; + + /// Why the exchange was rejected, or empty when accepted. + std::string add(const Exchange& e) + { + if (!e.well_formed()) + return "malformed exchange"; + if (e.rtt_ns() > kMaxRttNs) + return "round trip too long"; + const std::optional previous = m_current; + m_newest_ns = e.t4; + const ClockMapping newest = mapping_of(e, e.t4); + if (previous && disagrees(*previous, newest)) + { + m_discontinuity = true; + m_recent.clear(); + m_recent.push_back(e); + m_current = mapping_of(e, e.t4); + } + else + { + m_recent.push_back(e); + if (m_recent.size() > kWindow) + m_recent.pop_front(); + const Exchange* best = &m_recent.front(); + for (const Exchange& c : m_recent) + { + const int64_t u = uncertainty(c, e.t4); + const int64_t bu = uncertainty(*best, e.t4); + if (u < bu || (u == bu && c.t4 > best->t4)) + best = &c; + } + m_current = mapping_of(*best, e.t4); + } + return {}; + } + + /// The mapping at `now_ns` with its uncertainty aged to then, or nullopt when there is none or it is stale. + std::optional mapping(int64_t now_ns) const + { + if (!m_current || !m_newest_ns || now_ns - *m_newest_ns > kStaleAfterNs) + return std::nullopt; + ClockMapping aged = *m_current; + const int64_t age = std::max(0, now_ns - aged.accepted_at_ns); + aged.uncertainty_ns = + std::max(aged.uncertainty_ns, aged.rtt_ns / 2 + static_cast(kDriftAllowance * age)); + return aged; + } + + /// Set when the clocks jumped apart; frames already mapped keep their mapping. The owner counts the event and + /// clears it; the estimator has already restarted from the exchange that showed it. + bool discontinuity() const + { + return m_discontinuity; + } + void clear_discontinuity() + { + m_discontinuity = false; + } + +private: + static int64_t uncertainty(const Exchange& e, int64_t now_ns) + { + return e.rtt_ns() / 2 + static_cast(kDriftAllowance * std::max(0, now_ns - e.t4)); + } + + static ClockMapping mapping_of(const Exchange& e, int64_t now_ns) + { + return ClockMapping{ e.offset_ns(), e.rtt_ns(), uncertainty(e, now_ns), e.t4 }; + } + + static bool disagrees(const ClockMapping& previous, const ClockMapping& candidate) + { + const int64_t elapsed = std::max(0, candidate.accepted_at_ns - previous.accepted_at_ns); + const double allowed = static_cast(previous.uncertainty_ns + candidate.uncertainty_ns) + + kDriftAllowance * static_cast(elapsed); + const int64_t delta = candidate.offset_ns - previous.offset_ns; + return static_cast(delta < 0 ? -delta : delta) > allowed; + } + + std::deque m_recent; + std::optional m_current; + std::optional m_newest_ns; + bool m_discontinuity = false; +}; + +} // namespace video_stage diff --git a/src/video_stage/wire/inc/video_stage/clock_sync.hpp b/src/video_stage/wire/inc/video_stage/clock_sync.hpp new file mode 100644 index 0000000000..f084e23c59 --- /dev/null +++ b/src/video_stage/wire/inc/video_stage/clock_sync.hpp @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// The publisher's half of the PC's four-timestamp clock exchange: parse a request, build the reply. All times are +// CLOCK_MONOTONIC nanoseconds, the clock the PC maps. Wire format (little-endian), shared with +// examples/egocap/backend/egocap_web/livekit_clock_sync.py and the egocap emulator: +// request, 13 bytes: 'Q', u32 sequence, i64 t1 (the requester's clock) +// reply, 29 bytes: 'R', u32 sequence, i64 t1 (both echoed), i64 t2 (receipt), i64 t3 (reply send) +// No LiveKit type appears here, so the format is testable on its own. + +#pragma once + +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace sensing +{ + +inline constexpr const char* kClockSyncTopic = "isaaccapture.clock-sync"; +inline constexpr std::size_t kClockSyncRequestSize = 13; +inline constexpr std::size_t kClockSyncReplySize = 29; +/// Byte offset of t3 in a reply, so it can be written as late as possible, just before the send. +inline constexpr std::size_t kClockSyncReplyT3Offset = 21; + +struct ClockSyncRequest +{ + std::uint32_t sequence = 0; + std::int64_t t1 = 0; +}; + +inline std::int64_t monotonic_ns() +{ + timespec ts{}; + clock_gettime(CLOCK_MONOTONIC, &ts); + return static_cast(ts.tv_sec) * 1'000'000'000LL + ts.tv_nsec; +} + +inline void put_le(std::vector& out, std::size_t at, std::uint64_t value, std::size_t bytes) +{ + for (std::size_t i = 0; i < bytes; ++i) + out[at + i] = static_cast((value >> (8 * i)) & 0xFF); +} + +inline std::uint64_t get_le(const std::vector& in, std::size_t at, std::size_t bytes) +{ + std::uint64_t value = 0; + for (std::size_t i = 0; i < bytes; ++i) + value |= static_cast(in[at + i]) << (8 * i); + return value; +} + +/// The request in `data`, or nullopt for anything that is not exactly a request. +inline std::optional parse_clock_sync_request(const std::vector& data) +{ + if (data.size() != kClockSyncRequestSize || data[0] != 'Q') + return std::nullopt; + ClockSyncRequest request; + request.sequence = static_cast(get_le(data, 1, 4)); + request.t1 = static_cast(get_le(data, 5, 8)); + return request; +} + +inline std::vector build_clock_sync_reply(const ClockSyncRequest& request, std::int64_t t2, std::int64_t t3) +{ + std::vector reply(kClockSyncReplySize); + reply[0] = 'R'; + put_le(reply, 1, request.sequence, 4); + put_le(reply, 5, static_cast(request.t1), 8); + put_le(reply, 13, static_cast(t2), 8); + put_le(reply, kClockSyncReplyT3Offset, static_cast(t3), 8); + return reply; +} + +/// The requester's half of the same exchange, for a peer that sends requests (the PC video stage over its control +/// link). +inline std::vector build_clock_sync_request(const ClockSyncRequest& request) +{ + std::vector out(kClockSyncRequestSize); + out[0] = 'Q'; + put_le(out, 1, request.sequence, 4); + put_le(out, 5, static_cast(request.t1), 8); + return out; +} + +struct ClockSyncReply +{ + std::uint32_t sequence = 0; + std::int64_t t1 = 0; + std::int64_t t2 = 0; + std::int64_t t3 = 0; +}; + +/// The reply in `data`, or nullopt for anything that is not exactly a reply. +inline std::optional parse_clock_sync_reply(const std::vector& data) +{ + if (data.size() != kClockSyncReplySize || data[0] != 'R') + return std::nullopt; + ClockSyncReply reply; + reply.sequence = static_cast(get_le(data, 1, 4)); + reply.t1 = static_cast(get_le(data, 5, 8)); + reply.t2 = static_cast(get_le(data, 13, 8)); + reply.t3 = static_cast(get_le(data, kClockSyncReplyT3Offset, 8)); + return reply; +} + +/// Writes t3 into an already built reply. +inline void set_clock_sync_reply_t3(std::vector& reply, std::int64_t t3) +{ + put_le(reply, kClockSyncReplyT3Offset, static_cast(t3), 8); +} + +} // namespace sensing +} // namespace plugins diff --git a/src/video_stage/wire/inc/video_stage/encoded_video_frame.hpp b/src/video_stage/wire/inc/video_stage/encoded_video_frame.hpp new file mode 100644 index 0000000000..cc13bc95b7 --- /dev/null +++ b/src/video_stage/wire/inc/video_stage/encoded_video_frame.hpp @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace video_stage +{ + +/// One complete encoded H.264 access unit, with what accompanies it. +struct EncodedVideoFrame +{ + /// Annex-B bytes: an IDR carries its SPS/PPS in front of it. + const uint8_t* data = nullptr; + size_t size = 0; + /// The unit is an IDR, decodable on its own. + bool keyframe = false; + /// CLOCK_MONOTONIC microseconds at capture: the host timeline, not sensor time. + uint64_t timestamp_us = 0; + /// The frame's application timestamp in microseconds, on the clock its stream declares. Zero means the publisher + /// cannot vouch for a time, and the frame carries none. + uint64_t user_timestamp_us = 0; + /// The camera's own frame sequence. + uint32_t frame_id = 0; + /// The sensor's capture time on its own clock, carried as user data so a consumer can pair + /// frames from different sensors. + uint64_t capture_tsc = 0; +}; + +} // namespace video_stage diff --git a/src/video_stage/wire/inc/video_stage/frame_correlator.hpp b/src/video_stage/wire/inc/video_stage/frame_correlator.hpp new file mode 100644 index 0000000000..d18bbef440 --- /dev/null +++ b/src/video_stage/wire/inc/video_stage/frame_correlator.hpp @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace video_stage +{ + +/// What is known about a frame that went into the encoder. +struct InFlightFrame +{ + std::uint32_t sequence = 0; + std::uint64_t capture_tsc = 0; + /// The application timestamp to publish with the frame, in microseconds. + std::uint64_t stamp_us = 0; + /// The stamp given to the encoder with the input, in microseconds: the key a returned unit is matched by. + std::int64_t submitted_us = 0; +}; + +/** + * @brief Ties each encoded unit back to the frame it came from, and says when it cannot. + * + * Stamp mode credits a unit to a frame only when the stamp the encoder hands back equals one submitted; older frames + * the encoder never returned are skipped and counted, and a unit matching nothing is reported as not identified (the + * oldest in-flight entry is dropped so the queue stays bounded). Fifo mode assumes one unit per input in order and + * cannot be verified; it is an explicit choice for approximate timing. Worker thread only. + */ +class FrameCorrelator +{ +public: + enum class Mode + { + Stamp, + Fifo, + }; + + struct Result + { + InFlightFrame frame; + bool identified = true; + /// Inputs older than the match that the encoder did not return a unit for. + std::uint64_t skipped = 0; + }; + + explicit FrameCorrelator(Mode mode, std::size_t max_in_flight = 12) : m_mode(mode), m_max(max_in_flight) + { + } + + void submitted(const InFlightFrame& frame) + { + m_in_flight.push_back(frame); + // The encoder holds a bounded number of frames, so a longer queue is stale metadata. + while (m_in_flight.size() > m_max) + m_in_flight.pop_front(); + } + + /// The frame a unit stamped `unit_stamp_us` came from; nullopt when nothing is in flight. + std::optional returned(std::int64_t unit_stamp_us) + { + if (m_in_flight.empty()) + return std::nullopt; + Result result; + if (m_mode == Mode::Stamp) + { + const auto match = std::find_if(m_in_flight.begin(), m_in_flight.end(), + [&](const InFlightFrame& f) { return f.submitted_us == unit_stamp_us; }); + if (match != m_in_flight.end()) + { + result.skipped = static_cast(std::distance(m_in_flight.begin(), match)); + result.frame = *match; + m_in_flight.erase(m_in_flight.begin(), std::next(match)); + return result; + } + result.identified = false; + } + result.frame = m_in_flight.front(); + m_in_flight.pop_front(); + return result; + } + + bool empty() const + { + return m_in_flight.empty(); + } + std::size_t in_flight() const + { + return m_in_flight.size(); + } + +private: + Mode m_mode; + std::size_t m_max; + std::deque m_in_flight; +}; + +} // namespace video_stage diff --git a/src/video_stage/wire/inc/video_stage/h264_access_units.hpp b/src/video_stage/wire/inc/video_stage/h264_access_units.hpp new file mode 100644 index 0000000000..5acdc4960c --- /dev/null +++ b/src/video_stage/wire/inc/video_stage/h264_access_units.hpp @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace video_stage +{ + +/// One access unit of an Annex-B stream, as a range of the input. +struct AccessUnitRange +{ + std::size_t offset = 0; + std::size_t size = 0; + /// The unit contains an IDR slice. + bool idr = false; +}; + +/** + * @brief Split an Annex-B H.264 stream into access units. + * + * A unit starts at the first NAL after a picture's slices, or at a picture whose first slice follows another + * picture directly; further slices of one picture stay with it. End-of-sequence, end-of-stream and filler NALs + * belong to the picture before them. Parameter sets and SEI stay with the picture that follows. + */ +std::vector split_access_units(const std::uint8_t* data, std::size_t size); + +} // namespace video_stage diff --git a/src/video_stage/wire/inc/video_stage/h264_sps.hpp b/src/video_stage/wire/inc/video_stage/h264_sps.hpp new file mode 100644 index 0000000000..7d1b72ccce --- /dev/null +++ b/src/video_stage/wire/inc/video_stage/h264_sps.hpp @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +namespace video_stage +{ +namespace h264 +{ + +/// The fields of an H.264 sequence parameter set that decide how a decoder buffers pictures, and where the VUI's +/// bitstream_restriction sits so it can be rewritten. +struct SpsInfo +{ + uint32_t profile_idc = 0; + uint32_t level_idc = 0; + uint32_t chroma_format_idc = 1; + uint32_t pic_order_cnt_type = 0; + uint32_t max_num_ref_frames = 0; + uint32_t width = 0; ///< coded width in luma pixels + uint32_t height = 0; ///< coded height in luma pixels + bool frame_mbs_only = true; + bool vui_present = false; + bool full_range = false; + /// The stream states where its chroma samples sit (vertical offsets, 0 = the default left/co-sited siting that + /// the example operation assumes). Absent means the default. + bool chroma_loc_present = false; + uint32_t chroma_loc_top = 0; + uint32_t chroma_loc_bottom = 0; + bool bitstream_restriction = false; + /// Valid when bitstream_restriction is set: how many pictures may precede one in decode order and follow it in + /// display order, and how many a decoder must hold. + uint32_t max_num_reorder_frames = 0; + uint32_t max_dec_frame_buffering = 0; + + /// One line for a log. + std::string describe() const; +}; + +/// The first SPS NAL (type 7) in an Annex-B access unit, parsed; nullopt if there is none or it is not understood. +std::optional parse_sps(const uint8_t* annexb, size_t size); + +/// A copy of the access unit whose SPS declares that the stream never reorders pictures: bitstream_restriction with +/// max_num_reorder_frames 0 and max_dec_frame_buffering equal to max_num_ref_frames (at least 1). Without that, a +/// decoder must assume the worst-case DPB for the level and holds back many pictures before showing the first. +/// Only call it for a stream that really has no B-frames. Returns nullopt if there is no SPS or it cannot be rewritten. +std::optional> declare_no_reordering(const uint8_t* annexb, size_t size); + +} // namespace h264 +} // namespace video_stage diff --git a/src/video_stage/wire/tests/CMakeLists.txt b/src/video_stage/wire/tests/CMakeLists.txt new file mode 100644 index 0000000000..c3bb897bf4 --- /dev/null +++ b/src/video_stage/wire/tests/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(video_stage_wire_tests + test_avs1.cpp + test_clock_exchange.cpp + test_clock_sync.cpp + test_encoded_video_frame.cpp + test_frame_correlator.cpp + test_h264_access_units.cpp + test_h264_sps.cpp +) + +target_link_libraries(video_stage_wire_tests PRIVATE video_stage::wire Catch2::Catch2WithMain) + +catch_discover_tests(video_stage_wire_tests) diff --git a/src/video_stage/wire/tests/test_avs1.cpp b/src/video_stage/wire/tests/test_avs1.cpp new file mode 100644 index 0000000000..af38f27637 --- /dev/null +++ b/src/video_stage/wire/tests/test_avs1.cpp @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include +#include + +namespace avs1 = video_stage::avs1; + +namespace +{ + +// Record offsets (after the 4-byte length prefix). +constexpr size_t kStream = 64; +constexpr size_t kSample = 64 + 40; +constexpr size_t kPayload = 64 + 40 + 72; + +uint64_t be(const std::vector& b, size_t off, size_t n) +{ + uint64_t v = 0; + for (size_t i = 0; i < n; ++i) + v = (v << 8) | b.at(off + i); + return v; +} + +void set_be(std::vector& b, size_t off, size_t n, uint64_t v) +{ + for (size_t i = 0; i < n; ++i) + b.at(off + n - 1 - i) = static_cast(v >> (8 * i)); +} + +const std::vector kAu = { 0, 0, 0, 1, 0x67, 1, 2, 0, 0, 0, 1, 0x68, 3, 0, 0, 1, 0x65, 9, 9, 9 }; + +avs1::Sample make_sample() +{ + avs1::Sample s; + s.codec = avs1::Codec::H264; + s.width = 1280; + s.height = 720; + s.fps_numerator = 60; + s.bundle_sequence = 0x0102030405060708ULL; + s.session_id = 0x1112131415161718ULL; + s.frame_number = 0x2122232425262728ULL; + s.frame_timestamp_ns = 0x3132333435363738ULL; + return s; +} + +// Encoded record without the length prefix. +std::vector valid_record(const std::vector& payload = kAu) +{ + std::vector out; + avs1::encode(make_sample(), payload.data(), payload.size(), out); + return std::vector(out.begin() + 4, out.end()); +} + +bool rejects(const std::vector& rec) +{ + std::string error; + const auto r = avs1::parse(rec.data(), rec.size(), error); + return !r.has_value() && !error.empty(); +} + +} // namespace + +TEST_CASE("AVS1 round trip keeps every field", "[video_stage_wire]") +{ + const avs1::Sample s = make_sample(); + std::vector out; + avs1::encode(s, kAu.data(), kAu.size(), out); + REQUIRE(avs1::read_length(out.data()) == out.size() - 4); + + std::string error; + const auto r = avs1::parse(out.data() + 4, out.size() - 4, error); + REQUIRE(r.has_value()); + CHECK(r->sample.codec == avs1::Codec::H264); + CHECK(r->sample.width == s.width); + CHECK(r->sample.height == s.height); + CHECK(r->sample.fps_numerator == s.fps_numerator); + CHECK(r->sample.bundle_sequence == s.bundle_sequence); + CHECK(r->sample.session_id == s.session_id); + CHECK(r->sample.frame_number == s.frame_number); + CHECK(r->sample.frame_timestamp_ns == s.frame_timestamp_ns); + REQUIRE(r->payload_size == kAu.size()); + CHECK(std::memcmp(r->payload, kAu.data(), kAu.size()) == 0); + CHECK(r->payload == out.data() + 4 + kPayload); +} + +TEST_CASE("AVS1 encode follows WIRE_FORMAT.md byte for byte", "[video_stage_wire]") +{ + const avs1::Sample s = make_sample(); + std::vector out; + avs1::encode(s, kAu.data(), kAu.size(), out); + REQUIRE(out.size() == 4 + 64 + 40 + 72 + kAu.size()); + CHECK(be(out, 0, 4) == out.size() - 4); + + const std::vector r(out.begin() + 4, out.end()); + // HEADER "!4sHHIHHQQQQIIQ" + CHECK(std::memcmp(r.data(), "AVS1", 4) == 0); + CHECK(be(r, 4, 2) == 1); + CHECK(be(r, 6, 2) == 0); + CHECK(be(r, 8, 4) == 64); + CHECK(be(r, 12, 2) == 1); + CHECK(be(r, 14, 2) == 1); + CHECK(be(r, 16, 8) == s.bundle_sequence); + CHECK(be(r, 24, 8) == s.session_id); + CHECK(be(r, 32, 8) == 0); + CHECK(be(r, 40, 8) == 0); + CHECK(be(r, 48, 4) == 112); + CHECK(be(r, 52, 4) == kAu.size()); + CHECK(be(r, 56, 8) == 0); + // STREAM "!HHBBHIIIIQHHI" + CHECK(be(r, kStream + 0, 2) == 0); + CHECK(be(r, kStream + 2, 2) == 1); + CHECK(r[kStream + 4] == 2); + CHECK(r[kStream + 5] == 1); + CHECK(be(r, kStream + 6, 2) == 0); + CHECK(be(r, kStream + 8, 4) == s.width); + CHECK(be(r, kStream + 12, 4) == s.height); + CHECK(be(r, kStream + 16, 4) == s.fps_numerator); + CHECK(be(r, kStream + 20, 4) == 1); + CHECK(be(r, kStream + 24, 8) != 0); + CHECK(be(r, kStream + 32, 2) == 0); + CHECK(be(r, kStream + 34, 2) == 1); + CHECK(be(r, kStream + 36, 4) == 0); + // SAMPLE "!HBBHHIIQQQQQQII" + CHECK(be(r, kSample + 0, 2) == 0); + CHECK(r[kSample + 2] == 0); + CHECK(r[kSample + 3] == 0); + CHECK(be(r, kSample + 4, 2) == 0x0005); + CHECK(be(r, kSample + 6, 2) == 0); + CHECK(be(r, kSample + 16, 8) == s.frame_number); + CHECK(be(r, kSample + 32, 8) == s.frame_timestamp_ns); + CHECK(be(r, kSample + 64, 4) == 0); + CHECK(be(r, kSample + 68, 4) == kAu.size()); + CHECK(std::memcmp(r.data() + kPayload, kAu.data(), kAu.size()) == 0); +} + +TEST_CASE("AVS1 read_length is big endian", "[video_stage_wire]") +{ + const uint8_t p[4] = { 0x01, 0x02, 0x03, 0x04 }; + CHECK(avs1::read_length(p) == 0x01020304U); + const uint8_t q[4] = { 0xFF, 0xFF, 0xFF, 0xFF }; + CHECK(avs1::read_length(q) == 0xFFFFFFFFU); +} + +TEST_CASE("AVS1 parse rejects malformed records", "[video_stage_wire]") +{ + const std::vector good = valid_record(); + { + std::string error; + REQUIRE(avs1::parse(good.data(), good.size(), error).has_value()); + } + + struct Mutation + { + const char* name; + size_t offset; + size_t bytes; + uint64_t value; + }; + const Mutation mutations[] = { + { "bad magic", 0, 1, 'X' }, + { "version 2", 4, 2, 2 }, + { "version 0", 4, 2, 0 }, + { "bundle flags", 6, 2, 1 }, + { "bundle flags TAI", 6, 2, 2 }, + { "header_bytes", 8, 4, 65 }, + { "stream_count 0", 12, 2, 0 }, + { "stream_count 2", 12, 2, 2 }, + { "sample_count 0", 14, 2, 0 }, + { "sample_count 2", 14, 2, 2 }, + { "descriptor_bytes small", 48, 4, 111 }, + { "descriptor_bytes large", 48, 4, 113 }, + { "payload_bytes small", 52, 4, kAu.size() - 1 }, + { "payload_bytes large", 52, 4, kAu.size() + 1 }, + { "sender field offset 32", 32, 8, 1 }, + { "sender field offset 40", 40, 8, 1 }, + { "sender field offset 56", 56, 8, 1 }, + { "stream absent", kStream + 2, 2, 0 }, + { "stream stereo flag", kStream + 2, 2, 3 }, + { "codec raw", kStream + 4, 1, 0 }, + { "codec hevc", kStream + 4, 1, 1 }, + { "codec 3", kStream + 4, 1, 3 }, + { "pixel format 0", kStream + 5, 1, 0 }, + { "pixel format 2", kStream + 5, 1, 2 }, + { "rotation 180", kStream + 6, 2, 180 }, + { "fps denominator 0", kStream + 20, 4, 0 }, + { "first_sample 1", kStream + 32, 2, 1 }, + { "stream sample_count 0", kStream + 34, 2, 0 }, + { "stream sample_count 2", kStream + 34, 2, 2 }, + { "stream reserved", kStream + 36, 4, 1 }, + { "sample unknown flag 0x10", kSample + 4, 2, 0x0015 }, + { "sample unknown flag 0x8000", kSample + 4, 2, 0x8005 }, + { "sample not present", kSample + 4, 2, 0x0004 }, + { "sample no timestamp valid", kSample + 4, 2, 0x0001 }, + { "sample flags clear", kSample + 4, 2, 0 }, + { "sample reserved", kSample + 6, 2, 1 }, + { "payload length 0", kSample + 68, 4, 0 }, + { "payload offset 1", kSample + 64, 4, 1 }, + { "payload length mismatch", kSample + 68, 4, kAu.size() - 1 }, + }; + for (const Mutation& m : mutations) + { + DYNAMIC_SECTION(m.name) + { + std::vector rec = good; + set_be(rec, m.offset, m.bytes, m.value); + CHECK(rejects(rec)); + } + } + + SECTION("optional sample flags are accepted") + { + std::vector rec = good; + set_be(rec, kSample + 4, 2, 0x000F); + std::string error; + CHECK(avs1::parse(rec.data(), rec.size(), error).has_value()); + } + SECTION("record size does not match payload_bytes") + { + std::vector rec = good; + rec.push_back(0); + CHECK(rejects(rec)); + rec = good; + rec.pop_back(); + CHECK(rejects(rec)); + } + SECTION("empty payload") + { + std::vector rec(good.begin(), good.begin() + kPayload); + set_be(rec, 52, 4, 0); + set_be(rec, kSample + 68, 4, 0); + CHECK(rejects(rec)); + } + SECTION("short record") + { + CHECK(rejects(std::vector{})); + CHECK(rejects(std::vector(good.begin(), good.begin() + 10))); + } +} + +TEST_CASE("AVS1 rejects every proper prefix of a valid record", "[video_stage_wire]") +{ + const std::vector good = valid_record(); + for (size_t n = 0; n < good.size(); ++n) + { + std::string error; + const auto r = avs1::parse(good.data(), n, error); + INFO("prefix length " << n); + REQUIRE_FALSE(r.has_value()); + REQUIRE_FALSE(error.empty()); + } +} + +TEST_CASE("AVS1 parse survives single-byte corruption anywhere", "[video_stage_wire]") +{ + const std::vector good = valid_record(); + uint32_t state = 12345; + for (int i = 0; i < 4000; ++i) + { + state = state * 1664525U + 1013904223U; + std::vector rec = good; + rec[(state >> 8) % rec.size()] ^= static_cast(1U + (state >> 24) % 255U); + std::string error; + const auto r = avs1::parse(rec.data(), rec.size(), error); + if (r) + { + REQUIRE(r->payload >= rec.data()); + REQUIRE(r->payload + r->payload_size <= rec.data() + rec.size()); + } + } +} + +TEST_CASE("AVS1 contains_idr", "[video_stage_wire]") +{ + SECTION("3-byte start code IDR") + { + const uint8_t au[] = { 0, 0, 1, 0x65, 1, 2, 3 }; + CHECK(avs1::contains_idr(au, sizeof(au))); + } + SECTION("4-byte start code IDR") + { + const uint8_t au[] = { 0, 0, 0, 1, 0x65, 1, 2, 3 }; + CHECK(avs1::contains_idr(au, sizeof(au))); + } + SECTION("SPS, PPS, IDR") + { + CHECK(avs1::contains_idr(kAu.data(), kAu.size())); + } + SECTION("IDR with other nal_ref_idc") + { + const uint8_t au[] = { 0, 0, 0, 1, 0x25, 1 }; + CHECK(avs1::contains_idr(au, sizeof(au))); + } + SECTION("SPS and PPS only") + { + const uint8_t au[] = { 0, 0, 0, 1, 0x67, 1, 2, 0, 0, 0, 1, 0x68, 3 }; + CHECK_FALSE(avs1::contains_idr(au, sizeof(au))); + } + SECTION("non-IDR slice") + { + const uint8_t au[] = { 0, 0, 0, 1, 0x41, 1, 2, 3 }; + CHECK_FALSE(avs1::contains_idr(au, sizeof(au))); + } + SECTION("empty and tiny inputs") + { + CHECK_FALSE(avs1::contains_idr(nullptr, 0)); + const uint8_t au[] = { 0, 0, 1 }; + CHECK_FALSE(avs1::contains_idr(au, sizeof(au))); + } +} diff --git a/src/video_stage/wire/tests/test_clock_exchange.cpp b/src/video_stage/wire/tests/test_clock_exchange.cpp new file mode 100644 index 0000000000..db80ccfc8b --- /dev/null +++ b/src/video_stage/wire/tests/test_clock_exchange.cpp @@ -0,0 +1,229 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include + +using video_stage::ClockEstimator; +using video_stage::Exchange; + +namespace +{ +constexpr int64_t MS = 1'000'000; +constexpr int64_t S = 1'000'000'000; + +// An exchange between a local clock and a remote clock running `offset` ahead of it. +Exchange make_exchange(int64_t t1, int64_t offset = 0, int64_t forward = MS, int64_t back = MS, int64_t processing = 0) +{ + const int64_t arrival = t1 + forward; + const int64_t departure = arrival + processing; + return Exchange{ t1, arrival + offset, departure + offset, departure + back }; +} +} // namespace + +TEST_CASE("symmetric delays recover offset, rtt excludes processing", "[Clock]") +{ + const Exchange ex = make_exchange(10 * S, 123'456'000, 3 * MS, 3 * MS, 40 * MS); + CHECK(ex.offset_ns() == 123'456'000); + CHECK(ex.rtt_ns() == 6 * MS); + + ClockEstimator est; + CHECK(est.add(ex).empty()); + const auto m = est.mapping(ex.t4); + REQUIRE(m); + CHECK(m->offset_ns == 123'456'000); + CHECK(m->rtt_ns == 6 * MS); + CHECK(m->uncertainty_ns == 3 * MS); + CHECK(m->to_local_ns(S + 123'456'000) == S); +} + +TEST_CASE("asymmetric delay error is half the difference, within uncertainty", "[Clock]") +{ + const int64_t pairs[][2] = { { 1 * MS, 9 * MS }, { 9 * MS, 1 * MS }, { 2 * MS, 2 * MS } }; + for (const auto& p : pairs) + { + const Exchange ex = make_exchange(S, 5 * MS, p[0], p[1], 7 * MS); + ClockEstimator est; + est.add(ex); + const auto m = est.mapping(ex.t4); + REQUIRE(m); + const int64_t error = m->offset_ns - 5 * MS; + CHECK(error == (p[0] - p[1]) / 2); + CHECK(std::abs(error) <= m->uncertainty_ns); + } +} + +TEST_CASE("negative offset is recovered", "[Clock]") +{ + const Exchange ex = make_exchange(20 * S, -250 * MS, 2 * MS, 2 * MS, MS); + ClockEstimator est; + est.add(ex); + CHECK(est.mapping(ex.t4)->offset_ns == -250 * MS); +} + +TEST_CASE("malformed and slow exchanges are rejected without changing the mapping", "[Clock]") +{ + const Exchange bad[] = { + make_exchange(S, 0, 200 * MS, 100 * MS), // rtt 300 ms + Exchange{ 100, 0, 0, 50 }, // t4 before t1 + Exchange{ 0, 100, 50, 1000 }, // t3 before t2 + Exchange{ 0, 0, 900, 800 }, // negative rtt + }; + for (const Exchange& b : bad) + { + ClockEstimator est; + est.add(make_exchange(S, MS)); + const auto before = est.mapping(2 * S); + REQUIRE(before); + CHECK_FALSE(est.add(b).empty()); + const auto after = est.mapping(2 * S); + REQUIRE(after); + CHECK(after->offset_ns == before->offset_ns); + CHECK(after->rtt_ns == before->rtt_ns); + CHECK(after->accepted_at_ns == before->accepted_at_ns); + CHECK(after->uncertainty_ns == before->uncertainty_ns); + CHECK_FALSE(est.discontinuity()); + } +} + +TEST_CASE("best of the window is the lowest rtt", "[Clock]") +{ + ClockEstimator est; + const int64_t delays[] = { 5 * MS, 3 * MS, 8 * MS, 2 * MS, 6 * MS }; + int64_t t = S; + for (int64_t d : delays) + { + est.add(make_exchange(t, MS, d, d)); + t += S; + } + CHECK(est.mapping(6 * S)->rtt_ns == 4 * MS); +} + +TEST_CASE("high rtt outlier does not replace the better mapping; uncertainty ages at read", "[Clock]") +{ + ClockEstimator est; + est.add(make_exchange(1 * S, 10 * MS, MS, MS)); + const auto good = est.mapping(2 * S); + REQUIRE(good); + est.add(make_exchange(2 * S, 10 * MS, 20 * MS, 20 * MS)); + const auto m = est.mapping(3 * S); + REQUIRE(m); + CHECK(m->rtt_ns == good->rtt_ns); + CHECK(m->accepted_at_ns == good->accepted_at_ns); + CHECK(m->uncertainty_ns == good->rtt_ns / 2 + static_cast(200e-6 * (3 * S - good->accepted_at_ns))); +} + +TEST_CASE("uncertainty ages between exchanges without a new one", "[Clock]") +{ + ClockEstimator est; + const Exchange ex = make_exchange(10 * S, 5 * MS, 2 * MS, 2 * MS); + est.add(ex); + const auto at_add = est.mapping(ex.t4); + REQUIRE(at_add); + CHECK(at_add->uncertainty_ns == ex.rtt_ns() / 2); + const auto later = est.mapping(ex.t4 + 3 * S); + REQUIRE(later); + CHECK(later->uncertainty_ns == ex.rtt_ns() / 2 + static_cast(200e-6 * 3 * S)); + CHECK(later->offset_ns == at_add->offset_ns); + CHECK(later->accepted_at_ns == at_add->accepted_at_ns); +} + +TEST_CASE("steady link keeps choosing the newest exchange", "[Clock]") +{ + ClockEstimator est; + for (int s = 1; s <= 30; ++s) + { + const Exchange ex = make_exchange(s * S, 7 * MS, 2 * MS, 2 * MS); + CHECK(est.add(ex).empty()); + const auto m = est.mapping(ex.t4); + REQUIRE(m); + CHECK(m->accepted_at_ns == ex.t4); + CHECK(m->uncertainty_ns == 2 * MS); + } + CHECK_FALSE(est.discontinuity()); +} + +TEST_CASE("mapping is none before any exchange and after the stale window", "[Clock]") +{ + ClockEstimator est; + CHECK_FALSE(est.mapping(0)); + const Exchange ex = make_exchange(S, MS); + est.add(ex); + CHECK(est.mapping(ex.t4)); + CHECK(est.mapping(ex.t4 + 5 * S)); + CHECK_FALSE(est.mapping(ex.t4 + 5 * S + 1)); + + // A new exchange brings it back. + est.add(make_exchange(70 * S, MS)); + CHECK(est.mapping(70 * S + 2 * MS)); +} + +TEST_CASE("rejected exchanges do not refresh staleness", "[Clock]") +{ + ClockEstimator est; + const Exchange ex = make_exchange(S, MS); + est.add(ex); + const int64_t late = ex.t4 + 5 * S + 1; + REQUIRE_FALSE(est.mapping(late)); + CHECK_FALSE(est.add(make_exchange(3 * S, MS, 200 * MS, 100 * MS)).empty()); + CHECK_FALSE(est.add(Exchange{ 6 * S, 0, 0, 5 * S }).empty()); + CHECK_FALSE(est.mapping(late)); +} + +TEST_CASE("a clock step is a discontinuity and restarts from the new exchange", "[Clock]") +{ + ClockEstimator est; + for (int s = 1; s <= 5; ++s) + est.add(make_exchange(s * S, 0, 2 * MS, 2 * MS)); + CHECK_FALSE(est.discontinuity()); + + // Same rtt as before: the step alone must flag it. + const Exchange step = make_exchange(6 * S, 50 * MS, 2 * MS, 2 * MS); + CHECK(est.add(step).empty()); + CHECK(est.discontinuity()); + const auto m = est.mapping(step.t4); + REQUIRE(m); + CHECK(m->offset_ns == 50 * MS); + CHECK(m->accepted_at_ns == step.t4); + + // Sticky until cleared; later good exchanges do not clear it. + est.add(make_exchange(7 * S, 50 * MS, MS, MS)); + CHECK(est.discontinuity()); + CHECK(est.mapping(7 * S + 2 * MS)->offset_ns == 50 * MS); + + est.clear_discontinuity(); + CHECK_FALSE(est.discontinuity()); + est.add(make_exchange(8 * S, 50 * MS, MS, MS)); + CHECK_FALSE(est.discontinuity()); +} + +TEST_CASE("a lower-rtt step does not mix old exchanges into the new mapping", "[Clock]") +{ + ClockEstimator est; + for (int s = 1; s <= 3; ++s) + est.add(make_exchange(s * S, 0, 2 * MS, 2 * MS)); + est.add(make_exchange(4 * S, 50 * MS, MS, MS)); + CHECK(est.discontinuity()); + CHECK(est.mapping(4 * S + 2 * MS)->offset_ns == 50 * MS); +} + +TEST_CASE("drift stays continuous and tracks within uncertainty plus drift", "[Clock]") +{ + ClockEstimator est; + const double ppm = 50.0; + for (int s = 1; s <= 30; ++s) + { + const int64_t t1 = s * S; + auto remote = [&](int64_t local) { return local + static_cast(local * ppm * 1e-6); }; + const Exchange ex{ t1, remote(t1 + MS), remote(t1 + MS), t1 + 2 * MS }; + CHECK(est.add(ex).empty()); + const auto m = est.mapping(ex.t4); + REQUIRE(m); + const double true_offset = ppm * 1e-6 * ex.t4; + const double drift = ppm * 1e-6 * (ex.t4 - m->accepted_at_ns); + CHECK(std::abs(m->offset_ns - true_offset) <= m->uncertainty_ns + drift + 2); + } + CHECK_FALSE(est.discontinuity()); +} diff --git a/src/video_stage/wire/tests/test_clock_sync.cpp b/src/video_stage/wire/tests/test_clock_sync.cpp new file mode 100644 index 0000000000..3e67150712 --- /dev/null +++ b/src/video_stage/wire/tests/test_clock_sync.cpp @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include + +using plugins::sensing::build_clock_sync_reply; +using plugins::sensing::ClockSyncRequest; +using plugins::sensing::kClockSyncReplySize; +using plugins::sensing::kClockSyncRequestSize; +using plugins::sensing::monotonic_ns; +using plugins::sensing::parse_clock_sync_request; +using plugins::sensing::set_clock_sync_reply_t3; + +namespace +{ + +using Bytes = std::vector; + +Bytes from_hex(const std::string& hex) +{ + Bytes out; + for (std::size_t i = 0; i + 1 < hex.size(); i += 2) + out.push_back(static_cast(std::stoul(hex.substr(i, 2), nullptr, 16))); + return out; +} + +// Golden bytes produced by egocap_web.livekit_clock_sync.encode_request and +// tools/livekit_orin_emulator.build_sync_reply for the inputs named beside each constant. +// sequence 1, t1 1, t2 2, t3 3 +const char* const kSimpleRequest = "51010000000100000000000000"; +const char* const kSimpleReply = "5201000000010000000000000002000000000000000300000000000000"; +// sequence 0xFFFFFFFF, t1 -1234567890123, t2 INT64_MAX, t3 INT64_MIN +const char* const kExtremeRequest = "51ffffffff35fb048ee0feffff"; +const char* const kExtremeReply = "52ffffffff35fb048ee0feffffffffffffffffff7f0000000000000080"; + +} // namespace + +TEST_CASE("parse_clock_sync_request accepts a well-formed request", "[clock_sync]") +{ + auto request = parse_clock_sync_request(from_hex(kSimpleRequest)); + REQUIRE(request.has_value()); + CHECK(request->sequence == 1); + CHECK(request->t1 == 1); + + request = parse_clock_sync_request(from_hex(kExtremeRequest)); + REQUIRE(request.has_value()); + CHECK(request->sequence == 0xFFFFFFFFu); + CHECK(request->t1 == -1234567890123LL); +} + +TEST_CASE("parse_clock_sync_request rejects anything else", "[clock_sync]") +{ + for (std::size_t size : { std::size_t{ 0 }, std::size_t{ 12 }, std::size_t{ 14 }, std::size_t{ 29 } }) + { + Bytes data(size, 0); + if (size > 0) + data[0] = 'Q'; + CHECK_FALSE(parse_clock_sync_request(data).has_value()); + } + + Bytes wrong_kind = from_hex(kSimpleRequest); + wrong_kind[0] = 'X'; + CHECK_FALSE(parse_clock_sync_request(wrong_kind).has_value()); + + CHECK_FALSE(parse_clock_sync_request(from_hex(kSimpleReply)).has_value()); + + Bytes reply_kind_at_request_size = from_hex(kSimpleRequest); + reply_kind_at_request_size[0] = 'R'; + CHECK_FALSE(parse_clock_sync_request(reply_kind_at_request_size).has_value()); +} + +TEST_CASE("build_clock_sync_reply lays out the documented fields", "[clock_sync]") +{ + const Bytes reply = build_clock_sync_reply(ClockSyncRequest{ 1, 1 }, 2, 3); + REQUIRE(reply.size() == kClockSyncReplySize); + CHECK(reply[0] == 'R'); + CHECK(reply == from_hex(kSimpleReply)); + + // Explicit offsets, little-endian. + CHECK(reply[1] == 1); + CHECK(reply[5] == 1); + CHECK(reply[13] == 2); + CHECK(reply[21] == 3); +} + +TEST_CASE("build_clock_sync_reply handles large and negative values", "[clock_sync]") +{ + const ClockSyncRequest request{ 0xFFFFFFFFu, -1234567890123LL }; + const Bytes reply = build_clock_sync_reply( + request, std::numeric_limits::max(), std::numeric_limits::min()); + CHECK(reply == from_hex(kExtremeReply)); +} + +TEST_CASE("clock sync request and reply round trip through parse", "[clock_sync]") +{ + const Bytes request_bytes = from_hex(kExtremeRequest); + REQUIRE(request_bytes.size() == kClockSyncRequestSize); + const auto request = parse_clock_sync_request(request_bytes); + REQUIRE(request.has_value()); + const Bytes reply = build_clock_sync_reply( + *request, std::numeric_limits::max(), std::numeric_limits::min()); + CHECK(reply == from_hex(kExtremeReply)); +} + +TEST_CASE("set_clock_sync_reply_t3 changes only the t3 bytes", "[clock_sync]") +{ + const Bytes before = build_clock_sync_reply(ClockSyncRequest{ 7, -5 }, 11, 13); + Bytes after = before; + set_clock_sync_reply_t3(after, 0x0102030405060708LL); + + REQUIRE(after.size() == before.size()); + for (std::size_t i = 0; i < before.size(); ++i) + { + if (i >= 21 && i <= 28) + CHECK(after[i] == static_cast(0x08 - (i - 21))); + else + CHECK(after[i] == before[i]); + } + CHECK(after == build_clock_sync_reply(ClockSyncRequest{ 7, -5 }, 11, 0x0102030405060708LL)); +} + +TEST_CASE("monotonic_ns does not go backwards", "[clock_sync]") +{ + const std::int64_t a = monotonic_ns(); + const std::int64_t b = monotonic_ns(); + CHECK(a > 0); + CHECK(b >= a); +} diff --git a/src/video_stage/wire/tests/test_encoded_video_frame.cpp b/src/video_stage/wire/tests/test_encoded_video_frame.cpp new file mode 100644 index 0000000000..7445a6c231 --- /dev/null +++ b/src/video_stage/wire/tests/test_encoded_video_frame.cpp @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +TEST_CASE("a default EncodedVideoFrame is empty and not a keyframe", "[EncodedVideoFrame]") +{ + const video_stage::EncodedVideoFrame frame; + CHECK(frame.data == nullptr); + CHECK(frame.size == 0); + CHECK_FALSE(frame.keyframe); + CHECK(frame.timestamp_us == 0); + CHECK(frame.user_timestamp_us == 0); + CHECK(frame.frame_id == 0); + CHECK(frame.capture_tsc == 0); +} diff --git a/src/video_stage/wire/tests/test_frame_correlator.cpp b/src/video_stage/wire/tests/test_frame_correlator.cpp new file mode 100644 index 0000000000..9fc7707171 --- /dev/null +++ b/src/video_stage/wire/tests/test_frame_correlator.cpp @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include + +using video_stage::FrameCorrelator; +using video_stage::InFlightFrame; + +namespace +{ + +InFlightFrame frame(std::uint32_t sequence, std::int64_t submitted_us) +{ + InFlightFrame f; + f.sequence = sequence; + f.capture_tsc = 1000 + sequence; + f.stamp_us = 5000 + sequence; + f.submitted_us = submitted_us; + return f; +} + +} // namespace + +TEST_CASE("stamp mode matches returned units in order", "[frame_correlator]") +{ + FrameCorrelator c(FrameCorrelator::Mode::Stamp); + for (std::uint32_t i = 0; i < 5; ++i) + c.submitted(frame(i, 100 + 33 * i)); + for (std::uint32_t i = 0; i < 5; ++i) + { + const auto r = c.returned(100 + 33 * i); + REQUIRE(r.has_value()); + CHECK(r->identified); + CHECK(r->skipped == 0); + CHECK(r->frame.sequence == i); + CHECK(r->frame.capture_tsc == 1000 + i); + } + CHECK(c.empty()); +} + +TEST_CASE("stamp mode counts a skipped input and keeps matching", "[frame_correlator]") +{ + FrameCorrelator c(FrameCorrelator::Mode::Stamp); + c.submitted(frame(0, 10)); + c.submitted(frame(1, 20)); + c.submitted(frame(2, 30)); + auto a = c.returned(10); + REQUIRE(a); + CHECK(a->frame.sequence == 0); + auto third = c.returned(30); + REQUIRE(third); + CHECK(third->identified); + CHECK(third->frame.sequence == 2); + CHECK(third->skipped == 1); + CHECK(c.empty()); + + c.submitted(frame(3, 40)); + auto later = c.returned(40); + REQUIRE(later); + CHECK(later->identified); + CHECK(later->frame.sequence == 3); + CHECK(later->skipped == 0); +} + +TEST_CASE("stamp mode reports a unit that matches nothing as not identified", "[frame_correlator]") +{ + FrameCorrelator c(FrameCorrelator::Mode::Stamp); + c.submitted(frame(0, 10)); + c.submitted(frame(1, 20)); + c.submitted(frame(2, 30)); + const auto r = c.returned(999); + REQUIRE(r); + CHECK_FALSE(r->identified); + CHECK(r->skipped == 0); + CHECK(r->frame.sequence == 0); // the oldest, removed and not credited + CHECK(c.in_flight() == 2); + + const auto next = c.returned(20); + REQUIRE(next); + CHECK(next->identified); + CHECK(next->frame.sequence == 1); + CHECK(next->skipped == 0); + const auto last = c.returned(30); + REQUIRE(last); + CHECK(last->identified); + CHECK(last->frame.sequence == 2); +} + +TEST_CASE("returned yields nothing when no frame is in flight", "[frame_correlator]") +{ + for (auto mode : { FrameCorrelator::Mode::Stamp, FrameCorrelator::Mode::Fifo }) + { + FrameCorrelator c(mode); + CHECK_FALSE(c.returned(1).has_value()); + c.submitted(frame(0, 1)); + REQUIRE(c.returned(1)); + CHECK_FALSE(c.returned(1).has_value()); + } +} + +TEST_CASE("the in-flight queue is bounded by dropping the oldest", "[frame_correlator]") +{ + FrameCorrelator c(FrameCorrelator::Mode::Stamp, 12); + for (std::uint32_t i = 0; i < 13; ++i) + c.submitted(frame(i, 100 + i)); + CHECK(c.in_flight() == 12); + + // The dropped frame's unit arrives late: it matches nothing. + const auto late = c.returned(100); + REQUIRE(late); + CHECK_FALSE(late->identified); + CHECK(late->frame.sequence == 1); // the oldest remaining was removed, not credited + + const auto next = c.returned(102); + REQUIRE(next); + CHECK(next->identified); + CHECK(next->frame.sequence == 2); + CHECK(next->skipped == 0); +} + +TEST_CASE("fifo mode takes the oldest regardless of stamp and never skips", "[frame_correlator]") +{ + FrameCorrelator c(FrameCorrelator::Mode::Fifo); + c.submitted(frame(0, 10)); + c.submitted(frame(1, 20)); + c.submitted(frame(2, 30)); + for (std::uint32_t i = 0; i < 3; ++i) + { + const auto r = c.returned(30 - 10 * static_cast(i)); // stamps deliberately out of order + REQUIRE(r); + CHECK(r->identified); + CHECK(r->skipped == 0); + CHECK(r->frame.sequence == i); + } + CHECK(c.empty()); +} + +TEST_CASE("stamp mode: random submit and return sequences with skips", "[frame_correlator]") +{ + std::mt19937 rng(12345); + for (int run = 0; run < 100; ++run) + { + FrameCorrelator c(FrameCorrelator::Mode::Stamp); + std::vector pending; // submitted, not yet returned or skipped + std::uint32_t next = 0; + std::int64_t stamp = 1000; + for (int step = 0; step < 200; ++step) + { + if (rng() % 2 == 0 && pending.size() < 12) + { + stamp += 1 + static_cast(rng() % 50); // distinct, increasing + const InFlightFrame f = frame(next++, stamp); + c.submitted(f); + pending.push_back(f); + } + else if (!pending.empty()) + { + // Return a random in-flight frame; those before it are skipped by the encoder. + const std::size_t k = rng() % pending.size(); + const auto r = c.returned(pending[k].submitted_us); + REQUIRE(r); + REQUIRE(r->identified); + CHECK(r->frame.submitted_us == pending[k].submitted_us); + CHECK(r->frame.sequence == pending[k].sequence); + CHECK(r->skipped == k); + pending.erase(pending.begin(), pending.begin() + static_cast(k) + 1); + } + } + CHECK(c.in_flight() == pending.size()); + } +} diff --git a/src/video_stage/wire/tests/test_h264_access_units.cpp b/src/video_stage/wire/tests/test_h264_access_units.cpp new file mode 100644 index 0000000000..661c693898 --- /dev/null +++ b/src/video_stage/wire/tests/test_h264_access_units.cpp @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include + +using video_stage::AccessUnitRange; +using video_stage::split_access_units; + +namespace +{ + +using Bytes = std::vector; + +const Bytes kStart4 = { 0, 0, 0, 1 }; +const Bytes kStart3 = { 0, 0, 1 }; + +/// A NAL with the given start code, header byte and payload (payloads here never contain 00 00 0x). +Bytes nal(const Bytes& start, std::uint8_t header, const Bytes& payload) +{ + Bytes out = start; + out.push_back(header); + out.insert(out.end(), payload.begin(), payload.end()); + return out; +} + +// first_mb_in_slice is the Exp-Golomb code at the start of the slice data: '1' is 0, '010' is 1, '011' is 2. +const Bytes kFirstSlice = { 0x88, 0x84, 0x21 }; +const Bytes kSecondSlice = { 0x48, 0x84, 0x21 }; +const Bytes kThirdSlice = { 0x68, 0x84, 0x21 }; + +Bytes sps(const Bytes& start = kStart4) +{ + return nal(start, 0x67, { 0x64, 0x00, 0x1f, 0xac }); +} +Bytes pps(const Bytes& start = kStart4) +{ + return nal(start, 0x68, { 0xeb, 0xc3, 0xcb }); +} +Bytes idr(const Bytes& start = kStart4, const Bytes& slice = kFirstSlice) +{ + return nal(start, 0x65, slice); +} +Bytes p_slice(const Bytes& start = kStart3, const Bytes& slice = kFirstSlice) +{ + return nal(start, 0x41, slice); +} +Bytes sei(const Bytes& start = kStart3) +{ + return nal(start, 0x06, { 0x05, 0x02, 0xaa, 0xbb, 0x80 }); +} +Bytes filler(const Bytes& start = kStart3) +{ + return nal(start, 0x0c, { 0xff, 0xff, 0x80 }); +} + +Bytes concat(std::initializer_list parts) +{ + Bytes out; + for (const auto& part : parts) + out.insert(out.end(), part.begin(), part.end()); + return out; +} + +std::vector split(const Bytes& stream) +{ + return split_access_units(stream.data(), stream.size()); +} + +} // namespace + +TEST_CASE("an empty or start-code-free input has no access units", "[h264_access_units]") +{ + CHECK(split_access_units(nullptr, 0).empty()); + CHECK(split(Bytes{}).empty()); + CHECK(split(Bytes{ 1, 2, 3, 4, 5, 6 }).empty()); + CHECK(split(Bytes{ 0, 0, 1 }).empty()); // a start code with no NAL after it +} + +TEST_CASE("parameter sets stay with the IDR that follows", "[h264_access_units]") +{ + const Bytes a = concat({ sps(), pps(), idr() }); + const Bytes b = p_slice(); + const Bytes c = p_slice(); + const auto units = split(concat({ a, b, c })); + + REQUIRE(units.size() == 3); + CHECK(units[0].offset == 0); + CHECK(units[0].size == a.size()); + CHECK(units[0].idr); + CHECK(units[1].offset == a.size()); + CHECK(units[1].size == b.size()); + CHECK_FALSE(units[1].idr); + CHECK(units[2].offset == a.size() + b.size()); + CHECK(units[2].size == c.size()); + CHECK_FALSE(units[2].idr); +} + +TEST_CASE("the units tile the stream exactly", "[h264_access_units]") +{ + const Bytes stream = concat({ sps(), pps(), idr(), p_slice(), sei(), p_slice(), p_slice() }); + const auto units = split(stream); + REQUIRE(units.size() == 4); + std::size_t expected_offset = 0; + for (const auto& unit : units) + { + CHECK(unit.offset == expected_offset); + expected_offset += unit.size; + } + CHECK(expected_offset == stream.size()); +} + +TEST_CASE("slices of one picture stay together", "[h264_access_units]") +{ + const Bytes first = + concat({ sps(), pps(), idr(kStart4, kFirstSlice), idr(kStart3, kSecondSlice), idr(kStart3, kThirdSlice) }); + const Bytes second = concat({ p_slice(kStart3, kFirstSlice), p_slice(kStart3, kSecondSlice) }); + const auto units = split(concat({ first, second })); + + REQUIRE(units.size() == 2); + CHECK(units[0].size == first.size()); + CHECK(units[0].idr); + CHECK(units[1].offset == first.size()); + CHECK(units[1].size == second.size()); + CHECK_FALSE(units[1].idr); +} + +TEST_CASE("a new first slice directly after a picture starts a new unit", "[h264_access_units]") +{ + const Bytes p1 = concat({ p_slice(kStart3, kFirstSlice), p_slice(kStart3, kSecondSlice) }); + const Bytes p2 = p_slice(kStart3, kFirstSlice); + const auto units = split(concat({ p1, p2 })); + REQUIRE(units.size() == 2); + CHECK(units[0].size == p1.size()); + CHECK(units[1].size == p2.size()); +} + +TEST_CASE("SEI and an access unit delimiter before a picture belong to that picture", "[h264_access_units]") +{ + const Bytes p0 = p_slice(); + const Bytes aud = nal(kStart3, 0x09, { 0xf0 }); + const Bytes next = concat({ aud, sei(), p_slice() }); + const auto units = split(concat({ p0, next })); + + REQUIRE(units.size() == 2); + CHECK(units[0].size == p0.size()); + CHECK(units[1].offset == p0.size()); + CHECK(units[1].size == next.size()); +} + +TEST_CASE("trailing filler, end of sequence and end of stream stay with the previous picture", "[h264_access_units]") +{ + SECTION("filler") + { + const Bytes first = concat({ p_slice(), filler() }); + const Bytes second = p_slice(); + const auto units = split(concat({ first, second })); + REQUIRE(units.size() == 2); + CHECK(units[0].size == first.size()); + CHECK(units[1].offset == first.size()); + } + SECTION("end of sequence then the next IDR's parameter sets") + { + const Bytes first = concat({ + sps(), + pps(), + idr(), + nal(kStart3, 0x0a, {}), + }); + const Bytes second = concat({ sps(), pps(), idr() }); + const auto units = split(concat({ first, second })); + REQUIRE(units.size() == 2); + CHECK(units[0].size == first.size()); + CHECK(units[1].offset == first.size()); + CHECK(units[1].idr); + } + SECTION("end of stream at the very end") + { + const Bytes stream = concat({ p_slice(), nal(kStart4, 0x0b, {}) }); + const auto units = split(stream); + REQUIRE(units.size() == 1); + CHECK(units[0].size == stream.size()); + } + SECTION("several fillers in a row") + { + const Bytes first = concat({ p_slice(), filler(), filler() }); + const auto units = split(concat({ first, p_slice() })); + REQUIRE(units.size() == 2); + CHECK(units[0].size == first.size()); + } +} + +TEST_CASE("three- and four-byte start codes both work, and a four-byte code's leading zero begins its unit", + "[h264_access_units]") +{ + const Bytes first = concat( + { nal(kStart3, 0x67, { 0x64, 0x00, 0x1f }), nal(kStart3, 0x68, { 0xeb }), nal(kStart3, 0x65, kFirstSlice) }); + const Bytes second = p_slice(kStart4); + const Bytes third = p_slice(kStart3); + const auto units = split(concat({ first, second, third })); + + REQUIRE(units.size() == 3); + CHECK(units[0].size == first.size()); + CHECK(units[1].offset == first.size()); // includes the zero byte of the 4-byte code + CHECK(units[1].size == second.size()); + CHECK(units[2].offset == first.size() + second.size()); + CHECK(units[2].size == third.size()); +} + +TEST_CASE("only an IDR slice marks a unit as a keyframe", "[h264_access_units]") +{ + const auto units = split(concat({ sps(), pps(), idr(), p_slice(), idr(kStart4), p_slice() })); + REQUIRE(units.size() == 4); + CHECK(units[0].idr); + CHECK_FALSE(units[1].idr); + CHECK(units[2].idr); + CHECK_FALSE(units[3].idr); +} + +TEST_CASE("a single NAL is one unit", "[h264_access_units]") +{ + const Bytes stream = p_slice(kStart4); + const auto units = split(stream); + REQUIRE(units.size() == 1); + CHECK(units[0].offset == 0); + CHECK(units[0].size == stream.size()); +} diff --git a/src/video_stage/wire/tests/test_h264_sps.cpp b/src/video_stage/wire/tests/test_h264_sps.cpp new file mode 100644 index 0000000000..e1917638c8 --- /dev/null +++ b/src/video_stage/wire/tests/test_h264_sps.cpp @@ -0,0 +1,353 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include + +using video_stage::h264::declare_no_reordering; +using video_stage::h264::parse_sps; +using video_stage::h264::SpsInfo; + +namespace +{ +// First SPS, PPS and the start of the first IDR slice of short ffmpeg streams (x264 unless noted), 3 frames each. +// Every SPS contains emulation-prevention bytes in its VUI timing fields. The IDR bytes are truncated: only their +// preservation matters here. +// a_high_720 +const std::vector a_high_720_sps = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x1f, 0xac, 0xb2, + 0x00, 0xa0, 0x0b, 0x76, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, + 0x20, 0x00, 0x00, 0x07, 0x81, 0xe3, 0x06, 0x49 }; +const std::vector a_high_720_pps = { 0x00, 0x00, 0x00, 0x01, 0x68, 0xeb, 0xc3, 0xcb, 0x22, 0xc0 }; +const std::vector a_high_720_idr = { 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x08, 0x7f, 0xd3, 0xd6, 0x67, 0x0b, + 0x58, 0xc9, 0x09, 0x0f, 0xd0, 0x9f, 0x5b, 0xe3, 0x26, 0xe9, 0xc3, 0xd4, + 0xb8, 0x70, 0x34, 0xe3, 0x55, 0x12, 0xc0, 0x2f, 0xf6, 0xfb, 0x3c, 0x4c, + 0x99, 0xf9, 0xe1, 0x86, 0x0b, 0xe5, 0x90, 0x5d, 0x3f, 0x68, 0xf2, 0xb6 }; +// b_high_2560 +const std::vector b_high_2560_sps = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x33, 0xac, 0xb2, + 0x00, 0x50, 0x01, 0xf3, 0x60, 0x22, 0x00, 0x00, 0x03, 0x00, + 0x02, 0x00, 0x00, 0x03, 0x00, 0x78, 0x1e, 0x30, 0x64, 0x90 }; +const std::vector b_high_2560_pps = { 0x00, 0x00, 0x00, 0x01, 0x68, 0xeb, 0xc3, 0xcb, 0x22, 0xc0 }; +const std::vector b_high_2560_idr = { 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x08, 0x7f, 0xd3, 0xd6, 0x67, 0x0b, + 0x58, 0xc9, 0x09, 0x0f, 0xd0, 0x9f, 0x5b, 0xe3, 0x26, 0xe9, 0xc3, 0xd4, + 0xb8, 0x70, 0x34, 0xe3, 0x55, 0x12, 0xce, 0x35, 0x37, 0xb3, 0xca, 0x63, + 0x32, 0x3d, 0xb7, 0x41, 0xc7, 0x2a, 0x82, 0xea, 0x04, 0xb9, 0x64, 0x3e }; +// c_base_720 +const std::vector c_base_720_sps = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f, 0xd9, 0x00, + 0x50, 0x05, 0xbb, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, 0x10, + 0x00, 0x00, 0x03, 0x03, 0xc0, 0xf1, 0x83, 0x24, 0x80 }; +const std::vector c_base_720_pps = { 0x00, 0x00, 0x00, 0x01, 0x68, 0xcb, 0x83, 0xcb, 0x20 }; +const std::vector c_base_720_idr = { 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x08, 0x7c, 0x42, 0x22, 0xb0, 0x60, + 0x42, 0x1f, 0x80, 0xea, 0x40, 0x03, 0xec, 0xcc, 0xb3, 0x20, 0x00, 0x2d, + 0x66, 0x69, 0x14, 0x68, 0x3e, 0x4e, 0xff, 0x70, 0x2a, 0x0a, 0x4c, 0xb3, + 0x6e, 0x2d, 0xfe, 0x03, 0x16, 0x67, 0x22, 0x8e, 0x1f, 0x37, 0x7f, 0xb0 }; +// d_cbr_2560 +const std::vector d_cbr_2560_sps = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x33, 0xac, 0xb2, 0x00, + 0x50, 0x01, 0xf3, 0x60, 0x22, 0x00, 0x00, 0x03, 0x00, 0x02, 0x00, + 0x00, 0x03, 0x00, 0x78, 0xcc, 0x00, 0x01, 0xe8, 0x48, 0x00, 0x10, + 0x4b, 0xed, 0x61, 0x80, 0x3c, 0x60, 0xc9, 0x20 }; +const std::vector d_cbr_2560_pps = { 0x00, 0x00, 0x00, 0x01, 0x68, 0xeb, 0xcc, 0xb2, 0x2c }; +const std::vector d_cbr_2560_idr = { 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x0b, 0x7f, 0xdb, 0xe5, 0x95, 0xbd, + 0xc6, 0x03, 0x72, 0x93, 0x37, 0x67, 0xad, 0x5c, 0xfd, 0xf5, 0x21, 0xdc, + 0xcb, 0x9f, 0xfd, 0xce, 0xc4, 0xe2, 0x96, 0x1a, 0x45, 0xb2, 0x75, 0x05, + 0x88, 0xac, 0xfb, 0x5f, 0xf2, 0x45, 0x4c, 0x6d, 0x93, 0x7f, 0xac, 0x86 }; +// e_bframes +const std::vector e_bframes_sps = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x1f, 0xac, 0xd9, + 0x40, 0x50, 0x05, 0xbb, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, + 0x10, 0x00, 0x00, 0x03, 0x03, 0xc0, 0xf1, 0x83, 0x19, 0x60 }; +const std::vector e_bframes_pps = { 0x00, 0x00, 0x00, 0x01, 0x68, 0xe9, 0x38, 0xf2, 0xc8, 0xb0 }; +const std::vector e_bframes_idr = { 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x00, 0x21, 0xff, 0xd3, 0xd6, 0x67, + 0x0b, 0x58, 0xc9, 0x09, 0x0f, 0xd0, 0x9f, 0x5b, 0xe3, 0x26, 0xe9, 0xc3, + 0xd4, 0xb8, 0x70, 0x34, 0xe3, 0x55, 0x12, 0xc0, 0x2f, 0xf6, 0xfb, 0x3c, + 0x4c, 0x99, 0xf9, 0xe1, 0x86, 0x0b, 0xe5, 0x90, 0x5d, 0x3f, 0x68, 0xf2 }; +// f_full +const std::vector f_full_sps = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x1f, 0xac, 0xb2, + 0x00, 0xa0, 0x0b, 0x76, 0x02, 0xd9, 0x00, 0x00, 0x03, 0x00, + 0x01, 0x00, 0x00, 0x03, 0x00, 0x3c, 0x0f, 0x18, 0x32, 0x48 }; +const std::vector f_full_pps = { 0x00, 0x00, 0x00, 0x01, 0x68, 0xeb, 0xc3, 0xcb, 0x22, 0xc0 }; +const std::vector f_full_idr = { 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x08, 0x7f, 0xd3, 0xd6, 0x67, 0x0b, + 0x58, 0xc9, 0x09, 0x0f, 0xd0, 0x9f, 0x5b, 0xe3, 0x26, 0xe9, 0xc3, 0xd4, + 0xb8, 0x70, 0x34, 0xe3, 0x55, 0x12, 0xc0, 0x2f, 0xf6, 0xfb, 0x3c, 0x4c, + 0x99, 0xf9, 0xe1, 0x86, 0x0b, 0xe5, 0x90, 0x5d, 0x3f, 0x68, 0xf2, 0xb6 }; +// g_nvenc +const std::vector g_nvenc_sps = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x40, 0x1f, 0x95, 0x90, 0x05, 0x00, 0x5b, + 0xb0, 0x11, 0x00, 0x00, 0x03, 0x03, 0xe8, 0x00, 0x00, 0xea, 0x60, 0xe0, 0x00, + 0x00, 0x16, 0xe3, 0x60, 0x00, 0x07, 0xa1, 0x20, 0xbb, 0xcb, 0x82, 0x80 }; +const std::vector g_nvenc_pps = { 0x00, 0x00, 0x00, 0x01, 0x68, 0xeb, 0x8f, 0x20 }; +const std::vector g_nvenc_idr = { 0x00, 0x00, 0x00, 0x01, 0x65, 0xb8, 0x04, 0x3f, 0xad, 0x6e, 0xda, 0xc4, + 0xf1, 0x5a, 0x80, 0x2e, 0xee, 0x6f, 0x27, 0x6e, 0x50, 0x1f, 0xdd, 0x44, + 0xcf, 0xce, 0x97, 0x18, 0xd2, 0x37, 0x20, 0xd0, 0x6c, 0x94, 0xee, 0xe0, + 0xe1, 0xc6, 0x50, 0x59, 0x74, 0xda, 0x8d, 0x62, 0xfa, 0x36, 0x69, 0xbc }; + +std::vector concat(const std::vector& a, const std::vector& b, const std::vector& c) +{ + std::vector out(a); + out.insert(out.end(), b.begin(), b.end()); + out.insert(out.end(), c.begin(), c.end()); + return out; +} + +struct Case +{ + const char* name; + const std::vector* sps; + const std::vector* pps; + const std::vector* idr; + uint32_t profile, level, width, height, refs, poc; + bool vui, full_range, restriction; + uint32_t reorder, dpb; +}; + +const std::vector& cases() +{ + static const std::vector c = { + { "x264 high 1280x720", &a_high_720_sps, &a_high_720_pps, &a_high_720_idr, 100, 31, 1280, 720, 3, 2, true, + false, true, 0, 3 }, + { "x264 high 2560x1984", &b_high_2560_sps, &b_high_2560_pps, &b_high_2560_idr, 100, 51, 2560, 1984, 3, 2, true, + false, true, 0, 3 }, + { "x264 baseline 1280x720", &c_base_720_sps, &c_base_720_pps, &c_base_720_idr, 66, 31, 1280, 720, 3, 2, true, + false, true, 0, 3 }, + { "x264 high CBR (nal-hrd) 2560x1984", &d_cbr_2560_sps, &d_cbr_2560_pps, &d_cbr_2560_idr, 100, 51, 2560, 1984, + 3, 2, true, false, true, 0, 3 }, + { "x264 high with B-frames", &e_bframes_sps, &e_bframes_pps, &e_bframes_idr, 100, 31, 1280, 720, 4, 0, true, + false, true, 2, 4 }, + { "x264 full range", &f_full_sps, &f_full_pps, &f_full_idr, 100, 31, 1280, 720, 3, 2, true, true, true, 0, 3 }, + // NVENC emits a VUI without bitstream_restriction: the case the rewrite exists for. + { "h264_nvenc main", &g_nvenc_sps, &g_nvenc_pps, &g_nvenc_idr, 77, 31, 1280, 720, 3, 2, true, false, false, 0, 0 }, + }; + return c; +} +} // namespace + +TEST_CASE("parse_sps reads the fields of real SPS NAL units", "[Sps]") +{ + for (const Case& c : cases()) + { + INFO(c.name); + const auto au = concat(*c.sps, *c.pps, *c.idr); + const auto info = parse_sps(au.data(), au.size()); + REQUIRE(info); + CHECK(info->profile_idc == c.profile); + CHECK(info->level_idc == c.level); + CHECK(info->width == c.width); + CHECK(info->height == c.height); + CHECK(info->max_num_ref_frames == c.refs); + CHECK(info->pic_order_cnt_type == c.poc); + CHECK(info->frame_mbs_only); + CHECK(info->vui_present == c.vui); + CHECK(info->full_range == c.full_range); + CHECK(info->bitstream_restriction == c.restriction); + if (c.restriction) + { + CHECK(info->max_num_reorder_frames == c.reorder); + CHECK(info->max_dec_frame_buffering == c.dpb); + } + CHECK_FALSE(info->describe().empty()); + } +} + +TEST_CASE("declare_no_reordering sets the restriction and changes nothing else", "[Sps]") +{ + for (const Case& c : cases()) + { + INFO(c.name); + const auto au = concat(*c.sps, *c.pps, *c.idr); + const auto before = parse_sps(au.data(), au.size()); + REQUIRE(before); + const auto out = declare_no_reordering(au.data(), au.size()); + REQUIRE(out); + const auto after = parse_sps(out->data(), out->size()); + REQUIRE(after); + + CHECK(after->bitstream_restriction); + CHECK(after->max_num_reorder_frames == 0); + CHECK(after->max_dec_frame_buffering == std::max(before->max_num_ref_frames, 1)); + CHECK(after->profile_idc == before->profile_idc); + CHECK(after->level_idc == before->level_idc); + CHECK(after->chroma_format_idc == before->chroma_format_idc); + CHECK(after->pic_order_cnt_type == before->pic_order_cnt_type); + CHECK(after->max_num_ref_frames == before->max_num_ref_frames); + CHECK(after->width == before->width); + CHECK(after->height == before->height); + CHECK(after->frame_mbs_only == before->frame_mbs_only); + CHECK(after->vui_present); + CHECK(after->full_range == before->full_range); + + // Everything after the SPS (PPS, IDR) is byte-identical. + const std::size_t rest = c.pps->size() + c.idr->size(); + REQUIRE(out->size() > rest); + CHECK(std::equal(au.end() - rest, au.end(), out->end() - rest)); + // The SPS keeps its start code and header byte. + CHECK(std::equal(c.sps->begin(), c.sps->begin() + 5, out->begin())); + + // Rewriting is idempotent. + const auto twice = declare_no_reordering(out->data(), out->size()); + REQUIRE(twice); + CHECK(*twice == *out); + } +} + +TEST_CASE("an access unit with a different NAL ahead of the SPS is handled", "[Sps]") +{ + const Case& c = cases()[6]; + const std::vector aud = { 0, 0, 0, 1, 0x09, 0xf0 }; + const auto body = concat(*c.sps, *c.pps, *c.idr); + const auto au = concat(aud, body, {}); + const auto info = parse_sps(au.data(), au.size()); + REQUIRE(info); + CHECK(info->width == 1280); + const auto out = declare_no_reordering(au.data(), au.size()); + REQUIRE(out); + CHECK(std::equal(aud.begin(), aud.end(), out->begin())); + const auto after = parse_sps(out->data(), out->size()); + REQUIRE(after); + CHECK(after->bitstream_restriction); + CHECK(std::equal(au.end() - (c.pps->size() + c.idr->size()), au.end(), out->end() - (c.pps->size() + c.idr->size()))); +} + +TEST_CASE("a three byte start code is accepted", "[Sps]") +{ + const Case& c = cases()[0]; + std::vector au(c.sps->begin() + 1, c.sps->end()); // 00 00 01 + NAL + const auto info = parse_sps(au.data(), au.size()); + REQUIRE(info); + CHECK(info->width == 1280); +} + +TEST_CASE("no SPS or a damaged SPS yields nullopt", "[Sps]") +{ + const std::vector none = { 0, 0, 0, 1, 0x68, 0xce, 0x3c, 0x80 }; + CHECK_FALSE(parse_sps(none.data(), none.size())); + CHECK_FALSE(declare_no_reordering(none.data(), none.size())); + CHECK_FALSE(parse_sps(none.data(), 0)); + + const Case& c = cases()[0]; + std::vector truncated(c.sps->begin(), c.sps->begin() + 8); + CHECK_FALSE(declare_no_reordering(truncated.data(), truncated.size())); +} + +namespace +{ +struct BitWriter +{ + std::vector bytes; + int used = 8; + void bit(unsigned v) + { + if (used == 8) + { + bytes.push_back(0); + used = 0; + } + bytes.back() = static_cast(bytes.back() | ((v & 1u) << (7 - used))); + ++used; + } + void bits(unsigned v, int n) + { + for (int i = n - 1; i >= 0; --i) + bit((v >> i) & 1u); + } + void ue(unsigned v) + { + const unsigned x = v + 1; + int len = 0; + while ((x >> len) > 1) + ++len; + bits(0, len); + bits(x, len + 1); + } +}; + +/// A baseline-profile 1280x720 SPS whose VUI carries only the chroma location field. `chroma_loc` is null for none. +std::vector synthetic_sps(const std::pair* chroma_loc) +{ + BitWriter w; + w.bits(66, 8); // profile_idc + w.bits(0, 8); // constraint flags + w.bits(31, 8); // level_idc + w.ue(0); // seq_parameter_set_id + w.ue(0); // log2_max_frame_num_minus4 + w.ue(2); // pic_order_cnt_type + w.ue(1); // max_num_ref_frames + w.bit(0); // gaps_in_frame_num_value_allowed_flag + w.ue(79); // pic_width_in_mbs_minus1 + w.ue(44); // pic_height_in_map_units_minus1 + w.bit(1); // frame_mbs_only_flag + w.bit(1); // direct_8x8_inference_flag + w.bit(0); // frame_cropping_flag + w.bit(1); // vui_parameters_present_flag + w.bit(0); // aspect_ratio_info_present_flag + w.bit(0); // overscan_info_present_flag + w.bit(0); // video_signal_type_present_flag + if (chroma_loc) + { + w.bit(1); + w.ue(chroma_loc->first); + w.ue(chroma_loc->second); + } + else + { + w.bit(0); + } + w.bit(0); // timing_info_present_flag + w.bit(0); // nal_hrd_parameters_present_flag + w.bit(0); // vcl_hrd_parameters_present_flag + w.bit(0); // pic_struct_present_flag + w.bit(0); // bitstream_restriction_flag + w.bit(1); // rbsp_stop_one_bit + std::vector out = { 0, 0, 0, 1, 0x67 }; + out.insert(out.end(), w.bytes.begin(), w.bytes.end()); + return out; +} +} // namespace + +TEST_CASE("chroma sample locations parse into chroma_loc_*", "[Sps]") +{ + const std::pair zero{ 0, 0 }; + const std::pair one{ 1, 1 }; + const std::pair mixed{ 2, 5 }; + + SECTION("absent") + { + const auto au = synthetic_sps(nullptr); + const auto info = parse_sps(au.data(), au.size()); + REQUIRE(info); + CHECK(info->width == 1280); + CHECK(info->height == 720); + CHECK(info->vui_present); + CHECK_FALSE(info->chroma_loc_present); + CHECK(info->chroma_loc_top == 0); + CHECK(info->chroma_loc_bottom == 0); + } + SECTION("present with (0, 0)") + { + const auto au = synthetic_sps(&zero); + const auto info = parse_sps(au.data(), au.size()); + REQUIRE(info); + CHECK(info->chroma_loc_present); + CHECK(info->chroma_loc_top == 0); + CHECK(info->chroma_loc_bottom == 0); + } + SECTION("present with (1, 1)") + { + const auto au = synthetic_sps(&one); + const auto info = parse_sps(au.data(), au.size()); + REQUIRE(info); + CHECK(info->chroma_loc_present); + CHECK(info->chroma_loc_top == 1); + CHECK(info->chroma_loc_bottom == 1); + CHECK(info->width == 1280); + } + SECTION("distinct top and bottom values") + { + const auto au = synthetic_sps(&mixed); + const auto info = parse_sps(au.data(), au.size()); + REQUIRE(info); + CHECK(info->chroma_loc_top == 2); + CHECK(info->chroma_loc_bottom == 5); + } +}