sensing: answer the PC's clock exchange and stamp frames with a documented time

src/plugins/sensing/app/main.cpp

@@ -124,44 +124,50 @@ static std::string default_platform_config()
 
 static void print_usage(const char* program_name)
 {
-    std::cout << "Usage: " << program_name << " --livekit-url=URL --livekit-token-file=PATH --add-stream=... [options]\n"
-              << "\n"
-              << "Publishes each SENSING sensor as a LiveKit camera track: SIPL or Argus capture, hardware H.264,\n"
-              << "sent as it is.\n"
-              << "\n"
-              << "Streams (repeat for each sensor):\n"
-              << "  --add-stream=sensor=N,livekit=NAME   camera N (SIPL or Argus device), published as track NAME\n"
-              << "  --list-sensors                       print the cameras, then exit\n"
-              << "  --camera-backend=auto|sipl|argus     auto picks the one backend that matches (default);\n"
-              << "                                       (a backend must also be built in)\n"
-              << "  --argus-mode=N                       Argus sensor mode index (default: 0)\n"
-              << "\n"
-              << "Platform:\n"
-              << "  --platform-config=PATH  platform JSON (default: configs/shw5g.json beside this\n"
-              << "                          binary)\n"
-              << "  --config-name=NAME      named config inside it (default: SHW5G_2)\n"
-              << "  --link-masks='M M'      one mask per deserializer (default: '0x0000 0x1100')\n"
-              << "  --nito-dir=DIR          ISP tuning directory (default: /var/nvidia/nvcam/settings/sipl)\n"
-              << "\n"
-              << "LiveKit:\n"
-              << "  --livekit-url=URL         server, e.g. wss://livekit.example.org\n"
-              << "  --livekit-token-file=PATH file holding the join token; it names the room and the\n"
-              << "                            participant identity\n"
-              << "  --livekit-fps=N           publication rate cap (default: 30)\n"
-              << "  --livekit-scale-div=N     reduce every track by this integer divisor (default: 1)\n"
-              << "  --livekit-bitrate=N       target bitrate of each track's encoder in bps (default: 8000000)\n"
-              << "  --gop=N                   IDR period in frames (default: fps*5)\n"
-              << "\n"
-              << "Other:\n"
-              << "  --gpu-id=N          CUDA device index (default: 0)\n"
-              << "  --isp0-buffers=N    ISP0 buffers per sensor (default: 6)\n"
-              << "  --full-range        full-range YUV\n"
-              << "  --swap-uv           swap the chroma planes\n"
-              << "\n"
-              << "Example:\n"
-              << "  " << program_name << " --livekit-url=wss://livekit.example.org \\\n"
-              << "      --livekit-token-file=join.token --add-stream=sensor=0,livekit=sensing-left \\\n"
-              << "      --add-stream=sensor=1,livekit=sensing-right\n";
+    std::cout
+        << "Usage: " << program_name << " --livekit-url=URL --livekit-token-file=PATH --add-stream=... [options]\n"
+        << "\n"
+        << "Publishes each SENSING sensor as a LiveKit camera track: SIPL or Argus capture, hardware H.264,\n"
+        << "sent as it is.\n"
+        << "\n"
+        << "Streams (repeat for each sensor):\n"
+        << "  --add-stream=sensor=N,livekit=NAME   camera N (SIPL or Argus device), published as track NAME\n"
+        << "  --list-sensors                       print the cameras, then exit\n"
+        << "  --camera-backend=auto|sipl|argus     auto picks the one backend that matches (default);\n"
+        << "                                       (a backend must also be built in)\n"
+        << "  --argus-mode=N                       Argus sensor mode index (default: 0)\n"
+        << "\n"
+        << "Platform:\n"
+        << "  --platform-config=PATH  platform JSON (default: configs/shw5g.json beside this\n"
+        << "                          binary)\n"
+        << "  --config-name=NAME      named config inside it (default: SHW5G_2)\n"
+        << "  --link-masks='M M'      one mask per deserializer (default: '0x0000 0x1100')\n"
+        << "  --nito-dir=DIR          ISP tuning directory (default: /var/nvidia/nvcam/settings/sipl)\n"
+        << "\n"
+        << "LiveKit:\n"
+        << "  --livekit-url=URL         server, e.g. wss://livekit.example.org\n"
+        << "  --livekit-token-file=PATH file holding the join token; it names the room and the\n"
+        << "                            participant identity\n"
+        << "  --livekit-fps=N           publication rate cap (default: 30)\n"
+        << "  --livekit-scale-div=N     reduce every track by this integer divisor (default: 1)\n"
+        << "  --timestamp-basis=B       what each frame's timestamp is: monotonic_us (default) is this host's\n"
+        << "                            monotonic clock when the frame was ready to encode; capture_monotonic_us\n"
+        << "                            (UNVERIFIED on hardware) is the sensor's capture event on that clock and fails\n"
+        << "                            the stream if it cannot be placed; processing_unix_us is Unix processing time.\n"
+        << "                            The streams file on the PC must declare the same basis.\n"
+        << "  --livekit-bitrate=N       target bitrate of each track's encoder in bps (default: 8000000)\n"
+        << "  --gop=N                   IDR period in frames (default: fps*5)\n"
+        << "\n"
+        << "Other:\n"
+        << "  --gpu-id=N          CUDA device index (default: 0)\n"
+        << "  --isp0-buffers=N    ISP0 buffers per sensor (default: 6)\n"
+        << "  --full-range        full-range YUV\n"
+        << "  --swap-uv           swap the chroma planes\n"
+        << "\n"
+        << "Example:\n"
+        << "  " << program_name << " --livekit-url=wss://livekit.example.org \\\n"
+        << "      --livekit-token-file=join.token --add-stream=sensor=0,livekit=sensing-left \\\n"
+        << "      --add-stream=sensor=1,livekit=sensing-right\n";
 }
 
 // =============================================================================
@@ -238,6 +244,22 @@ try
         {
             camera_config.livekit.fps = static_cast<uint32_t>(std::stoul(arg.substr(14)));
         }
+        else if (arg.find("--timestamp-basis=") == 0)
+        {
+            const std::string basis = arg.substr(18);
+            if (basis == "monotonic_us")
+                camera_config.livekit.timestamp_basis = TimestampBasis::FrameReadyMonotonic;
+            else if (basis == "capture_monotonic_us")
+                camera_config.livekit.timestamp_basis = TimestampBasis::SensorCaptureMonotonic;
+            else if (basis == "processing_unix_us")
+                camera_config.livekit.timestamp_basis = TimestampBasis::ProcessingUnix;
+            else
+            {
+                std::cerr << "Error: --timestamp-basis must be monotonic_us, capture_monotonic_us or processing_unix_us."
+                          << std::endl;
+                return 1;
+            }
+        }
         else if (arg.find("--livekit-bitrate=") == 0)
         {
             camera_config.livekit.bitrate_bps = static_cast<uint32_t>(std::stoul(arg.substr(18)));

src/plugins/sensing/core/CMakeLists.txt

@@ -5,6 +5,7 @@
 add_library(sensing_core STATIC
     sensing_camera.cpp
     camera_backend.cpp
+    sensor_clock.cpp
     capture_factory.cpp
     jetson_encoder.cpp
     livekit_video_output.cpp
@@ -16,6 +17,7 @@ add_library(sensing_core STATIC
     inc/sensing_core/capture_factory.hpp
     inc/sensing_core/jetson_encoder.hpp
     inc/sensing_core/sensing_camera.hpp
+    inc/sensing_core/sensor_clock.hpp
     inc/sensing_core/sensing_types.hpp
     inc/sensing_core/sipl_camera.hpp
     ${JETSON_MMAPI_SOURCES}

src/plugins/sensing/core/argus_capture.cpp

@@ -552,6 +552,7 @@ struct ArgusCapture::Impl : std::enable_shared_from_this<ArgusCapture::Impl>
         if (metadata)
         {
             frame.sensor_time = metadata->getSensorTimestamp();
+            frame.sensor_domain = SensorTimeDomain::MonotonicRawNs;
             frame.sequence = metadata->getCaptureId();
         }
         else

src/plugins/sensing/core/inc/sensing_core/camera_capture.hpp

@@ -6,6 +6,8 @@
 
 #pragma once
 
+#include "sensor_clock.hpp"
+
 #include <cstddef>
 #include <cstdint>
 #include <functional>
@@ -120,6 +122,11 @@ struct CaptureFrame
     /// The backend's own sensor timestamp, unscaled and in its own clock domain; never convert it to another
     /// domain here. Zero when the backend reports none.
     uint64_t sensor_time = 0;
+    /// What `sensor_time` counts; the claim is checked at run time by SensorClockMap, not trusted.
+    SensorTimeDomain sensor_domain = SensorTimeDomain::None;
+    /// CLOCK_MONOTONIC ns of the sensor's capture event, set by the LiveKit output when it publishes capture
+    /// time; zero otherwise.
+    uint64_t capture_monotonic_ns = 0;
     std::variant<CudaRgbaPixels, DmaBufNv12Pixels> pixels;
     FrameLease lease;
 };

src/plugins/sensing/core/inc/sensing_core/sensing_types.hpp

@@ -25,6 +25,17 @@ struct StreamConfig
 };
 
 /// Where and how the tracks are published.
+/// The clock and event a published frame's application timestamp is read from.
+enum class TimestampBasis
+{
+    /// CLOCK_MONOTONIC when the frame was ready to encode (`monotonic_us`); recordable.
+    FrameReadyMonotonic,
+    /// The sensor's capture event converted to CLOCK_MONOTONIC (`capture_monotonic_us`); unverified on hardware.
+    SensorCaptureMonotonic,
+    /// CLOCK_REALTIME when the frame finished conversion (`processing_unix_us`); not recordable.
+    ProcessingUnix,
+};
+
 struct LiveKitConfig
 {
     std::string url;
@@ -36,6 +47,8 @@ struct LiveKitConfig
     uint32_t scale_div = 1;
     /// Target bitrate of each track's hardware encoder; the VBR ceiling is half as much again.
     uint32_t bitrate_bps = 8'000'000;
+    /// What each frame's application timestamp is; the receiving side's streams file must declare the same basis.
+    TimestampBasis timestamp_basis = TimestampBasis::FrameReadyMonotonic;
 };
 
 struct SensingConfig

src/plugins/sensing/core/inc/sensing_core/sensor_clock.hpp

@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <stdexcept>
+#include <string>
+
+namespace plugins
+{
+namespace sensing
+{
+
+/// What a backend's `CaptureFrame::sensor_time` counts.
+enum class SensorTimeDomain
+{
+    /// The backend reports no sensor time.
+    None,
+    /// SIPL `frameCaptureTSC`: ticks of the ARM generic counter (assumed; checked at run time by SensorClockMap).
+    GenericCounterTicks,
+    /// Argus `getSensorTimestamp()`: nanoseconds on CLOCK_MONOTONIC_RAW (assumed; checked at run time).
+    MonotonicRawNs,
+};
+
+/// A source clock reading paired with CLOCK_MONOTONIC, read back to back.
+struct ClockPair
+{
+    /// The source clock's value (counter ticks or raw ns) at the midpoint of the bracket.
+    int64_t source = 0;
+    /// CLOCK_MONOTONIC ns at the same midpoint.
+    int64_t monotonic_ns = 0;
+    /// Width of the bracket the pair was read in; the pair is no more certain than this.
+    int64_t bracket_ns = 0;
+};
+
+/// Reads the ARM generic counter (CNTVCT_EL0) and CLOCK_MONOTONIC back to back, keeping the narrowest of a few
+/// attempts. Throws where the counter is not readable (not aarch64).
+ClockPair read_counter_pair();
+/// The generic counter's frequency in Hz (CNTFRQ_EL0).
+int64_t read_counter_hz();
+/// Reads CLOCK_MONOTONIC_RAW and CLOCK_MONOTONIC back to back, keeping the narrowest of a few attempts.
+ClockPair read_raw_pair();
+
+/**
+ * @brief Places a backend's sensor timestamp on this machine's CLOCK_MONOTONIC.
+ *
+ * One anchor pair (source clock, CLOCK_MONOTONIC) is read through a supported clock API and refreshed at a low
+ * rate, never per frame; a frame's capture time is the anchor plus the source-clock difference. The source's units
+ * are an assumption, not a fact (see SensorTimeDomain), so every conversion is checked: a capture time that is in
+ * the future of the frame's ready time, or more than `max_age_ns` before it, means the assumed domain or rate is
+ * wrong and the conversion throws. There is no fallback to a host-side stamp.
+ *
+ * Not thread-safe; one instance per capture thread.
+ */
+class SensorClockMap
+{
+public:
+    /// Reads one pair; injectable so the mapping is testable off-target.
+    using PairReader = std::function<ClockPair()>;
+
+    struct Config
+    {
+        SensorTimeDomain domain = SensorTimeDomain::None;
+        /// Counter frequency in Hz; required for GenericCounterTicks.
+        int64_t counter_hz = 0;
+        /// Anchor refresh period.
+        int64_t refresh_ns = 1'000'000'000;
+        /// Largest plausible gap between capture and the frame being ready.
+        int64_t max_age_ns = 2'000'000'000;
+    };
+
+    SensorClockMap(Config config, PairReader reader) : m_config(config), m_reader(std::move(reader))
+    {
+        if (m_config.domain == SensorTimeDomain::None)
+            throw std::invalid_argument("SensorClockMap needs a sensor time domain");
+        if (m_config.domain == SensorTimeDomain::GenericCounterTicks && m_config.counter_hz <= 0)
+            throw std::invalid_argument("SensorClockMap needs the counter frequency for counter ticks");
+    }
+
+    /**
+     * @brief CLOCK_MONOTONIC ns of the sensor's capture event.
+     * @param sensor_time The backend's raw sensor timestamp.
+     * @param ready_monotonic_ns CLOCK_MONOTONIC when the frame was ready (the plausibility reference).
+     * @param now_monotonic_ns CLOCK_MONOTONIC now, to decide when the anchor is due.
+     * @throws std::runtime_error when the value is not plausible for the assumed domain.
+     */
+    int64_t to_monotonic_ns(uint64_t sensor_time, int64_t ready_monotonic_ns, int64_t now_monotonic_ns)
+    {
+        if (!m_have_anchor || now_monotonic_ns - m_anchor_taken_ns >= m_config.refresh_ns)
+        {
+            m_anchor = m_reader();
+            m_have_anchor = true;
+            m_anchor_taken_ns = now_monotonic_ns;
+        }
+        const int64_t delta = to_ns(static_cast<int64_t>(sensor_time) - m_anchor.source);
+        const int64_t capture = m_anchor.monotonic_ns + delta;
+        if (capture > ready_monotonic_ns || ready_monotonic_ns - capture > m_config.max_age_ns)
+        {
+            throw std::runtime_error(
+                "sensor timestamp " + std::to_string(sensor_time) + " maps to " +
+                std::to_string(capture - ready_monotonic_ns) +
+                " ns relative to the frame being ready: the assumed sensor clock domain or rate is wrong");
+        }
+        return capture;
+    }
+
+    /// Uncertainty of the current anchor (its bracket); zero before the first conversion.
+    int64_t anchor_uncertainty_ns() const
+    {
+        return m_have_anchor ? m_anchor.bracket_ns : 0;
+    }
+
+private:
+    int64_t to_ns(int64_t source_delta) const
+    {
+        if (m_config.domain == SensorTimeDomain::MonotonicRawNs)
+            return source_delta;
+        // Split into whole seconds and remainder so the product cannot overflow 64 bits.
+        const int64_t hz = m_config.counter_hz;
+        const int64_t secs = source_delta / hz;
+        const int64_t rem = source_delta % hz;
+        return secs * 1'000'000'000 + rem * 1'000'000'000 / hz;
+    }
+
+    Config m_config;
+    PairReader m_reader;
+    ClockPair m_anchor{};
+    int64_t m_anchor_taken_ns = 0;
+    bool m_have_anchor = false;
+};
+
+} // namespace sensing
+} // namespace plugins

src/plugins/sensing/core/jetson_encoder.cpp

@@ -6,9 +6,12 @@
 #include "rgba_to_nv12.cuh"
 
 #include <linux/videodev2.h>
+#include <log_bridge/logger.hpp>
 
 #include <NvVideoEncoder.h>
+#include <atomic>
 #include <cerrno>
+#include <cstdlib>
 #include <cstring>
 #include <cuda_runtime.h>
 #include <deque>
@@ -47,6 +50,12 @@ struct JetsonEncoder::Impl
     EncoderConfig config;
     NvVideoEncoder* encoder = nullptr;
 
+    // SENSING_ENCODER_TIMESTAMP_PROBE=1 logs the first units' returned stamps beside the last stamp submitted, to
+    // show whether the driver copies the OUTPUT stamp to the unit. Diagnostic only: nothing consumes the result.
+    const bool timestamp_probe = std::getenv("SENSING_ENCODER_TIMESTAMP_PROBE") != nullptr;
+    std::atomic<int64_t> last_submitted_us{ 0 };
+    std::atomic<int> probe_logged{ 0 };
+
     // DMA-BUF input: a wrapper per output buffer carrying the fd the encoder reads, and the lease that keeps the
     // capture buffer alive until the plane returns that index. Members destroy after ~Impl deleted the encoder.
     struct DmaInput : NvBuffer
@@ -141,6 +150,12 @@ struct JetsonEncoder::Impl
             unit.timestamp_ns = static_cast<int64_t>(v4l2_buf->timestamp.tv_sec) * 1000000000LL +
                                 static_cast<int64_t>(v4l2_buf->timestamp.tv_usec) * 1000LL;
             unit.keyframe = (v4l2_buf->flags & V4L2_BUF_FLAG_KEYFRAME) != 0;
+            if (self->timestamp_probe && self->probe_logged.fetch_add(1) < 30)
+            {
+                isaaccapture::Logger::get("isaaccapture.plugins.sensing.JetsonEncoder")
+                    ->info("[ts-probe] unit stamp_us={} last_submitted_us={}", unit.timestamp_ns / 1000,
+                           self->last_submitted_us.load());
+            }
             std::lock_guard<std::mutex> lock(self->mutex);
             self->ready.push_back(std::move(unit));
         }
@@ -302,6 +317,7 @@ bool JetsonEncoder::submit(uintptr_t rgba_device_ptr, std::size_t row_pitch_byte
     // Microsecond granularity is all a timeval carries, well below the frame interval.
     v4l2_buf.timestamp.tv_sec = static_cast<time_t>(timestamp_ns / 1000000000LL);
     v4l2_buf.timestamp.tv_usec = static_cast<suseconds_t>((timestamp_ns % 1000000000LL) / 1000LL);
+    m_impl->last_submitted_us.store(timestamp_ns / 1000);
 
     check(enc->output_plane.qBuffer(v4l2_buf, nullptr), "output qBuffer");
     return true;
@@ -334,6 +350,7 @@ bool JetsonEncoder::submit_dmabuf(int dmabuf_fd, int64_t timestamp_ns, FrameLeas
     v4l2_buf.m.planes[0].bytesused = 1;
     v4l2_buf.timestamp.tv_sec = static_cast<time_t>(timestamp_ns / 1000000000LL);
     v4l2_buf.timestamp.tv_usec = static_cast<suseconds_t>((timestamp_ns % 1000000000LL) / 1000LL);
+    m_impl->last_submitted_us.store(timestamp_ns / 1000);
 
     check(m_impl->encoder->output_plane.qBuffer(v4l2_buf, &input), "output qBuffer");
     m_impl->dma_leases[index] = std::move(lease);