From c9ba562dec1e164c0cbe8516d7decba3b0bf2909 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sat, 26 Sep 2026 11:54:37 -0700 Subject: [PATCH] backends: per-node frame reporting for drive coordinate taps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LUIAppleBackend collects every rendered node's frame (global coordinate space, via onGeometryChange on the single LUIAnyNodeView choke point) and emits the full map once per coalesced layout flush through onFramesReport while frameReportingEnabled is on. emitFramesSnapshot() pushes the current table for late-attaching drivers. The Flutter backend does the same via a _FrameProbe element table and a post-frame measure loop that re-measures every frame while enabled — scrolling moves nodes without rebuilds. Feeds drive's live-attach 'tap x y' hit-testing; hosts forward the map to the live socket as {"frames":[[id,x,y,w,h],...]}. --- .../LUIAppleBackend/LUIAppleBackend.swift | 52 ++++++++ .../LUIAppleBackend/LUISwiftUIRoot.swift | 31 ++++- platform/flutter/lib/lui_flutter_backend.dart | 124 ++++++++++++++++-- 3 files changed, 190 insertions(+), 17 deletions(-) diff --git a/platform/apple/Sources/LUIAppleBackend/LUIAppleBackend.swift b/platform/apple/Sources/LUIAppleBackend/LUIAppleBackend.swift index a4c54d67..22fe3d89 100644 --- a/platform/apple/Sources/LUIAppleBackend/LUIAppleBackend.swift +++ b/platform/apple/Sources/LUIAppleBackend/LUIAppleBackend.swift @@ -375,6 +375,58 @@ public final class LUIAppleBackend { public private(set) var generation = 0 public var onEvent: ((LUIEvent) -> Void)? + // MARK: - Node frame reporting + + /// Gates emission of `onFramesReport`. Frames are collected + /// unconditionally (a dict write per layout change) so a driver can + /// attach mid-session and still see every node's latest geometry. + public var frameReportingEnabled = false { + didSet { + if frameReportingEnabled && !oldValue { + scheduleFramesReport() + } + } + } + + /// Called once per coalesced layout flush with the full node-id → + /// frame map in the scene's global coordinate space. Feeds drive's + /// live-attach `tap x y` hit-testing; set `frameReportingEnabled` to + /// start receiving reports. + public var onFramesReport: (([Int: CGRect]) -> Void)? + + private var nodeFrames: [Int: CGRect] = [:] + private var framesReportScheduled = false + + func reportNodeFrame(_ nodeID: Int, _ rect: CGRect) { + if nodeFrames[nodeID] == rect { return } + nodeFrames[nodeID] = rect + scheduleFramesReport() + } + + func removeNodeFrame(_ nodeID: Int) { + if nodeFrames.removeValue(forKey: nodeID) != nil { + scheduleFramesReport() + } + } + + /// Push the current frame table to `onFramesReport` immediately — + /// for a driver/handler that attached after the last layout flush. + public func emitFramesSnapshot() { + guard frameReportingEnabled else { return } + onFramesReport?(nodeFrames) + } + + private func scheduleFramesReport() { + guard !framesReportScheduled else { return } + framesReportScheduled = true + DispatchQueue.main.async { [weak self] in + guard let self else { return } + self.framesReportScheduled = false + guard self.frameReportingEnabled else { return } + self.onFramesReport?(self.nodeFrames) + } + } + private var tree = LUIRetainedTree() private var models: [Int: LUINodeModel] = [:] private var extensionModels: [Int: LUIExtensionNodeModel] = [:] diff --git a/platform/apple/Sources/LUIAppleBackend/LUISwiftUIRoot.swift b/platform/apple/Sources/LUIAppleBackend/LUISwiftUIRoot.swift index 0eb331b7..48888ab2 100644 --- a/platform/apple/Sources/LUIAppleBackend/LUISwiftUIRoot.swift +++ b/platform/apple/Sources/LUIAppleBackend/LUISwiftUIRoot.swift @@ -488,11 +488,34 @@ struct LUIAnyNodeView: View, Equatable { @ViewBuilder var body: some View { - if let model = backend.model(id: nodeID) { - LUINodeView(model: model, backend: backend) - } else if let model = backend.extensionModel(id: nodeID) { - LUIExtensionNodeView(model: model, backend: backend) + Group { + if let model = backend.model(id: nodeID) { + LUINodeView(model: model, backend: backend) + } else if let model = backend.extensionModel(id: nodeID) { + LUIExtensionNodeView(model: model, backend: backend) + } } + .modifier(LUIFrameReportModifier(nodeID: nodeID, backend: backend)) + } +} + +/// Feeds `LUIAppleBackend.reportNodeFrame` with each node's frame in the +/// scene's global coordinate space. Collection is unconditional (cheap +/// dict writes) so `frameReportingEnabled` can turn on mid-session. +private struct LUIFrameReportModifier: ViewModifier { + let nodeID: Int + let backend: LUIAppleBackend + + func body(content: Content) -> some View { + content + .onGeometryChange(for: CGRect.self) { proxy in + proxy.frame(in: .global) + } action: { rect in + backend.reportNodeFrame(nodeID, rect) + } + .onDisappear { + backend.removeNodeFrame(nodeID) + } } } diff --git a/platform/flutter/lib/lui_flutter_backend.dart b/platform/flutter/lib/lui_flutter_backend.dart index ae1f4709..acee0444 100644 --- a/platform/flutter/lib/lui_flutter_backend.dart +++ b/platform/flutter/lib/lui_flutter_backend.dart @@ -396,6 +396,51 @@ final class _LUIAppearDispatcherState extends State<_LUIAppearDispatcher> { Widget build(BuildContext context) => widget.child; } +/// Registers the node's element in the backend's frame-probe table so +/// `_measureFrames` can locate its render object each frame. Transparent +/// to layout. +final class _FrameProbe extends StatefulWidget { + const _FrameProbe({ + super.key, + required this.backend, + required this.node, + required this.child, + }); + + final LUIFlutterBackend backend; + final int node; + final Widget child; + + @override + State<_FrameProbe> createState() => _FrameProbeState(); +} + +final class _FrameProbeState extends State<_FrameProbe> { + @override + void initState() { + super.initState(); + widget.backend._frameProbes[widget.node] = context; + } + + @override + void didUpdateWidget(_FrameProbe oldWidget) { + super.didUpdateWidget(oldWidget); + if (oldWidget.node != widget.node) { + widget.backend._frameProbes.remove(oldWidget.node); + widget.backend._frameProbes[widget.node] = context; + } + } + + @override + void dispose() { + widget.backend._frameProbes.remove(widget.node); + super.dispose(); + } + + @override + Widget build(BuildContext context) => widget.child; +} + final class LUIFlutterBackend { LUIFlutterBackend({ this.onEvent, @@ -408,6 +453,51 @@ final class LUIFlutterBackend { final void Function(LUIEvent event)? onEvent; final Map appIcons; + + /// Called once per frame while [frameReportingEnabled] is on, with the + /// full node-id → rect map in global coordinates — feeds drive's + /// live-attach `tap x y` hit-testing. + void Function(Map frames)? onFrames; + + final Map _frameProbes = {}; + Map? _lastFrames; + bool _frameReportingEnabled = false; + bool _frameMeasureScheduled = false; + + bool get frameReportingEnabled => _frameReportingEnabled; + set frameReportingEnabled(bool value) { + if (value == _frameReportingEnabled) return; + _frameReportingEnabled = value; + if (value) _scheduleFrameMeasure(); + } + + void _scheduleFrameMeasure() { + if (_frameMeasureScheduled) return; + _frameMeasureScheduled = true; + SchedulerBinding.instance.addPostFrameCallback((_) { + _frameMeasureScheduled = false; + _measureFrames(); + // A persistent loop: scrolling moves nodes without rebuilds, so + // frames must be re-measured every frame while reporting is on. + if (_frameReportingEnabled) _scheduleFrameMeasure(); + }); + } + + void _measureFrames() { + final frames = {}; + for (final entry in _frameProbes.entries) { + final renderObject = entry.value.findRenderObject(); + if (renderObject is RenderBox && renderObject.attached) { + frames[entry.key] = + renderObject.localToGlobal(Offset.zero) & renderObject.size; + } + } + if (!mapEquals(frames, _lastFrames)) { + _lastFrames = frames; + onFrames?.call(frames); + } + } + final LUIFlutterExtensionRegistry _extensionRegistry; final _LUITooltipSession _tooltipSession = _LUITooltipSession(); Map _states = {}; @@ -1013,24 +1103,32 @@ final class LUIFlutterBackend { Widget widget({required int node}) { final extensionHandle = _extensionHandles[node]; if (extensionHandle != null) { - return ListenableBuilder( + return _FrameProbe( key: nodeKey(node), - listenable: extensionHandle, - builder: (context, _) => _buildExtensionNode(node), + backend: this, + node: node, + child: ListenableBuilder( + listenable: extensionHandle, + builder: (context, _) => _buildExtensionNode(node), + ), ); } final handle = _requireHandle(node); - return ListenableBuilder( + return _FrameProbe( key: nodeKey(node), - listenable: handle, - // The Builder under _withTheme gives _buildNode a context that already - // carries this node's tokens, so a themed scope covers the node - // itself, not only its descendants. - builder: (context, _) => _withTheme( - context, - node, - Builder( - builder: (inner) => _withAppear(node, _buildNode(inner, node)), + backend: this, + node: node, + child: ListenableBuilder( + listenable: handle, + // The Builder under _withTheme gives _buildNode a context that already + // carries this node's tokens, so a themed scope covers the node + // itself, not only its descendants. + builder: (context, _) => _withTheme( + context, + node, + Builder( + builder: (inner) => _withAppear(node, _buildNode(inner, node)), + ), ), ), );