diff --git a/.changes/web-video-renderer-lifecycle b/.changes/web-video-renderer-lifecycle new file mode 100644 index 000000000..953fbbe83 --- /dev/null +++ b/.changes/web-video-renderer-lifecycle @@ -0,0 +1 @@ +patch type="fixed" "Prevent stale video renderer attachments and listener leaks during async initialization, track changes, and disposal" diff --git a/lib/src/widgets/video_track_renderer.dart b/lib/src/widgets/video_track_renderer.dart index 6c7ec792a..8ae82b0bb 100644 --- a/lib/src/widgets/video_track_renderer.dart +++ b/lib/src/widgets/video_track_renderer.dart @@ -12,59 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -import 'dart:async'; -import 'dart:math'; - -import 'package:flutter/foundation.dart' show ValueListenable, kIsWeb; import 'package:flutter/widgets.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; -import '../events.dart'; -import '../extensions.dart'; -import '../internal/events.dart'; -import '../logger.dart'; -import '../managers/event.dart'; -import '../support/platform.dart'; import '../track/local/local.dart'; -import '../track/local/video.dart'; -import '../track/options.dart'; -import '../track/video_track_view_registration.dart'; import '../types/other.dart'; +import 'video_track_renderer_common.dart'; -enum VideoViewMirrorMode { - auto, - off, - mirror, -} - -enum VideoRenderMode { - /// Let the SDK choose the rendering backend. Currently resolves to - /// [texture] on all platforms, but the resolution may change in a - /// future release. - auto, - - /// Render frames into a Flutter texture. - texture, - - /// Render with a native platform view. Supported on iOS and macOS, - /// other platforms fall back to [texture]. - platformView, -} +import 'video_track_renderer_native.dart' + if (dart.library.js_interop) 'video_track_renderer_web.dart' + as implementation; -enum VideoViewFit { - contain, - cover, -} - -extension VideoViewFitExt on VideoViewFit { - rtc.RTCVideoViewObjectFit toRTCType() { - if (this == VideoViewFit.cover) { - return rtc.RTCVideoViewObjectFit.RTCVideoViewObjectFitCover; - } - return rtc.RTCVideoViewObjectFit.RTCVideoViewObjectFitContain; - } -} +export 'video_track_renderer_common.dart' show VideoRenderMode, VideoViewFit, VideoViewFitExt, VideoViewMirrorMode; /// Widget that renders a [VideoTrack]. class VideoTrackRenderer extends StatefulWidget { @@ -72,7 +32,13 @@ class VideoTrackRenderer extends StatefulWidget { final VideoViewFit fit; final VideoViewMirrorMode mirrorMode; final VideoRenderMode renderMode; + + /// An existing renderer borrowed by this widget. The caller remains responsible + /// for disposing it; the widget detaches it but never disposes it. final rtc.RTCVideoRenderer? cachedRenderer; + + /// Whether to dispose a renderer created by this widget when it is released. + /// Defaults to true. This has no effect on [cachedRenderer]. final bool autoDisposeRenderer; /// wrap the video view in a Center widget (if [fit] is [VideoViewFit.contain]) @@ -99,317 +65,24 @@ class VideoTrackRenderer extends StatefulWidget { this.autoCenter = true, this.adaptiveStreamPixelDensity = AdaptiveStreamPixelDensity.auto, this.placeholderBuilder, - Key? key, - }) : super(key: key); + super.key, + }); @override - State createState() => _VideoTrackRendererState(); + State createState() => _VideoTrackRendererState(); } class _VideoTrackRendererState extends State { - rtc.VideoRenderer? _renderer; - // for flutter web only. - bool _rendererReadyForWeb = false; - double? _aspectRatio; - EventsListener? _listener; - // Used to compute visibility information - late VideoTrackViewRegistration _viewRegistration; - - bool _usesPlatformView(VideoRenderMode renderMode) => - renderMode == VideoRenderMode.platformView && [PlatformType.iOS, PlatformType.macOS].contains(lkPlatform()); - - bool get _shouldUsePlatformView => _usesPlatformView(widget.renderMode); - - double? get _rendererAspectRatio { - final renderer = _renderer; - if (renderer != null && renderer is ValueListenable) { - return (renderer as ValueListenable).value.aspectRatio; - } - return null; - } - - Future _initializeRenderer() async { - if (_shouldUsePlatformView) { - return null; - } - // A leftover platform view controller is owned by its RTCVideoPlatFormView - // widget, which disposes it on unmount. Only drop our reference here. - if (_renderer != null && _renderer is! rtc.RTCVideoRenderer) { - _releaseRenderer(dispose: false); - } - if (_renderer == null) { - final cachedRenderer = widget.cachedRenderer; - if (cachedRenderer != null) { - _renderer = cachedRenderer; - } else { - _renderer = rtc.RTCVideoRenderer(); - await _renderer!.initialize(); - } - } - await _attach(); - return _renderer!; - } - - void setZoom(double zoomLevel) async { - final videoTrack = _renderer?.srcObject!.getVideoTracks().first; - if (videoTrack == null) return; - await rtc.Helper.setZoom(videoTrack, zoomLevel); - } - - void onViewFinderTap(TapDownDetails details, BoxConstraints constraints) { - final videoTrack = _renderer?.srcObject!.getVideoTracks().first; - if (videoTrack == null) return; - - final point = Point( - details.localPosition.dx / constraints.maxWidth, - details.localPosition.dy / constraints.maxHeight, - ); - - // Don't wait here as it will slow down the UI unnecessarily. - unawaited(rtc.Helper.setFocusPoint(videoTrack, point)); - unawaited(rtc.Helper.setExposurePoint(videoTrack, point)); - } - - /// Detaches the current renderer and drops our reference to it. - /// Pass [dispose] only for renderers this widget created and owns. - void _releaseRenderer({required bool dispose}) { - final renderer = _renderer; - _renderer = null; - try { - renderer?.onResize = null; - renderer?.srcObject = null; - if (dispose) { - unawaited(renderer?.dispose()); - } - } catch (e) { - logger.warning('Got error releasing renderer: $e'); - } - } - - @override - void initState() { - super.initState(); - if (!_shouldUsePlatformView && widget.cachedRenderer != null) { - _renderer = widget.cachedRenderer; - } - _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity); - if (kIsWeb) { - unawaited(() async { - await _initializeRenderer(); - if (!mounted) return; - setState(() => _rendererReadyForWeb = true); - }()); - } - } - - @override - void dispose() { - widget.track.removeViewRegistration(_viewRegistration); - unawaited(_listener?.dispose()); - if (widget.autoDisposeRenderer) { - _releaseRenderer(dispose: true); - } - super.dispose(); - } - - Future _attach() async { - _renderer?.srcObject = widget.track.mediaStream; - await _listener?.dispose(); - _listener = widget.track.createListener() - ..on((event) { - if (!mounted) return; - _renderer?.srcObject = event.stream; - }) - ..on((event) { - if (!mounted) return; - // force recompute of mirror mode - setState(() {}); - }); - _renderer?.onResize = () { - if (mounted) { - setState(() { - _aspectRatio = _rendererAspectRatio; - }); - } - }; - } - @override - void didUpdateWidget(covariant VideoTrackRenderer oldWidget) { - super.didUpdateWidget(oldWidget); - if (_usesPlatformView(oldWidget.renderMode) != _shouldUsePlatformView) { - // Only dispose texture renderers we created ourselves. Platform view - // controllers belong to RTCVideoPlatFormView and a cachedRenderer - // belongs to the caller, who may hand it back on a later switch. - final ownsRenderer = _renderer is rtc.RTCVideoRenderer && !identical(_renderer, oldWidget.cachedRenderer); - unawaited(_listener?.dispose()); - _listener = null; - _aspectRatio = null; - _releaseRenderer(dispose: ownsRenderer && oldWidget.autoDisposeRenderer); - } - - if (widget.track != oldWidget.track) { - oldWidget.track.removeViewRegistration(_viewRegistration); - _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity); - unawaited(() async { - await _attach(); - }()); - } else if (widget.adaptiveStreamPixelDensity != oldWidget.adaptiveStreamPixelDensity) { - _viewRegistration.pixelDensity = widget.adaptiveStreamPixelDensity; - } - - if ([BrowserType.safari, BrowserType.firefox].contains(lkBrowser()) && oldWidget.key != widget.key) { - _renderer?.srcObject = widget.track.mediaStream; - } - } - - Widget _videoViewForWeb() => !_rendererReadyForWeb - ? (widget.placeholderBuilder?.call(context) ?? const SizedBox.shrink()) - : Builder( - key: _viewRegistration.key, - builder: (ctx) { - // let it render before notifying build - WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) { - widget.track.onVideoViewBuild?.call(); - }); - return rtc.RTCVideoView( - _renderer! as rtc.RTCVideoRenderer, - mirror: _shouldMirror(), - filterQuality: FilterQuality.medium, - objectFit: widget.fit.toRTCType(), - placeholderBuilder: widget.placeholderBuilder, - ); - }, - ); - - Widget _videoRendererView() { - if (_shouldUsePlatformView) { - return rtc.RTCVideoPlatFormView( - mirror: _shouldMirror(), - objectFit: widget.fit.toRTCType(), - onViewReady: (controller) async { - _renderer = controller; - _renderer?.srcObject = widget.track.mediaStream; - await _attach(); - }, - ); - } - return rtc.RTCVideoView( - _renderer! as rtc.RTCVideoRenderer, - mirror: _shouldMirror(), - filterQuality: FilterQuality.medium, - objectFit: widget.fit.toRTCType(), - placeholderBuilder: widget.placeholderBuilder, - ); - } - - Widget _videoViewForNative() => FutureBuilder( - future: _initializeRenderer(), - builder: (context, snapshot) { - if ((snapshot.hasData && _renderer != null) || _shouldUsePlatformView) { - return Builder( - key: _viewRegistration.key, - builder: (ctx) { - // let it render before notifying build - WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) { - widget.track.onVideoViewBuild?.call(); - }); - - if (!lkPlatformIsMobile() || widget.track is! LocalVideoTrack) { - return _videoRendererView(); - } - return LayoutBuilder( - builder: (BuildContext context, BoxConstraints constraints) { - return GestureDetector( - onScaleStart: (details) {}, - onScaleUpdate: (details) { - if (details.scale != 1.0) { - setZoom(details.scale); - } - }, - onTapDown: (TapDownDetails details) => onViewFinderTap(details, constraints), - child: _videoRendererView(), - ); - }, - ); - }, - ); - } - return widget.placeholderBuilder?.call(context) ?? const SizedBox.shrink(); - }, + Widget build(BuildContext context) => implementation.buildVideoTrackRenderer( + track: widget.track, + fit: widget.fit, + mirrorMode: widget.mirrorMode, + renderMode: widget.renderMode, + cachedRenderer: widget.cachedRenderer, + autoDisposeRenderer: widget.autoDisposeRenderer, + autoCenter: widget.autoCenter, + adaptiveStreamPixelDensity: widget.adaptiveStreamPixelDensity, + placeholderBuilder: widget.placeholderBuilder, ); - - // FutureBuilder will cause flickering for flutter web. so using - // different rendering methods for web and native. - @override - Widget build(BuildContext context) { - final child = kIsWeb ? _videoViewForWeb() : _videoViewForNative(); - - if (widget.fit == VideoViewFit.cover) { - return child; - } - - final videoView = LayoutBuilder( - builder: (BuildContext context, BoxConstraints constraints) { - if (!constraints.hasBoundedWidth && !constraints.hasBoundedHeight) { - return child; - } - if (_aspectRatio == null) { - return child; - } - - bool fixHeight; - if (!constraints.hasBoundedWidth) { - fixHeight = true; - } else if (!constraints.hasBoundedHeight) { - fixHeight = false; - } else { - // both width and height are bound, figure out which to fix based on aspect ratios - final constraintsAspectRatio = constraints.maxWidth / constraints.maxHeight; - fixHeight = constraintsAspectRatio > _aspectRatio!; - } - final double width; - final double height; - if (fixHeight) { - height = constraints.maxHeight; - width = height * _aspectRatio!; - } else { - width = constraints.maxWidth; - height = width / _aspectRatio!; - } - return SizedBox(width: width, height: height, child: child); - }, - ); - - if (widget.autoCenter) { - return Center(child: videoView); - } else { - return videoView; - } - } - - bool _shouldMirror() { - // off for screen share - if (widget.track.source == TrackSource.screenShareVideo) return false; - // on - if (widget.mirrorMode == VideoViewMirrorMode.mirror) return true; - // auto - if (widget.mirrorMode == VideoViewMirrorMode.auto) { - final track = widget.track; - if (track is LocalVideoTrack) { - final settings = track.mediaStreamTrack.getSettings(); - final facingMode = settings['facingMode']; - if (facingMode != null) { - return facingMode == 'user'; - } - final options = track.currentOptions; - if (options is CameraCaptureOptions) { - // mirror if front camera - return options.cameraPosition == CameraPosition.front; - } - } - } - // default to false - return false; - } } diff --git a/lib/src/widgets/video_track_renderer_common.dart b/lib/src/widgets/video_track_renderer_common.dart new file mode 100644 index 000000000..f765526d8 --- /dev/null +++ b/lib/src/widgets/video_track_renderer_common.dart @@ -0,0 +1,71 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; + +import '../track/local/local.dart'; +import '../track/local/video.dart'; +import '../track/options.dart'; +import '../types/other.dart'; + +enum VideoViewMirrorMode { + auto, + off, + mirror, +} + +enum VideoRenderMode { + /// Let the SDK choose the rendering backend. Currently resolves to + /// [texture] on all platforms, but the resolution may change in a + /// future release. + auto, + + /// Render frames into a Flutter texture. + texture, + + /// Render with a native platform view. Supported on iOS and macOS, + /// other platforms fall back to [texture]. + platformView, +} + +enum VideoViewFit { + contain, + cover, +} + +extension VideoViewFitExt on VideoViewFit { + rtc.RTCVideoViewObjectFit toRTCType() { + if (this == VideoViewFit.cover) { + return rtc.RTCVideoViewObjectFit.RTCVideoViewObjectFitCover; + } + return rtc.RTCVideoViewObjectFit.RTCVideoViewObjectFitContain; + } +} + +rtc.RTCVideoRenderer Function() videoTrackRendererFactory = rtc.RTCVideoRenderer.new; + +bool Function(VideoRenderMode)? videoTrackRendererPlatformViewOverride; + +bool shouldMirror(VideoTrack track, VideoViewMirrorMode mirrorMode) { + if (track.source == TrackSource.screenShareVideo) return false; + if (mirrorMode == VideoViewMirrorMode.mirror) return true; + if (mirrorMode == VideoViewMirrorMode.auto && track is LocalVideoTrack) { + final settings = track.mediaStreamTrack.getSettings(); + final facingMode = settings['facingMode']; + if (facingMode != null) return facingMode == 'user'; + final options = track.currentOptions; + if (options is CameraCaptureOptions) return options.cameraPosition == CameraPosition.front; + } + return false; +} diff --git a/lib/src/widgets/video_track_renderer_native.dart b/lib/src/widgets/video_track_renderer_native.dart new file mode 100644 index 000000000..9dd24a3ac --- /dev/null +++ b/lib/src/widgets/video_track_renderer_native.dart @@ -0,0 +1,320 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:async'; +import 'dart:math'; + +import 'package:flutter/foundation.dart' show ValueListenable; +import 'package:flutter/widgets.dart'; + +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; + +import '../events.dart'; +import '../extensions.dart'; +import '../internal/events.dart'; +import '../logger.dart'; +import '../managers/event.dart'; +import '../support/platform.dart'; +import '../track/local/local.dart'; +import '../track/local/video.dart'; +import '../track/video_track_view_registration.dart'; +import '../types/other.dart' hide ConnectionState; +import 'video_track_renderer_common.dart'; + +Widget buildVideoTrackRenderer({ + required VideoTrack track, + required VideoViewFit fit, + required VideoViewMirrorMode mirrorMode, + required VideoRenderMode renderMode, + required rtc.RTCVideoRenderer? cachedRenderer, + required bool autoDisposeRenderer, + required bool autoCenter, + required AdaptiveStreamPixelDensity adaptiveStreamPixelDensity, + required WidgetBuilder? placeholderBuilder, +}) => _NativeVideoTrackRenderer( + track: track, + fit: fit, + mirrorMode: mirrorMode, + renderMode: renderMode, + cachedRenderer: cachedRenderer, + autoDisposeRenderer: autoDisposeRenderer, + autoCenter: autoCenter, + adaptiveStreamPixelDensity: adaptiveStreamPixelDensity, + placeholderBuilder: placeholderBuilder, +); + +class _NativeVideoTrackRenderer extends StatefulWidget { + const _NativeVideoTrackRenderer({ + required this.track, + required this.fit, + required this.mirrorMode, + required this.renderMode, + required this.cachedRenderer, + required this.autoDisposeRenderer, + required this.autoCenter, + required this.adaptiveStreamPixelDensity, + required this.placeholderBuilder, + }); + + final VideoTrack track; + final VideoViewFit fit; + final VideoViewMirrorMode mirrorMode; + final VideoRenderMode renderMode; + final rtc.RTCVideoRenderer? cachedRenderer; + final bool autoDisposeRenderer; + final bool autoCenter; + final AdaptiveStreamPixelDensity adaptiveStreamPixelDensity; + final WidgetBuilder? placeholderBuilder; + + @override + State<_NativeVideoTrackRenderer> createState() => _NativeVideoTrackRendererState(); +} + +class _NativeVideoTrackRendererState extends State<_NativeVideoTrackRenderer> { + rtc.VideoRenderer? _renderer; + bool _ownsRenderer = false; + bool _disposed = false; + int _generation = 0; + int _platformViewGeneration = 0; + Future? _initializing; + Future? _rendererFuture; + double? _aspectRatio; + EventsListener? _listener; + late VideoTrackViewRegistration _viewRegistration; + + bool _usesPlatformView(VideoRenderMode renderMode) => + videoTrackRendererPlatformViewOverride?.call(renderMode) ?? + (renderMode == VideoRenderMode.platformView && [PlatformType.iOS, PlatformType.macOS].contains(lkPlatform())); + + bool get _shouldUsePlatformView => _usesPlatformView(widget.renderMode); + + double? get _rendererAspectRatio { + final renderer = _renderer; + if (renderer is ValueListenable) { + return (renderer as ValueListenable).value.aspectRatio; + } + return null; + } + + Future _startRenderer() async { + final generation = _generation; + if (_shouldUsePlatformView) return null; + if (_renderer == null) { + final cached = widget.cachedRenderer; + if (cached != null) { + _renderer = cached; + } else { + final renderer = videoTrackRendererFactory(); + _renderer = renderer; + _ownsRenderer = true; + _initializing = renderer.initialize(); + } + } + final renderer = _renderer!; + await _initializing; + if (identical(renderer, _renderer)) _initializing = null; + if (_disposed || generation != _generation || !identical(renderer, _renderer)) return null; + await _attach(generation, renderer); + if (_disposed || generation != _generation || !identical(renderer, _renderer)) return null; + return renderer; + } + + void _scheduleRenderer() => _rendererFuture = _startRenderer(); + + void setZoom(double zoomLevel) async { + final videoTrack = _renderer?.srcObject!.getVideoTracks().first; + if (videoTrack == null) return; + await rtc.Helper.setZoom(videoTrack, zoomLevel); + } + + void onViewFinderTap(TapDownDetails details, BoxConstraints constraints) { + final videoTrack = _renderer?.srcObject!.getVideoTracks().first; + if (videoTrack == null) return; + final point = Point( + details.localPosition.dx / constraints.maxWidth, + details.localPosition.dy / constraints.maxHeight, + ); + unawaited(rtc.Helper.setFocusPoint(videoTrack, point)); + unawaited(rtc.Helper.setExposurePoint(videoTrack, point)); + } + + void _releaseRenderer({required bool dispose}) { + final renderer = _renderer; + _renderer = null; + final ownsRenderer = _ownsRenderer; + _ownsRenderer = false; + final initializing = _initializing; + _initializing = null; + renderer?.onResize = null; + if (renderer == null) return; + if (initializing == null) { + try { + renderer.srcObject = null; + } catch (e) { + logger.warning('Got error detaching renderer: $e'); + } + if (dispose && ownsRenderer) unawaited(renderer.dispose()); + } else { + unawaited(() async { + try { + await initializing; + renderer.srcObject = null; + } catch (e) { + logger.warning('Got error detaching renderer: $e'); + } finally { + if (dispose && ownsRenderer) await renderer.dispose(); + } + }()); + } + } + + @override + void initState() { + super.initState(); + _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity); + _scheduleRenderer(); + } + + @override + void dispose() { + _disposed = true; + _generation++; + _platformViewGeneration++; + widget.track.removeViewRegistration(_viewRegistration); + final listener = _listener; + _listener = null; + unawaited(listener?.dispose()); + _releaseRenderer(dispose: widget.autoDisposeRenderer); + super.dispose(); + } + + Future _attach(int generation, rtc.VideoRenderer renderer) async { + final oldListener = _listener; + _listener = null; + await oldListener?.dispose(); + if (_disposed || generation != _generation || !identical(renderer, _renderer)) return; + final track = widget.track; + renderer.srcObject = track.mediaStream; + _listener = track.createListener() + ..on((event) { + if (_disposed || generation != _generation || !identical(renderer, _renderer)) return; + renderer.srcObject = event.stream; + }) + ..on((event) { + if (_disposed || generation != _generation || !mounted) return; + setState(() {}); + }); + renderer.onResize = () { + if (_disposed || generation != _generation || !identical(renderer, _renderer) || !mounted) return; + setState(() => _aspectRatio = _rendererAspectRatio); + }; + } + + @override + void didUpdateWidget(covariant _NativeVideoTrackRenderer oldWidget) { + super.didUpdateWidget(oldWidget); + final modeChanged = _usesPlatformView(oldWidget.renderMode) != _shouldUsePlatformView; + final cachedChanged = !_shouldUsePlatformView && !identical(oldWidget.cachedRenderer, widget.cachedRenderer); + final trackChanged = !identical(oldWidget.track, widget.track); + if (modeChanged || cachedChanged || trackChanged) _generation++; + final platformViewRetired = modeChanged || (trackChanged && _shouldUsePlatformView); + if (platformViewRetired) _platformViewGeneration++; + if (platformViewRetired || cachedChanged) { + final listener = _listener; + _listener = null; + unawaited(listener?.dispose()); + _aspectRatio = null; + _releaseRenderer(dispose: oldWidget.autoDisposeRenderer); + } + if (trackChanged) { + oldWidget.track.removeViewRegistration(_viewRegistration); + _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity); + } else if (widget.adaptiveStreamPixelDensity != oldWidget.adaptiveStreamPixelDensity) { + _viewRegistration.pixelDensity = widget.adaptiveStreamPixelDensity; + } + if (modeChanged || cachedChanged || (trackChanged && !_shouldUsePlatformView)) { + _scheduleRenderer(); + } + } + + Widget _videoRendererView() { + if (_shouldUsePlatformView) { + final platformViewGeneration = _platformViewGeneration; + return rtc.RTCVideoPlatFormView( + mirror: shouldMirror(widget.track, widget.mirrorMode), + objectFit: widget.fit.toRTCType(), + placeholderBuilder: widget.placeholderBuilder, + onViewReady: (controller) async { + if (_disposed || platformViewGeneration != _platformViewGeneration || !_shouldUsePlatformView) return; + _renderer = controller; + await _attach(_generation, controller); + }, + ); + } + return rtc.RTCVideoView( + _renderer! as rtc.RTCVideoRenderer, + mirror: shouldMirror(widget.track, widget.mirrorMode), + filterQuality: FilterQuality.medium, + objectFit: widget.fit.toRTCType(), + placeholderBuilder: widget.placeholderBuilder, + ); + } + + @override + Widget build(BuildContext context) { + final child = FutureBuilder( + future: _rendererFuture, + builder: (context, snapshot) { + if ((snapshot.connectionState == ConnectionState.done && + snapshot.hasData && + identical(snapshot.data, _renderer)) || + _shouldUsePlatformView) { + return Builder( + key: _viewRegistration.key, + builder: (context) { + WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) { + widget.track.onVideoViewBuild?.call(); + }); + if (!lkPlatformIsMobile() || widget.track is! LocalVideoTrack) return _videoRendererView(); + return LayoutBuilder( + builder: (context, constraints) => GestureDetector( + onScaleStart: (details) {}, + onScaleUpdate: (details) { + if (details.scale != 1.0) setZoom(details.scale); + }, + onTapDown: (details) => onViewFinderTap(details, constraints), + child: _videoRendererView(), + ), + ); + }, + ); + } + return widget.placeholderBuilder?.call(context) ?? const SizedBox.shrink(); + }, + ); + if (widget.fit == VideoViewFit.cover) return child; + final videoView = LayoutBuilder( + builder: (context, constraints) { + if (!constraints.hasBoundedWidth && !constraints.hasBoundedHeight || _aspectRatio == null) return child; + final fixHeight = + !constraints.hasBoundedWidth || + constraints.hasBoundedHeight && constraints.maxWidth / constraints.maxHeight > _aspectRatio!; + final width = fixHeight ? constraints.maxHeight * _aspectRatio! : constraints.maxWidth; + final height = fixHeight ? constraints.maxHeight : constraints.maxWidth / _aspectRatio!; + return SizedBox(width: width, height: height, child: child); + }, + ); + return widget.autoCenter ? Center(child: videoView) : videoView; + } +} diff --git a/lib/src/widgets/video_track_renderer_web.dart b/lib/src/widgets/video_track_renderer_web.dart new file mode 100644 index 000000000..7d39b643e --- /dev/null +++ b/lib/src/widgets/video_track_renderer_web.dart @@ -0,0 +1,355 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:async'; + +import 'package:flutter/widgets.dart'; + +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; + +import '../events.dart'; +import '../extensions.dart'; +import '../internal/events.dart'; +import '../logger.dart'; +import '../managers/event.dart'; +import '../support/platform.dart'; +import '../track/local/local.dart'; +import '../track/video_track_view_registration.dart'; +import '../types/other.dart'; +import 'video_track_renderer_common.dart'; + +Widget buildVideoTrackRenderer({ + required VideoTrack track, + required VideoViewFit fit, + required VideoViewMirrorMode mirrorMode, + required VideoRenderMode renderMode, + required rtc.RTCVideoRenderer? cachedRenderer, + required bool autoDisposeRenderer, + required bool autoCenter, + required AdaptiveStreamPixelDensity adaptiveStreamPixelDensity, + required WidgetBuilder? placeholderBuilder, +}) => _WebVideoTrackRenderer( + track: track, + fit: fit, + mirrorMode: mirrorMode, + renderMode: renderMode, + cachedRenderer: cachedRenderer, + autoDisposeRenderer: autoDisposeRenderer, + autoCenter: autoCenter, + adaptiveStreamPixelDensity: adaptiveStreamPixelDensity, + placeholderBuilder: placeholderBuilder, +); + +class _WebVideoTrackRenderer extends StatefulWidget { + const _WebVideoTrackRenderer({ + required this.track, + required this.fit, + required this.mirrorMode, + required this.renderMode, + required this.cachedRenderer, + required this.autoDisposeRenderer, + required this.autoCenter, + required this.adaptiveStreamPixelDensity, + required this.placeholderBuilder, + }); + + final VideoTrack track; + final VideoViewFit fit; + final VideoViewMirrorMode mirrorMode; + final VideoRenderMode renderMode; + final rtc.RTCVideoRenderer? cachedRenderer; + final bool autoDisposeRenderer; + final bool autoCenter; + final AdaptiveStreamPixelDensity adaptiveStreamPixelDensity; + final WidgetBuilder? placeholderBuilder; + + @override + State<_WebVideoTrackRenderer> createState() => _WebVideoTrackRendererState(); +} + +class _WebVideoTrackRendererState extends State<_WebVideoTrackRenderer> { + rtc.RTCVideoRenderer? _renderer; + bool _ownsRenderer = false; + bool _disposed = false; + bool _trackMuted = false; + String? _originalVisibility; + int _generation = 0; + Future? _initializing; + Future? _rendererFuture; + bool _rendererReadyForWeb = false; + double? _aspectRatio; + EventsListener? _listener; + late VideoTrackViewRegistration _viewRegistration; + + double? get _rendererAspectRatio => _renderer?.value.aspectRatio; + + Future _startRenderer() async { + final generation = _generation; + if (_renderer == null) { + final cached = widget.cachedRenderer; + if (cached != null) { + _renderer = cached; + } else { + final renderer = videoTrackRendererFactory(); + _renderer = renderer; + _ownsRenderer = true; + _initializing = renderer.initialize(); + } + } + final renderer = _renderer!; + await _initializing; + if (identical(renderer, _renderer)) _initializing = null; + if (_disposed || generation != _generation || !identical(renderer, _renderer)) { + return null; + } + await _attach(generation, renderer); + if (_disposed || generation != _generation || !identical(renderer, _renderer)) { + return null; + } + return renderer; + } + + void _scheduleRenderer() { + final future = _rendererFuture = _startRenderer(); + unawaited(() async { + final renderer = await future; + if (renderer == null || !mounted || !identical(future, _rendererFuture)) { + return; + } + setState(() => _rendererReadyForWeb = true); + }()); + } + + void _releaseRenderer({required bool dispose}) { + final renderer = _renderer; + _renderer = null; + final ownsRenderer = _ownsRenderer; + _ownsRenderer = false; + final initializing = _initializing; + _initializing = null; + renderer?.onResize = null; + if (renderer == null) return; + final originalVisibility = _originalVisibility; + _originalVisibility = null; + if (originalVisibility != null) { + // The analyzer resolves the native renderer, but this file runs only on web. + // flutter_webrtc/lib/src/web/rtc_video_renderer.dart + // ignore: undefined_method + renderer.findHtmlView()?.style.visibility = originalVisibility; + } + if (initializing == null) { + try { + renderer.srcObject = null; + } catch (e) { + logger.warning('Got error detaching renderer: $e'); + } + if (dispose && ownsRenderer) unawaited(renderer.dispose()); + } else { + unawaited(() async { + try { + await initializing; + renderer.srcObject = null; + } catch (e) { + logger.warning('Got error detaching renderer: $e'); + } finally { + if (dispose && ownsRenderer) await renderer.dispose(); + } + }()); + } + } + + @override + void initState() { + super.initState(); + _viewRegistration = widget.track.addViewRegistration( + pixelDensity: widget.adaptiveStreamPixelDensity, + ); + _scheduleRenderer(); + } + + @override + void dispose() { + _disposed = true; + _generation++; + widget.track.removeViewRegistration(_viewRegistration); + final listener = _listener; + _listener = null; + unawaited(listener?.dispose()); + _releaseRenderer(dispose: widget.autoDisposeRenderer); + super.dispose(); + } + + void _setTrackMuted(rtc.RTCVideoRenderer renderer, bool muted) { + setState(() => _trackMuted = muted); + _setVideoElementVisible(renderer, !muted); + } + + void _setVideoElementVisible(rtc.RTCVideoRenderer renderer, bool visible) { + // Keep both the HTML element and its MediaStream attached. Detaching + // srcObject makes Safari reconfigure video compositing for all tiles. + // CSS visibility hides only this element while preserving its layout, + // decoder, and the other participants' video surfaces. + // + // The analyzer resolves the native renderer, but this file runs only on web. + // flutter_webrtc/lib/src/web/rtc_video_renderer.dart + // ignore: undefined_method + final element = renderer.findHtmlView(); + if (element == null) return; + _originalVisibility ??= element.style.visibility; + element.style.visibility = visible ? 'visible' : 'hidden'; + } + + Future _attach(int generation, rtc.RTCVideoRenderer renderer) async { + final oldListener = _listener; + _listener = null; + await oldListener?.dispose(); + if (_disposed || generation != _generation || !identical(renderer, _renderer)) { + return; + } + + final track = widget.track; + _trackMuted = track.muted; + renderer.srcObject = track.mediaStream; + _setVideoElementVisible(renderer, !_trackMuted); + + _listener = track.createListener() + ..on((event) { + if (_disposed || generation != _generation || !identical(renderer, _renderer)) { + return; + } + + // Keep the media stream attached even while hidden. Safari retains + // stable composition for the other HtmlElementView video tiles this + // way, and the element becomes visible immediately on unmute. + renderer.srcObject = event.stream; + _setVideoElementVisible(renderer, !_trackMuted); + }) + ..on((event) { + if (_disposed || generation != _generation || !identical(renderer, _renderer)) { + return; + } + _setTrackMuted(renderer, event.muted); + }) + ..on((event) { + if (_disposed || generation != _generation || !mounted) return; + setState(() {}); + }); + + renderer.onResize = () { + if (_disposed || generation != _generation || !identical(renderer, _renderer) || !mounted) { + return; + } + setState(() => _aspectRatio = _rendererAspectRatio); + }; + } + + @override + void didUpdateWidget(covariant _WebVideoTrackRenderer oldWidget) { + super.didUpdateWidget(oldWidget); + final cachedChanged = !identical( + oldWidget.cachedRenderer, + widget.cachedRenderer, + ); + final trackChanged = !identical(oldWidget.track, widget.track); + + if (cachedChanged || trackChanged) { + _generation++; + _rendererReadyForWeb = false; + if (trackChanged) _trackMuted = widget.track.muted; + } + + if (cachedChanged) { + final listener = _listener; + _listener = null; + unawaited(listener?.dispose()); + _aspectRatio = null; + _releaseRenderer(dispose: oldWidget.autoDisposeRenderer); + } + + if (trackChanged) { + oldWidget.track.removeViewRegistration(_viewRegistration); + _viewRegistration = widget.track.addViewRegistration( + pixelDensity: widget.adaptiveStreamPixelDensity, + ); + } else if (widget.adaptiveStreamPixelDensity != oldWidget.adaptiveStreamPixelDensity) { + _viewRegistration.pixelDensity = widget.adaptiveStreamPixelDensity; + } + + if (cachedChanged || trackChanged) { + _scheduleRenderer(); + } + + if (!cachedChanged && + [BrowserType.safari, BrowserType.firefox].contains(lkBrowser()) && + oldWidget.key != widget.key) { + final renderer = _renderer; + if (renderer != null) { + renderer.srcObject = widget.track.mediaStream; + _setVideoElementVisible(renderer, !_trackMuted); + } + } + } + + @override + Widget build(BuildContext context) { + final child = !_rendererReadyForWeb + ? (widget.placeholderBuilder?.call(context) ?? const SizedBox.shrink()) + : Builder( + key: _viewRegistration.key, + builder: (context) { + final generation = _generation; + final track = widget.track; + final renderer = _renderer!; + WidgetsBindingCompatible.instance?.addPostFrameCallback((_) { + if (!_disposed && + generation == _generation && + identical(track, widget.track) && + identical(renderer, _renderer)) { + _setVideoElementVisible(renderer, !_trackMuted); + track.onVideoViewBuild?.call(); + } + }); + return Opacity( + opacity: _trackMuted ? 0 : 1, + child: rtc.RTCVideoView( + _renderer!, + mirror: shouldMirror(widget.track, widget.mirrorMode), + filterQuality: FilterQuality.medium, + objectFit: widget.fit.toRTCType(), + placeholderBuilder: widget.placeholderBuilder, + ), + ); + }, + ); + + if (widget.fit == VideoViewFit.cover) return child; + + final videoView = LayoutBuilder( + builder: (context, constraints) { + if ((!constraints.hasBoundedWidth && !constraints.hasBoundedHeight) || _aspectRatio == null) { + return child; + } + + final fixHeight = + !constraints.hasBoundedWidth || + (constraints.hasBoundedHeight && constraints.maxWidth / constraints.maxHeight > _aspectRatio!); + final width = fixHeight ? constraints.maxHeight * _aspectRatio! : constraints.maxWidth; + final height = fixHeight ? constraints.maxHeight : constraints.maxWidth / _aspectRatio!; + return SizedBox(width: width, height: height, child: child); + }, + ); + + return widget.autoCenter ? Center(child: videoView) : videoView; + } +} diff --git a/test/widgets/video_track_renderer_lifecycle_test.dart b/test/widgets/video_track_renderer_lifecycle_test.dart new file mode 100644 index 000000000..50cdc95de --- /dev/null +++ b/test/widgets/video_track_renderer_lifecycle_test.dart @@ -0,0 +1,393 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:async'; + +import 'package:flutter/foundation.dart' show debugDefaultTargetPlatformOverride, kIsWeb; +import 'package:flutter/widgets.dart'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:mockito/mockito.dart'; + +import 'package:livekit_client/src/events.dart'; +import 'package:livekit_client/src/internal/events.dart'; +import 'package:livekit_client/src/managers/event.dart'; +import 'package:livekit_client/src/track/remote/video.dart'; +import 'package:livekit_client/src/types/other.dart'; +import 'package:livekit_client/src/widgets/video_track_renderer.dart'; +import 'package:livekit_client/src/widgets/video_track_renderer_common.dart'; + +class _Stream extends Mock implements rtc.MediaStream {} + +class _MediaTrack extends Mock implements rtc.MediaStreamTrack {} + +class _Renderer extends rtc.RTCVideoRenderer { + final initialized = Completer(); + rtc.MediaStream? _stream; + int initializationCount = 0; + int disposeCount = 0; + + @override + Future initialize() { + initializationCount++; + return initialized.future; + } + + @override + rtc.MediaStream? get srcObject => _stream; + + @override + set srcObject(rtc.MediaStream? value) => _stream = value; + + @override + Future dispose() async { + disposeCount++; + await super.dispose(); + } +} + +class _PlatformController extends rtc.RTCVideoPlatformViewController { + _PlatformController() : super(1); + rtc.MediaStream? _stream; + bool retired = false; + + @override + rtc.MediaStream? get srcObject => _stream; + + @override + set srcObject(rtc.MediaStream? value) { + if (retired) throw StateError('Controller is retired'); + _stream = value; + } +} + +class _Listener extends EventsListener { + _Listener(super.emitter); + Completer? beforeDispose; + int disposeCount = 0; + + @override + Future dispose() async { + disposeCount++; + if (beforeDispose != null) await beforeDispose!.future; + return super.dispose(); + } +} + +class _Track extends RemoteVideoTrack { + _Track(this.stream) : super(TrackSource.camera, stream, _MediaTrack()); + final rtc.MediaStream stream; + final created = <_Listener>[]; + + @override + EventsListener createListener({bool synchronized = false}) { + final listener = _Listener(events); + created.add(listener); + return listener; + } + + void updateStream(rtc.MediaStream stream) => events.emit(TrackStreamUpdatedEvent(track: this, stream: stream)); +} + +void main() { + late _Renderer renderer; + setUp(() { + renderer = _Renderer(); + videoTrackRendererFactory = () => renderer; + }); + tearDown(() { + videoTrackRendererFactory = rtc.RTCVideoRenderer.new; + videoTrackRendererPlatformViewOverride = null; + debugDefaultTargetPlatformOverride = null; + }); + + Widget view( + _Track track, { + rtc.RTCVideoRenderer? cached, + bool autoDispose = true, + VideoRenderMode renderMode = VideoRenderMode.auto, + }) => WidgetsApp( + color: Color(0xff000000), + builder: (context, child) => SizedBox( + width: 100, + height: 100, + child: VideoTrackRenderer( + track, + key: const ValueKey('video'), + cachedRenderer: cached, + autoDisposeRenderer: autoDispose, + renderMode: renderMode, + ), + ), + ); + + test('VideoTrackRenderer remains assignable to StatefulWidget', () { + final StatefulWidget widget = VideoTrackRenderer(_Track(_Stream())); + expect(widget, isA()); + }); + + testWidgets('dispose during initialize never attaches and disposes the owned renderer once', (tester) async { + final track = _Track(_Stream()); + await tester.pumpWidget(view(track)); + expect(renderer.initializationCount, 1); + await tester.pumpWidget(const SizedBox()); + renderer.initialized.complete(); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(renderer.srcObject, isNull); + expect(track.created, isEmpty); + expect(renderer.disposeCount, 1); + expect(tester.takeException(), isNull); + }); + + testWidgets('track replacement during initialization attaches only the latest stream', (tester) async { + final a = _Track(_Stream()); + final b = _Track(_Stream()); + await tester.pumpWidget(view(a)); + await tester.pumpWidget(view(b)); + renderer.initialized.complete(); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(renderer.initializationCount, 1); + expect(a.created, isEmpty); + expect(b.created.length, 1); + expect(renderer.srcObject, same(b.stream)); + await tester.pumpWidget(const SizedBox()); + await tester.pump(const Duration(milliseconds: 100)); + }); + + testWidgets('rapid replacement leaves only the latest listener and stream', (tester) async { + final a = _Track(_Stream()); + final b = _Track(_Stream()); + final c = _Track(_Stream()); + renderer.initialized.complete(); + await tester.pumpWidget(view(a)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(renderer.srcObject, same(a.stream)); + final gate = a.created.single.beforeDispose = Completer(); + await tester.pumpWidget(view(b)); + await tester.pumpWidget(view(c)); + gate.complete(); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(renderer.initializationCount, 1); + expect(renderer.srcObject, same(c.stream)); + expect(a.created.single.disposeCount, 1); + expect(b.created, isEmpty); + expect(c.created.single.isDisposed, isFalse); + a.updateStream(_Stream()); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(renderer.srcObject, same(c.stream)); + final updated = _Stream(); + c.updateStream(updated); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(renderer.srcObject, same(updated)); + await tester.pumpWidget(const SizedBox()); + expect(c.created.single.disposeCount, 1); + await tester.pump(const Duration(milliseconds: 100)); + }); + + testWidgets('disposing while replacing listener cannot install a new one', (tester) async { + final a = _Track(_Stream()); + final b = _Track(_Stream()); + renderer.initialized.complete(); + await tester.pumpWidget(view(a)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + final gate = a.created.single.beforeDispose = Completer(); + await tester.pumpWidget(view(b)); + await tester.pumpWidget(const SizedBox()); + gate.complete(); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(b.created, isEmpty); + expect(renderer.srcObject, isNull); + expect(renderer.disposeCount, 1); + expect(a.created.single.disposeCount, 1); + expect(tester.takeException(), isNull); + }); + + testWidgets('cached renderer is borrowed, even when replaced or autoDispose is true', (tester) async { + final a = _Track(_Stream()); + final b = _Track(_Stream()); + final cached = _Renderer()..initialized.complete(); + final replacement = _Renderer()..initialized.complete(); + await tester.pumpWidget(view(a, cached: cached)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + await tester.pumpWidget(view(b, cached: cached)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(cached.srcObject, same(b.stream)); + await tester.pumpWidget(view(b, cached: replacement)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(cached.disposeCount, 0); + expect(cached.srcObject, isNull); + expect(replacement.srcObject, same(b.stream)); + await tester.pumpWidget(const SizedBox()); + expect(replacement.disposeCount, 0); + expect(replacement.srcObject, isNull); + await tester.pump(const Duration(milliseconds: 100)); + }); + + testWidgets('cached to owned renderer waits for initialization before building video view', (tester) async { + final track = _Track(_Stream()); + final cached = _Renderer()..initialized.complete(); + await tester.pumpWidget(view(track, cached: cached)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(find.byType(rtc.RTCVideoView), findsOneWidget); + + await tester.pumpWidget(view(track)); + expect(renderer.initializationCount, 1); + expect(find.byType(rtc.RTCVideoView), findsNothing); + expect(cached.disposeCount, 0); + renderer.initialized.complete(); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(find.byType(rtc.RTCVideoView), findsOneWidget); + expect(renderer.srcObject, same(track.stream)); + await tester.pumpWidget(const SizedBox()); + await tester.pump(const Duration(milliseconds: 100)); + }); + + testWidgets('platform view ignores readiness from a retired view after track replacement', (tester) async { + if (kIsWeb) return; // RTCVideoPlatFormView is only supported on iOS and macOS. + videoTrackRendererPlatformViewOverride = (mode) => mode == VideoRenderMode.platformView; + debugDefaultTargetPlatformOverride = TargetPlatform.android; + final a = _Track(_Stream()); + final b = _Track(_Stream()); + final oldController = _PlatformController(); + final controller = _PlatformController(); + + await tester.pumpWidget(view(a, renderMode: VideoRenderMode.platformView)); + final ready = tester.widget(find.byType(rtc.RTCVideoPlatFormView)).onViewReady!; + await tester.pumpWidget(view(b, renderMode: VideoRenderMode.platformView)); + oldController.retired = true; + ready(oldController); + final newReady = tester.widget(find.byType(rtc.RTCVideoPlatFormView)).onViewReady!; + newReady(controller); + ready(oldController); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + + expect(controller.srcObject, same(b.stream)); + expect(oldController.srcObject, isNull); + expect(tester.takeException(), isNull); + expect(a.created, isEmpty); + expect(b.created.single.isDisposed, isFalse); + await tester.pumpWidget(const SizedBox()); + debugDefaultTargetPlatformOverride = null; + }); + + testWidgets('platform view reattaches on track changes but ignores cached renderer changes', (tester) async { + if (kIsWeb) return; // RTCVideoPlatFormView is only supported on iOS and macOS. + videoTrackRendererPlatformViewOverride = (mode) => mode == VideoRenderMode.platformView; + debugDefaultTargetPlatformOverride = TargetPlatform.android; + final a = _Track(_Stream()); + final b = _Track(_Stream()); + final controller = _PlatformController(); + final cachedA = _Renderer()..initialized.complete(); + final cachedB = _Renderer()..initialized.complete(); + + await tester.pumpWidget(view(a, cached: cachedA, renderMode: VideoRenderMode.platformView)); + tester.widget(find.byType(rtc.RTCVideoPlatFormView)).onViewReady!(controller); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(controller.srcObject, same(a.stream)); + expect(a.created.length, 1); + + await tester.pumpWidget(view(a, cached: cachedB, renderMode: VideoRenderMode.platformView)); + expect(controller.srcObject, same(a.stream)); + expect(a.created.length, 1); + expect(a.created.single.disposeCount, 0); + + await tester.pumpWidget(view(b, cached: cachedB, renderMode: VideoRenderMode.platformView)); + expect(controller.srcObject, isNull); + controller.retired = true; + final replacementController = _PlatformController(); + tester.widget(find.byType(rtc.RTCVideoPlatFormView)).onViewReady!(replacementController); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(replacementController.srcObject, same(b.stream)); + expect(tester.takeException(), isNull); + expect(a.created.single.disposeCount, 1); + expect(b.created.single.isDisposed, isFalse); + a.updateStream(_Stream()); + await tester.pump(); + expect(replacementController.srcObject, same(b.stream)); + await tester.pumpWidget(const SizedBox()); + expect(replacementController.srcObject, isNull); + expect(cachedA.disposeCount, 0); + expect(cachedB.disposeCount, 0); + debugDefaultTargetPlatformOverride = null; + }); + + testWidgets('render mode switch retires owned texture without disposing a platform controller', (tester) async { + if (kIsWeb) return; // Platform views are native-only. + videoTrackRendererPlatformViewOverride = (mode) => mode == VideoRenderMode.platformView; + debugDefaultTargetPlatformOverride = TargetPlatform.android; + final track = _Track(_Stream()); + renderer.initialized.complete(); + await tester.pumpWidget(view(track)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(renderer.srcObject, same(track.stream)); + + await tester.pumpWidget(view(track, renderMode: VideoRenderMode.platformView)); + expect(renderer.disposeCount, 1); + expect(renderer.srcObject, isNull); + final ready = tester.widget(find.byType(rtc.RTCVideoPlatFormView)).onViewReady!; + final controller = _PlatformController(); + ready(controller); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(controller.srcObject, same(track.stream)); + + final cached = _Renderer()..initialized.complete(); + await tester.pumpWidget(view(track, cached: cached)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(controller.srcObject, isNull); + expect(cached.srcObject, same(track.stream)); + ready(controller); // The old view's delayed callback must not resurrect its controller. + await tester.pump(); + expect(controller.srcObject, isNull); + expect(cached.srcObject, same(track.stream)); + await tester.pumpWidget(const SizedBox()); + debugDefaultTargetPlatformOverride = null; + }); + + testWidgets('rebuild and autoDispose false do not initialize or dispose twice', (tester) async { + final track = _Track(_Stream()); + renderer.initialized.complete(); + await tester.pumpWidget(view(track, autoDispose: false)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + await tester.pumpWidget(view(track, autoDispose: false)); + await tester.runAsync(() async => await Future.delayed(Duration.zero)); + await tester.pumpAndSettle(); + expect(renderer.initializationCount, 1); + expect(track.created.length, 1); + await tester.pumpWidget(const SizedBox()); + expect(renderer.disposeCount, 0); + expect(renderer.srcObject, isNull); + await tester.pump(const Duration(milliseconds: 100)); + }); +} diff --git a/test/widgets/video_track_renderer_web_test.dart b/test/widgets/video_track_renderer_web_test.dart new file mode 100644 index 000000000..f7d3ff182 --- /dev/null +++ b/test/widgets/video_track_renderer_web_test.dart @@ -0,0 +1,174 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@TestOn('browser') +library; + +import 'dart:ui' as ui; + +import 'package:flutter/rendering.dart'; +import 'package:flutter/widgets.dart'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:mockito/mockito.dart'; +import 'package:web/web.dart' as web; + +import 'package:livekit_client/src/track/remote/video.dart'; +import 'package:livekit_client/src/types/other.dart'; +import 'package:livekit_client/src/widgets/video_track_renderer.dart'; + +class _Stream extends Mock implements rtc.MediaStream {} + +class _MediaTrack extends Mock implements rtc.MediaStreamTrack {} + +class _Renderer extends rtc.RTCVideoRenderer { + final video = web.HTMLVideoElement(); + rtc.MediaStream? _stream; + + // The analyzer resolves the native renderer; the browser renderer defines this method. + web.HTMLVideoElement? findHtmlView() => video; + + @override + bool get renderVideo => _stream != null; + + @override + rtc.MediaStream? get srcObject => _stream; + + @override + set srcObject(rtc.MediaStream? value) => _stream = value; +} + +RemoteVideoTrack _track() => RemoteVideoTrack(TrackSource.camera, _Stream(), _MediaTrack()); + +Widget _view(RemoteVideoTrack track, _Renderer renderer) => WidgetsApp( + color: Color(0xff000000), + builder: (context, child) => SizedBox( + width: 100, + height: 100, + child: VideoTrackRenderer(track, cachedRenderer: renderer), + ), +); + +void main() { + testWidgets( + 'muting hides a retained Flutter video frame without unmounting it', + (tester) async { + final renderer = _Renderer(); + await renderer.initialize(); + final track = _track(); + final boundaryKey = GlobalKey(); + + await tester.pumpWidget( + WidgetsApp( + color: const Color(0xff000000), + builder: (context, child) => RepaintBoundary( + key: boundaryKey, + child: ColoredBox( + color: const Color(0xffffffff), + child: VideoTrackRenderer(track, cachedRenderer: renderer, fit: VideoViewFit.cover), + ), + ), + ), + ); + await tester.pumpAndSettle(); + + // Seed the default flutter_webrtc backend's retained frame, as if a video + // frame had already been captured before the track was muted. + final recorder = ui.PictureRecorder(); + Canvas(recorder).drawColor(const Color(0xffff0000), BlendMode.src); + final picture = recorder.endRecording(); + final frame = await tester.runAsync(() => picture.toImage(100, 100)); + picture.dispose(); + final videoState = tester.state(find.byType(rtc.RTCVideoView)) as dynamic; + videoState.capturedFrame = frame; + renderer.notifyListeners(); + await tester.pump(); + + Future> pixel() async { + final boundary = boundaryKey.currentContext!.findRenderObject()! as RenderRepaintBoundary; + final image = await boundary.toImage(); + final bytes = await image.toByteData(format: ui.ImageByteFormat.rawRgba); + image.dispose(); + return bytes!.buffer.asUint8List().sublist(0, 4); + } + + expect(await tester.runAsync(pixel), [255, 0, 0, 255]); + track.updateMuted(true); + await tester.pumpAndSettle(); + expect(await tester.runAsync(pixel), [255, 255, 255, 255]); + expect(tester.state(find.byType(rtc.RTCVideoView)), same(videoState)); + expect(renderer.srcObject, same(track.mediaStream)); + + track.updateMuted(false); + await tester.pumpAndSettle(); + expect(await tester.runAsync(pixel), [255, 0, 0, 255]); + expect(tester.state(find.byType(rtc.RTCVideoView)), same(videoState)); + expect(renderer.srcObject, same(track.mediaStream)); + + await tester.pumpWidget(const SizedBox()); + await tester.pump(const Duration(milliseconds: 100)); + await renderer.dispose(); + }, + // The HTML-element backend does not paint retained frames into Flutter. + skip: const bool.fromEnvironment('WEBRTC_USE_HTML_ELEMENT_VIEW'), + ); + + testWidgets('initially muted track stays attached but hidden until unmuted', (tester) async { + final renderer = _Renderer(); + await renderer.initialize(); + final track = _track()..updateMuted(true); + + await tester.pumpWidget(_view(track, renderer)); + await tester.pumpAndSettle(); + expect(renderer.srcObject, same(track.mediaStream)); + expect(renderer.video.style.visibility, 'hidden'); + + track.updateMuted(false); + await tester.pumpAndSettle(); + expect(renderer.video.style.visibility, 'visible'); + expect(renderer.srcObject, same(track.mediaStream)); + + await tester.pumpWidget(const SizedBox()); + await tester.pump(const Duration(milliseconds: 100)); + await renderer.dispose(); + }); + + testWidgets('borrowed renderer visibility is restored on replacement and disposal', (tester) async { + final first = _Renderer(); + final second = _Renderer(); + await first.initialize(); + await second.initialize(); + first.video.style.visibility = 'collapse'; + second.video.style.visibility = ''; + final track = _track()..updateMuted(true); + + await tester.pumpWidget(_view(track, first)); + await tester.pumpAndSettle(); + expect(first.video.style.visibility, 'hidden'); + + await tester.pumpWidget(_view(track, second)); + await tester.pumpAndSettle(); + expect(first.video.style.visibility, 'collapse'); + expect(first.srcObject, isNull); + expect(second.video.style.visibility, 'hidden'); + + await tester.pumpWidget(const SizedBox()); + expect(second.video.style.visibility, ''); + expect(second.srcObject, isNull); + await tester.pump(const Duration(milliseconds: 100)); + await first.dispose(); + await second.dispose(); + }); +}