diff --git a/.changes/ignore-replaced-socket-callbacks b/.changes/ignore-replaced-socket-callbacks new file mode 100644 index 000000000..44a4eb339 --- /dev/null +++ b/.changes/ignore-replaced-socket-callbacks @@ -0,0 +1 @@ +patch type="fixed" "A late close from a signal socket the SDK already replaced no longer starts another reconnect. In a browser it arrives up to seconds later, which made a room resume every 2 s after its first reconnect. A signal connect that only opens after disconnect() or a newer connect() is closed instead of taking over the connection" \ No newline at end of file diff --git a/lib/src/connection_check/checks/reconnect.dart b/lib/src/connection_check/checks/reconnect.dart index ce9266607..a64dc6363 100644 --- a/lib/src/connection_check/checks/reconnect.dart +++ b/lib/src/connection_check/checks/reconnect.dart @@ -44,9 +44,8 @@ class ReconnectCheck extends Checker { }); try { - // Forcefully close the underlying signal WebSocket; the engine should - // notice and resume the session on its own. - await engine.signalClient.cleanUp(); + // Simulate a signal loss and wait for the room to resume. + await engine.signalClient.simulateSignalDrop(); await reconnectCompleter.future.timeout( const Duration(seconds: 5), diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index 0fd4200ed..60bdc0cc6 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -354,6 +354,10 @@ class Engine extends Disposable with EventsEmittable { } catch (error) { logger.fine('Connect Error $error'); + // A superseded attempt reports nothing, the connection that replaced it + // is the one the room has. + if (error is SignalConnectSupersededException) rethrow; + // during a reconnect this connect() runs inside restartConnection and // attemptReconnect owns disconnect emission, emitting here as well // would produce two events for one failure diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index 74e4a1b74..d661c60a1 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -1353,12 +1353,12 @@ extension RoomDebugMethods on Room { bool? leaveRequestFullReconnect, }) async { if (signalReconnect != null && signalReconnect) { - await engine.signalClient.cleanUp(); + await engine.signalClient.simulateSignalDrop(); return; } if (fullReconnect != null && fullReconnect) { engine.fullReconnectOnNext = true; - await engine.signalClient.cleanUp(); + await engine.signalClient.simulateSignalDrop(); return; } engine.signalClient.sendSimulateScenario( @@ -1375,7 +1375,7 @@ extension RoomDebugMethods on Room { // The server only acts on the next resume, so start one now. Mirrors the // post action in client-sdk-js. if ((disconnectSignalOnResume ?? false) || (disconnectSignalOnResumeNoMessages ?? false)) { - await engine.signalClient.cleanUp(); + await engine.signalClient.simulateSignalDrop(); } } } diff --git a/lib/src/core/signal_client.dart b/lib/src/core/signal_client.dart index a4e8eb4ab..d3fb00c2a 100644 --- a/lib/src/core/signal_client.dart +++ b/lib/src/core/signal_client.dart @@ -43,6 +43,16 @@ class SignalClient extends Disposable with EventsEmittable { ConnectionState get connectionState => _connectionState; final WebSocketConnector _wsConnector; + + // Id of the socket whose callbacks are honored. connect() gives each new + // socket the next id, and cleanUp() advances it too so the socket it disposes + // stops being current. A real socket reports its close after dispose() + // returns, a few milliseconds later on a good path but seconds later in a + // browser when the server keeps TCP open after the close handshake. That + // report used to be attributed to whatever connection was current by then, + // clobbering its state and starting a reconnect against a healthy socket, + // over and over when every reconnect replaced the socket again. + int _socketId = 0; LiveKitWebSocket? _ws; final _queue = Queue(); @@ -144,6 +154,8 @@ class SignalClient extends Disposable with EventsEmittable { logger.fine('SignalClient connecting with url: ${rtcUri.redactingAccessToken}'); + // Id of the socket this attempt opens, set once the previous one is gone. + int? socketId; try { if (reconnect == true) { _connectionState = ConnectionState.reconnecting; @@ -154,25 +166,38 @@ class SignalClient extends Disposable with EventsEmittable { } // Clean up existing socket await cleanUp(); - // Attempt to connect - var future = _wsConnector( + // Attempt to connect. Callbacks carry the id of the socket they belong to + // so a late report from a replaced socket is ignored. + final id = socketId = ++_socketId; + final ws = await _wsConnector( rtcUri, options: WebSocketEventHandlers( - onData: _onSocketData, - onDispose: _onSocketDispose, - onError: _onSocketError, + onData: (data) => _ifCurrentSocket(id, () => _onSocketData(data)), + onDispose: () => _ifCurrentSocket(id, _onSocketDispose), + onError: (error) => _ifCurrentSocket(id, () => _onSocketError(error)), ), headers: { 'Authorization': 'Bearer $token', }, networkOptions: roomOptions.networkOptions, - ); - future = future.timeout(connectOptions.timeouts.connection); - _ws = await future; + ).timeout(connectOptions.timeouts.connection); + if (id != _socketId) { + // disconnect() or a newer connect() took over while this socket was + // opening. Installing it would take the place of the current one. + await ws.dispose(); + throw const SignalConnectSupersededException(); + } + _ws = ws; // Successful connection _connectionState = ConnectionState.connected; events.emit(const SignalConnectedEvent()); } catch (socketError) { + // An attempt that something else replaced leaves the state and the + // events to the connection that replaced it. + if (socketId != null && socketId != _socketId) { + if (socketError is SignalConnectSupersededException) rethrow; + throw const SignalConnectSupersededException(); + } if (socketError is CertificatePinningException) { // In reconnect mode the engine owns state and event emission, // emitting here would race its reconnect handling. @@ -242,11 +267,15 @@ class SignalClient extends Disposable with EventsEmittable { @internal Future cleanUp() async { logger.fine('[${objectId}] cleanUp()'); + _socketId++; _connectionState = ConnectionState.disconnected; - await _ws?.dispose(); + // Detached before the await, so a connect() that runs while the old + // socket is still closing keeps its own socket, queue and pings. + final ws = _ws; _ws = null; _queue.clear(); _clearPingInterval(); + await ws?.dispose(); } void _sendRequest( @@ -409,6 +438,31 @@ class SignalClient extends Disposable with EventsEmittable { } } + void _ifCurrentSocket(int socketId, void Function() action) { + if (socketId != _socketId) { + logger.fine('[$objectId] ignoring a callback from a socket that was already replaced'); + return; + } + action(); + } + + /// Drops the signal socket the way a server side close does: the socket goes + /// away and the engine sees a disconnect it has to recover from. For the + /// debug scenarios and the connection checker. [cleanUp] alone is a quiet + /// teardown, the disposed socket's own close is ignored. + @internal + Future simulateSignalDrop() async { + if (_ws == null) { + logger.fine('[$objectId] simulateSignalDrop(): no signal socket to drop'); + return; + } + // The loss is reported before the socket has finished closing, which can + // take a while, so it is charged to the socket that was dropped. + final dropped = cleanUp(); + _onSocketDispose(); + await dropped; + } + void _onSocketError(dynamic error) { logger.warning('received websocket error $error'); } @@ -629,3 +683,14 @@ extension SignalClientInternalMethods on SignalClient { @internal void clearQueue() => _queue.clear(); } + +/// Thrown by a [SignalClient.connect] whose socket was no longer wanted by the +/// time it opened, because disconnect() or a newer connect() took over. The +/// connection that took over owns the state and the events. +@internal +class SignalConnectSupersededException implements Exception { + const SignalConnectSupersededException(); + + @override + String toString() => 'SignalConnectSupersededException: the signal connection was superseded while it was opening'; +} diff --git a/test/core/stale_socket_close_test.dart b/test/core/stale_socket_close_test.dart new file mode 100644 index 000000000..b934695ad --- /dev/null +++ b/test/core/stale_socket_close_test.dart @@ -0,0 +1,254 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@Timeout(Duration(seconds: 10)) +library; + +import 'dart:async'; + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/livekit_client.dart'; +import 'package:livekit_client/src/connection_check/checks/reconnect.dart'; +import 'package:livekit_client/src/core/signal_client.dart'; +import 'package:livekit_client/src/proto/livekit_models.pb.dart' as lk_models; +import 'package:livekit_client/src/proto/livekit_rtc.pb.dart' as lk_rtc; +import 'package:livekit_client/src/support/websocket.dart'; +import '../mock/e2e_container.dart'; +import '../mock/websocket_mock.dart'; + +const exampleUri = 'ws://www.example.com'; +const token = 'token'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late E2EContainer container; + + setUp(() { + container = E2EContainer(); + }); + + tearDown(() async { + await container.dispose(); + }); + + test('a close reported by a replaced socket does not disturb the next connection', () async { + // the real sockets report their close some time after dispose() + container.wsConnector.holdCloses = true; + await container.connectRoom(); + await container.room.disconnect(); + + final events = []; + container.room.events.on(events.add); + container.room.events.on(events.add); + container.room.events.on(events.add); + await container.connectRoom(); + // the first socket reports its close only now, with the second one up + container.wsConnector.releaseCloses(); + await _settle(); + + expect(container.room.connectionState, ConnectionState.connected); + expect(events, isEmpty, reason: 'the stale close belongs to the old socket and must be ignored'); + }); + + for (final (name, action) in [ + ('resume', lk_rtc.LeaveRequest_Action.RESUME), + ('full reconnect', lk_rtc.LeaveRequest_Action.RECONNECT), + ]) { + test('a late close of the socket a $name replaced does not start another reconnect', () async { + // A browser reports the close when the server drops TCP, which can be + // seconds after the close handshake. By then the reconnect is done and + // the new socket is current. + container.wsConnector.holdCloses = true; + await container.connectRoom(); + + final events = []; + final sub = container.room.events.listen(events.add); + final replaced = container.wsConnector.handlers; + + // the server asks for a reconnect while the socket is still open + container.wsConnector.onData(_leave(action, lk_models.DisconnectReason.MIGRATION)); + await _waitForNewSocket(container, replaced); + final reconnected = container.room.events.waitFor(duration: const Duration(seconds: 5)); + if (action == lk_rtc.LeaveRequest_Action.RESUME) { + container.wsConnector.onData(lk_rtc.SignalResponse(reconnect: lk_rtc.ReconnectResponse()).writeToBuffer()); + } else { + await container.answerJoin(); + } + await reconnected; + final current = container.wsConnector.handlers; + + // the replaced socket reports its close only now + container.wsConnector.releaseCloses(); + await _settle(); + await sub(); + + expect(container.room.connectionState, ConnectionState.connected); + expect( + identical(container.wsConnector.handlers, current), + isTrue, + reason: 'the close of the replaced socket must not open another connection', + ); + expect(events.whereType(), hasLength(1)); + expect(events.whereType(), hasLength(1)); + expect(events.whereType(), isEmpty); + }); + } + + test('data a replaced socket delivers late is ignored', () async { + await container.connectRoom(); + final replaced = container.wsConnector.handlers; + container.wsConnector.onData(_leave(lk_rtc.LeaveRequest_Action.RESUME, lk_models.DisconnectReason.MIGRATION)); + await _waitForNewSocket(container, replaced); + final reconnected = container.room.events.waitFor(duration: const Duration(seconds: 5)); + container.wsConnector.onData(lk_rtc.SignalResponse(reconnect: lk_rtc.ReconnectResponse()).writeToBuffer()); + await reconnected; + + final disconnected = []; + container.room.events.on(disconnected.add); + // the old socket still had a Leave on its way + replaced!.onData!(_leave(lk_rtc.LeaveRequest_Action.DISCONNECT, lk_models.DisconnectReason.CLIENT_INITIATED)); + await _settle(); + + expect(container.room.connectionState, ConnectionState.connected); + expect(disconnected, isEmpty); + }); + + test('a connect that opens after disconnect() and a rejoin does not take over the rejoined session', () async { + final gate = Completer(); + container.wsConnector.connectGate = gate.future; + final disconnected = []; + container.room.events.on(disconnected.add); + + final stale = container.room.connect(exampleUri, token).then((_) => null, onError: (Object e) => e); + await Future.delayed(const Duration(milliseconds: 10)); + await container.room.disconnect(); + container.wsConnector.connectGate = null; + await container.connectRoom(); + final rejoined = container.wsConnector.sockets.last; + + // the first connect's socket opens only now + gate.complete(); + final staleError = await stale.timeout(const Duration(seconds: 2)); + await _settle(); + final staleSocket = container.wsConnector.sockets.last; + + expect(staleError, isA()); + expect(staleSocket, isNot(same(rejoined))); + expect(staleSocket.isDisposed, isTrue, reason: 'a socket nobody wants any more is closed'); + expect(container.room.connectionState, ConnectionState.connected); + expect(disconnected, hasLength(1), reason: 'only the disconnect() the app asked for'); + + final before = rejoined.sent.length; + await container.client.sendLeave(); + expect(rejoined.sent, hasLength(before + 1), reason: 'requests go out on the rejoined session'); + expect(staleSocket.sent, isEmpty); + }); + + test('a cleanUp still closing the old socket leaves the next connection alone', () async { + // a dart:io close waits for the server to answer the close frame + final connector = MockWebSocketConnector()..disposeLatency = const Duration(milliseconds: 100); + final client = SignalClient(connector.connect); + Future connect() => + client.connect(exampleUri, token, connectOptions: const ConnectOptions(), roomOptions: const RoomOptions()); + try { + await connect(); + final closing = client.cleanUp(); + await connect(); + final current = connector.sockets.last; + await closing; + + final before = current.sent.length; + await client.sendLeave(); + expect(current.sent, hasLength(before + 1), reason: 'the first cleanUp() must not detach the new socket'); + } finally { + await client.dispose(); + } + }); + + test('simulating a signal drop with no signal socket does nothing', () async { + final events = []; + container.room.events.on(events.add); + container.room.events.on(events.add); + + await container.room.sendSimulateScenario(signalReconnect: true); + await _settle(); + + expect(events, isEmpty); + expect(container.wsConnector.sockets, isEmpty); + }); + + test('the signalReconnect debug scenario still drops the signal and starts a reconnect', () async { + await container.connectRoom(); + final events = []; + container.room.events.on(events.add); + + await container.room.sendSimulateScenario(signalReconnect: true); + + expect( + await _resumeStarted(container), + isTrue, + reason: 'the engine must re-open the signal connection as a resume', + ); + expect(events, isNotEmpty); + }); + + test('the disconnectSignalOnResume debug action still starts a resume', () async { + container.wsConnector.closeLatency = const Duration(milliseconds: 50); + await container.connectRoom(); + + await container.room.sendSimulateScenario(disconnectSignalOnResume: true); + + expect(await _resumeStarted(container), isTrue, reason: 'the server only acts on the next resume'); + }); + + test('the connection checker reconnect check succeeds once the room resumes', () async { + container.wsConnector.closeLatency = const Duration(milliseconds: 50); + await container.connectRoom(); + final replaced = container.wsConnector.handlers; + + final check = ReconnectCheck(exampleUri, token, room: container.room); + final running = check.run(); + await _waitForNewSocket(container, replaced); + container.wsConnector.onData(lk_rtc.SignalResponse(reconnect: lk_rtc.ReconnectResponse()).writeToBuffer()); + final info = await running; + + expect(info.status, CheckStatus.success, reason: info.logs.map((l) => l.message).join(', ')); + }); +} + +List _leave(lk_rtc.LeaveRequest_Action action, lk_models.DisconnectReason reason) => lk_rtc.SignalResponse( + leave: lk_rtc.LeaveRequest(action: action, reason: reason), +).writeToBuffer(); + +/// Lets the events of something that just happened run their course. +Future _settle() => Future.delayed(const Duration(milliseconds: 100)); + +Future _waitForNewSocket(E2EContainer container, WebSocketEventHandlers? previous) async { + final deadline = DateTime.now().add(const Duration(seconds: 3)); + while (identical(container.wsConnector.handlers, previous) && DateTime.now().isBefore(deadline)) { + await Future.delayed(const Duration(milliseconds: 5)); + } + expect(identical(container.wsConnector.handlers, previous), isFalse, reason: 'the SDK never opened a new socket'); +} + +Future _resumeStarted(E2EContainer container) async { + final deadline = DateTime.now().add(const Duration(seconds: 3)); + while (DateTime.now().isBefore(deadline)) { + if (container.wsConnector.uri?.queryParameters['reconnect'] == '1') return true; + await Future.delayed(const Duration(milliseconds: 5)); + } + return false; +} diff --git a/test/mock/websocket_mock.dart b/test/mock/websocket_mock.dart index 33fbadbf9..81b46cd41 100644 --- a/test/mock/websocket_mock.dart +++ b/test/mock/websocket_mock.dart @@ -12,16 +12,41 @@ // See the License for the specific language governing permissions and // limitations under the License. +import 'dart:async'; + import 'package:livekit_client/src/options.dart'; import 'package:livekit_client/src/support/websocket.dart'; class MockWebSocket extends LiveKitWebSocket { - MockWebSocket([this.onSend]); + /// Like the real sockets, the close is reported to the SDK after dispose() + /// has returned, see [MockWebSocketConnector.closeLatency] and + /// [MockWebSocketConnector.holdCloses]. + MockWebSocket(this._connector, this.handlers) { + onDispose(() async { + final connector = _connector; + if (connector.disposeLatency > Duration.zero) await Future.delayed(connector.disposeLatency); + void reportClose() => handlers?.onDispose?.call(); + if (connector.holdCloses) { + connector._heldCloses.add(reportClose); + } else { + unawaited(Future.delayed(connector.closeLatency, reportClose)); + } + }); + } + + final MockWebSocketConnector _connector; - final void Function(List data)? onSend; + /// The callbacks the SDK gave this socket. + final WebSocketEventHandlers? handlers; + + /// Every message the SDK sent over this socket. + final List> sent = []; @override - void send(List data) => onSend?.call(data); + void send(List data) { + sent.add(data); + _connector.sentData.add(data); + } } class MockWebSocketConnector { @@ -31,6 +56,29 @@ class MockWebSocketConnector { NetworkOptions? networkOptions; Object? connectError; + /// Delay between the SDK disposing a socket and that socket reporting its close. + Duration closeLatency = Duration.zero; + + /// How long dispose() takes, like a dart:io close that waits on the server. + Duration disposeLatency = Duration.zero; + + /// While true, the closes of disposed sockets are held until [releaseCloses], + /// so a test decides exactly when an old socket reports its close. + bool holdCloses = false; + final _heldCloses = []; + + /// Reports every close held by [holdCloses]. + void releaseCloses() { + final held = [..._heldCloses]; + _heldCloses.clear(); + for (final reportClose in held) { + reportClose(); + } + } + + /// Every socket this connector opened, oldest first. + final List sockets = []; + /// When true, [connectError] is thrown once and then cleared, so the next /// connect succeeds. Models a first attempt that fails and a retry that gets /// through. @@ -70,6 +118,8 @@ class MockWebSocketConnector { } handlers = options; - return MockWebSocket(sentData.add); + final socket = MockWebSocket(this, options); + sockets.add(socket); + return socket; } }