From 20135bb792b14146223ad6b5f422562d048d89ab Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 17 Sep 2026 00:20:53 +0800 Subject: [PATCH 1/6] fix: ignore callbacks from a signal socket that was already replaced A real socket reports its close a few milliseconds after dispose() returns. That report went through the same handler as a live close, so when it landed during the next connect it reset the signal state to disconnected and started a reconnect against a socket that was fine. Each socket's callbacks now carry a generation, bumped on cleanUp() and on every connect, and a callback from an older generation is dropped. The signalReconnect and fullReconnect debug scenarios relied on that late close to trigger the engine's recovery. They now call simulateSignalDrop(), which disposes the socket and raises the disconnect explicitly. The mock socket reports its close after dispose() like the real ones, so the race is reproducible in tests. --- .changes/ignore-replaced-socket-callbacks | 1 + lib/src/core/room.dart | 4 +- lib/src/core/signal_client.dart | 33 +++++++++-- test/core/stale_socket_close_test.dart | 72 +++++++++++++++++++++++ test/mock/websocket_mock.dart | 20 ++++++- 5 files changed, 121 insertions(+), 9 deletions(-) create mode 100644 .changes/ignore-replaced-socket-callbacks create mode 100644 test/core/stale_socket_close_test.dart diff --git a/.changes/ignore-replaced-socket-callbacks b/.changes/ignore-replaced-socket-callbacks new file mode 100644 index 000000000..5ff119d14 --- /dev/null +++ b/.changes/ignore-replaced-socket-callbacks @@ -0,0 +1 @@ +patch type="fixed" "Events from a signal socket the SDK already replaced are ignored, so a connect right after a disconnect can no longer be knocked into a reconnect by a late close from the old socket" \ No newline at end of file diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index 7e30d983c..41dac78c7 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -1256,12 +1256,12 @@ extension RoomDebugMethods on Room { bool? leaveRequestFullReconnect, }) async { if (signalReconnect != null && signalReconnect) { - await engine.signalClient.cleanUp(); + await engine.signalClient.simulateSignalDrop(); return; } if (fullReconnect != null && fullReconnect) { engine.fullReconnectOnNext = true; - await engine.signalClient.cleanUp(); + await engine.signalClient.simulateSignalDrop(); return; } engine.signalClient.sendSimulateScenario( diff --git a/lib/src/core/signal_client.dart b/lib/src/core/signal_client.dart index 497342e48..a1a9d8c9e 100644 --- a/lib/src/core/signal_client.dart +++ b/lib/src/core/signal_client.dart @@ -43,6 +43,12 @@ class SignalClient extends Disposable with EventsEmittable { ConnectionState get connectionState => _connectionState; final WebSocketConnector _wsConnector; + + // Bumped whenever the current socket is replaced or disposed. A real socket + // reports its close a few milliseconds after dispose() returns, and that + // report used to be attributed to whatever connection was current by then, + // clobbering its state and starting a reconnect against a healthy socket. + int _socketGeneration = 0; LiveKitWebSocket? _ws; final _queue = Queue(); @@ -154,13 +160,15 @@ class SignalClient extends Disposable with EventsEmittable { } // Clean up existing socket await cleanUp(); - // Attempt to connect + // Attempt to connect. Callbacks carry the generation of the socket they + // belong to so a late report from a replaced socket is ignored. + final generation = ++_socketGeneration; var future = _wsConnector( rtcUri, options: WebSocketEventHandlers( - onData: _onSocketData, - onDispose: _onSocketDispose, - onError: _onSocketError, + onData: (data) => _ifCurrentSocket(generation, () => _onSocketData(data)), + onDispose: () => _ifCurrentSocket(generation, _onSocketDispose), + onError: (error) => _ifCurrentSocket(generation, () => _onSocketError(error)), ), headers: { 'Authorization': 'Bearer $token', @@ -242,6 +250,7 @@ class SignalClient extends Disposable with EventsEmittable { @internal Future cleanUp() async { logger.fine('[${objectId}] cleanUp()'); + _socketGeneration++; _connectionState = ConnectionState.disconnected; await _ws?.dispose(); _ws = null; @@ -409,6 +418,22 @@ class SignalClient extends Disposable with EventsEmittable { } } + void _ifCurrentSocket(int generation, void Function() action) { + if (generation != _socketGeneration) { + logger.fine('[$objectId] ignoring a callback from a socket that was already replaced'); + return; + } + action(); + } + + /// Debug only. Drops the signal socket the way a server side close does: the + /// socket goes away and the engine sees a disconnect it has to recover from. + @internal + Future simulateSignalDrop() async { + await cleanUp(); + _onSocketDispose(); + } + void _onSocketError(dynamic error) { logger.warning('received websocket error $error'); } diff --git a/test/core/stale_socket_close_test.dart b/test/core/stale_socket_close_test.dart new file mode 100644 index 000000000..df2c4f250 --- /dev/null +++ b/test/core/stale_socket_close_test.dart @@ -0,0 +1,72 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@Timeout(Duration(seconds: 10)) +library; + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/livekit_client.dart'; +import '../mock/e2e_container.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late E2EContainer container; + + setUp(() { + container = E2EContainer(); + }); + + tearDown(() async { + await container.dispose(); + }); + + test('a close reported by a replaced socket does not disturb the next connection', () async { + // the real sockets report their close a few milliseconds after dispose() + container.wsConnector.closeLatency = const Duration(milliseconds: 50); + await container.connectRoom(); + await container.room.disconnect(); + + // connect again before the first socket has reported its close + final events = []; + container.room.events.on(events.add); + container.room.events.on(events.add); + container.room.events.on(events.add); + await container.connectRoom(); + await Future.delayed(const Duration(milliseconds: 150)); + + expect(container.room.connectionState, ConnectionState.connected); + expect(events, isEmpty, reason: 'the stale close belongs to the old socket and must be ignored'); + }); + + test('the signalReconnect debug scenario still drops the signal and starts a reconnect', () async { + await container.connectRoom(); + final events = []; + container.room.events.on(events.add); + + await container.room.sendSimulateScenario(signalReconnect: true); + + final deadline = DateTime.now().add(const Duration(seconds: 3)); + while (!(container.wsConnector.uri?.queryParameters['reconnect'] == '1') && DateTime.now().isBefore(deadline)) { + await Future.delayed(const Duration(milliseconds: 5)); + } + expect( + container.wsConnector.uri?.queryParameters['reconnect'], + '1', + reason: 'the engine must re-open the signal connection as a resume', + ); + expect(events, isNotEmpty); + }); +} diff --git a/test/mock/websocket_mock.dart b/test/mock/websocket_mock.dart index 0917de753..2fe3cea27 100644 --- a/test/mock/websocket_mock.dart +++ b/test/mock/websocket_mock.dart @@ -1,3 +1,8 @@ +import 'dart:async'; + +import 'package:livekit_client/src/options.dart'; +import 'package:livekit_client/src/support/websocket.dart'; + // Copyright 2024 LiveKit, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,10 +17,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -import 'package:livekit_client/src/options.dart'; -import 'package:livekit_client/src/support/websocket.dart'; class MockWebSocket extends LiveKitWebSocket { + /// Like the real sockets, the close is reported to the SDK after dispose() + /// has returned, [closeLatency] later. + MockWebSocket(WebSocketEventHandlers? options, Duration closeLatency) { + onDispose(() async { + unawaited(Future.delayed(closeLatency, () => options?.onDispose?.call())); + }); + } + @override void send(List data) {} } @@ -27,6 +38,9 @@ class MockWebSocketConnector { NetworkOptions? networkOptions; Object? connectError; + /// Delay between the SDK disposing a socket and that socket reporting its close. + Duration closeLatency = Duration.zero; + WebSocketOnData get onData => handlers!.onData!; WebSocketOnDispose get onDispose => handlers!.onDispose!; @@ -49,6 +63,6 @@ class MockWebSocketConnector { } handlers = options; - return MockWebSocket(); + return MockWebSocket(options, closeLatency); } } From 299ab52ea646523687372dd619685ae27f0f394e Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 17 Oct 2026 05:28:40 +0800 Subject: [PATCH 2/6] fix: the reconnect check and the resume debug action still drop the socket cleanUp() now ignores the close the disposed socket reports afterwards, so callers that relied on that late close to make the engine recover stopped working. The connection checker's reconnect check always reported that no reconnect was attempted, and the disconnectSignalOnResume debug actions never started the resume the server is waiting for. Both now call simulateSignalDrop(), like the signalReconnect and fullReconnect scenarios already do. --- .../connection_check/checks/reconnect.dart | 2 +- lib/src/core/room.dart | 2 +- lib/src/core/signal_client.dart | 6 ++-- test/core/stale_socket_close_test.dart | 34 +++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/lib/src/connection_check/checks/reconnect.dart b/lib/src/connection_check/checks/reconnect.dart index ce9266607..e20179bae 100644 --- a/lib/src/connection_check/checks/reconnect.dart +++ b/lib/src/connection_check/checks/reconnect.dart @@ -46,7 +46,7 @@ class ReconnectCheck extends Checker { try { // Forcefully close the underlying signal WebSocket; the engine should // notice and resume the session on its own. - await engine.signalClient.cleanUp(); + await engine.signalClient.simulateSignalDrop(); await reconnectCompleter.future.timeout( const Duration(seconds: 5), diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index 41dac78c7..8dd4f3e11 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -1278,7 +1278,7 @@ extension RoomDebugMethods on Room { // The server only acts on the next resume, so start one now. Mirrors the // post action in client-sdk-js. if ((disconnectSignalOnResume ?? false) || (disconnectSignalOnResumeNoMessages ?? false)) { - await engine.signalClient.cleanUp(); + await engine.signalClient.simulateSignalDrop(); } } } diff --git a/lib/src/core/signal_client.dart b/lib/src/core/signal_client.dart index a1a9d8c9e..3ee05731b 100644 --- a/lib/src/core/signal_client.dart +++ b/lib/src/core/signal_client.dart @@ -426,8 +426,10 @@ class SignalClient extends Disposable with EventsEmittable { action(); } - /// Debug only. Drops the signal socket the way a server side close does: the - /// socket goes away and the engine sees a disconnect it has to recover from. + /// Drops the signal socket the way a server side close does: the socket goes + /// away and the engine sees a disconnect it has to recover from. For the + /// debug scenarios and the connection checker. [cleanUp] alone is a quiet + /// teardown, the disposed socket's own close is ignored. @internal Future simulateSignalDrop() async { await cleanUp(); diff --git a/test/core/stale_socket_close_test.dart b/test/core/stale_socket_close_test.dart index df2c4f250..431f9bea1 100644 --- a/test/core/stale_socket_close_test.dart +++ b/test/core/stale_socket_close_test.dart @@ -18,6 +18,7 @@ library; import 'package:flutter_test/flutter_test.dart'; import 'package:livekit_client/livekit_client.dart'; +import 'package:livekit_client/src/connection_check/checks/reconnect.dart'; import '../mock/e2e_container.dart'; void main() { @@ -69,4 +70,37 @@ void main() { ); expect(events, isNotEmpty); }); + + test('the disconnectSignalOnResume debug action still starts a resume', () async { + // the real sockets report their close a few milliseconds after dispose() + container.wsConnector.closeLatency = const Duration(milliseconds: 50); + await container.connectRoom(); + + await container.room.sendSimulateScenario(disconnectSignalOnResume: true); + + expect(await _resumeStarted(container), isTrue, reason: 'the server only acts on the next resume'); + }); + + test('the connection checker reconnect check still sees a resume', () async { + container.wsConnector.closeLatency = const Duration(milliseconds: 50); + await container.connectRoom(); + + final check = ReconnectCheck('ws://www.example.com', 'token', room: container.room); + final info = await check.run(); + + expect( + info.logs.map((l) => l.message), + isNot(contains('Did not attempt to reconnect')), + reason: 'the check drops the signal socket and expects the engine to notice', + ); + }); +} + +Future _resumeStarted(E2EContainer container) async { + final deadline = DateTime.now().add(const Duration(seconds: 3)); + while (DateTime.now().isBefore(deadline)) { + if (container.wsConnector.uri?.queryParameters['reconnect'] == '1') return true; + await Future.delayed(const Duration(milliseconds: 5)); + } + return false; } From ef6abd1c641d7a60ab4d3aa2227da69b2409443a Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 17 Oct 2026 06:02:24 +0800 Subject: [PATCH 3/6] refactor: call the socket counter an id _socketGeneration becomes _socketId. The value is compared to decide whether a callback came from the current socket, which an id describes more directly than a generation. No behavior change. --- lib/src/core/signal_client.dart | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/src/core/signal_client.dart b/lib/src/core/signal_client.dart index 3ee05731b..4053b89be 100644 --- a/lib/src/core/signal_client.dart +++ b/lib/src/core/signal_client.dart @@ -44,11 +44,13 @@ class SignalClient extends Disposable with EventsEmittable { final WebSocketConnector _wsConnector; - // Bumped whenever the current socket is replaced or disposed. A real socket - // reports its close a few milliseconds after dispose() returns, and that - // report used to be attributed to whatever connection was current by then, - // clobbering its state and starting a reconnect against a healthy socket. - int _socketGeneration = 0; + // Id of the socket whose callbacks are honored. connect() gives each new + // socket the next id, and cleanUp() advances it too so the socket it disposes + // stops being current. A real socket reports its close a few milliseconds + // after dispose() returns, and that report used to be attributed to whatever + // connection was current by then, clobbering its state and starting a + // reconnect against a healthy socket. + int _socketId = 0; LiveKitWebSocket? _ws; final _queue = Queue(); @@ -160,15 +162,15 @@ class SignalClient extends Disposable with EventsEmittable { } // Clean up existing socket await cleanUp(); - // Attempt to connect. Callbacks carry the generation of the socket they - // belong to so a late report from a replaced socket is ignored. - final generation = ++_socketGeneration; + // Attempt to connect. Callbacks carry the id of the socket they belong to + // so a late report from a replaced socket is ignored. + final socketId = ++_socketId; var future = _wsConnector( rtcUri, options: WebSocketEventHandlers( - onData: (data) => _ifCurrentSocket(generation, () => _onSocketData(data)), - onDispose: () => _ifCurrentSocket(generation, _onSocketDispose), - onError: (error) => _ifCurrentSocket(generation, () => _onSocketError(error)), + onData: (data) => _ifCurrentSocket(socketId, () => _onSocketData(data)), + onDispose: () => _ifCurrentSocket(socketId, _onSocketDispose), + onError: (error) => _ifCurrentSocket(socketId, () => _onSocketError(error)), ), headers: { 'Authorization': 'Bearer $token', @@ -250,7 +252,7 @@ class SignalClient extends Disposable with EventsEmittable { @internal Future cleanUp() async { logger.fine('[${objectId}] cleanUp()'); - _socketGeneration++; + _socketId++; _connectionState = ConnectionState.disconnected; await _ws?.dispose(); _ws = null; @@ -418,8 +420,8 @@ class SignalClient extends Disposable with EventsEmittable { } } - void _ifCurrentSocket(int generation, void Function() action) { - if (generation != _socketGeneration) { + void _ifCurrentSocket(int socketId, void Function() action) { + if (socketId != _socketId) { logger.fine('[$objectId] ignoring a callback from a socket that was already replaced'); return; } From f116f502769c3ec362ef7b1c42879d28ad9bbe5e Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:13:09 +0800 Subject: [PATCH 4/6] test: a late close of the socket a resume or full reconnect replaced The close of a replaced socket can reach the client seconds after the reconnect that replaced it, a browser reports it only once the server drops TCP. On main it was taken for a loss of the new socket and every reconnect started the next one. Both shapes fail with the socket id check removed. --- lib/src/core/signal_client.dart | 10 +++-- test/core/stale_socket_close_test.dart | 60 +++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/lib/src/core/signal_client.dart b/lib/src/core/signal_client.dart index 3b47536ca..30da66532 100644 --- a/lib/src/core/signal_client.dart +++ b/lib/src/core/signal_client.dart @@ -46,10 +46,12 @@ class SignalClient extends Disposable with EventsEmittable { // Id of the socket whose callbacks are honored. connect() gives each new // socket the next id, and cleanUp() advances it too so the socket it disposes - // stops being current. A real socket reports its close a few milliseconds - // after dispose() returns, and that report used to be attributed to whatever - // connection was current by then, clobbering its state and starting a - // reconnect against a healthy socket. + // stops being current. A real socket reports its close after dispose() + // returns, a few milliseconds later on a good path but seconds later in a + // browser when the server keeps TCP open after the close handshake. That + // report used to be attributed to whatever connection was current by then, + // clobbering its state and starting a reconnect against a healthy socket, + // over and over when every reconnect replaced the socket again. int _socketId = 0; LiveKitWebSocket? _ws; diff --git a/test/core/stale_socket_close_test.dart b/test/core/stale_socket_close_test.dart index 431f9bea1..44285b4c6 100644 --- a/test/core/stale_socket_close_test.dart +++ b/test/core/stale_socket_close_test.dart @@ -19,6 +19,9 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:livekit_client/livekit_client.dart'; import 'package:livekit_client/src/connection_check/checks/reconnect.dart'; +import 'package:livekit_client/src/proto/livekit_models.pb.dart' as lk_models; +import 'package:livekit_client/src/proto/livekit_rtc.pb.dart' as lk_rtc; +import 'package:livekit_client/src/support/websocket.dart'; import '../mock/e2e_container.dart'; void main() { @@ -35,7 +38,7 @@ void main() { }); test('a close reported by a replaced socket does not disturb the next connection', () async { - // the real sockets report their close a few milliseconds after dispose() + // the real sockets report their close some time after dispose() container.wsConnector.closeLatency = const Duration(milliseconds: 50); await container.connectRoom(); await container.room.disconnect(); @@ -52,6 +55,53 @@ void main() { expect(events, isEmpty, reason: 'the stale close belongs to the old socket and must be ignored'); }); + for (final (name, action) in [ + ('resume', lk_rtc.LeaveRequest_Action.RESUME), + ('full reconnect', lk_rtc.LeaveRequest_Action.RECONNECT), + ]) { + test('a late close of the socket a $name replaced does not start another reconnect', () async { + // A browser reports the close when the server drops TCP, which can be + // seconds after the close handshake. By then the reconnect is done and + // the new socket is current. + const closeLatency = Duration(milliseconds: 300); + container.wsConnector.closeLatency = closeLatency; + await container.connectRoom(); + + final events = []; + final sub = container.room.events.listen(events.add); + final replaced = container.wsConnector.handlers; + + // the server asks for a reconnect while the socket is still open + container.wsConnector.onData( + lk_rtc.SignalResponse( + leave: lk_rtc.LeaveRequest(action: action, reason: lk_models.DisconnectReason.MIGRATION), + ).writeToBuffer(), + ); + await _waitForNewSocket(container, replaced); + if (action == lk_rtc.LeaveRequest_Action.RESUME) { + container.wsConnector.onData(lk_rtc.SignalResponse(reconnect: lk_rtc.ReconnectResponse()).writeToBuffer()); + } else { + await container.answerJoin(); + } + await container.room.events.waitFor(duration: const Duration(seconds: 5)); + final current = container.wsConnector.handlers; + + // let the replaced socket report its close, and give a reconnect it + // might start time to open a socket + await Future.delayed(closeLatency * 3); + await sub(); + + expect(container.room.connectionState, ConnectionState.connected); + expect( + identical(container.wsConnector.handlers, current), + isTrue, + reason: 'the close of the replaced socket must not open another connection', + ); + expect(events.whereType(), hasLength(1)); + expect(events.whereType(), isEmpty); + }); + } + test('the signalReconnect debug scenario still drops the signal and starts a reconnect', () async { await container.connectRoom(); final events = []; @@ -96,6 +146,14 @@ void main() { }); } +Future _waitForNewSocket(E2EContainer container, WebSocketEventHandlers? previous) async { + final deadline = DateTime.now().add(const Duration(seconds: 3)); + while (identical(container.wsConnector.handlers, previous) && DateTime.now().isBefore(deadline)) { + await Future.delayed(const Duration(milliseconds: 5)); + } + expect(identical(container.wsConnector.handlers, previous), isFalse, reason: 'the SDK never opened a new socket'); +} + Future _resumeStarted(E2EContainer container) async { final deadline = DateTime.now().add(const Duration(seconds: 3)); while (DateTime.now().isBefore(deadline)) { From f8a72be48f74a7faaf62f906310cd93279775c4a Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:26:29 +0800 Subject: [PATCH 5/6] chore: describe the reconnect loop in the changeset --- .changes/ignore-replaced-socket-callbacks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/ignore-replaced-socket-callbacks b/.changes/ignore-replaced-socket-callbacks index 5ff119d14..69e93e364 100644 --- a/.changes/ignore-replaced-socket-callbacks +++ b/.changes/ignore-replaced-socket-callbacks @@ -1 +1 @@ -patch type="fixed" "Events from a signal socket the SDK already replaced are ignored, so a connect right after a disconnect can no longer be knocked into a reconnect by a late close from the old socket" \ No newline at end of file +patch type="fixed" "A late close from a signal socket the SDK already replaced no longer starts another reconnect. In a browser it arrives up to seconds later, which made a room resume every 2 s after its first reconnect" \ No newline at end of file From 9552119c9fdc1253fbe0f1bc34c540c4f400c29c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:43:39 +0800 Subject: [PATCH 6/6] fix: a superseded signal connect closes its socket, and cleanUp() detaches the old socket first A connect whose socket opened only after disconnect() or a newer connect() still installed itself as the current socket, so requests went to the wrong socket and the room could report connected after disconnect() returned. It now closes that socket and fails with SignalConnectSupersededException, which leaves the shared state and events alone and is not a reason to fail over to another region. cleanUp() detached the socket only after awaiting its close, so a connect() that ran while a dart:io close waited on the server lost its own socket, queue and ping timers when that cleanUp() finished. It now detaches first and awaits the close after. simulateSignalDrop() does nothing without a socket and reports the loss before the socket has finished closing. The connection checker comment matches what it does now. Tests: the mock holds old closes until the test releases them, so the late close tests no longer depend on timing, the checker test answers the resume and expects success, and there are new tests for late data from a replaced socket, the superseded connect, the cleanUp() race and a drop with no socket. Each fails without its change. Found in review (Codex). --- .changes/ignore-replaced-socket-callbacks | 2 +- .../connection_check/checks/reconnect.dart | 3 +- lib/src/core/engine.dart | 4 + lib/src/core/signal_client.dart | 54 +++++-- test/core/stale_socket_close_test.dart | 148 ++++++++++++++---- test/mock/websocket_mock.dart | 51 +++++- 6 files changed, 214 insertions(+), 48 deletions(-) diff --git a/.changes/ignore-replaced-socket-callbacks b/.changes/ignore-replaced-socket-callbacks index 69e93e364..44a4eb339 100644 --- a/.changes/ignore-replaced-socket-callbacks +++ b/.changes/ignore-replaced-socket-callbacks @@ -1 +1 @@ -patch type="fixed" "A late close from a signal socket the SDK already replaced no longer starts another reconnect. In a browser it arrives up to seconds later, which made a room resume every 2 s after its first reconnect" \ No newline at end of file +patch type="fixed" "A late close from a signal socket the SDK already replaced no longer starts another reconnect. In a browser it arrives up to seconds later, which made a room resume every 2 s after its first reconnect. A signal connect that only opens after disconnect() or a newer connect() is closed instead of taking over the connection" \ No newline at end of file diff --git a/lib/src/connection_check/checks/reconnect.dart b/lib/src/connection_check/checks/reconnect.dart index e20179bae..a64dc6363 100644 --- a/lib/src/connection_check/checks/reconnect.dart +++ b/lib/src/connection_check/checks/reconnect.dart @@ -44,8 +44,7 @@ class ReconnectCheck extends Checker { }); try { - // Forcefully close the underlying signal WebSocket; the engine should - // notice and resume the session on its own. + // Simulate a signal loss and wait for the room to resume. await engine.signalClient.simulateSignalDrop(); await reconnectCompleter.future.timeout( diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index 0fd4200ed..60bdc0cc6 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -354,6 +354,10 @@ class Engine extends Disposable with EventsEmittable { } catch (error) { logger.fine('Connect Error $error'); + // A superseded attempt reports nothing, the connection that replaced it + // is the one the room has. + if (error is SignalConnectSupersededException) rethrow; + // during a reconnect this connect() runs inside restartConnection and // attemptReconnect owns disconnect emission, emitting here as well // would produce two events for one failure diff --git a/lib/src/core/signal_client.dart b/lib/src/core/signal_client.dart index 30da66532..d3fb00c2a 100644 --- a/lib/src/core/signal_client.dart +++ b/lib/src/core/signal_client.dart @@ -154,6 +154,8 @@ class SignalClient extends Disposable with EventsEmittable { logger.fine('SignalClient connecting with url: ${rtcUri.redactingAccessToken}'); + // Id of the socket this attempt opens, set once the previous one is gone. + int? socketId; try { if (reconnect == true) { _connectionState = ConnectionState.reconnecting; @@ -166,25 +168,36 @@ class SignalClient extends Disposable with EventsEmittable { await cleanUp(); // Attempt to connect. Callbacks carry the id of the socket they belong to // so a late report from a replaced socket is ignored. - final socketId = ++_socketId; - var future = _wsConnector( + final id = socketId = ++_socketId; + final ws = await _wsConnector( rtcUri, options: WebSocketEventHandlers( - onData: (data) => _ifCurrentSocket(socketId, () => _onSocketData(data)), - onDispose: () => _ifCurrentSocket(socketId, _onSocketDispose), - onError: (error) => _ifCurrentSocket(socketId, () => _onSocketError(error)), + onData: (data) => _ifCurrentSocket(id, () => _onSocketData(data)), + onDispose: () => _ifCurrentSocket(id, _onSocketDispose), + onError: (error) => _ifCurrentSocket(id, () => _onSocketError(error)), ), headers: { 'Authorization': 'Bearer $token', }, networkOptions: roomOptions.networkOptions, - ); - future = future.timeout(connectOptions.timeouts.connection); - _ws = await future; + ).timeout(connectOptions.timeouts.connection); + if (id != _socketId) { + // disconnect() or a newer connect() took over while this socket was + // opening. Installing it would take the place of the current one. + await ws.dispose(); + throw const SignalConnectSupersededException(); + } + _ws = ws; // Successful connection _connectionState = ConnectionState.connected; events.emit(const SignalConnectedEvent()); } catch (socketError) { + // An attempt that something else replaced leaves the state and the + // events to the connection that replaced it. + if (socketId != null && socketId != _socketId) { + if (socketError is SignalConnectSupersededException) rethrow; + throw const SignalConnectSupersededException(); + } if (socketError is CertificatePinningException) { // In reconnect mode the engine owns state and event emission, // emitting here would race its reconnect handling. @@ -256,10 +269,13 @@ class SignalClient extends Disposable with EventsEmittable { logger.fine('[${objectId}] cleanUp()'); _socketId++; _connectionState = ConnectionState.disconnected; - await _ws?.dispose(); + // Detached before the await, so a connect() that runs while the old + // socket is still closing keeps its own socket, queue and pings. + final ws = _ws; _ws = null; _queue.clear(); _clearPingInterval(); + await ws?.dispose(); } void _sendRequest( @@ -436,8 +452,15 @@ class SignalClient extends Disposable with EventsEmittable { /// teardown, the disposed socket's own close is ignored. @internal Future simulateSignalDrop() async { - await cleanUp(); + if (_ws == null) { + logger.fine('[$objectId] simulateSignalDrop(): no signal socket to drop'); + return; + } + // The loss is reported before the socket has finished closing, which can + // take a while, so it is charged to the socket that was dropped. + final dropped = cleanUp(); _onSocketDispose(); + await dropped; } void _onSocketError(dynamic error) { @@ -660,3 +683,14 @@ extension SignalClientInternalMethods on SignalClient { @internal void clearQueue() => _queue.clear(); } + +/// Thrown by a [SignalClient.connect] whose socket was no longer wanted by the +/// time it opened, because disconnect() or a newer connect() took over. The +/// connection that took over owns the state and the events. +@internal +class SignalConnectSupersededException implements Exception { + const SignalConnectSupersededException(); + + @override + String toString() => 'SignalConnectSupersededException: the signal connection was superseded while it was opening'; +} diff --git a/test/core/stale_socket_close_test.dart b/test/core/stale_socket_close_test.dart index 44285b4c6..b934695ad 100644 --- a/test/core/stale_socket_close_test.dart +++ b/test/core/stale_socket_close_test.dart @@ -15,14 +15,21 @@ @Timeout(Duration(seconds: 10)) library; +import 'dart:async'; + import 'package:flutter_test/flutter_test.dart'; import 'package:livekit_client/livekit_client.dart'; import 'package:livekit_client/src/connection_check/checks/reconnect.dart'; +import 'package:livekit_client/src/core/signal_client.dart'; import 'package:livekit_client/src/proto/livekit_models.pb.dart' as lk_models; import 'package:livekit_client/src/proto/livekit_rtc.pb.dart' as lk_rtc; import 'package:livekit_client/src/support/websocket.dart'; import '../mock/e2e_container.dart'; +import '../mock/websocket_mock.dart'; + +const exampleUri = 'ws://www.example.com'; +const token = 'token'; void main() { TestWidgetsFlutterBinding.ensureInitialized(); @@ -39,17 +46,18 @@ void main() { test('a close reported by a replaced socket does not disturb the next connection', () async { // the real sockets report their close some time after dispose() - container.wsConnector.closeLatency = const Duration(milliseconds: 50); + container.wsConnector.holdCloses = true; await container.connectRoom(); await container.room.disconnect(); - // connect again before the first socket has reported its close final events = []; container.room.events.on(events.add); container.room.events.on(events.add); container.room.events.on(events.add); await container.connectRoom(); - await Future.delayed(const Duration(milliseconds: 150)); + // the first socket reports its close only now, with the second one up + container.wsConnector.releaseCloses(); + await _settle(); expect(container.room.connectionState, ConnectionState.connected); expect(events, isEmpty, reason: 'the stale close belongs to the old socket and must be ignored'); @@ -63,8 +71,7 @@ void main() { // A browser reports the close when the server drops TCP, which can be // seconds after the close handshake. By then the reconnect is done and // the new socket is current. - const closeLatency = Duration(milliseconds: 300); - container.wsConnector.closeLatency = closeLatency; + container.wsConnector.holdCloses = true; await container.connectRoom(); final events = []; @@ -72,23 +79,20 @@ void main() { final replaced = container.wsConnector.handlers; // the server asks for a reconnect while the socket is still open - container.wsConnector.onData( - lk_rtc.SignalResponse( - leave: lk_rtc.LeaveRequest(action: action, reason: lk_models.DisconnectReason.MIGRATION), - ).writeToBuffer(), - ); + container.wsConnector.onData(_leave(action, lk_models.DisconnectReason.MIGRATION)); await _waitForNewSocket(container, replaced); + final reconnected = container.room.events.waitFor(duration: const Duration(seconds: 5)); if (action == lk_rtc.LeaveRequest_Action.RESUME) { container.wsConnector.onData(lk_rtc.SignalResponse(reconnect: lk_rtc.ReconnectResponse()).writeToBuffer()); } else { await container.answerJoin(); } - await container.room.events.waitFor(duration: const Duration(seconds: 5)); + await reconnected; final current = container.wsConnector.handlers; - // let the replaced socket report its close, and give a reconnect it - // might start time to open a socket - await Future.delayed(closeLatency * 3); + // the replaced socket reports its close only now + container.wsConnector.releaseCloses(); + await _settle(); await sub(); expect(container.room.connectionState, ConnectionState.connected); @@ -97,11 +101,95 @@ void main() { isTrue, reason: 'the close of the replaced socket must not open another connection', ); + expect(events.whereType(), hasLength(1)); expect(events.whereType(), hasLength(1)); expect(events.whereType(), isEmpty); }); } + test('data a replaced socket delivers late is ignored', () async { + await container.connectRoom(); + final replaced = container.wsConnector.handlers; + container.wsConnector.onData(_leave(lk_rtc.LeaveRequest_Action.RESUME, lk_models.DisconnectReason.MIGRATION)); + await _waitForNewSocket(container, replaced); + final reconnected = container.room.events.waitFor(duration: const Duration(seconds: 5)); + container.wsConnector.onData(lk_rtc.SignalResponse(reconnect: lk_rtc.ReconnectResponse()).writeToBuffer()); + await reconnected; + + final disconnected = []; + container.room.events.on(disconnected.add); + // the old socket still had a Leave on its way + replaced!.onData!(_leave(lk_rtc.LeaveRequest_Action.DISCONNECT, lk_models.DisconnectReason.CLIENT_INITIATED)); + await _settle(); + + expect(container.room.connectionState, ConnectionState.connected); + expect(disconnected, isEmpty); + }); + + test('a connect that opens after disconnect() and a rejoin does not take over the rejoined session', () async { + final gate = Completer(); + container.wsConnector.connectGate = gate.future; + final disconnected = []; + container.room.events.on(disconnected.add); + + final stale = container.room.connect(exampleUri, token).then((_) => null, onError: (Object e) => e); + await Future.delayed(const Duration(milliseconds: 10)); + await container.room.disconnect(); + container.wsConnector.connectGate = null; + await container.connectRoom(); + final rejoined = container.wsConnector.sockets.last; + + // the first connect's socket opens only now + gate.complete(); + final staleError = await stale.timeout(const Duration(seconds: 2)); + await _settle(); + final staleSocket = container.wsConnector.sockets.last; + + expect(staleError, isA()); + expect(staleSocket, isNot(same(rejoined))); + expect(staleSocket.isDisposed, isTrue, reason: 'a socket nobody wants any more is closed'); + expect(container.room.connectionState, ConnectionState.connected); + expect(disconnected, hasLength(1), reason: 'only the disconnect() the app asked for'); + + final before = rejoined.sent.length; + await container.client.sendLeave(); + expect(rejoined.sent, hasLength(before + 1), reason: 'requests go out on the rejoined session'); + expect(staleSocket.sent, isEmpty); + }); + + test('a cleanUp still closing the old socket leaves the next connection alone', () async { + // a dart:io close waits for the server to answer the close frame + final connector = MockWebSocketConnector()..disposeLatency = const Duration(milliseconds: 100); + final client = SignalClient(connector.connect); + Future connect() => + client.connect(exampleUri, token, connectOptions: const ConnectOptions(), roomOptions: const RoomOptions()); + try { + await connect(); + final closing = client.cleanUp(); + await connect(); + final current = connector.sockets.last; + await closing; + + final before = current.sent.length; + await client.sendLeave(); + expect(current.sent, hasLength(before + 1), reason: 'the first cleanUp() must not detach the new socket'); + } finally { + await client.dispose(); + } + }); + + test('simulating a signal drop with no signal socket does nothing', () async { + final events = []; + container.room.events.on(events.add); + container.room.events.on(events.add); + + await container.room.sendSimulateScenario(signalReconnect: true); + await _settle(); + + expect(events, isEmpty); + expect(container.wsConnector.sockets, isEmpty); + }); + test('the signalReconnect debug scenario still drops the signal and starts a reconnect', () async { await container.connectRoom(); final events = []; @@ -109,20 +197,15 @@ void main() { await container.room.sendSimulateScenario(signalReconnect: true); - final deadline = DateTime.now().add(const Duration(seconds: 3)); - while (!(container.wsConnector.uri?.queryParameters['reconnect'] == '1') && DateTime.now().isBefore(deadline)) { - await Future.delayed(const Duration(milliseconds: 5)); - } expect( - container.wsConnector.uri?.queryParameters['reconnect'], - '1', + await _resumeStarted(container), + isTrue, reason: 'the engine must re-open the signal connection as a resume', ); expect(events, isNotEmpty); }); test('the disconnectSignalOnResume debug action still starts a resume', () async { - // the real sockets report their close a few milliseconds after dispose() container.wsConnector.closeLatency = const Duration(milliseconds: 50); await container.connectRoom(); @@ -131,21 +214,28 @@ void main() { expect(await _resumeStarted(container), isTrue, reason: 'the server only acts on the next resume'); }); - test('the connection checker reconnect check still sees a resume', () async { + test('the connection checker reconnect check succeeds once the room resumes', () async { container.wsConnector.closeLatency = const Duration(milliseconds: 50); await container.connectRoom(); + final replaced = container.wsConnector.handlers; - final check = ReconnectCheck('ws://www.example.com', 'token', room: container.room); - final info = await check.run(); + final check = ReconnectCheck(exampleUri, token, room: container.room); + final running = check.run(); + await _waitForNewSocket(container, replaced); + container.wsConnector.onData(lk_rtc.SignalResponse(reconnect: lk_rtc.ReconnectResponse()).writeToBuffer()); + final info = await running; - expect( - info.logs.map((l) => l.message), - isNot(contains('Did not attempt to reconnect')), - reason: 'the check drops the signal socket and expects the engine to notice', - ); + expect(info.status, CheckStatus.success, reason: info.logs.map((l) => l.message).join(', ')); }); } +List _leave(lk_rtc.LeaveRequest_Action action, lk_models.DisconnectReason reason) => lk_rtc.SignalResponse( + leave: lk_rtc.LeaveRequest(action: action, reason: reason), +).writeToBuffer(); + +/// Lets the events of something that just happened run their course. +Future _settle() => Future.delayed(const Duration(milliseconds: 100)); + Future _waitForNewSocket(E2EContainer container, WebSocketEventHandlers? previous) async { final deadline = DateTime.now().add(const Duration(seconds: 3)); while (identical(container.wsConnector.handlers, previous) && DateTime.now().isBefore(deadline)) { diff --git a/test/mock/websocket_mock.dart b/test/mock/websocket_mock.dart index 3a2ca7aeb..81b46cd41 100644 --- a/test/mock/websocket_mock.dart +++ b/test/mock/websocket_mock.dart @@ -19,17 +19,34 @@ import 'package:livekit_client/src/support/websocket.dart'; class MockWebSocket extends LiveKitWebSocket { /// Like the real sockets, the close is reported to the SDK after dispose() - /// has returned, [closeLatency] later. - MockWebSocket(WebSocketEventHandlers? options, Duration closeLatency, [this.onSend]) { + /// has returned, see [MockWebSocketConnector.closeLatency] and + /// [MockWebSocketConnector.holdCloses]. + MockWebSocket(this._connector, this.handlers) { onDispose(() async { - unawaited(Future.delayed(closeLatency, () => options?.onDispose?.call())); + final connector = _connector; + if (connector.disposeLatency > Duration.zero) await Future.delayed(connector.disposeLatency); + void reportClose() => handlers?.onDispose?.call(); + if (connector.holdCloses) { + connector._heldCloses.add(reportClose); + } else { + unawaited(Future.delayed(connector.closeLatency, reportClose)); + } }); } - final void Function(List data)? onSend; + final MockWebSocketConnector _connector; + + /// The callbacks the SDK gave this socket. + final WebSocketEventHandlers? handlers; + + /// Every message the SDK sent over this socket. + final List> sent = []; @override - void send(List data) => onSend?.call(data); + void send(List data) { + sent.add(data); + _connector.sentData.add(data); + } } class MockWebSocketConnector { @@ -42,6 +59,26 @@ class MockWebSocketConnector { /// Delay between the SDK disposing a socket and that socket reporting its close. Duration closeLatency = Duration.zero; + /// How long dispose() takes, like a dart:io close that waits on the server. + Duration disposeLatency = Duration.zero; + + /// While true, the closes of disposed sockets are held until [releaseCloses], + /// so a test decides exactly when an old socket reports its close. + bool holdCloses = false; + final _heldCloses = []; + + /// Reports every close held by [holdCloses]. + void releaseCloses() { + final held = [..._heldCloses]; + _heldCloses.clear(); + for (final reportClose in held) { + reportClose(); + } + } + + /// Every socket this connector opened, oldest first. + final List sockets = []; + /// When true, [connectError] is thrown once and then cleared, so the next /// connect succeeds. Models a first attempt that fails and a retry that gets /// through. @@ -81,6 +118,8 @@ class MockWebSocketConnector { } handlers = options; - return MockWebSocket(options, closeLatency, sentData.add); + final socket = MockWebSocket(this, options); + sockets.add(socket); + return socket; } }