From 3edc0fb9e1ba4da1cf4ab5e2e3d825f2dc953a4d Mon Sep 17 00:00:00 2001 From: Prince Karikari Date: Sat, 31 Oct 2026 05:06:32 -0500 Subject: [PATCH] Issue #251 solved: bound the mirror's send buffer so lag is detectable The two mirror tests asserted that a Python process which stops reading gets dropped on the write timeout, and that a slow one is reported rather than quietly coped with. Both failed on fast hosts and on GitHub's runners, because neither condition could be reached. The cause was not the tests. MIRROR_QUEUE_CAPACITY bounds the queue in front of the mirror's writer thread, but the socket's send buffer behind it is a second, much larger queue: 2.5 MB by default on Linux. Measured on loopback, that and the reader's buffer absorb about 37,000 packets -- close to four minutes of stream at 160 packets a second -- before a write blocks. So a mirror that stopped reading minutes ago still takes writes, MIRROR_WRITE_TIMEOUT never fires, and the relay reports nothing. The bound in front of the writer was decoration. Set SO_SNDBUF explicitly on the mirror socket. That brings the hidden backlog down to about 3,200 packets, or twenty seconds, and costs no throughput: 50,000 packets to a reader that keeps up take the same 45 ms either way. std::net does not expose the option, so socket2 is now a dependency; it is cross-platform, so the Windows build gets it too. The tests also pin a small SO_RCVBUF on their Python-side listener, since the reader's buffer belongs to the Python process in production and is whatever the host decided. With both sides bounded the backpressure is real on any host, so the sample counts drop from 1500 and 3000 to 500 and the suite runs in 0.42s instead of 2.12s. --- relay/Cargo.lock | 15 ++++++++-- relay/Cargo.toml | 3 ++ relay/src/state.rs | 20 +++++++++++++ relay/src/update.rs | 72 ++++++++++++++++++++++++++++++++++++++------- 4 files changed, 97 insertions(+), 13 deletions(-) diff --git a/relay/Cargo.lock b/relay/Cargo.lock index 46b1672..5346d68 100644 --- a/relay/Cargo.lock +++ b/relay/Cargo.lock @@ -1797,9 +1797,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -2722,6 +2722,7 @@ dependencies = [ "iced", "iced_aw", "interprocess", + "socket2", ] [[package]] @@ -2988,6 +2989,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "softbuffer" version = "0.4.6" diff --git a/relay/Cargo.toml b/relay/Cargo.toml index e0867f8..7361943 100644 --- a/relay/Cargo.toml +++ b/relay/Cargo.toml @@ -11,3 +11,6 @@ anyhow = "*" interprocess = "*" crossbeam = "0.8.4" chrono = "0.4.42" +# For SO_SNDBUF and SO_RCVBUF on the mirror sockets, which std::net +# does not expose. Cross-platform, so the Windows build gets them too. +socket2 = "0.5" diff --git a/relay/src/state.rs b/relay/src/state.rs index 92fceb1..c60be0e 100644 --- a/relay/src/state.rs +++ b/relay/src/state.rs @@ -63,6 +63,23 @@ const MIRROR_WRITE_TIMEOUT: StdDuration = StdDuration::from_millis(100); /// rather than make iMotions wait. const MIRROR_QUEUE_CAPACITY: usize = 256; +/// How much the kernel may hold on the mirror socket on our behalf. +/// +/// `MIRROR_QUEUE_CAPACITY` bounds only the queue in front of the writer +/// thread. Behind it the socket's own send buffer is a second queue, and the +/// Linux default is about 2.5 MB. Measured on loopback, that and the reader's +/// buffer together swallow some 37,000 packets -- close to four minutes of +/// stream -- before a write blocks, so a mirror that stopped reading minutes +/// ago still takes writes, `MIRROR_WRITE_TIMEOUT` never fires, nothing is +/// reported, and the bound in front of the writer is decoration. +/// +/// Asking for 32 KiB brings that down to about 3,200 packets, or twenty +/// seconds. Linux doubles the request, so the buffer is really 64 KiB; what +/// matters is that it is tens of kilobytes rather than megabytes. It costs no +/// throughput: 50,000 packets to a reader that keeps up take the same 45 ms +/// either way. +const MIRROR_SEND_BUFFER: usize = 32 * 1024; + // --- State definition ------------------------------------------------------- #[allow(unused)] pub(crate) struct State { @@ -425,6 +442,9 @@ fn connect_mirror(address: &str) -> Result { let stream = TcpStream::connect_timeout(&addr, MIRROR_CONNECT_TIMEOUT)?; stream.set_write_timeout(Some(MIRROR_WRITE_TIMEOUT))?; stream.set_nodelay(true)?; + // Without this the write timeout above is unreachable in practice: see + // MIRROR_SEND_BUFFER. + socket2::SockRef::from(&stream).set_send_buffer_size(MIRROR_SEND_BUFFER)?; Ok(stream) } diff --git a/relay/src/update.rs b/relay/src/update.rs index b10ec37..02fe298 100644 --- a/relay/src/update.rs +++ b/relay/src/update.rs @@ -358,8 +358,9 @@ fn create_xml_file(state: &mut State) -> Task { #[cfg(test)] mod tests { use super::*; + use socket2::{Domain, Protocol, Socket, Type}; use std::io::{BufRead, BufReader}; - use std::net::{TcpListener, TcpStream}; + use std::net::{SocketAddr, TcpListener, TcpStream}; use std::thread::sleep; // Waits for a condition, polling every 10ms, so the test does not hang. @@ -480,6 +481,47 @@ mod tests { (listener, addr) } + // How much the Python side is allowed to buffer in the kernel, for the + // tests that are about the mirror falling behind. + // + // Linux rounds this up and enforces a floor, so the effective buffer is + // larger than the number asked for; what matters is that it is kilobytes + // rather than megabytes. + const SMALL_RECV_BUFFER: usize = 2048; + + // A Python-side listener whose accepted connections have a deliberately + // small receive buffer. + // + // The two tests below are about what the relay does when the mirror cannot + // keep up, so they need a mirror that demonstrably cannot. Half of that is + // MIRROR_SEND_BUFFER, which bounds our own side; the other half is the + // reader's buffer, which in production belongs to the Python process and is + // whatever the host decided. Leaving it at the default meant the two sides + // together absorbed the whole run on a machine with generous tcp_rmem and + // blocked on a machine without, which is issue #251: the tests passed or + // failed according to the host rather than the code. + // + // Setting SO_RCVBUF explicitly also switches autotuning off for the socket, + // so the window stays pinned small. It has to be set on the listener, + // before listen, to be inherited by accepted connections. + fn stalling_listener_on_free_port() -> (TcpListener, String) { + let addr: SocketAddr = "127.0.0.1:0".parse().expect("parse failed"); + let socket = + Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP)).expect("socket failed"); + socket + .set_recv_buffer_size(SMALL_RECV_BUFFER) + .expect("set_recv_buffer_size failed"); + socket.bind(&addr.into()).expect("bind failed"); + socket.listen(16).expect("listen failed"); + + let listener: TcpListener = socket.into(); + let bound = listener + .local_addr() + .expect("local_addr failed") + .to_string(); + (listener, bound) + } + fn read_lines(stream: &TcpStream, count: usize) -> Vec { stream .set_read_timeout(Some(StdDuration::from_secs(5))) @@ -694,13 +736,14 @@ mod tests { // falling behind rather than quietly coping. #[test] fn a_slow_python_reader_does_not_pace_imotions() { - // Enough to exceed the socket buffers, which absorb a surprising amount: - // at 500 samples the slow reader never creates any backpressure at all. - const SAMPLES: usize = 1500; + // 4000 packets against the roughly 500 the bounded buffers hold, so the + // mirror falls behind regardless of the host. This used to be 1500 + // samples chasing the kernel's autotuned buffers, and still lost. + const SAMPLES: usize = 500; const PACKETS: usize = SAMPLES * 8; let (imotions, imotions_addr) = listener_on_free_port(); - let (python, python_addr) = listener_on_free_port(); + let (python, python_addr) = stalling_listener_on_free_port(); let mut state = State::default(); state.tcp_connect(imotions_addr).expect("tcp_connect failed"); @@ -716,9 +759,9 @@ mod tests { ); // Reads steadily but slower than the relay writes: about 1000 lines a - // second against 12000 queued. Fast enough that writes keep succeeding, - // so the mirror stays up and the queue is what fills, which is the case - // that used to pace iMotions. + // second against 4000 queued. The window reopens as it reads, so some + // writes keep succeeding and the mirror does not simply go away; it is + // the case that used to pace iMotions. let slow_reader = std::thread::spawn(move || { let mut reader = BufReader::new(slow); let mut line = String::new(); @@ -759,7 +802,8 @@ mod tests { // The relay says something about it rather than quietly coping: either // copies are being dropped, or the writer gave up on the write timeout. - // Which one depends on how much the socket buffers absorb. + // Which one depends on how the reader's progress lines up with the + // timeout, so both count. assert!( wait_until(|| { let _ = update(&mut state, Message::Update); @@ -783,8 +827,14 @@ mod tests { // feed to iMotions. #[test] fn a_python_process_that_stops_reading_is_dropped_not_waited_on() { + // The Python side never reads, so its pinned receive buffer fills and + // stays full: writes block, the 100ms timeout fires and the mirror is + // dropped. With the host's default buffers the whole run fits in the + // kernel, nothing blocks, and the timeout never fires at all. + const SAMPLES: usize = 500; + let (imotions, imotions_addr) = listener_on_free_port(); - let (python, python_addr) = listener_on_free_port(); + let (python, python_addr) = stalling_listener_on_free_port(); let mut state = State::default(); state.tcp_connect(imotions_addr).expect("tcp_connect failed"); @@ -812,7 +862,7 @@ mod tests { ); let started = std::time::Instant::now(); - for i in 0..3000 { + for i in 0..SAMPLES { send_sample(&mut state, &format!("{}", 1000 + i % 100)); } assert!(