diff --git a/relay/Cargo.lock b/relay/Cargo.lock index 46b1672..5346d68 100644 --- a/relay/Cargo.lock +++ b/relay/Cargo.lock @@ -1797,9 +1797,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -2722,6 +2722,7 @@ dependencies = [ "iced", "iced_aw", "interprocess", + "socket2", ] [[package]] @@ -2988,6 +2989,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "softbuffer" version = "0.4.6" diff --git a/relay/Cargo.toml b/relay/Cargo.toml index e0867f8..7361943 100644 --- a/relay/Cargo.toml +++ b/relay/Cargo.toml @@ -11,3 +11,6 @@ anyhow = "*" interprocess = "*" crossbeam = "0.8.4" chrono = "0.4.42" +# For SO_SNDBUF and SO_RCVBUF on the mirror sockets, which std::net +# does not expose. Cross-platform, so the Windows build gets them too. +socket2 = "0.5" diff --git a/relay/src/state.rs b/relay/src/state.rs index 92fceb1..c60be0e 100644 --- a/relay/src/state.rs +++ b/relay/src/state.rs @@ -63,6 +63,23 @@ const MIRROR_WRITE_TIMEOUT: StdDuration = StdDuration::from_millis(100); /// rather than make iMotions wait. const MIRROR_QUEUE_CAPACITY: usize = 256; +/// How much the kernel may hold on the mirror socket on our behalf. +/// +/// `MIRROR_QUEUE_CAPACITY` bounds only the queue in front of the writer +/// thread. Behind it the socket's own send buffer is a second queue, and the +/// Linux default is about 2.5 MB. Measured on loopback, that and the reader's +/// buffer together swallow some 37,000 packets -- close to four minutes of +/// stream -- before a write blocks, so a mirror that stopped reading minutes +/// ago still takes writes, `MIRROR_WRITE_TIMEOUT` never fires, nothing is +/// reported, and the bound in front of the writer is decoration. +/// +/// Asking for 32 KiB brings that down to about 3,200 packets, or twenty +/// seconds. Linux doubles the request, so the buffer is really 64 KiB; what +/// matters is that it is tens of kilobytes rather than megabytes. It costs no +/// throughput: 50,000 packets to a reader that keeps up take the same 45 ms +/// either way. +const MIRROR_SEND_BUFFER: usize = 32 * 1024; + // --- State definition ------------------------------------------------------- #[allow(unused)] pub(crate) struct State { @@ -425,6 +442,9 @@ fn connect_mirror(address: &str) -> Result { let stream = TcpStream::connect_timeout(&addr, MIRROR_CONNECT_TIMEOUT)?; stream.set_write_timeout(Some(MIRROR_WRITE_TIMEOUT))?; stream.set_nodelay(true)?; + // Without this the write timeout above is unreachable in practice: see + // MIRROR_SEND_BUFFER. + socket2::SockRef::from(&stream).set_send_buffer_size(MIRROR_SEND_BUFFER)?; Ok(stream) } diff --git a/relay/src/update.rs b/relay/src/update.rs index b10ec37..02fe298 100644 --- a/relay/src/update.rs +++ b/relay/src/update.rs @@ -358,8 +358,9 @@ fn create_xml_file(state: &mut State) -> Task { #[cfg(test)] mod tests { use super::*; + use socket2::{Domain, Protocol, Socket, Type}; use std::io::{BufRead, BufReader}; - use std::net::{TcpListener, TcpStream}; + use std::net::{SocketAddr, TcpListener, TcpStream}; use std::thread::sleep; // Waits for a condition, polling every 10ms, so the test does not hang. @@ -480,6 +481,47 @@ mod tests { (listener, addr) } + // How much the Python side is allowed to buffer in the kernel, for the + // tests that are about the mirror falling behind. + // + // Linux rounds this up and enforces a floor, so the effective buffer is + // larger than the number asked for; what matters is that it is kilobytes + // rather than megabytes. + const SMALL_RECV_BUFFER: usize = 2048; + + // A Python-side listener whose accepted connections have a deliberately + // small receive buffer. + // + // The two tests below are about what the relay does when the mirror cannot + // keep up, so they need a mirror that demonstrably cannot. Half of that is + // MIRROR_SEND_BUFFER, which bounds our own side; the other half is the + // reader's buffer, which in production belongs to the Python process and is + // whatever the host decided. Leaving it at the default meant the two sides + // together absorbed the whole run on a machine with generous tcp_rmem and + // blocked on a machine without, which is issue #251: the tests passed or + // failed according to the host rather than the code. + // + // Setting SO_RCVBUF explicitly also switches autotuning off for the socket, + // so the window stays pinned small. It has to be set on the listener, + // before listen, to be inherited by accepted connections. + fn stalling_listener_on_free_port() -> (TcpListener, String) { + let addr: SocketAddr = "127.0.0.1:0".parse().expect("parse failed"); + let socket = + Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP)).expect("socket failed"); + socket + .set_recv_buffer_size(SMALL_RECV_BUFFER) + .expect("set_recv_buffer_size failed"); + socket.bind(&addr.into()).expect("bind failed"); + socket.listen(16).expect("listen failed"); + + let listener: TcpListener = socket.into(); + let bound = listener + .local_addr() + .expect("local_addr failed") + .to_string(); + (listener, bound) + } + fn read_lines(stream: &TcpStream, count: usize) -> Vec { stream .set_read_timeout(Some(StdDuration::from_secs(5))) @@ -694,13 +736,14 @@ mod tests { // falling behind rather than quietly coping. #[test] fn a_slow_python_reader_does_not_pace_imotions() { - // Enough to exceed the socket buffers, which absorb a surprising amount: - // at 500 samples the slow reader never creates any backpressure at all. - const SAMPLES: usize = 1500; + // 4000 packets against the roughly 500 the bounded buffers hold, so the + // mirror falls behind regardless of the host. This used to be 1500 + // samples chasing the kernel's autotuned buffers, and still lost. + const SAMPLES: usize = 500; const PACKETS: usize = SAMPLES * 8; let (imotions, imotions_addr) = listener_on_free_port(); - let (python, python_addr) = listener_on_free_port(); + let (python, python_addr) = stalling_listener_on_free_port(); let mut state = State::default(); state.tcp_connect(imotions_addr).expect("tcp_connect failed"); @@ -716,9 +759,9 @@ mod tests { ); // Reads steadily but slower than the relay writes: about 1000 lines a - // second against 12000 queued. Fast enough that writes keep succeeding, - // so the mirror stays up and the queue is what fills, which is the case - // that used to pace iMotions. + // second against 4000 queued. The window reopens as it reads, so some + // writes keep succeeding and the mirror does not simply go away; it is + // the case that used to pace iMotions. let slow_reader = std::thread::spawn(move || { let mut reader = BufReader::new(slow); let mut line = String::new(); @@ -759,7 +802,8 @@ mod tests { // The relay says something about it rather than quietly coping: either // copies are being dropped, or the writer gave up on the write timeout. - // Which one depends on how much the socket buffers absorb. + // Which one depends on how the reader's progress lines up with the + // timeout, so both count. assert!( wait_until(|| { let _ = update(&mut state, Message::Update); @@ -783,8 +827,14 @@ mod tests { // feed to iMotions. #[test] fn a_python_process_that_stops_reading_is_dropped_not_waited_on() { + // The Python side never reads, so its pinned receive buffer fills and + // stays full: writes block, the 100ms timeout fires and the mirror is + // dropped. With the host's default buffers the whole run fits in the + // kernel, nothing blocks, and the timeout never fires at all. + const SAMPLES: usize = 500; + let (imotions, imotions_addr) = listener_on_free_port(); - let (python, python_addr) = listener_on_free_port(); + let (python, python_addr) = stalling_listener_on_free_port(); let mut state = State::default(); state.tcp_connect(imotions_addr).expect("tcp_connect failed"); @@ -812,7 +862,7 @@ mod tests { ); let started = std::time::Instant::now(); - for i in 0..3000 { + for i in 0..SAMPLES { send_sample(&mut state, &format!("{}", 1000 + i % 100)); } assert!(