From c904dd8e3a3449a14afec0236b172deb142caf34 Mon Sep 17 00:00:00 2001 From: Utkarsh Dalal Date: Mon, 16 Nov 2026 11:12:15 +0530 Subject: [PATCH 1/3] Wake only as many single-object waiters as a signal op can satisfy A semaphore release, auto-reset event set or mutex unlock woke every waiter on the object. Each wake cost the signaling thread one futex syscall while holding the region lock, and every extra woken waiter then took the region lock twice, failed to acquire and parked again. Unity's job system releases a semaphore to its parked workers many times per frame: on The Forest (Snapdragon 8 Elite) the main thread made ~41k syscalls/s and spent about a quarter of its time in NtReleaseSemaphore -> sem_release. Now: - A single-object, non-alertable waiter registers its node with registered = 2 (REG_SINGLE). Readers only ever test registered against 0, so the shared layout and older builds are unaffected in both directions. - wake_object takes a budget: the release count for a semaphore, 1 for an auto-reset event, a freed mutex or an abandoned mutex. It wakes at most that many REG_SINGLE waiters. A woken single-object waiter always retries the acquire before any timeout or exit, so each unit lands on a thread that takes the object or finds it already taken. - The budget is charged only when futex_wake reports a thread woken, so a waiter that is spinning, about to park or dead costs nothing and the next one is woken too. - Multi-object, wait-all and alertable waiters are always woken, as before. Close, pulse, manual-reset events and dead-process reaping still wake everyone. - Waiters are woken oldest first, so a contended object cannot starve an early waiter. - NTSYNC_WAKE_ALL=1 restores wake-all. - New stats: threads_woken, wakes_deferred. Tests: release count, oldest-first order, auto vs manual events, mutex handoff, and multi-object and alertable waiters. Two stress tests release exactly as many units as consumers need, with thieves and mixed waiter kinds, so a lost wakeup fails them. The full suite and the process-death soaks pass on device (8 Elite, 3 runs). --- src/core.rs | 151 +++++++++++++++++++++----- src/lib.rs | 307 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 430 insertions(+), 28 deletions(-) diff --git a/src/core.rs b/src/core.rs index 3652f11..4b9b2ed 100644 --- a/src/core.rs +++ b/src/core.rs @@ -472,11 +472,13 @@ fn futex_wait(addr: *const AtomicU32, expected: u32, timeout: Option) } } -fn futex_wake(addr: *const AtomicU32) { +/// Returns how many threads the kernel woke: 1 if the waiter was parked on +/// the word, 0 if it was not (spinning, between its seq sample and the +/// futex call, or gone). +fn futex_wake(addr: *const AtomicU32) -> u32 { // Each head node is slept on by exactly one waiter, so wake one. - unsafe { - libc::syscall(libc::SYS_futex, addr as *const u32, libc::FUTEX_WAKE, 1, 0, 0, 0); - } + let ret = unsafe { libc::syscall(libc::SYS_futex, addr as *const u32, libc::FUTEX_WAKE, 1, 0, 0, 0) }; + if ret > 0 { ret as u32 } else { 0 } } /// Pop a node from the free list, or hand out a never-before-used node @@ -547,13 +549,27 @@ fn node_detach(r: &Region, idx: u32) { } } +/// `registered` values. Readers only ever test it against 0, so a builder +/// that predates REG_SINGLE treats both values as "on the list", and this +/// one treats an older builder's 1 as a waiter it must always wake: the +/// shared layout is unchanged in both directions. +const REG_MULTI: u32 = 1; +/// The node is the only one of its waiter: a single-object, non-alertable +/// wait. Such a waiter is satisfied by this object alone and, once woken, +/// always attempts to acquire it, so a signal op may wake only as many of +/// them as it can satisfy (see wake_object). +const REG_SINGLE: u32 = 2; + +/// Wake budget meaning "every waiter". +const WAKE_ALL: u32 = u32::MAX; + /// Register `node` on object `obj_idx`'s waiter list. Caller holds the lock. -fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32) { +fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32, single: bool) { let node = r.node_ptr(idx); let slot = r.slot_ptr(obj_idx); unsafe { let old_head = (*slot).waiter_head; - (*node).registered = 1; + (*node).registered = if single { REG_SINGLE } else { REG_MULTI }; (*node).obj = obj_idx; (*node).wait_word = wait_word; (*node).obj_prev = NIL; @@ -565,26 +581,66 @@ fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32) { } } -/// Wake every waiter registered on an object (kernel: wake_up(&obj->q)). -/// Caller holds the lock. Each waiter is detached and its head seq bumped; -/// it re-checks conditions when scheduled. -fn wake_object(r: &Region, obj_idx: u32) { +/// Wake the waiters registered on an object. Caller holds the lock. +/// +/// `budget` caps how many single-object waiters (REG_SINGLE) are woken: +/// the units a signal op made available (semaphore release count, 1 for an +/// auto-reset event or a freed mutex). Waking more of them only makes the +/// extras run, fail to acquire, take the region lock twice and park again, +/// all while the signaler pays one futex syscall per waiter - Unity's job +/// system, which releases a semaphore to N parked workers many times per +/// frame, spent about a quarter of its main thread there. A woken +/// single-object waiter always retries the acquire (wait() re-registers and +/// calls try_satisfy under the lock before any timeout or exit), so each +/// unit of budget lands on a thread that will either take the object or +/// find it already taken. The budget is charged only when futex_wake reports +/// a thread actually woken: a waiter that is spinning, about to park, or +/// dead costs no budget, and the next one is woken too. +/// +/// Every other waiter (several objects, wait-all, alertable) is always +/// woken, as before: it may be woken and then not take this object, and +/// waking only it could leave the object signaled with others asleep. +/// WAKE_ALL wakes everyone (close, pulse, manual-reset events, reaping). +/// +/// Waiters are woken oldest first (node_register pushes at the head), so a +/// contended mutex or semaphore cannot starve an early waiter. +fn wake_object(r: &Region, obj_idx: u32, budget: u32) { let debug = debug_enabled(); + let mut left = if wake_all_forced() { WAKE_ALL } else { budget }; let slot = r.slot_ptr(obj_idx); let mut idx = unsafe { (*slot).waiter_head }; + let mut guard = 0; + while idx != NIL && guard < NODE_COUNT { + let next = unsafe { (*r.node_ptr(idx)).obj_next }; + if next == NIL { + break; + } + idx = next; + guard += 1; + } while idx != NIL { let node = r.node_ptr(idx); - let (next, word) = unsafe { ((*node).obj_next, (*node).wait_word) }; + let (prev, word, single) = + unsafe { ((*node).obj_prev, (*node).wait_word, (*node).registered == REG_SINGLE) }; + if single && left == 0 { + stat_bump(16); // wakes_deferred + idx = prev; + continue; + } node_detach(r, idx); let head = r.node_ptr(word); - unsafe { + let woken = unsafe { if debug { (*head).wake_ts.store(monotonic_ns(), Ordering::Relaxed); } (*head).seq.fetch_add(1, Ordering::Release); - futex_wake(&(*head).seq); + futex_wake(&(*head).seq) + }; + stat_add(15, woken as u64); // threads_woken + if single && woken > 0 && left != WAKE_ALL { + left -= 1; } - idx = next; + idx = prev; } } @@ -592,14 +648,14 @@ fn wake_object(r: &Region, obj_idx: u32) { /// The race with a waiter that registers just after our NIL load is closed /// by the waiter re-checking acquirability under the region lock before it /// sleeps (registration and wake walks are mutually exclusive). -fn wake_waiters_if_any(r: &Region, obj_idx: u32) { +fn wake_waiters_if_any(r: &Region, obj_idx: u32, budget: u32) { let slot = unsafe { &*r.slot_ptr(obj_idx) }; if ld32(&slot.waiter_head) == NIL { return; } stat_bump(8); // wake_walks if let Ok(_g) = r.lock_guard() { - wake_object(r, obj_idx); + wake_object(r, obj_idx, budget); } } @@ -917,7 +973,7 @@ pub fn close(handle: u32) -> bool { (&*(&(*slot).state as *const u32 as *const AtomicU32)).store(SLOT_FREE, Ordering::Release); (*slot).generation = (*slot).generation.wrapping_add(1); } - wake_object(r, handle & INDEX_MASK); + wake_object(r, handle & INDEX_MASK, WAKE_ALL); true } @@ -937,7 +993,7 @@ pub fn sem_release(handle: u32, count: u32) -> Result { }; match cas64(w0, w, pack_sem(sum, max)) { Ok(_) => { - wake_waiters_if_any(r, handle & INDEX_MASK); + wake_waiters_if_any(r, handle & INDEX_MASK, count); return Ok(cur); } Err(_) => continue, @@ -958,7 +1014,10 @@ pub fn event_set(handle: u32) -> Result { } match cas64(w0, w, w | 1) { Ok(_) => { - wake_waiters_if_any(r, handle & INDEX_MASK); + // An auto-reset event satisfies one waiter; a manual-reset + // one stays signaled and satisfies all of them. + let manual = (w >> 32) & 1 != 0; + wake_waiters_if_any(r, handle & INDEX_MASK, if manual { WAKE_ALL } else { 1 }); return Ok(0); } Err(_) => continue, @@ -999,7 +1058,7 @@ pub fn event_pulse(handle: u32) -> Result { st64(&(*slot).w0, ld64(&(*slot).w0) & !1u64); prev }; - wake_object(r, handle & INDEX_MASK); + wake_object(r, handle & INDEX_MASK, WAKE_ALL); Ok(prev) } @@ -1027,7 +1086,7 @@ pub fn mutex_unlock(handle: u32, owner: u32) -> Result { match cas64(w0, w, new) { Ok(_) => { if freed { - wake_waiters_if_any(r, handle & INDEX_MASK); + wake_waiters_if_any(r, handle & INDEX_MASK, 1); } return Ok(count); } @@ -1053,7 +1112,7 @@ pub fn mutex_kill(handle: u32, owner: u32) -> Result<(), Error> { } match cas64(w0, w, pack_mutex(0, 0, true)) { Ok(_) => { - wake_waiters_if_any(r, handle & INDEX_MASK); + wake_waiters_if_any(r, handle & INDEX_MASK, 1); return Ok(()); } Err(_) => continue, @@ -1243,7 +1302,7 @@ fn debug_log(msg: &str) { mod stats { use std::sync::atomic::{AtomicU64, Ordering}; - pub const NAMES: [&str; 15] = [ + pub const NAMES: [&str; 17] = [ "wait_calls", // total wait_any/wait_all calls "wait_fast_ok", // satisfied without any lock "wait_blocked", // actually slept on a futex @@ -1259,12 +1318,15 @@ mod stats { "wake_lat_us_max", // worst wake latency, us "spin_wins", // waits satisfied during the pre-sleep spin "spin_exhausted", // spins that still ended in a futex sleep + "threads_woken", // waiters a wake walk actually woke (futex_wake > 0) + "wakes_deferred", // single-object waiters left parked: budget spent ]; - pub static COUNTERS: [AtomicU64; 15] = [ + pub static COUNTERS: [AtomicU64; 17] = [ AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), - AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), + AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), + AtomicU64::new(0), ]; #[inline] @@ -1609,15 +1671,17 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert } } let head = nodes[0]; + // One node in all (one object, no alert event): see REG_SINGLE. + let single = n_nodes == 1; for (i, h) in handles.iter().enumerate() { if unsafe { (*r.node_ptr(nodes[i])).registered } == 0 { - node_register(r, nodes[i], h & INDEX_MASK, head); + node_register(r, nodes[i], h & INDEX_MASK, head, single); } } if alert != 0 { let an = nodes[handles.len()]; if unsafe { (*r.node_ptr(an)).registered } == 0 { - node_register(r, an, alert & INDEX_MASK, head); + node_register(r, an, alert & INDEX_MASK, head, false); } } @@ -2034,7 +2098,7 @@ fn reap_dead_locked(r: &Region, cands: &[DeadCand], alive: &[bool]) -> u32 { { (*slot).state = SLOT_FREE; (*slot).generation = (*slot).generation.wrapping_add(1); - wake_object(r, c.index); + wake_object(r, c.index, WAKE_ALL); freed += 1; } } @@ -2085,6 +2149,13 @@ fn sweep_interval_ns() -> u64 { /// opted in. Capped at 100k to bound silly values. Each iteration is a /// handful of atomic loads (~10-20ns), so e.g. 200 iters costs ~2-4us /// of CPU in exchange for every avoided sleep/wake pair. +/// NTSYNC_WAKE_ALL=1 makes every signal op wake all of an object's waiters, +/// as before budgeted wakes existed. Read once. +fn wake_all_forced() -> bool { + static FORCED: OnceLock = OnceLock::new(); + *FORCED.get_or_init(|| std::env::var("NTSYNC_WAKE_ALL").is_ok_and(|v| v == "1")) +} + fn spin_max_iters() -> u32 { static ITERS: OnceLock = OnceLock::new(); *ITERS.get_or_init(|| { @@ -2396,6 +2467,30 @@ pub(crate) fn test_reap_slot_candidate(cand: (u32, u32, u32), alive: bool) -> u3 reap_dead_locked(r, &[c], &[alive]) } +/// Tests: (single-object, other) waiter nodes registered on `handle`. +#[cfg(test)] +pub(crate) fn test_waiter_counts(handle: u32) -> (u32, u32) { + let Ok(r) = region() else { return (0, 0) }; + let Ok(_g) = r.lock_guard() else { return (0, 0) }; + let Some(slot) = r.slot(handle) else { return (0, 0) }; + let (mut single, mut other) = (0, 0); + let mut idx = unsafe { (*slot).waiter_head }; + let mut guard = 0; + while idx != NIL && guard <= NODE_COUNT { + guard += 1; + let node = r.node_ptr(idx); + unsafe { + match (*node).registered { + REG_SINGLE => single += 1, + 0 => {} + _ => other += 1, + } + idx = (*node).obj_next; + } + } + (single, other) +} + /// Tests: (index, pid) of the first node registered on `handle`'s waiter /// list, i.e. of the process currently blocked on it. #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index a4453b2..1768c9b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,6 +72,313 @@ mod tests { assert!(close(h)); } + // ---- Budgeted wakes: a signal op wakes only as many single-object + // waiters as it can satisfy, oldest first; everyone else as before. ---- + + fn wait_until(what: &str, mut cond: impl FnMut() -> bool) { + let start = std::time::Instant::now(); + while !cond() { + assert!(start.elapsed() < Duration::from_secs(10), "timed out waiting for {what}"); + thread::sleep(Duration::from_millis(1)); + } + } + + /// Park `n` single-object waiters on `h`, one at a time so their order + /// on the list is known. Each pushes its id to `order` once it acquires. + fn park_single_waiters( + h: u32, + n: usize, + owner_base: u32, + order: &Arc>>, + ) -> Vec> { + let mut joins = Vec::new(); + for i in 0..n { + let order = order.clone(); + joins.push(thread::spawn(move || { + let out = wait_any(&[h], owner_base + i as u32, None, 0); + assert!(matches!(out, WaitOutcome::Signaled { index: 0, .. })); + order.lock().unwrap().push(i); + })); + wait_until("waiter to park", || test_waiter_counts(h).0 as usize == i + 1); + } + joins + } + + #[test] + fn semaphore_release_wakes_only_released_count() { + setup(); + let h = create_semaphore(0, 10).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let joins = park_single_waiters(h, 4, 9100, &order); + + sem_release(h, 1).unwrap(); + wait_until("one waiter to acquire", || order.lock().unwrap().len() == 1); + thread::sleep(Duration::from_millis(50)); + assert_eq!(order.lock().unwrap().len(), 1, "release(1) satisfied more than one waiter"); + assert_eq!(test_waiter_counts(h), (3, 0), "the other waiters must stay parked"); + + sem_release(h, 2).unwrap(); + wait_until("two more waiters", || order.lock().unwrap().len() == 3); + thread::sleep(Duration::from_millis(50)); + assert_eq!(test_waiter_counts(h), (1, 0)); + + sem_release(h, 1).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(read_sem(h).unwrap().0, 0); + assert!(close(h)); + } + + #[test] + fn single_waiters_are_woken_oldest_first() { + setup(); + let h = create_semaphore(0, 10).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let joins = park_single_waiters(h, 5, 9200, &order); + for k in 1..=5 { + sem_release(h, 1).unwrap(); + wait_until("next waiter", || order.lock().unwrap().len() == k); + } + for j in joins { + j.join().unwrap(); + } + assert_eq!(*order.lock().unwrap(), vec![0, 1, 2, 3, 4]); + assert!(close(h)); + } + + #[test] + fn auto_event_wakes_one_manual_event_wakes_all() { + setup(); + let auto = create_event(false, false).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let joins = park_single_waiters(auto, 3, 9300, &order); + event_set(auto).unwrap(); + wait_until("one auto waiter", || order.lock().unwrap().len() == 1); + thread::sleep(Duration::from_millis(50)); + assert_eq!(order.lock().unwrap().len(), 1); + assert_eq!(test_waiter_counts(auto), (2, 0)); + // A set that lands while the event is still signaled is a no-op, as + // on Windows; keep setting until everyone got through. + let start = std::time::Instant::now(); + while order.lock().unwrap().len() < 3 { + let _ = event_set(auto); + thread::sleep(Duration::from_millis(1)); + assert!(start.elapsed() < Duration::from_secs(10), "auto-event waiter stranded"); + } + for j in joins { + j.join().unwrap(); + } + assert!(close(auto)); + + let manual = create_event(true, false).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let joins = park_single_waiters(manual, 3, 9310, &order); + event_set(manual).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(order.lock().unwrap().len(), 3); + assert!(close(manual)); + } + + #[test] + fn mutex_unlock_hands_over_to_one_waiter() { + setup(); + let m = create_mutex(9400, 1).unwrap(); + let inside = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let done = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let mut joins = Vec::new(); + for i in 0..4u32 { + let (inside, done) = (inside.clone(), done.clone()); + joins.push(thread::spawn(move || { + let me = 9401 + i; + assert!(is_signaled(wait_any(&[m], me, None, 0), 0)); + assert_eq!(inside.fetch_add(1, Ordering::SeqCst), 0, "two owners at once"); + thread::sleep(Duration::from_millis(2)); + inside.fetch_sub(1, Ordering::SeqCst); + mutex_unlock(m, me).unwrap(); + done.fetch_add(1, Ordering::SeqCst); + })); + } + wait_until("all mutex waiters parked", || test_waiter_counts(m).0 == 4); + mutex_unlock(m, 9400).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(done.load(Ordering::SeqCst), 4); + assert!(close(m)); + } + + #[test] + fn multi_object_and_alertable_waiters_are_always_woken() { + setup(); + let sem = create_semaphore(0, 10).unwrap(); + let other = create_event(true, false).unwrap(); + let alert = create_event(true, false).unwrap(); + let got = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let mut joins = Vec::new(); + { + let got = got.clone(); + joins.push(thread::spawn(move || { + assert!(is_signaled(wait_any(&[sem, other], 9500, None, 0), 0)); + got.fetch_add(1, Ordering::SeqCst); + })); + } + wait_until("multi waiter", || test_waiter_counts(sem).1 == 1); + { + let got = got.clone(); + joins.push(thread::spawn(move || { + assert!(is_signaled(wait_any(&[sem], 9501, None, alert), 0)); + got.fetch_add(1, Ordering::SeqCst); + })); + } + wait_until("alertable waiter", || test_waiter_counts(sem).1 == 2); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + joins.extend(park_single_waiters(sem, 2, 9510, &order)); + assert_eq!(test_waiter_counts(sem), (2, 2)); + + // Release one unit: both non-single waiters are woken (one takes it, + // the other parks again) and both single ones stay parked. + sem_release(sem, 1).unwrap(); + wait_until("one acquire", || got.load(Ordering::SeqCst) + order.lock().unwrap().len() as u32 == 1); + thread::sleep(Duration::from_millis(50)); + sem_release(sem, 3).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(got.load(Ordering::SeqCst) + order.lock().unwrap().len() as u32, 4); + for h in [sem, other, alert] { + assert!(close(h)); + } + } + + /// No lost wakeups under contention. Exactly as many units are released + /// as consumers need, so a wake spent on a thread that never takes its + /// unit would leave a consumer parked forever: every consumer must + /// finish within the deadline. Thieves steal units with zero-timeout + /// waits and give them back, so woken waiters regularly lose the race. + #[test] + fn stress_budgeted_wakes_never_strand_a_waiter() { + setup(); + const PER: u32 = 300; + let sem = create_semaphore(0, 100_000).unwrap(); + let other = create_event(true, false).unwrap(); + let alert = create_event(true, false).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let taken = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let mut consumers = Vec::new(); + // 5 single-object consumers, 1 multi-object, 1 alertable. + for i in 0..7u32 { + let taken = taken.clone(); + consumers.push(thread::spawn(move || { + for _ in 0..PER { + let out = match i { + 5 => wait_any(&[sem, other], 9600 + i, None, 0), + 6 => wait_any(&[sem], 9600 + i, None, alert), + _ => wait_any(&[sem], 9600 + i, None, 0), + }; + assert!(is_signaled(out, 0), "consumer {i}: {out:?}"); + taken.fetch_add(1, Ordering::Relaxed); + } + })); + } + let mut thieves = Vec::new(); + for t in 0..2u32 { + let stop = stop.clone(); + thieves.push(thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + if is_signaled(wait_any(&[sem], 9700 + t, Some(Duration::ZERO), 0), 0) { + std::hint::spin_loop(); + sem_release(sem, 1).unwrap(); + } + std::thread::yield_now(); + } + })); + } + let total = PER * 7; + let mut released = 0; + let mut k = 0u32; + while released < total { + let n = (1 + k % 4).min(total - released); + sem_release(sem, n).unwrap(); + released += n; + k += 1; + if k % 16 == 0 { + thread::sleep(Duration::from_micros(200)); + } + } + let start = std::time::Instant::now(); + while taken.load(Ordering::Relaxed) < total { + assert!( + start.elapsed() < Duration::from_secs(20), + "lost wakeup: {} of {total} units taken, sem count {:?}, waiters {:?}", + taken.load(Ordering::Relaxed), + read_sem(sem), + test_waiter_counts(sem) + ); + thread::sleep(Duration::from_millis(5)); + } + for c in consumers { + c.join().unwrap(); + } + stop.store(true, Ordering::Relaxed); + for t in thieves { + t.join().unwrap(); + } + assert_eq!(read_sem(sem).unwrap().0, 0); + for h in [sem, other, alert] { + assert!(close(h)); + } + } + + /// Mutex and auto-reset event handoffs under contention: mutual + /// exclusion holds and nobody is left parked. + #[test] + fn stress_mutex_and_auto_event_handoff() { + setup(); + let m = create_mutex(0, 0).unwrap(); + let ev = create_event(false, false).unwrap(); + let inside = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let mut joins = Vec::new(); + for i in 0..6u32 { + let inside = inside.clone(); + joins.push(thread::spawn(move || { + let me = 9800 + i; + for _ in 0..500 { + assert!(is_signaled(wait_any(&[m], me, None, 0), 0)); + assert_eq!(inside.fetch_add(1, Ordering::SeqCst), 0, "two mutex owners"); + inside.fetch_sub(1, Ordering::SeqCst); + mutex_unlock(m, me).unwrap(); + } + })); + } + // Auto-event ping-pong: each consumer waits, the producer sets until + // every wait has been satisfied. + let got = Arc::new(std::sync::atomic::AtomicU32::new(0)); + for i in 0..4u32 { + let got = got.clone(); + joins.push(thread::spawn(move || { + for _ in 0..200 { + assert!(is_signaled(wait_any(&[ev], 9900 + i, None, 0), 0)); + got.fetch_add(1, Ordering::SeqCst); + } + })); + } + let start = std::time::Instant::now(); + while got.load(Ordering::SeqCst) < 800 { + let _ = event_set(ev); + std::thread::yield_now(); + assert!(start.elapsed() < Duration::from_secs(20), "auto-event waiter stranded at {}", got.load(Ordering::SeqCst)); + } + for j in joins { + j.join().unwrap(); + } + for h in [m, ev] { + assert!(close(h)); + } + } + #[test] fn handle_zero_is_never_allocated() { setup(); From f4656096ecd1f4550b4a09829bf066ac8a572274 Mon Sep 17 00:00:00 2001 From: Utkarsh Dalal Date: Mon, 16 Nov 2026 12:03:05 +0530 Subject: [PATCH 2/3] Budget wakes for alertable and multi-object wait-any waiters too Wine waits alertably unless the caller asks otherwise, so the workers of a Unity job system registered an alert node next to the semaphore node and were never counted as single-object waiters: every release still woke all of them (1 deferred wake in 18.8 M signal ops on The Forest). A wait-any waiter with several nodes now registers its object nodes as REG_ANY, which wake_object budgets like REG_SINGLE. Such a waiter can return without the object it was woken for (its alert fired, an earlier object in its list was signaled, or a handle went invalid). It then wakes the next waiter in its place, so the unit is never left in the object with waiters asleep. Wait-all waiters and alert nodes are always woken, as before. Tests: two deterministic ones park a decoy ahead of a consumer and make the decoy return on its alert or on another object; both fail with the forwarding removed. The stress test now fires alerts and a second object at its consumers. The Unity simulation's workers wait alertably and it reports the cost of sem_release. --- src/core.rs | 115 ++++++++++++++++++++++++++++--------- src/engine_sim.rs | 15 ++++- src/lib.rs | 143 +++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 230 insertions(+), 43 deletions(-) diff --git a/src/core.rs b/src/core.rs index 4b9b2ed..4844838 100644 --- a/src/core.rs +++ b/src/core.rs @@ -559,17 +559,24 @@ const REG_MULTI: u32 = 1; /// always attempts to acquire it, so a signal op may wake only as many of /// them as it can satisfy (see wake_object). const REG_SINGLE: u32 = 2; +/// An object node of a wait-any waiter that has other nodes too (several +/// objects, or an alert event). It is woken within the budget like a +/// REG_SINGLE one, but such a waiter can return without taking the object it +/// was woken for (the alert fired, or an earlier object in its list was +/// signaled). It then passes the wake on to the next waiter: see the `owed` +/// mask in wait(). +const REG_ANY: u32 = 3; /// Wake budget meaning "every waiter". const WAKE_ALL: u32 = u32::MAX; /// Register `node` on object `obj_idx`'s waiter list. Caller holds the lock. -fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32, single: bool) { +fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32, reg: u32) { let node = r.node_ptr(idx); let slot = r.slot_ptr(obj_idx); unsafe { let old_head = (*slot).waiter_head; - (*node).registered = if single { REG_SINGLE } else { REG_MULTI }; + (*node).registered = reg; (*node).obj = obj_idx; (*node).wait_word = wait_word; (*node).obj_prev = NIL; @@ -583,24 +590,26 @@ fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32, single: boo /// Wake the waiters registered on an object. Caller holds the lock. /// -/// `budget` caps how many single-object waiters (REG_SINGLE) are woken: +/// `budget` caps how many wait-any waiters (REG_SINGLE, REG_ANY) are woken: /// the units a signal op made available (semaphore release count, 1 for an /// auto-reset event or a freed mutex). Waking more of them only makes the /// extras run, fail to acquire, take the region lock twice and park again, /// all while the signaler pays one futex syscall per waiter - Unity's job /// system, which releases a semaphore to N parked workers many times per -/// frame, spent about a quarter of its main thread there. A woken -/// single-object waiter always retries the acquire (wait() re-registers and -/// calls try_satisfy under the lock before any timeout or exit), so each -/// unit of budget lands on a thread that will either take the object or -/// find it already taken. The budget is charged only when futex_wake reports -/// a thread actually woken: a waiter that is spinning, about to park, or -/// dead costs no budget, and the next one is woken too. +/// frame, spent about a fifth of its main thread there. Wine waits +/// alertably by default, so those workers are REG_ANY, not REG_SINGLE. /// -/// Every other waiter (several objects, wait-all, alertable) is always -/// woken, as before: it may be woken and then not take this object, and -/// waking only it could leave the object signaled with others asleep. -/// WAKE_ALL wakes everyone (close, pulse, manual-reset events, reaping). +/// A woken wait-any waiter retries the acquire (wait() re-registers and +/// calls try_satisfy under the lock before any timeout), so each unit of +/// budget lands on a thread that takes the object, finds it already taken, +/// or - REG_ANY only - returns for another reason and wakes the next waiter +/// in its place. The budget is charged only when futex_wake reports a +/// thread actually woken: a waiter that is spinning, about to park, or dead +/// costs no budget, and the next one is woken too. +/// +/// Wait-all waiters and alert-event nodes (REG_MULTI) are always woken, as +/// before. WAKE_ALL wakes everyone (close, pulse, manual-reset events, +/// reaping). /// /// Waiters are woken oldest first (node_register pushes at the head), so a /// contended mutex or semaphore cannot starve an early waiter. @@ -620,9 +629,9 @@ fn wake_object(r: &Region, obj_idx: u32, budget: u32) { } while idx != NIL { let node = r.node_ptr(idx); - let (prev, word, single) = - unsafe { ((*node).obj_prev, (*node).wait_word, (*node).registered == REG_SINGLE) }; - if single && left == 0 { + let (prev, word, reg) = unsafe { ((*node).obj_prev, (*node).wait_word, (*node).registered) }; + let budgeted = reg == REG_SINGLE || reg == REG_ANY; + if budgeted && left == 0 { stat_bump(16); // wakes_deferred idx = prev; continue; @@ -637,13 +646,31 @@ fn wake_object(r: &Region, obj_idx: u32, budget: u32) { futex_wake(&(*head).seq) }; stat_add(15, woken as u64); // threads_woken - if single && woken > 0 && left != WAKE_ALL { + if budgeted && woken > 0 && left != WAKE_ALL { left -= 1; } idx = prev; } } +/// Pass on the wakes a wait-any waiter was given for objects it is leaving +/// without having taken: bit i of `owed` is handles[i]. Each was one unit of +/// some signal op's budget, so one more waiter is woken per object. If the +/// unit is gone by now the woken waiter just parks again. Caller holds the +/// lock. +fn forward_wakes(r: &Region, handles: &[u32], owed: u64, taken: Option) { + let mut bits = owed; + while bits != 0 { + let i = bits.trailing_zeros(); + bits &= bits - 1; + if taken == Some(i) { + continue; + } + stat_bump(17); // wakes_forwarded + wake_object(r, handles[i as usize] & INDEX_MASK, 1); + } +} + /// Lock-free fast check followed by a locked wake walk, used by signal ops. /// The race with a waiter that registers just after our NIL load is closed /// by the waiter re-checking acquirability under the region lock before it @@ -1302,7 +1329,7 @@ fn debug_log(msg: &str) { mod stats { use std::sync::atomic::{AtomicU64, Ordering}; - pub const NAMES: [&str; 17] = [ + pub const NAMES: [&str; 18] = [ "wait_calls", // total wait_any/wait_all calls "wait_fast_ok", // satisfied without any lock "wait_blocked", // actually slept on a futex @@ -1319,14 +1346,15 @@ mod stats { "spin_wins", // waits satisfied during the pre-sleep spin "spin_exhausted", // spins that still ended in a futex sleep "threads_woken", // waiters a wake walk actually woke (futex_wake > 0) - "wakes_deferred", // single-object waiters left parked: budget spent + "wakes_deferred", // wait-any waiters left parked: budget spent + "wakes_forwarded", // wakes passed on by a waiter that left without the object ]; - pub static COUNTERS: [AtomicU64; 17] = [ + pub static COUNTERS: [AtomicU64; 18] = [ AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), - AtomicU64::new(0), + AtomicU64::new(0), AtomicU64::new(0), ]; #[inline] @@ -1563,6 +1591,10 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert let mut entry_seqs = [0u64; MAX_WAIT_COUNT]; let n_nodes = handles.len() + (alert != 0) as usize; let mut nodes = NodeSet::new(); + // Object nodes this thread detached itself after its last park (timeout, + // EINTR, or a wake that came through another node). Every other node + // found detached at the top of the loop was detached by a signaler. + let mut self_detached = 0u64; // ---- Fast path: validate and try to acquire without any lock. ---- let fast_valid = handles.iter().enumerate().all(|(i, h)| { @@ -1609,6 +1641,21 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert return WaitOutcome::Invalid; }; + // Objects whose signaler woke this wait-any waiter since it last + // parked (see REG_ANY). Leaving without taking one of them passes + // its wake on; a failed acquire below clears the debt, because the + // unit is then gone and this thread parks on the object again. + let mut owed = 0u64; + if !all && n_nodes > 1 && !nodes.is_empty() { + for i in 0..handles.len() { + if unsafe { (*r.node_ptr(nodes[i])).registered } == 0 { + owed |= 1 << i; + } + } + owed &= !self_detached; + } + self_detached = 0; + // Authoritative validation (also catches objects closed mid-wait). let mut valid = true; for (i, h) in handles.iter().enumerate() { @@ -1634,6 +1681,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert } if !valid { waiter_cleanup(r, &mut nodes); + forward_wakes(r, handles, owed, None); return WaitOutcome::Invalid; } @@ -1644,6 +1692,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert let slot = r.slot(alert).expect("validated above"); if ld64(unsafe { &(*slot).w0 }) as u32 != 0 { waiter_cleanup(r, &mut nodes); + forward_wakes(r, handles, owed, None); return WaitOutcome::Signaled { index: handles.len() as u32, owner_dead: false, @@ -1671,17 +1720,22 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert } } let head = nodes[0]; - // One node in all (one object, no alert event): see REG_SINGLE. - let single = n_nodes == 1; + let reg = if all { + REG_MULTI + } else if n_nodes == 1 { + REG_SINGLE + } else { + REG_ANY + }; for (i, h) in handles.iter().enumerate() { if unsafe { (*r.node_ptr(nodes[i])).registered } == 0 { - node_register(r, nodes[i], h & INDEX_MASK, head, single); + node_register(r, nodes[i], h & INDEX_MASK, head, reg); } } if alert != 0 { let an = nodes[handles.len()]; if unsafe { (*r.node_ptr(an)).registered } == 0 { - node_register(r, an, alert & INDEX_MASK, head, false); + node_register(r, an, alert & INDEX_MASK, head, REG_MULTI); } } @@ -1697,6 +1751,9 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert // Authoritative acquire attempt. if let Some(outcome) = try_satisfy(r, handles, owner, &entry_seqs, all) { waiter_cleanup(r, &mut nodes); + if let WaitOutcome::Signaled { index, .. } = outcome { + forward_wakes(r, handles, owed, Some(index)); + } return outcome; } @@ -1880,7 +1937,10 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert .any(|&idx| unsafe { ld32(&(*r.node_ptr(idx)).registered) } != 0); if any_registered { if let Ok(_g) = r.lock_guard() { - for &idx in nodes.iter() { + for (i, &idx) in nodes.iter().enumerate() { + if i < handles.len() && unsafe { (*r.node_ptr(idx)).registered } != 0 { + self_detached |= 1 << i; + } node_detach(r, idx); } } @@ -2468,6 +2528,7 @@ pub(crate) fn test_reap_slot_candidate(cand: (u32, u32, u32), alive: bool) -> u3 } /// Tests: (single-object, other) waiter nodes registered on `handle`. +/// "Other" counts REG_ANY and REG_MULTI nodes alike. #[cfg(test)] pub(crate) fn test_waiter_counts(handle: u32) -> (u32, u32) { let Ok(r) = region() else { return (0, 0) }; diff --git a/src/engine_sim.rs b/src/engine_sim.rs index 5f4c125..04f5bde 100644 --- a/src/engine_sim.rs +++ b/src/engine_sim.rs @@ -269,7 +269,9 @@ fn sim_ue45_frame_graph() { // ------------------------------------------------------------------ Unity --- /// Unity job system: per-job events, `Complete()` waits, and the contended -/// graphics-device mutex. +/// graphics-device mutex. Workers wait alertably, as every Wine wait does +/// unless the caller asks otherwise, so the cost of a release with several +/// workers parked shows up in "sem_release". #[test] #[ignore] fn sim_unity_job_system() { @@ -286,13 +288,16 @@ fn sim_unity_job_system() { let next_job = Arc::new(AtomicU32::new(0)); let mut workers = Vec::new(); + let mut alerts = Vec::new(); for _ in 0..WORKERS { let stop = stop.clone(); let events = job_events.clone(); let next_job = next_job.clone(); + let alert = create_event(true, false).unwrap(); + alerts.push(alert); workers.push(thread::spawn(move || { while !stop.load(Ordering::Relaxed) { - match wait_any(&[job_sem], me(), Some(Duration::from_millis(50)), 0) { + match wait_any(&[job_sem], me(), Some(Duration::from_millis(50)), alert) { WaitOutcome::Signaled { .. } => {} WaitOutcome::Timeout => continue, _ => break, @@ -314,11 +319,14 @@ fn sim_unity_job_system() { let mut frames = Frames::new(FPS); let budget = Duration::from_secs_f64(1.0 / FPS as f64); let mut complete = Hist::with_capacity(8192); + let mut release = Hist::with_capacity(8192); for _ in 0..FRAMES { let start = Instant::now(); next_job.store(0, Ordering::Relaxed); for _ in 0..JOBS { + let w = Instant::now(); let _ = sem_release(job_sem, 1); + release.push(w.elapsed()); } // JobHandle.Complete(): one wait per job, in dispatch order. for &event in job_events.iter() { @@ -339,11 +347,12 @@ fn sim_unity_job_system() { let _ = sem_release(job_sem, WORKERS as u32); frames.report("Unity job system", FPS); complete.report("Complete() wait"); + release.report("sem_release"); for w in workers { w.join().unwrap(); } assert_eq!(frames.total as usize, FRAMES); - for e in job_events { + for e in job_events.into_iter().chain(alerts) { let _ = close(e); } let _ = close(job_sem); diff --git a/src/lib.rs b/src/lib.rs index 1768c9b..625e193 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,8 +72,8 @@ mod tests { assert!(close(h)); } - // ---- Budgeted wakes: a signal op wakes only as many single-object - // waiters as it can satisfy, oldest first; everyone else as before. ---- + // ---- Budgeted wakes: a signal op wakes only as many wait-any waiters + // as it can satisfy, oldest first; wait-all waiters as before. ---- fn wait_until(what: &str, mut cond: impl FnMut() -> bool) { let start = std::time::Instant::now(); @@ -211,7 +211,7 @@ mod tests { } #[test] - fn multi_object_and_alertable_waiters_are_always_woken() { + fn multi_object_and_alertable_waiters_are_woken_within_budget() { setup(); let sem = create_semaphore(0, 10).unwrap(); let other = create_event(true, false).unwrap(); @@ -238,11 +238,13 @@ mod tests { joins.extend(park_single_waiters(sem, 2, 9510, &order)); assert_eq!(test_waiter_counts(sem), (2, 2)); - // Release one unit: both non-single waiters are woken (one takes it, - // the other parks again) and both single ones stay parked. + // Release one unit: only the oldest waiter (the multi-object one) is + // woken and takes it; the other three stay parked. sem_release(sem, 1).unwrap(); - wait_until("one acquire", || got.load(Ordering::SeqCst) + order.lock().unwrap().len() as u32 == 1); + wait_until("one acquire", || got.load(Ordering::SeqCst) == 1); thread::sleep(Duration::from_millis(50)); + assert_eq!(test_waiter_counts(sem), (2, 1), "the other waiters must stay parked"); + assert!(order.lock().unwrap().is_empty()); sem_release(sem, 3).unwrap(); for j in joins { j.join().unwrap(); @@ -253,6 +255,100 @@ mod tests { } } + /// A wait-all waiter may need more than this object, so it is woken + /// whatever the budget. + #[test] + fn wait_all_waiters_are_always_woken() { + setup(); + let sem = create_semaphore(0, 10).unwrap(); + let gate = create_event(true, true).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut joins = park_single_waiters(sem, 1, 9550, &order); + joins.push(thread::spawn(move || { + assert!(matches!(wait_all(&[sem, gate], 9551, None, 0), WaitOutcome::Signaled { .. })); + })); + wait_until("wait-all waiter", || test_waiter_counts(sem) == (1, 1)); + // Two units for two waiters: the single one is the oldest and uses + // the whole budget of the first release if it is the only one woken. + sem_release(sem, 1).unwrap(); + sem_release(sem, 1).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(read_sem(sem).unwrap().0, 0); + for h in [sem, gate] { + assert!(close(h)); + } + } + + /// A wait-any waiter woken for a semaphore unit that returns for another + /// reason (its alert fired, or an earlier object in its list was + /// signaled) must pass the wake on, or the unit sits in the semaphore + /// with the next waiter asleep. `alertable` picks the reason. + fn decoy_passes_its_wake_on(alertable: bool, owner_base: u32) { + setup(); + let mut passed_on = 0; + for round in 0..200u32 { + let sem = create_semaphore(0, 10).unwrap(); + let ev = create_event(true, false).unwrap(); + let owner = owner_base + round * 2; + // The decoy parks first, so it is the oldest waiter and the one + // a release of one unit wakes. + let decoy = thread::spawn(move || { + let out = if alertable { + wait_any(&[sem], owner, None, ev) + } else { + wait_any(&[ev, sem], owner, None, 0) + }; + let sem_index = if alertable { 0 } else { 1 }; + assert!(matches!(out, WaitOutcome::Signaled { .. }), "decoy: {out:?}"); + is_signaled(out, sem_index) + }); + wait_until("decoy to park", || test_waiter_counts(sem).1 == 1); + let consumer = thread::spawn(move || { + assert!(is_signaled(wait_any(&[sem], owner + 1, None, 0), 0)); + }); + wait_until("consumer to park", || test_waiter_counts(sem).0 == 1); + + // The release wakes only the decoy. The event lands before the + // decoy gets to run, so it returns on the event and leaves the + // unit behind. + sem_release(sem, 1).unwrap(); + event_set(ev).unwrap(); + if decoy.join().unwrap() { + // The decoy won the race and took the unit itself. + sem_release(sem, 1).unwrap(); + } else { + passed_on += 1; + } + let start = std::time::Instant::now(); + while !consumer.is_finished() { + assert!( + start.elapsed() < Duration::from_secs(10), + "round {round}: consumer left parked, sem count {:?}", + read_sem(sem) + ); + thread::sleep(Duration::from_micros(200)); + } + consumer.join().unwrap(); + assert_eq!(read_sem(sem).unwrap().0, 0); + for h in [sem, ev] { + assert!(close(h)); + } + } + assert!(passed_on > 0, "no round had the decoy return on the event"); + } + + #[test] + fn alerted_waiter_passes_its_wake_on() { + decoy_passes_its_wake_on(true, 9800); + } + + #[test] + fn waiter_satisfied_by_another_object_passes_its_wake_on() { + decoy_passes_its_wake_on(false, 10300); + } + /// No lost wakeups under contention. Exactly as many units are released /// as consumers need, so a wake spent on a thread that never takes its /// unit would leave a consumer parked forever: every consumer must @@ -268,17 +364,27 @@ mod tests { let stop = Arc::new(AtomicBool::new(false)); let taken = Arc::new(std::sync::atomic::AtomicU32::new(0)); let mut consumers = Vec::new(); - // 5 single-object consumers, 1 multi-object, 1 alertable. + // 4 single-object consumers, 2 multi-object, 2 alertable (one is both). for i in 0..7u32 { let taken = taken.clone(); consumers.push(thread::spawn(move || { for _ in 0..PER { - let out = match i { - 5 => wait_any(&[sem, other], 9600 + i, None, 0), - 6 => wait_any(&[sem], 9600 + i, None, alert), - _ => wait_any(&[sem], 9600 + i, None, 0), - }; - assert!(is_signaled(out, 0), "consumer {i}: {out:?}"); + // Consumer 4 returns on `other` and consumers 5 and 6 on + // their alert whenever the toggler fires them; they + // then wait again, having passed their wake on. + loop { + let (out, sem_index) = match i { + 4 => (wait_any(&[other, sem], 9600 + i, None, 0), 1), + 5 => (wait_any(&[sem, other], 9600 + i, None, alert), 0), + 6 => (wait_any(&[sem], 9600 + i, None, alert), 0), + _ => (wait_any(&[sem], 9600 + i, None, 0), 0), + }; + assert!(matches!(out, WaitOutcome::Signaled { .. }), "consumer {i}: {out:?}"); + if is_signaled(out, sem_index) { + break; + } + std::thread::yield_now(); + } taken.fetch_add(1, Ordering::Relaxed); } })); @@ -296,6 +402,17 @@ mod tests { } })); } + for (t, ev) in [other, alert].into_iter().enumerate() { + let stop = stop.clone(); + thieves.push(thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + event_set(ev).unwrap(); + std::hint::spin_loop(); + event_reset(ev).unwrap(); + thread::sleep(Duration::from_micros(50 + 70 * t as u64)); + } + })); + } let total = PER * 7; let mut released = 0; let mut k = 0u32; From 6342614affcff9658e1e297b5d0e63c1430cb306 Mon Sep 17 00:00:00 2001 From: Utkarsh Dalal Date: Mon, 16 Nov 2026 13:08:41 +0530 Subject: [PATCH 3/3] Issue release wakes after the region unlock; host lock flag in place of SIGUSR1 masking The cost of a semaphore release on Unity's job semaphore was the region lock around each wake, not the number of threads woken. - Wakes after the unlock: wake walks detach nodes and bump their seq under the lock and collect the futex words (up to 16, inline beyond that); the FUTEX_WAKEs are issued after the unlock, so a woken waiter no longer runs into the lock its waker still holds. A budgeted wake is charged at detach; if its futex_wake finds nobody parked, a second locked walk wakes one more waiter per such unit (what the inline walk did within the same walk). Applies to signal ops and to wakes a waiter passes on; close, pulse and reaping stay inline. - ntsync_set_lock_flag_hook: a host whose SIGUSR1 handler suspends threads (Wine) can give each thread a flag the library sets while it holds or waits for the region lock. The handler marks the signal pending and returns while the flag is set; the library swaps the flag to 0 after the unlock and re-raises SIGUSR1 if it was pending. This replaces two pthread_sigmask syscalls per acquisition. Without the hook, or for threads it returns NULL for, masking stays as before. - Region lock: trylock first, so lock_contended counts contention without NTSYNC_DEBUG and the debug timedlock runs only when the lock is busy; the debug lock-owner tid comes from bionic's cached gettid(). - Stats: wakes_unparked, wake_rewalks, sigusr1_deferred appended to the stats line. - sim_unity_release_cost: 7 alertable workers on one semaphore, 300 single-unit releases per 60 fps frame in bursts, per-job completion events; reports main-thread CPU per release, wake walks and lock acquisitions, and per-thread CPU and switches. - Tests: a budgeted wake that lands on a dead waiter goes to the next one; SIGUSR1 delivered to a thread inside the region lock is deferred, delivered once, and does not wedge the lock, with a negative control that wedges when the handler ignores the flag. --- README.md | 16 ++- include/ntsync_user.h | 19 +++ src/core.rs | 307 ++++++++++++++++++++++++++++++++++------- src/engine_sim.rs | 308 ++++++++++++++++++++++++++++++++++++++++++ src/ffi.rs | 8 ++ src/lib.rs | 165 ++++++++++++++++++++++ 6 files changed, 775 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 2cbfc3c..47fe508 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,24 @@ mutex consistent. set this when the host process never installs a SIGUSR1 handler (i.e. not under Wine); it measurably reduces lock-path CPU under contention (~13% per-op in the multi-process benchmark). + A host that does install such a handler can avoid both syscalls with + `ntsync_set_lock_flag_hook` instead: the library sets a per-thread flag + the host owns while the thread holds the lock, the host's SIGUSR1 handler + marks the signal pending and returns while the flag is set, and the + library re-raises SIGUSR1 right after the unlock (protocol in + `include/ntsync_user.h`). Threads for which the hook returns NULL keep + the masking. +- `NTSYNC_WAKE_ALL=1` — every signal op wakes all of an object's waiters + instead of only as many wait-any waiters as it made units available. Kept + for comparison; the default is cheaper for the releasing thread once its + wakes are issued after the unlock (see `sim_unity_release_cost`). - `NTSYNC_DEBUG=1` — log to stderr, stdout and `$TMPDIR/ntsync_debug.log` (the file is the reliable channel for in-game diagnostics); dump per-process stats every 10 s (waits, fast-path hits, lock acquisitions, - wake walks, signal→scheduled wake latency `wake_lat_cnt/us_sum/us_max`); + wake walks, signal→scheduled wake latency `wake_lat_cnt/us_sum/us_max`, + budgeted wakes that found nobody parked `wakes_unparked` and the second + walks that re-issued them `wake_rewalks`, SIGUSR1s held back by the + lock-flag hook `sigusr1_deferred`); emit a "stuck wait" dump with per-object state (including creator pid liveness) when a wait stalls. - Object lifetime: an object is reaped only when its creator **and every diff --git a/include/ntsync_user.h b/include/ntsync_user.h index 5578fca..06acc12 100644 --- a/include/ntsync_user.h +++ b/include/ntsync_user.h @@ -95,6 +95,25 @@ int32_t ntsync_init(const char *path); * process exits. Returns the number of freed objects or a negative errno. */ int32_t ntsync_sweep_dead(void); +/* Optional SIGUSR1 cooperation, for hosts whose SIGUSR1 handler suspends the + * thread (Wine). A thread suspended while it holds the shared region lock + * would wedge every process on it, so by default the library blocks SIGUSR1 + * around each lock acquisition: two sigprocmask syscalls per acquisition. + * A host can instead give each thread an int the library sets while the + * thread holds (or waits for) the lock: + * - get_flag returns the calling thread's int (NULL: fall back to masking + * for this acquisition). It is called in normal context only. + * - Before taking the lock the library stores NTSYNC_LOCK_FLAG_HELD; after + * releasing it, it atomically swaps the int to 0 and, if + * NTSYNC_LOCK_FLAG_SIGNAL_PENDING was set, re-raises SIGUSR1. + * - The host's SIGUSR1 handler must, before doing anything else, check the + * int: if nonzero, atomically OR in NTSYNC_LOCK_FLAG_SIGNAL_PENDING and + * return without suspending. + * Pass NULL to uninstall. Returns 0. */ +#define NTSYNC_LOCK_FLAG_HELD 1 +#define NTSYNC_LOCK_FLAG_SIGNAL_PENDING 2 +int32_t ntsync_set_lock_flag_hook(int *(*get_flag)(void)); + /* Create objects. Return 0 and store the handle, or a negative errno. * Handles are never 0: slot 0 is permanently reserved because 0 is the * "no alert" sentinel in ntsync_wait_args.alert. */ diff --git a/src/core.rs b/src/core.rs index 4844838..9852d79 100644 --- a/src/core.rs +++ b/src/core.rs @@ -33,7 +33,7 @@ //! ntsync_sweep_dead(). use std::ffi::CString; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicI32, AtomicU32, AtomicU64, Ordering}; use std::sync::OnceLock; use std::time::Duration; @@ -355,26 +355,38 @@ impl Region { let mut waited = std::time::Duration::ZERO; // Wine suspends threads with SIGUSR1; if it lands while we hold the // shared region mutex the holder stays parked and every process - // (including wineserver) wedges on the lock. Defer it until release. - // Consumers without a SIGUSR1 handler can set - // NTSYNC_NO_SIGUSR1_BLOCK=1 to skip the two pthread_sigmask - // syscalls per lock acquisition. - let old_mask = if sigusr1_block_enabled() { - let mut old_mask = unsafe { std::mem::zeroed::() }; - let mut block = unsafe { std::mem::zeroed::() }; - unsafe { - libc::sigemptyset(&mut block); - libc::sigaddset(&mut block, libc::SIGUSR1); - libc::pthread_sigmask(libc::SIG_BLOCK, &block, &mut old_mask); + // (including wineserver) wedges on the lock. Defer it until release: + // through the host's per-thread lock flag when it installed one + // (ntsync_set_lock_flag_hook: its SIGUSR1 handler holds the signal + // back while the flag is set, no syscalls), else by blocking SIGUSR1 + // for the duration (two pthread_sigmask syscalls). Consumers without + // a SIGUSR1 handler can set NTSYNC_NO_SIGUSR1_BLOCK=1 to skip both. + let defer = match lock_flag() { + Some(flag) => { + flag.store(LOCK_FLAG_HELD, Ordering::SeqCst); + Defer::Flag(flag) } - Some(old_mask) - } else { - None + None if sigusr1_block_enabled() => { + let mut old_mask = unsafe { std::mem::zeroed::() }; + let mut block = unsafe { std::mem::zeroed::() }; + unsafe { + libc::sigemptyset(&mut block); + libc::sigaddset(&mut block, libc::SIGUSR1); + libc::pthread_sigmask(libc::SIG_BLOCK, &block, &mut old_mask); + } + Defer::Mask(old_mask) + } + None => Defer::Nothing, }; - let mut tries = 0u32; + // A failed trylock is the contention count; it costs one CAS, which + // pthread_mutex_lock would try first anyway. + let first = unsafe { libc::pthread_mutex_trylock(lock) }; + let mut tries = if first == libc::EBUSY { 1u32 } else { 0 }; loop { tries += 1; - let ret = if debug { + let ret = if tries == 1 { + first + } else if debug { // Poll with a timeout so a stuck holder can be logged. let mut ts = unsafe { std::mem::zeroed::() }; unsafe { libc::clock_gettime(libc::CLOCK_REALTIME, &mut ts) }; @@ -401,7 +413,7 @@ impl Region { (*h).lock_owner_tid = current_tid(); } } - return Ok(RegionGuard { region: self, old_mask }); + return Ok(RegionGuard { region: self, defer }); } if ret == libc::ETIMEDOUT { waited += std::time::Duration::from_secs(5); @@ -417,20 +429,83 @@ impl Region { continue; } if ret != libc::EINTR { - if let Some(old_mask) = old_mask { - unsafe { - libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut()) - }; - } + defer.end(); return Err(-ret); } } } } +/// How a region-lock holder keeps SIGUSR1 (Wine's suspend) from parking it +/// inside the lock. +enum Defer { + /// No SIGUSR1 handler to worry about (NTSYNC_NO_SIGUSR1_BLOCK=1). + Nothing, + /// SIGUSR1 blocked; the previous mask is restored on release. + Mask(libc::sigset_t), + /// The host's per-thread lock flag is set; its handler defers SIGUSR1 + /// while it is, and marks it pending. + Flag(&'static AtomicI32), +} + +impl Defer { + /// Called once the lock is released (or was never taken). + fn end(&self) { + match self { + Defer::Nothing => {} + Defer::Mask(old_mask) => unsafe { + libc::pthread_sigmask(libc::SIG_SETMASK, old_mask, std::ptr::null_mut()); + }, + Defer::Flag(flag) => { + // Atomic swap: a SIGUSR1 that lands before it sees the flag + // set and leaves PENDING for this read; one after it finds the + // flag clear and suspends the thread right there, outside the + // lock. Either way it is delivered exactly once. + if flag.swap(0, Ordering::SeqCst) & LOCK_FLAG_SIGNAL_PENDING != 0 { + stat_bump(20); // sigusr1_deferred + unsafe { libc::raise(libc::SIGUSR1) }; + } + } + } + } +} + +/// ntsync_set_lock_flag_hook protocol: the library stores LOCK_FLAG_HELD in +/// the thread's flag before it takes the region lock and swaps it back to 0 +/// after the unlock. The host's SIGUSR1 handler, finding it nonzero, ORs in +/// LOCK_FLAG_SIGNAL_PENDING and returns without suspending; the library then +/// re-raises SIGUSR1 at the swap. +pub const LOCK_FLAG_HELD: i32 = 1; +pub const LOCK_FLAG_SIGNAL_PENDING: i32 = 2; + +/// Host callback returning the calling thread's lock flag (or null to fall +/// back to masking SIGUSR1 for this acquisition). +pub type LockFlagHook = unsafe extern "C" fn() -> *mut i32; + +static LOCK_FLAG_HOOK: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Install (or with None remove) the host's lock-flag callback. +pub fn set_lock_flag_hook(hook: Option) { + LOCK_FLAG_HOOK.store(hook.map_or(0, |f| f as usize), Ordering::SeqCst); +} + +fn lock_flag() -> Option<&'static AtomicI32> { + let f = LOCK_FLAG_HOOK.load(Ordering::Relaxed); + if f == 0 { + return None; + } + let hook: LockFlagHook = unsafe { std::mem::transmute::(f) }; + let p = unsafe { hook() }; + if p.is_null() { + None + } else { + Some(unsafe { &*(p as *const AtomicI32) }) + } +} + struct RegionGuard<'a> { region: &'a Region, - old_mask: Option, + defer: Defer, } impl Drop for RegionGuard<'_> { @@ -441,10 +516,8 @@ impl Drop for RegionGuard<'_> { (*h).lock_owner_tid = 0; let lock = &(*h).lock as *const libc::pthread_mutex_t as *mut _; libc::pthread_mutex_unlock(lock); - if let Some(old_mask) = &self.old_mask { - libc::pthread_sigmask(libc::SIG_SETMASK, old_mask, std::ptr::null_mut()); - } } + self.defer.end(); } } @@ -613,7 +686,7 @@ fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32, reg: u32) { /// /// Waiters are woken oldest first (node_register pushes at the head), so a /// contended mutex or semaphore cannot starve an early waiter. -fn wake_object(r: &Region, obj_idx: u32, budget: u32) { +fn wake_object(r: &Region, obj_idx: u32, budget: u32, mut later: Option<&mut PendingWakes<'_>>) { let debug = debug_enabled(); let mut left = if wake_all_forced() { WAKE_ALL } else { budget }; let slot = r.slot_ptr(obj_idx); @@ -638,27 +711,126 @@ fn wake_object(r: &Region, obj_idx: u32, budget: u32) { } node_detach(r, idx); let head = r.node_ptr(word); - let woken = unsafe { + unsafe { if debug { (*head).wake_ts.store(monotonic_ns(), Ordering::Relaxed); } (*head).seq.fetch_add(1, Ordering::Release); - futex_wake(&(*head).seq) - }; - stat_add(15, woken as u64); // threads_woken - if budgeted && woken > 0 && left != WAKE_ALL { - left -= 1; + } + if let Some(pw) = later.as_deref_mut().filter(|pw| pw.n < PENDING_WAKES) { + // Woken after the unlock (PendingWakes::flush), so the thread + // does not run straight into the lock we still hold. The unit is + // charged now; flush gives it back if nobody was parked. + pw.push(obj_idx, word, budgeted && left != WAKE_ALL); + if budgeted && left != WAKE_ALL { + left -= 1; + } + } else { + let woken = futex_wake(unsafe { &(*head).seq }); + stat_add(15, woken as u64); // threads_woken + if budgeted && woken > 0 && left != WAKE_ALL { + left -= 1; + } } idx = prev; } } +/// Most wakes one locked walk defers past the unlock; any beyond are woken +/// inline. A semaphore release wakes at most its count, an event or mutex 1. +const PENDING_WAKES: usize = 16; + +/// Futex wakes collected under the region lock and issued after it is +/// dropped (flush, or Drop: declare it before the guard so it drops after). +/// +/// A late FUTEX_WAKE is safe: the node was detached and its seq bumped under +/// the lock, so its waiter cannot sleep through it (futex_wait returns EAGAIN +/// on the changed seq); if the waiter already left and the node was reused, +/// the new owner sees a spurious wake, re-checks under the lock and parks +/// again. +/// +/// A budgeted wake is charged when its node is detached. If futex_wake then +/// reports nobody parked on the word, that waiter was spinning, between its +/// seq sample and futex_wait (it retries and takes the unit itself), or gone +/// (a dead process or a terminated thread, which never will). The walk +/// cannot tell these apart, so it does what the inline walk did: wakes one +/// more waiter for each such unit, in a second locked walk. Over-waking is +/// safe; stranding a unit is not. +struct PendingWakes<'a> { + r: &'a Region, + n: usize, + obj: [u32; PENDING_WAKES], + word: [u32; PENDING_WAKES], + budgeted: u32, // bit i: entry i was charged to a budget +} + +impl<'a> PendingWakes<'a> { + fn new(r: &'a Region) -> Self { + PendingWakes { r, n: 0, obj: [0; PENDING_WAKES], word: [0; PENDING_WAKES], budgeted: 0 } + } + + fn push(&mut self, obj: u32, word: u32, budgeted: bool) { + self.obj[self.n] = obj; + self.word[self.n] = word; + if budgeted { + self.budgeted |= 1 << self.n; + } + self.n += 1; + } + + /// Issue the collected wakes. Call with the region lock released. + fn flush(&mut self) { + let r = self.r; + while self.n > 0 { + // Units whose budgeted wake found nobody parked, per object. + let mut redo = [(0u32, 0u32); PENDING_WAKES]; + let mut nredo = 0; + for i in 0..self.n { + let woken = futex_wake(unsafe { &(*r.node_ptr(self.word[i])).seq }); + stat_add(15, woken as u64); // threads_woken + if woken == 0 && self.budgeted & (1 << i) != 0 { + stat_bump(18); // wakes_unparked + match redo[..nredo].iter_mut().find(|(o, _)| *o == self.obj[i]) { + Some(e) => e.1 += 1, + None => { + redo[nredo] = (self.obj[i], 1); + nredo += 1; + } + } + } + } + self.n = 0; + self.budgeted = 0; + if nredo == 0 { + return; + } + let slot_has_waiters = + |o: u32| ld32(unsafe { &(*r.slot_ptr(o)).waiter_head }) != NIL; + if !redo[..nredo].iter().any(|&(o, _)| slot_has_waiters(o)) { + return; + } + stat_bump(19); // wake_rewalks + if let Ok(_g) = r.lock_guard() { + for &(o, units) in &redo[..nredo] { + wake_object(r, o, units, Some(self)); + } + } + } + } +} + +impl Drop for PendingWakes<'_> { + fn drop(&mut self) { + self.flush(); + } +} + /// Pass on the wakes a wait-any waiter was given for objects it is leaving /// without having taken: bit i of `owed` is handles[i]. Each was one unit of /// some signal op's budget, so one more waiter is woken per object. If the /// unit is gone by now the woken waiter just parks again. Caller holds the /// lock. -fn forward_wakes(r: &Region, handles: &[u32], owed: u64, taken: Option) { +fn forward_wakes(r: &Region, handles: &[u32], owed: u64, taken: Option, later: &mut PendingWakes<'_>) { let mut bits = owed; while bits != 0 { let i = bits.trailing_zeros(); @@ -667,7 +839,7 @@ fn forward_wakes(r: &Region, handles: &[u32], owed: u64, taken: Option) { continue; } stat_bump(17); // wakes_forwarded - wake_object(r, handles[i as usize] & INDEX_MASK, 1); + wake_object(r, handles[i as usize] & INDEX_MASK, 1, Some(&mut *later)); } } @@ -681,13 +853,25 @@ fn wake_waiters_if_any(r: &Region, obj_idx: u32, budget: u32) { return; } stat_bump(8); // wake_walks + let mut later = PendingWakes::new(r); if let Ok(_g) = r.lock_guard() { - wake_object(r, obj_idx, budget); + wake_object(r, obj_idx, budget, Some(&mut later)); } + later.flush(); } fn current_tid() -> u32 { - unsafe { libc::syscall(libc::SYS_gettid) as u32 } + // bionic's gettid() returns the tid cached in the thread struct; the raw + // syscall cost one kernel entry per region-lock acquisition under + // NTSYNC_DEBUG. + #[cfg(target_os = "android")] + unsafe { + libc::gettid() as u32 + } + #[cfg(not(target_os = "android"))] + unsafe { + libc::syscall(libc::SYS_gettid) as u32 + } } fn open_and_map(path: &str) -> Result { @@ -1000,7 +1184,7 @@ pub fn close(handle: u32) -> bool { (&*(&(*slot).state as *const u32 as *const AtomicU32)).store(SLOT_FREE, Ordering::Release); (*slot).generation = (*slot).generation.wrapping_add(1); } - wake_object(r, handle & INDEX_MASK, WAKE_ALL); + wake_object(r, handle & INDEX_MASK, WAKE_ALL, None); true } @@ -1085,7 +1269,7 @@ pub fn event_pulse(handle: u32) -> Result { st64(&(*slot).w0, ld64(&(*slot).w0) & !1u64); prev }; - wake_object(r, handle & INDEX_MASK, WAKE_ALL); + wake_object(r, handle & INDEX_MASK, WAKE_ALL, None); Ok(prev) } @@ -1329,7 +1513,7 @@ fn debug_log(msg: &str) { mod stats { use std::sync::atomic::{AtomicU64, Ordering}; - pub const NAMES: [&str; 18] = [ + pub const NAMES: [&str; 21] = [ "wait_calls", // total wait_any/wait_all calls "wait_fast_ok", // satisfied without any lock "wait_blocked", // actually slept on a futex @@ -1348,13 +1532,17 @@ mod stats { "threads_woken", // waiters a wake walk actually woke (futex_wake > 0) "wakes_deferred", // wait-any waiters left parked: budget spent "wakes_forwarded", // wakes passed on by a waiter that left without the object + "wakes_unparked", // budgeted wakes issued after the unlock that found nobody parked + "wake_rewalks", // second locked walks to re-issue those units + "sigusr1_deferred", // SIGUSR1s the host held back during a region lock, re-raised on release ]; - pub static COUNTERS: [AtomicU64; 18] = [ + pub static COUNTERS: [AtomicU64; 21] = [ + AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), - AtomicU64::new(0), AtomicU64::new(0), + AtomicU64::new(0), ]; #[inline] @@ -1630,6 +1818,10 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert // re-validates authoritatively (a close racing the fast path fails it). loop { + // Wakes passed on below are issued after the guard drops: `later` is + // declared first, so it drops (and flushes) after `_g` on every + // return from this iteration. + let mut later = PendingWakes::new(r); // Take the region lock for the validate/register cycle. Registration // and signalers' wake walks are mutually exclusive, and we re-check // acquirability under the lock before sleeping, so no wakeup is lost. @@ -1681,7 +1873,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert } if !valid { waiter_cleanup(r, &mut nodes); - forward_wakes(r, handles, owed, None); + forward_wakes(r, handles, owed, None, &mut later); return WaitOutcome::Invalid; } @@ -1692,7 +1884,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert let slot = r.slot(alert).expect("validated above"); if ld64(unsafe { &(*slot).w0 }) as u32 != 0 { waiter_cleanup(r, &mut nodes); - forward_wakes(r, handles, owed, None); + forward_wakes(r, handles, owed, None, &mut later); return WaitOutcome::Signaled { index: handles.len() as u32, owner_dead: false, @@ -1752,7 +1944,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert if let Some(outcome) = try_satisfy(r, handles, owner, &entry_seqs, all) { waiter_cleanup(r, &mut nodes); if let WaitOutcome::Signaled { index, .. } = outcome { - forward_wakes(r, handles, owed, Some(index)); + forward_wakes(r, handles, owed, Some(index), &mut later); } return outcome; } @@ -1777,6 +1969,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert // or fails immediately with EAGAIN - never a missed wakeup. let seq = unsafe { (*r.node_ptr(head)).seq.load(Ordering::Acquire) }; drop(_g); + later.flush(); // Bounded spin before sleeping (NTSYNC_SPIN_ITERS, default 0 = off): // a signal that lands within the window is acquired lock-free, @@ -2158,7 +2351,7 @@ fn reap_dead_locked(r: &Region, cands: &[DeadCand], alive: &[bool]) -> u32 { { (*slot).state = SLOT_FREE; (*slot).generation = (*slot).generation.wrapping_add(1); - wake_object(r, c.index, WAKE_ALL); + wake_object(r, c.index, WAKE_ALL, None); freed += 1; } } @@ -2265,6 +2458,18 @@ fn spin_tail_budget_ns() -> u64 { }) } +/// Snapshot of the per-process counters (they count with or without +/// NTSYNC_DEBUG; only the dump thread needs it). +#[cfg(test)] +pub(crate) fn test_counter(i: usize) -> u64 { + stats::COUNTERS[i].load(Ordering::Relaxed) +} + +#[cfg(test)] +pub(crate) fn test_counters() -> Vec<(&'static str, u64)> { + stats::NAMES.iter().enumerate().map(|(i, n)| (*n, stats::COUNTERS[i].load(Ordering::Relaxed))).collect() +} + /// Largest burst a thread may bank: its allowance for 10 ms of wall time. /// The bucket must not hold more than a few frames' worth, or a thread /// spending faster than it refills goes unthrottled for seconds and the @@ -2634,6 +2839,14 @@ pub(crate) fn test_sweep_clock_ns() -> u64 { /// Tests: force the auto-sweep rate limiter to consider the interval /// elapsed, so the next signal op sweeps regardless of wall time. +#[cfg(test)] +pub(crate) fn test_touch_sweep_clock() { + if let Ok(r) = region() { + let cell = unsafe { &*(&r.header().sweep_ns as *const u64 as *const AtomicU64) }; + cell.store(monotonic_ns(), Ordering::Release); + } +} + #[cfg(test)] pub(crate) fn test_reset_sweep_clock() { if let Ok(r) = region() { diff --git a/src/engine_sim.rs b/src/engine_sim.rs index 04f5bde..65d0dd1 100644 --- a/src/engine_sim.rs +++ b/src/engine_sim.rs @@ -752,3 +752,311 @@ fn sim_primitive_costs() { let _ = close(sem); let _ = close(m); } + +// ------------------------------------------------- Unity release cost --- + +/// Thread CPU time (CLOCK_THREAD_CPUTIME_ID) in ns. +fn thread_cpu_ns() -> u64 { + let mut ts = unsafe { std::mem::zeroed::() }; + unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut ts) }; + ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64 +} + +/// (user+sys ns, voluntary switches, involuntary switches) of the calling thread. +fn thread_rusage() -> (u64, u64, u64) { + let mut ru: libc::rusage = unsafe { std::mem::zeroed() }; + unsafe { libc::getrusage(1 /* RUSAGE_THREAD */, &mut ru) }; + let ns = |t: libc::timeval| t.tv_sec as u64 * 1_000_000_000 + t.tv_usec as u64 * 1000; + (ns(ru.ru_utime) + ns(ru.ru_stime), ru.ru_nvcsw as u64, ru.ru_nivcsw as u64) +} + +fn pin_to(cpu: usize) { + unsafe { + let mut set: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_SET(cpu, &mut set); + libc::sched_setaffinity(0, std::mem::size_of::(), &set); + } +} + +thread_local! { + static SIM_FLAG: std::sync::atomic::AtomicI32 = const { std::sync::atomic::AtomicI32::new(0) }; +} + +unsafe extern "C" fn sim_lock_flag() -> *mut i32 { + SIM_FLAG.with(|f| f.as_ptr()) +} + +fn walks_now() -> u64 { + test_counter(8) +} + +fn env_u64(name: &str, default: u64) -> u64 { + std::env::var(name).ok().and_then(|v| v.parse().ok()).unwrap_or(default) +} + +fn spin_us(us: u64) { + if us == 0 { + return; + } + let t = Instant::now(); + let d = Duration::from_micros(us); + while t.elapsed() < d { + std::hint::spin_loop(); + } +} + +/// The Forest's Unity job system as measured in game on the Odin 3: seven +/// workers wait alertably (wait-any with an alert event, as Wine does) on one +/// job semaphore; the main thread releases it one unit at a time, about 300 +/// times per 60 fps frame, in bursts with its own work in between, and waits +/// for each job's completion event (JobHandle.Complete). Each acquired unit is +/// one job of a few microseconds. +/// +/// The judge for the release path is the main thread's CPU per release and +/// its syscalls per release (run under `simpleperf stat --per-thread -e +/// raw_syscalls:sys_enter`; the thread is named `sim-main`), with worker CPU +/// and switches alongside. Knobs (env): SIM_WORKERS 7, SIM_FPS 60, +/// SIM_RELEASES 300 per frame, SIM_BURST 10 releases per burst, SIM_GAP_US +/// main work between two releases of a burst, SIM_MAIN_US main work after a +/// burst before its completion waits, SIM_JOB_US worker work per job, +/// SIM_SECS run length, SIM_PIN 1 = main on the last core, workers on the +/// others. +#[test] +#[ignore] +fn sim_unity_release_cost() { + setup(); + let workers_n = env_u64("SIM_WORKERS", 7) as usize; + let fps = env_u64("SIM_FPS", 60); + let per_frame = env_u64("SIM_RELEASES", 300) as usize; + let burst = env_u64("SIM_BURST", 10).max(1) as usize; + let gap_us = env_u64("SIM_GAP_US", 5); + let main_us = env_u64("SIM_MAIN_US", 150); + let job_us = env_u64("SIM_JOB_US", 60); + let secs = env_u64("SIM_SECS", 8); + let pin = env_u64("SIM_PIN", 1) == 1; + let ncpu = thread::available_parallelism().map(|n| n.get()).unwrap_or(8); + + // SIM_FLAG_HOOK=1: install a host lock-flag callback, as Wine does, so + // lock acquisitions use the flag instead of masking SIGUSR1. + if env_u64("SIM_FLAG_HOOK", 0) == 1 { + set_lock_flag_hook(Some(sim_lock_flag)); + } + let job_sem = create_semaphore(0, 1 << 20).unwrap(); + let done: Arc> = Arc::new((0..burst).map(|_| create_event(false, false).unwrap()).collect()); + let next_job = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + + let mut workers = Vec::new(); + let mut alerts = Vec::new(); + // Threads report over channels and then stay parked until the process + // exits, so simpleperf's per-thread counters are still open at the end. + let (wtx, wrx) = std::sync::mpsc::channel(); + for w in 0..workers_n { + let (stop, done, next_job, wtx) = (stop.clone(), done.clone(), next_job.clone(), wtx.clone()); + let alert = create_event(true, false).unwrap(); + alerts.push(alert); + workers.push( + thread::Builder::new() + .name(format!("sim-w{w}")) + .spawn(move || { + if pin { + pin_to(w % (ncpu - 1)); + } + let r0 = thread_rusage(); + let mut jobs = 0u64; + loop { + match wait_any(&[job_sem], me(), Some(Duration::from_millis(100)), alert) { + WaitOutcome::Signaled { index: 0, .. } => {} + WaitOutcome::Timeout => { + if stop.load(Ordering::Relaxed) { + break; + } + continue; + } + _ => break, + } + if stop.load(Ordering::Relaxed) { + break; + } + let j = next_job.fetch_add(1, Ordering::Relaxed) as usize; + spin_us(job_us); + let _ = event_set(done[j % done.len()]); + jobs += 1; + } + let r1 = thread_rusage(); + let _ = wtx.send((jobs, r1.0 - r0.0, r1.1 - r0.1, r1.2 - r0.2)); + loop { + thread::park(); + } + }) + .unwrap(), + ); + } + // SIM_HOT=1: one SCHED_IDLE spinner per worker core keeps those cores out + // of deep idle, as the game's other threads do, so a woken worker runs + // within a few microseconds (it preempts the spinner) instead of after an + // idle-exit. + let hot = env_u64("SIM_HOT", 0) == 1; + if hot { + for c in 0..ncpu - 1 { + let stop = stop.clone(); + thread::Builder::new() + .name(format!("sim-bg{c}")) + .spawn(move || { + pin_to(c); + let p = libc::sched_param { sched_priority: 0 }; + unsafe { libc::sched_setscheduler(0, 5 /* SCHED_IDLE */, &p) }; + while !stop.load(Ordering::Relaxed) { + std::hint::spin_loop(); + } + }) + .unwrap(); + } + } + thread::sleep(Duration::from_millis(200)); // let every worker park + + let c0 = test_counters(); + let p0 = cpu_time(); + let (mtx, mrx) = std::sync::mpsc::channel(); + let main = { + let done = done.clone(); + thread::Builder::new() + .name("sim-main".into()) + .spawn(move || { + if pin { + pin_to(ncpu - 1); + } + let r0 = thread_rusage(); + let frame = Duration::from_secs_f64(1.0 / fps as f64); + let end = Instant::now() + Duration::from_secs(secs); + let (mut releases, mut rel_cpu, mut rel_wall, mut frames) = (0u64, 0u64, 0u64, 0u64); + // Releases that took a wake walk vs not, and their wall time + // (walks are counted process-wide, but during a release + // section only this thread signals). + let (mut walked, mut walked_ns) = (0u64, 0u64); + while Instant::now() < end { + let fstart = Instant::now(); + let mut left = per_frame; + while left > 0 { + let n = left.min(burst); + left -= n; + // Release section: priced by thread CPU, minus the + // main thread's own spin work in between. + let c = thread_cpu_ns(); + let mut spun = Duration::ZERO; + let mut inside = Duration::ZERO; + for _ in 0..n { + let s = Instant::now(); + spin_us(gap_us); + let w0 = walks_now(); + let t = Instant::now(); + spun += t - s; + let _ = sem_release(job_sem, 1); + let el = t.elapsed(); + inside += el; + if walks_now() != w0 { + walked += 1; + walked_ns += el.as_nanos() as u64; + } + } + rel_cpu += (thread_cpu_ns() - c).saturating_sub(spun.as_nanos() as u64); + rel_wall += inside.as_nanos() as u64; + releases += n as u64; + spin_us(main_us); + for &e in done.iter().take(n) { + let _ = wait_any(&[e], me(), Some(Duration::from_secs(2)), 0); + } + } + frames += 1; + if let Some(rest) = frame.checked_sub(fstart.elapsed()) { + thread::sleep(rest); + } + } + let r1 = thread_rusage(); + println!( + "UNITYSIM main: walked_releases={walked} ({:.3}/release) walk_wall_ns={:.0} nowalk_wall_ns={:.0}", + walked as f64 / releases as f64, + walked_ns as f64 / walked.max(1) as f64, + (rel_wall - walked_ns) as f64 / (releases - walked).max(1) as f64 + ); + let _ = mtx.send((releases, rel_cpu, rel_wall, frames, r1.0 - r0.0, r1.1 - r0.1, r1.2 - r0.2)); + loop { + thread::park(); + } + }) + .unwrap() + }; + let (releases, rel_cpu, rel_wall, frames, main_cpu, main_vcs, main_ivcs) = mrx.recv().unwrap(); + let _ = main; + let c1 = test_counters(); + let p1 = cpu_time(); + stop.store(true, Ordering::Relaxed); + let _ = sem_release(job_sem, workers_n as u32); + let mut w_jobs = 0; + let mut w_cpu = 0; + let mut w_vcs = 0; + let mut w_ivcs = 0; + drop(wtx); + let _ = workers; + for _ in 0..workers_n { + let (j, c, v, i) = wrx.recv().unwrap(); + w_jobs += j; + w_cpu += c; + w_vcs += v; + w_ivcs += i; + } + let d = |name: &str| { + let a = c0.iter().find(|(n, _)| *n == name).unwrap().1; + let b = c1.iter().find(|(n, _)| *n == name).unwrap().1; + (b - a) as f64 + }; + let wall = secs as f64; + let ld = |name: &str| -> f64 { + let i = c0.iter().position(|(n, _)| *n == name).unwrap(); + (c1[i].1 - c0[i].1) as f64 + }; + if ld("wake_lat_cnt") > 0.0 { + println!("UNITYSIM wake_lat_avg_us={:.1}", ld("wake_lat_us_sum") / ld("wake_lat_cnt")); + } + println!( + "UNITYSIM flag_hook={} hot={hot} workers={workers_n} fps={fps} per_frame={per_frame} burst={burst} gap_us={gap_us} main_us={main_us} job_us={job_us} secs={secs} wake_all={} no_sigmask={} debug={}", + std::env::var("SIM_FLAG_HOOK").unwrap_or_default(), + std::env::var("NTSYNC_WAKE_ALL").unwrap_or_default(), + std::env::var("NTSYNC_NO_SIGUSR1_BLOCK").unwrap_or_default(), + std::env::var("NTSYNC_DEBUG").unwrap_or_default(), + ); + println!( + "UNITYSIM frames={frames} fps_got={:.1} releases={releases} rel_cpu_ns={:.0} rel_wall_ns={:.0} main_cpu_pct={:.1} main_vcs_s={:.0} main_ivcs_s={:.0} workers_cpu_pct={:.1} worker_vcs_s={:.0} worker_ivcs_s={:.0} proc_cpu_pct={:.1} jobs={w_jobs}", + frames as f64 / wall, + rel_cpu as f64 / releases as f64, + rel_wall as f64 / releases as f64, + main_cpu as f64 / wall / 1e7, + main_vcs as f64 / wall, + main_ivcs as f64 / wall, + w_cpu as f64 / wall / 1e7, + w_vcs as f64 / wall / workers_n as f64, + w_ivcs as f64 / wall / workers_n as f64, + (p1 - p0).as_secs_f64() / wall * 100.0, + ); + let ops = d("signal_ops"); + println!( + "UNITYSIM walks_per_op={:.3} woken_per_walk={:.3} woken_per_op={:.3} blocked_per_wait={:.3} lock_acq_per_rel={:.3} contended_per_rel={:.3} walks_s={:.0} lock_acq_s={:.0} contended_s={:.0} deferred_s={:.0} forwarded={:.0} zero_wakes_s={:.0} rewalks_s={:.0}", + d("wake_walks") / ops, + d("threads_woken") / d("wake_walks"), + d("threads_woken") / ops, + d("wait_blocked") / d("wait_calls"), + d("lock_acq") / releases as f64, + d("lock_contended") / releases as f64, + d("wake_walks") / wall, + d("lock_acq") / wall, + d("lock_contended") / wall, + d("wakes_deferred") / wall, + d("wakes_forwarded"), + c1.iter().find(|(n, _)| *n == "wakes_unparked").map_or(0.0, |_| d("wakes_unparked")) / wall, + c1.iter().find(|(n, _)| *n == "wake_rewalks").map_or(0.0, |_| d("wake_rewalks")) / wall, + ); + for e in done.iter().copied().chain(alerts) { + let _ = close(e); + } + let _ = close(job_sem); +} diff --git a/src/ffi.rs b/src/ffi.rs index c7c9154..78ce6d6 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -130,6 +130,14 @@ pub unsafe extern "C" fn ntsync_init(path: *const std::ffi::c_char) -> i32 { } } +/// Install the host's per-thread lock-flag callback (NULL removes it). See +/// ntsync_user.h for the protocol its SIGUSR1 handler must follow. Returns 0. +#[no_mangle] +pub extern "C" fn ntsync_set_lock_flag_hook(hook: Option) -> i32 { + core::set_lock_flag_hook(hook); + 0 +} + /// Free all objects whose creator process no longer exists. Returns the /// number of freed objects, or a negative errno. #[no_mangle] diff --git a/src/lib.rs b/src/lib.rs index 625e193..e5fbfd7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -349,6 +349,171 @@ mod tests { decoy_passes_its_wake_on(false, 10300); } + /// A budgeted wake is charged when its node is detached and issued after + /// the unlock. If the oldest waiter belongs to a process that died, the + /// wake finds nobody parked, and the unit must go to the next waiter + /// instead of staying in the semaphore with that waiter asleep. + #[test] + fn dead_waiter_does_not_strand_a_unit() { + setup(); + let sem = create_semaphore(0, 10).unwrap(); + // Keep the auto-sweep from purging the dead node before the release. + test_touch_sweep_clock(); + let pid = unsafe { libc::fork() }; + if pid == 0 { + let _ = wait_any(&[sem], 11001, None, 0); + unsafe { libc::_exit(0) }; + } + wait_until("child waiter to park", || test_waiter_counts(sem).0 == 1); + unsafe { + libc::kill(pid, libc::SIGKILL); + libc::waitpid(pid, std::ptr::null_mut(), 0); + } + let live = thread::spawn(move || { + assert!(is_signaled(wait_any(&[sem], 11002, Some(Duration::from_secs(10)), 0), 0)); + }); + wait_until("live waiter to park", || test_waiter_counts(sem).0 == 2); + sem_release(sem, 1).unwrap(); + let start = std::time::Instant::now(); + while !live.is_finished() { + assert!(start.elapsed() < Duration::from_secs(5), "unit stranded behind a dead waiter"); + thread::sleep(Duration::from_millis(1)); + } + live.join().unwrap(); + assert_eq!(read_sem(sem).unwrap().0, 0); + assert!(close(sem)); + } + + /// Host cooperation for SIGUSR1 (ntsync_set_lock_flag_hook), modelled on + /// Wine: the handler "suspends" the thread (blocks until resumed) unless + /// the thread's lock flag is set, in which case it marks the signal + /// pending and returns; the library re-raises it after the unlock. + mod usr1 { + use super::super::core::*; + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; + + pub static TARGET: AtomicU32 = AtomicU32::new(0); + pub static FLAG: AtomicI32 = AtomicI32::new(0); + pub static SUSPENDS: AtomicU32 = AtomicU32::new(0); + pub static SUSPENDED: AtomicBool = AtomicBool::new(false); + pub static RESUME: AtomicBool = AtomicBool::new(false); + /// Negative control: the handler suspends regardless of the flag. + pub static IGNORE_FLAG: AtomicBool = AtomicBool::new(false); + + pub unsafe extern "C" fn lock_flag() -> *mut i32 { + if unsafe { libc::gettid() } as u32 == TARGET.load(Ordering::SeqCst) { + FLAG.as_ptr() + } else { + std::ptr::null_mut() + } + } + + pub extern "C" fn handler(_: i32) { + if !IGNORE_FLAG.load(Ordering::SeqCst) && FLAG.load(Ordering::SeqCst) != 0 { + FLAG.fetch_or(LOCK_FLAG_SIGNAL_PENDING, Ordering::SeqCst); + return; + } + SUSPENDS.fetch_add(1, Ordering::SeqCst); + SUSPENDED.store(true, Ordering::SeqCst); + let ts = libc::timespec { tv_sec: 0, tv_nsec: 50_000 }; + while !RESUME.load(Ordering::SeqCst) { + unsafe { libc::nanosleep(&ts, std::ptr::null_mut()) }; + } + RESUME.store(false, Ordering::SeqCst); + SUSPENDED.store(false, Ordering::SeqCst); + } + + pub fn install() { + unsafe { + let mut sa: libc::sigaction = std::mem::zeroed(); + sa.sa_sigaction = handler as extern "C" fn(i32) as usize; + sa.sa_flags = libc::SA_RESTART; + libc::sigemptyset(&mut sa.sa_mask); + libc::sigaction(libc::SIGUSR1, &sa, std::ptr::null_mut()); + } + set_lock_flag_hook(Some(lock_flag)); + } + } + + /// Suspend a thread that is busy taking and dropping the region lock, + /// `rounds` times, and each time check that another thread can still + /// take the lock while it is suspended. Returns (wedged rounds, + /// signals the library deferred). + fn suspend_lock_holder(rounds: u32, ignore_flag: bool) -> (u32, u64) { + setup(); + usr1::install(); + usr1::IGNORE_FLAG.store(ignore_flag, Ordering::SeqCst); + usr1::SUSPENDS.store(0, Ordering::SeqCst); + let stop = Arc::new(AtomicBool::new(false)); + let (tx, rx) = std::sync::mpsc::channel(); + let busy = { + let stop = stop.clone(); + thread::spawn(move || { + let tid = unsafe { libc::gettid() } as u32; + usr1::TARGET.store(tid, Ordering::SeqCst); + tx.send(tid).unwrap(); + while !stop.load(Ordering::Relaxed) { + // create and close each take the region lock. + let e = create_event(false, false).unwrap(); + assert!(close(e)); + } + }) + }; + let tid = rx.recv().unwrap(); + let deferred0 = test_counter(20); + let mut wedged = 0; + for round in 0..rounds { + unsafe { libc::syscall(libc::SYS_tgkill, libc::getpid(), tid as i32, libc::SIGUSR1) }; + let start = std::time::Instant::now(); + while !usr1::SUSPENDED.load(Ordering::SeqCst) { + assert!(start.elapsed() < Duration::from_secs(5), "round {round}: SIGUSR1 lost"); + std::hint::spin_loop(); + } + // Another thread must get the region lock while the target is + // suspended. + let probe = thread::spawn(|| { + let e = create_event(false, false).unwrap(); + assert!(close(e)); + }); + let start = std::time::Instant::now(); + while !probe.is_finished() && start.elapsed() < Duration::from_millis(300) { + thread::sleep(Duration::from_micros(200)); + } + if !probe.is_finished() { + wedged += 1; + } + usr1::RESUME.store(true, Ordering::SeqCst); + while usr1::SUSPENDED.load(Ordering::SeqCst) { + std::hint::spin_loop(); + } + probe.join().unwrap(); + } + stop.store(true, Ordering::Relaxed); + busy.join().unwrap(); + usr1::TARGET.store(0, Ordering::SeqCst); + usr1::IGNORE_FLAG.store(false, Ordering::SeqCst); + set_lock_flag_hook(None); + assert_eq!(usr1::SUSPENDS.load(Ordering::SeqCst), rounds, "every SIGUSR1 suspends exactly once"); + (wedged, test_counter(20) - deferred0) + } + + #[test] + fn sigusr1_in_region_lock_is_deferred_not_lost() { + let (wedged, deferred) = suspend_lock_holder(300, false); + eprintln!("300 suspends: {wedged} wedged, {deferred} deferred inside the lock"); + assert_eq!(wedged, 0, "a thread suspended inside the region lock wedged it"); + assert!(deferred > 0, "no SIGUSR1 landed inside the lock; the test proved nothing"); + } + + /// Negative control for the test above: a handler that ignores the flag + /// suspends the thread inside the lock and wedges it. + #[test] + fn sigusr1_in_region_lock_wedges_without_deferral() { + let (wedged, _) = suspend_lock_holder(300, true); + eprintln!("300 suspends, flag ignored: {wedged} wedged"); + assert!(wedged > 0, "no round wedged; the deferral test cannot tell a fix from luck"); + } + /// No lost wakeups under contention. Exactly as many units are released /// as consumers need, so a wake spent on a thread that never takes its /// unit would leave a consumer parked forever: every consumer must