diff --git a/README.md b/README.md index 2cbfc3c..47fe508 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,24 @@ mutex consistent. set this when the host process never installs a SIGUSR1 handler (i.e. not under Wine); it measurably reduces lock-path CPU under contention (~13% per-op in the multi-process benchmark). + A host that does install such a handler can avoid both syscalls with + `ntsync_set_lock_flag_hook` instead: the library sets a per-thread flag + the host owns while the thread holds the lock, the host's SIGUSR1 handler + marks the signal pending and returns while the flag is set, and the + library re-raises SIGUSR1 right after the unlock (protocol in + `include/ntsync_user.h`). Threads for which the hook returns NULL keep + the masking. +- `NTSYNC_WAKE_ALL=1` — every signal op wakes all of an object's waiters + instead of only as many wait-any waiters as it made units available. Kept + for comparison; the default is cheaper for the releasing thread once its + wakes are issued after the unlock (see `sim_unity_release_cost`). - `NTSYNC_DEBUG=1` — log to stderr, stdout and `$TMPDIR/ntsync_debug.log` (the file is the reliable channel for in-game diagnostics); dump per-process stats every 10 s (waits, fast-path hits, lock acquisitions, - wake walks, signal→scheduled wake latency `wake_lat_cnt/us_sum/us_max`); + wake walks, signal→scheduled wake latency `wake_lat_cnt/us_sum/us_max`, + budgeted wakes that found nobody parked `wakes_unparked` and the second + walks that re-issued them `wake_rewalks`, SIGUSR1s held back by the + lock-flag hook `sigusr1_deferred`); emit a "stuck wait" dump with per-object state (including creator pid liveness) when a wait stalls. - Object lifetime: an object is reaped only when its creator **and every diff --git a/include/ntsync_user.h b/include/ntsync_user.h index 5578fca..06acc12 100644 --- a/include/ntsync_user.h +++ b/include/ntsync_user.h @@ -95,6 +95,25 @@ int32_t ntsync_init(const char *path); * process exits. Returns the number of freed objects or a negative errno. */ int32_t ntsync_sweep_dead(void); +/* Optional SIGUSR1 cooperation, for hosts whose SIGUSR1 handler suspends the + * thread (Wine). A thread suspended while it holds the shared region lock + * would wedge every process on it, so by default the library blocks SIGUSR1 + * around each lock acquisition: two sigprocmask syscalls per acquisition. + * A host can instead give each thread an int the library sets while the + * thread holds (or waits for) the lock: + * - get_flag returns the calling thread's int (NULL: fall back to masking + * for this acquisition). It is called in normal context only. + * - Before taking the lock the library stores NTSYNC_LOCK_FLAG_HELD; after + * releasing it, it atomically swaps the int to 0 and, if + * NTSYNC_LOCK_FLAG_SIGNAL_PENDING was set, re-raises SIGUSR1. + * - The host's SIGUSR1 handler must, before doing anything else, check the + * int: if nonzero, atomically OR in NTSYNC_LOCK_FLAG_SIGNAL_PENDING and + * return without suspending. + * Pass NULL to uninstall. Returns 0. */ +#define NTSYNC_LOCK_FLAG_HELD 1 +#define NTSYNC_LOCK_FLAG_SIGNAL_PENDING 2 +int32_t ntsync_set_lock_flag_hook(int *(*get_flag)(void)); + /* Create objects. Return 0 and store the handle, or a negative errno. * Handles are never 0: slot 0 is permanently reserved because 0 is the * "no alert" sentinel in ntsync_wait_args.alert. */ diff --git a/src/core.rs b/src/core.rs index 3652f11..9852d79 100644 --- a/src/core.rs +++ b/src/core.rs @@ -33,7 +33,7 @@ //! ntsync_sweep_dead(). use std::ffi::CString; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicI32, AtomicU32, AtomicU64, Ordering}; use std::sync::OnceLock; use std::time::Duration; @@ -355,26 +355,38 @@ impl Region { let mut waited = std::time::Duration::ZERO; // Wine suspends threads with SIGUSR1; if it lands while we hold the // shared region mutex the holder stays parked and every process - // (including wineserver) wedges on the lock. Defer it until release. - // Consumers without a SIGUSR1 handler can set - // NTSYNC_NO_SIGUSR1_BLOCK=1 to skip the two pthread_sigmask - // syscalls per lock acquisition. - let old_mask = if sigusr1_block_enabled() { - let mut old_mask = unsafe { std::mem::zeroed::() }; - let mut block = unsafe { std::mem::zeroed::() }; - unsafe { - libc::sigemptyset(&mut block); - libc::sigaddset(&mut block, libc::SIGUSR1); - libc::pthread_sigmask(libc::SIG_BLOCK, &block, &mut old_mask); + // (including wineserver) wedges on the lock. Defer it until release: + // through the host's per-thread lock flag when it installed one + // (ntsync_set_lock_flag_hook: its SIGUSR1 handler holds the signal + // back while the flag is set, no syscalls), else by blocking SIGUSR1 + // for the duration (two pthread_sigmask syscalls). Consumers without + // a SIGUSR1 handler can set NTSYNC_NO_SIGUSR1_BLOCK=1 to skip both. + let defer = match lock_flag() { + Some(flag) => { + flag.store(LOCK_FLAG_HELD, Ordering::SeqCst); + Defer::Flag(flag) } - Some(old_mask) - } else { - None + None if sigusr1_block_enabled() => { + let mut old_mask = unsafe { std::mem::zeroed::() }; + let mut block = unsafe { std::mem::zeroed::() }; + unsafe { + libc::sigemptyset(&mut block); + libc::sigaddset(&mut block, libc::SIGUSR1); + libc::pthread_sigmask(libc::SIG_BLOCK, &block, &mut old_mask); + } + Defer::Mask(old_mask) + } + None => Defer::Nothing, }; - let mut tries = 0u32; + // A failed trylock is the contention count; it costs one CAS, which + // pthread_mutex_lock would try first anyway. + let first = unsafe { libc::pthread_mutex_trylock(lock) }; + let mut tries = if first == libc::EBUSY { 1u32 } else { 0 }; loop { tries += 1; - let ret = if debug { + let ret = if tries == 1 { + first + } else if debug { // Poll with a timeout so a stuck holder can be logged. let mut ts = unsafe { std::mem::zeroed::() }; unsafe { libc::clock_gettime(libc::CLOCK_REALTIME, &mut ts) }; @@ -401,7 +413,7 @@ impl Region { (*h).lock_owner_tid = current_tid(); } } - return Ok(RegionGuard { region: self, old_mask }); + return Ok(RegionGuard { region: self, defer }); } if ret == libc::ETIMEDOUT { waited += std::time::Duration::from_secs(5); @@ -417,20 +429,83 @@ impl Region { continue; } if ret != libc::EINTR { - if let Some(old_mask) = old_mask { - unsafe { - libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut()) - }; - } + defer.end(); return Err(-ret); } } } } +/// How a region-lock holder keeps SIGUSR1 (Wine's suspend) from parking it +/// inside the lock. +enum Defer { + /// No SIGUSR1 handler to worry about (NTSYNC_NO_SIGUSR1_BLOCK=1). + Nothing, + /// SIGUSR1 blocked; the previous mask is restored on release. + Mask(libc::sigset_t), + /// The host's per-thread lock flag is set; its handler defers SIGUSR1 + /// while it is, and marks it pending. + Flag(&'static AtomicI32), +} + +impl Defer { + /// Called once the lock is released (or was never taken). + fn end(&self) { + match self { + Defer::Nothing => {} + Defer::Mask(old_mask) => unsafe { + libc::pthread_sigmask(libc::SIG_SETMASK, old_mask, std::ptr::null_mut()); + }, + Defer::Flag(flag) => { + // Atomic swap: a SIGUSR1 that lands before it sees the flag + // set and leaves PENDING for this read; one after it finds the + // flag clear and suspends the thread right there, outside the + // lock. Either way it is delivered exactly once. + if flag.swap(0, Ordering::SeqCst) & LOCK_FLAG_SIGNAL_PENDING != 0 { + stat_bump(20); // sigusr1_deferred + unsafe { libc::raise(libc::SIGUSR1) }; + } + } + } + } +} + +/// ntsync_set_lock_flag_hook protocol: the library stores LOCK_FLAG_HELD in +/// the thread's flag before it takes the region lock and swaps it back to 0 +/// after the unlock. The host's SIGUSR1 handler, finding it nonzero, ORs in +/// LOCK_FLAG_SIGNAL_PENDING and returns without suspending; the library then +/// re-raises SIGUSR1 at the swap. +pub const LOCK_FLAG_HELD: i32 = 1; +pub const LOCK_FLAG_SIGNAL_PENDING: i32 = 2; + +/// Host callback returning the calling thread's lock flag (or null to fall +/// back to masking SIGUSR1 for this acquisition). +pub type LockFlagHook = unsafe extern "C" fn() -> *mut i32; + +static LOCK_FLAG_HOOK: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Install (or with None remove) the host's lock-flag callback. +pub fn set_lock_flag_hook(hook: Option) { + LOCK_FLAG_HOOK.store(hook.map_or(0, |f| f as usize), Ordering::SeqCst); +} + +fn lock_flag() -> Option<&'static AtomicI32> { + let f = LOCK_FLAG_HOOK.load(Ordering::Relaxed); + if f == 0 { + return None; + } + let hook: LockFlagHook = unsafe { std::mem::transmute::(f) }; + let p = unsafe { hook() }; + if p.is_null() { + None + } else { + Some(unsafe { &*(p as *const AtomicI32) }) + } +} + struct RegionGuard<'a> { region: &'a Region, - old_mask: Option, + defer: Defer, } impl Drop for RegionGuard<'_> { @@ -441,10 +516,8 @@ impl Drop for RegionGuard<'_> { (*h).lock_owner_tid = 0; let lock = &(*h).lock as *const libc::pthread_mutex_t as *mut _; libc::pthread_mutex_unlock(lock); - if let Some(old_mask) = &self.old_mask { - libc::pthread_sigmask(libc::SIG_SETMASK, old_mask, std::ptr::null_mut()); - } } + self.defer.end(); } } @@ -472,11 +545,13 @@ fn futex_wait(addr: *const AtomicU32, expected: u32, timeout: Option) } } -fn futex_wake(addr: *const AtomicU32) { +/// Returns how many threads the kernel woke: 1 if the waiter was parked on +/// the word, 0 if it was not (spinning, between its seq sample and the +/// futex call, or gone). +fn futex_wake(addr: *const AtomicU32) -> u32 { // Each head node is slept on by exactly one waiter, so wake one. - unsafe { - libc::syscall(libc::SYS_futex, addr as *const u32, libc::FUTEX_WAKE, 1, 0, 0, 0); - } + let ret = unsafe { libc::syscall(libc::SYS_futex, addr as *const u32, libc::FUTEX_WAKE, 1, 0, 0, 0) }; + if ret > 0 { ret as u32 } else { 0 } } /// Pop a node from the free list, or hand out a never-before-used node @@ -547,13 +622,34 @@ fn node_detach(r: &Region, idx: u32) { } } +/// `registered` values. Readers only ever test it against 0, so a builder +/// that predates REG_SINGLE treats both values as "on the list", and this +/// one treats an older builder's 1 as a waiter it must always wake: the +/// shared layout is unchanged in both directions. +const REG_MULTI: u32 = 1; +/// The node is the only one of its waiter: a single-object, non-alertable +/// wait. Such a waiter is satisfied by this object alone and, once woken, +/// always attempts to acquire it, so a signal op may wake only as many of +/// them as it can satisfy (see wake_object). +const REG_SINGLE: u32 = 2; +/// An object node of a wait-any waiter that has other nodes too (several +/// objects, or an alert event). It is woken within the budget like a +/// REG_SINGLE one, but such a waiter can return without taking the object it +/// was woken for (the alert fired, or an earlier object in its list was +/// signaled). It then passes the wake on to the next waiter: see the `owed` +/// mask in wait(). +const REG_ANY: u32 = 3; + +/// Wake budget meaning "every waiter". +const WAKE_ALL: u32 = u32::MAX; + /// Register `node` on object `obj_idx`'s waiter list. Caller holds the lock. -fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32) { +fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32, reg: u32) { let node = r.node_ptr(idx); let slot = r.slot_ptr(obj_idx); unsafe { let old_head = (*slot).waiter_head; - (*node).registered = 1; + (*node).registered = reg; (*node).obj = obj_idx; (*node).wait_word = wait_word; (*node).obj_prev = NIL; @@ -565,16 +661,54 @@ fn node_register(r: &Region, idx: u32, obj_idx: u32, wait_word: u32) { } } -/// Wake every waiter registered on an object (kernel: wake_up(&obj->q)). -/// Caller holds the lock. Each waiter is detached and its head seq bumped; -/// it re-checks conditions when scheduled. -fn wake_object(r: &Region, obj_idx: u32) { +/// Wake the waiters registered on an object. Caller holds the lock. +/// +/// `budget` caps how many wait-any waiters (REG_SINGLE, REG_ANY) are woken: +/// the units a signal op made available (semaphore release count, 1 for an +/// auto-reset event or a freed mutex). Waking more of them only makes the +/// extras run, fail to acquire, take the region lock twice and park again, +/// all while the signaler pays one futex syscall per waiter - Unity's job +/// system, which releases a semaphore to N parked workers many times per +/// frame, spent about a fifth of its main thread there. Wine waits +/// alertably by default, so those workers are REG_ANY, not REG_SINGLE. +/// +/// A woken wait-any waiter retries the acquire (wait() re-registers and +/// calls try_satisfy under the lock before any timeout), so each unit of +/// budget lands on a thread that takes the object, finds it already taken, +/// or - REG_ANY only - returns for another reason and wakes the next waiter +/// in its place. The budget is charged only when futex_wake reports a +/// thread actually woken: a waiter that is spinning, about to park, or dead +/// costs no budget, and the next one is woken too. +/// +/// Wait-all waiters and alert-event nodes (REG_MULTI) are always woken, as +/// before. WAKE_ALL wakes everyone (close, pulse, manual-reset events, +/// reaping). +/// +/// Waiters are woken oldest first (node_register pushes at the head), so a +/// contended mutex or semaphore cannot starve an early waiter. +fn wake_object(r: &Region, obj_idx: u32, budget: u32, mut later: Option<&mut PendingWakes<'_>>) { let debug = debug_enabled(); + let mut left = if wake_all_forced() { WAKE_ALL } else { budget }; let slot = r.slot_ptr(obj_idx); let mut idx = unsafe { (*slot).waiter_head }; + let mut guard = 0; + while idx != NIL && guard < NODE_COUNT { + let next = unsafe { (*r.node_ptr(idx)).obj_next }; + if next == NIL { + break; + } + idx = next; + guard += 1; + } while idx != NIL { let node = r.node_ptr(idx); - let (next, word) = unsafe { ((*node).obj_next, (*node).wait_word) }; + let (prev, word, reg) = unsafe { ((*node).obj_prev, (*node).wait_word, (*node).registered) }; + let budgeted = reg == REG_SINGLE || reg == REG_ANY; + if budgeted && left == 0 { + stat_bump(16); // wakes_deferred + idx = prev; + continue; + } node_detach(r, idx); let head = r.node_ptr(word); unsafe { @@ -582,9 +716,130 @@ fn wake_object(r: &Region, obj_idx: u32) { (*head).wake_ts.store(monotonic_ns(), Ordering::Relaxed); } (*head).seq.fetch_add(1, Ordering::Release); - futex_wake(&(*head).seq); } - idx = next; + if let Some(pw) = later.as_deref_mut().filter(|pw| pw.n < PENDING_WAKES) { + // Woken after the unlock (PendingWakes::flush), so the thread + // does not run straight into the lock we still hold. The unit is + // charged now; flush gives it back if nobody was parked. + pw.push(obj_idx, word, budgeted && left != WAKE_ALL); + if budgeted && left != WAKE_ALL { + left -= 1; + } + } else { + let woken = futex_wake(unsafe { &(*head).seq }); + stat_add(15, woken as u64); // threads_woken + if budgeted && woken > 0 && left != WAKE_ALL { + left -= 1; + } + } + idx = prev; + } +} + +/// Most wakes one locked walk defers past the unlock; any beyond are woken +/// inline. A semaphore release wakes at most its count, an event or mutex 1. +const PENDING_WAKES: usize = 16; + +/// Futex wakes collected under the region lock and issued after it is +/// dropped (flush, or Drop: declare it before the guard so it drops after). +/// +/// A late FUTEX_WAKE is safe: the node was detached and its seq bumped under +/// the lock, so its waiter cannot sleep through it (futex_wait returns EAGAIN +/// on the changed seq); if the waiter already left and the node was reused, +/// the new owner sees a spurious wake, re-checks under the lock and parks +/// again. +/// +/// A budgeted wake is charged when its node is detached. If futex_wake then +/// reports nobody parked on the word, that waiter was spinning, between its +/// seq sample and futex_wait (it retries and takes the unit itself), or gone +/// (a dead process or a terminated thread, which never will). The walk +/// cannot tell these apart, so it does what the inline walk did: wakes one +/// more waiter for each such unit, in a second locked walk. Over-waking is +/// safe; stranding a unit is not. +struct PendingWakes<'a> { + r: &'a Region, + n: usize, + obj: [u32; PENDING_WAKES], + word: [u32; PENDING_WAKES], + budgeted: u32, // bit i: entry i was charged to a budget +} + +impl<'a> PendingWakes<'a> { + fn new(r: &'a Region) -> Self { + PendingWakes { r, n: 0, obj: [0; PENDING_WAKES], word: [0; PENDING_WAKES], budgeted: 0 } + } + + fn push(&mut self, obj: u32, word: u32, budgeted: bool) { + self.obj[self.n] = obj; + self.word[self.n] = word; + if budgeted { + self.budgeted |= 1 << self.n; + } + self.n += 1; + } + + /// Issue the collected wakes. Call with the region lock released. + fn flush(&mut self) { + let r = self.r; + while self.n > 0 { + // Units whose budgeted wake found nobody parked, per object. + let mut redo = [(0u32, 0u32); PENDING_WAKES]; + let mut nredo = 0; + for i in 0..self.n { + let woken = futex_wake(unsafe { &(*r.node_ptr(self.word[i])).seq }); + stat_add(15, woken as u64); // threads_woken + if woken == 0 && self.budgeted & (1 << i) != 0 { + stat_bump(18); // wakes_unparked + match redo[..nredo].iter_mut().find(|(o, _)| *o == self.obj[i]) { + Some(e) => e.1 += 1, + None => { + redo[nredo] = (self.obj[i], 1); + nredo += 1; + } + } + } + } + self.n = 0; + self.budgeted = 0; + if nredo == 0 { + return; + } + let slot_has_waiters = + |o: u32| ld32(unsafe { &(*r.slot_ptr(o)).waiter_head }) != NIL; + if !redo[..nredo].iter().any(|&(o, _)| slot_has_waiters(o)) { + return; + } + stat_bump(19); // wake_rewalks + if let Ok(_g) = r.lock_guard() { + for &(o, units) in &redo[..nredo] { + wake_object(r, o, units, Some(self)); + } + } + } + } +} + +impl Drop for PendingWakes<'_> { + fn drop(&mut self) { + self.flush(); + } +} + +/// Pass on the wakes a wait-any waiter was given for objects it is leaving +/// without having taken: bit i of `owed` is handles[i]. Each was one unit of +/// some signal op's budget, so one more waiter is woken per object. If the +/// unit is gone by now the woken waiter just parks again. Caller holds the +/// lock. +fn forward_wakes(r: &Region, handles: &[u32], owed: u64, taken: Option, later: &mut PendingWakes<'_>) { + let mut bits = owed; + while bits != 0 { + let i = bits.trailing_zeros(); + bits &= bits - 1; + if taken == Some(i) { + continue; + } + stat_bump(17); // wakes_forwarded + wake_object(r, handles[i as usize] & INDEX_MASK, 1, Some(&mut *later)); } } @@ -592,19 +847,31 @@ fn wake_object(r: &Region, obj_idx: u32) { /// The race with a waiter that registers just after our NIL load is closed /// by the waiter re-checking acquirability under the region lock before it /// sleeps (registration and wake walks are mutually exclusive). -fn wake_waiters_if_any(r: &Region, obj_idx: u32) { +fn wake_waiters_if_any(r: &Region, obj_idx: u32, budget: u32) { let slot = unsafe { &*r.slot_ptr(obj_idx) }; if ld32(&slot.waiter_head) == NIL { return; } stat_bump(8); // wake_walks + let mut later = PendingWakes::new(r); if let Ok(_g) = r.lock_guard() { - wake_object(r, obj_idx); + wake_object(r, obj_idx, budget, Some(&mut later)); } + later.flush(); } fn current_tid() -> u32 { - unsafe { libc::syscall(libc::SYS_gettid) as u32 } + // bionic's gettid() returns the tid cached in the thread struct; the raw + // syscall cost one kernel entry per region-lock acquisition under + // NTSYNC_DEBUG. + #[cfg(target_os = "android")] + unsafe { + libc::gettid() as u32 + } + #[cfg(not(target_os = "android"))] + unsafe { + libc::syscall(libc::SYS_gettid) as u32 + } } fn open_and_map(path: &str) -> Result { @@ -917,7 +1184,7 @@ pub fn close(handle: u32) -> bool { (&*(&(*slot).state as *const u32 as *const AtomicU32)).store(SLOT_FREE, Ordering::Release); (*slot).generation = (*slot).generation.wrapping_add(1); } - wake_object(r, handle & INDEX_MASK); + wake_object(r, handle & INDEX_MASK, WAKE_ALL, None); true } @@ -937,7 +1204,7 @@ pub fn sem_release(handle: u32, count: u32) -> Result { }; match cas64(w0, w, pack_sem(sum, max)) { Ok(_) => { - wake_waiters_if_any(r, handle & INDEX_MASK); + wake_waiters_if_any(r, handle & INDEX_MASK, count); return Ok(cur); } Err(_) => continue, @@ -958,7 +1225,10 @@ pub fn event_set(handle: u32) -> Result { } match cas64(w0, w, w | 1) { Ok(_) => { - wake_waiters_if_any(r, handle & INDEX_MASK); + // An auto-reset event satisfies one waiter; a manual-reset + // one stays signaled and satisfies all of them. + let manual = (w >> 32) & 1 != 0; + wake_waiters_if_any(r, handle & INDEX_MASK, if manual { WAKE_ALL } else { 1 }); return Ok(0); } Err(_) => continue, @@ -999,7 +1269,7 @@ pub fn event_pulse(handle: u32) -> Result { st64(&(*slot).w0, ld64(&(*slot).w0) & !1u64); prev }; - wake_object(r, handle & INDEX_MASK); + wake_object(r, handle & INDEX_MASK, WAKE_ALL, None); Ok(prev) } @@ -1027,7 +1297,7 @@ pub fn mutex_unlock(handle: u32, owner: u32) -> Result { match cas64(w0, w, new) { Ok(_) => { if freed { - wake_waiters_if_any(r, handle & INDEX_MASK); + wake_waiters_if_any(r, handle & INDEX_MASK, 1); } return Ok(count); } @@ -1053,7 +1323,7 @@ pub fn mutex_kill(handle: u32, owner: u32) -> Result<(), Error> { } match cas64(w0, w, pack_mutex(0, 0, true)) { Ok(_) => { - wake_waiters_if_any(r, handle & INDEX_MASK); + wake_waiters_if_any(r, handle & INDEX_MASK, 1); return Ok(()); } Err(_) => continue, @@ -1243,7 +1513,7 @@ fn debug_log(msg: &str) { mod stats { use std::sync::atomic::{AtomicU64, Ordering}; - pub const NAMES: [&str; 15] = [ + pub const NAMES: [&str; 21] = [ "wait_calls", // total wait_any/wait_all calls "wait_fast_ok", // satisfied without any lock "wait_blocked", // actually slept on a futex @@ -1259,12 +1529,20 @@ mod stats { "wake_lat_us_max", // worst wake latency, us "spin_wins", // waits satisfied during the pre-sleep spin "spin_exhausted", // spins that still ended in a futex sleep + "threads_woken", // waiters a wake walk actually woke (futex_wake > 0) + "wakes_deferred", // wait-any waiters left parked: budget spent + "wakes_forwarded", // wakes passed on by a waiter that left without the object + "wakes_unparked", // budgeted wakes issued after the unlock that found nobody parked + "wake_rewalks", // second locked walks to re-issue those units + "sigusr1_deferred", // SIGUSR1s the host held back during a region lock, re-raised on release ]; - pub static COUNTERS: [AtomicU64; 15] = [ + pub static COUNTERS: [AtomicU64; 21] = [ + AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), + AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), - AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), + AtomicU64::new(0), ]; #[inline] @@ -1501,6 +1779,10 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert let mut entry_seqs = [0u64; MAX_WAIT_COUNT]; let n_nodes = handles.len() + (alert != 0) as usize; let mut nodes = NodeSet::new(); + // Object nodes this thread detached itself after its last park (timeout, + // EINTR, or a wake that came through another node). Every other node + // found detached at the top of the loop was detached by a signaler. + let mut self_detached = 0u64; // ---- Fast path: validate and try to acquire without any lock. ---- let fast_valid = handles.iter().enumerate().all(|(i, h)| { @@ -1536,6 +1818,10 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert // re-validates authoritatively (a close racing the fast path fails it). loop { + // Wakes passed on below are issued after the guard drops: `later` is + // declared first, so it drops (and flushes) after `_g` on every + // return from this iteration. + let mut later = PendingWakes::new(r); // Take the region lock for the validate/register cycle. Registration // and signalers' wake walks are mutually exclusive, and we re-check // acquirability under the lock before sleeping, so no wakeup is lost. @@ -1547,6 +1833,21 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert return WaitOutcome::Invalid; }; + // Objects whose signaler woke this wait-any waiter since it last + // parked (see REG_ANY). Leaving without taking one of them passes + // its wake on; a failed acquire below clears the debt, because the + // unit is then gone and this thread parks on the object again. + let mut owed = 0u64; + if !all && n_nodes > 1 && !nodes.is_empty() { + for i in 0..handles.len() { + if unsafe { (*r.node_ptr(nodes[i])).registered } == 0 { + owed |= 1 << i; + } + } + owed &= !self_detached; + } + self_detached = 0; + // Authoritative validation (also catches objects closed mid-wait). let mut valid = true; for (i, h) in handles.iter().enumerate() { @@ -1572,6 +1873,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert } if !valid { waiter_cleanup(r, &mut nodes); + forward_wakes(r, handles, owed, None, &mut later); return WaitOutcome::Invalid; } @@ -1582,6 +1884,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert let slot = r.slot(alert).expect("validated above"); if ld64(unsafe { &(*slot).w0 }) as u32 != 0 { waiter_cleanup(r, &mut nodes); + forward_wakes(r, handles, owed, None, &mut later); return WaitOutcome::Signaled { index: handles.len() as u32, owner_dead: false, @@ -1609,15 +1912,22 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert } } let head = nodes[0]; + let reg = if all { + REG_MULTI + } else if n_nodes == 1 { + REG_SINGLE + } else { + REG_ANY + }; for (i, h) in handles.iter().enumerate() { if unsafe { (*r.node_ptr(nodes[i])).registered } == 0 { - node_register(r, nodes[i], h & INDEX_MASK, head); + node_register(r, nodes[i], h & INDEX_MASK, head, reg); } } if alert != 0 { let an = nodes[handles.len()]; if unsafe { (*r.node_ptr(an)).registered } == 0 { - node_register(r, an, alert & INDEX_MASK, head); + node_register(r, an, alert & INDEX_MASK, head, REG_MULTI); } } @@ -1633,6 +1943,9 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert // Authoritative acquire attempt. if let Some(outcome) = try_satisfy(r, handles, owner, &entry_seqs, all) { waiter_cleanup(r, &mut nodes); + if let WaitOutcome::Signaled { index, .. } = outcome { + forward_wakes(r, handles, owed, Some(index), &mut later); + } return outcome; } @@ -1656,6 +1969,7 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert // or fails immediately with EAGAIN - never a missed wakeup. let seq = unsafe { (*r.node_ptr(head)).seq.load(Ordering::Acquire) }; drop(_g); + later.flush(); // Bounded spin before sleeping (NTSYNC_SPIN_ITERS, default 0 = off): // a signal that lands within the window is acquired lock-free, @@ -1816,7 +2130,10 @@ fn wait(handles: &[u32], owner: u32, timeout: Option, all: bool, alert .any(|&idx| unsafe { ld32(&(*r.node_ptr(idx)).registered) } != 0); if any_registered { if let Ok(_g) = r.lock_guard() { - for &idx in nodes.iter() { + for (i, &idx) in nodes.iter().enumerate() { + if i < handles.len() && unsafe { (*r.node_ptr(idx)).registered } != 0 { + self_detached |= 1 << i; + } node_detach(r, idx); } } @@ -2034,7 +2351,7 @@ fn reap_dead_locked(r: &Region, cands: &[DeadCand], alive: &[bool]) -> u32 { { (*slot).state = SLOT_FREE; (*slot).generation = (*slot).generation.wrapping_add(1); - wake_object(r, c.index); + wake_object(r, c.index, WAKE_ALL, None); freed += 1; } } @@ -2085,6 +2402,13 @@ fn sweep_interval_ns() -> u64 { /// opted in. Capped at 100k to bound silly values. Each iteration is a /// handful of atomic loads (~10-20ns), so e.g. 200 iters costs ~2-4us /// of CPU in exchange for every avoided sleep/wake pair. +/// NTSYNC_WAKE_ALL=1 makes every signal op wake all of an object's waiters, +/// as before budgeted wakes existed. Read once. +fn wake_all_forced() -> bool { + static FORCED: OnceLock = OnceLock::new(); + *FORCED.get_or_init(|| std::env::var("NTSYNC_WAKE_ALL").is_ok_and(|v| v == "1")) +} + fn spin_max_iters() -> u32 { static ITERS: OnceLock = OnceLock::new(); *ITERS.get_or_init(|| { @@ -2134,6 +2458,18 @@ fn spin_tail_budget_ns() -> u64 { }) } +/// Snapshot of the per-process counters (they count with or without +/// NTSYNC_DEBUG; only the dump thread needs it). +#[cfg(test)] +pub(crate) fn test_counter(i: usize) -> u64 { + stats::COUNTERS[i].load(Ordering::Relaxed) +} + +#[cfg(test)] +pub(crate) fn test_counters() -> Vec<(&'static str, u64)> { + stats::NAMES.iter().enumerate().map(|(i, n)| (*n, stats::COUNTERS[i].load(Ordering::Relaxed))).collect() +} + /// Largest burst a thread may bank: its allowance for 10 ms of wall time. /// The bucket must not hold more than a few frames' worth, or a thread /// spending faster than it refills goes unthrottled for seconds and the @@ -2396,6 +2732,31 @@ pub(crate) fn test_reap_slot_candidate(cand: (u32, u32, u32), alive: bool) -> u3 reap_dead_locked(r, &[c], &[alive]) } +/// Tests: (single-object, other) waiter nodes registered on `handle`. +/// "Other" counts REG_ANY and REG_MULTI nodes alike. +#[cfg(test)] +pub(crate) fn test_waiter_counts(handle: u32) -> (u32, u32) { + let Ok(r) = region() else { return (0, 0) }; + let Ok(_g) = r.lock_guard() else { return (0, 0) }; + let Some(slot) = r.slot(handle) else { return (0, 0) }; + let (mut single, mut other) = (0, 0); + let mut idx = unsafe { (*slot).waiter_head }; + let mut guard = 0; + while idx != NIL && guard <= NODE_COUNT { + guard += 1; + let node = r.node_ptr(idx); + unsafe { + match (*node).registered { + REG_SINGLE => single += 1, + 0 => {} + _ => other += 1, + } + idx = (*node).obj_next; + } + } + (single, other) +} + /// Tests: (index, pid) of the first node registered on `handle`'s waiter /// list, i.e. of the process currently blocked on it. #[cfg(test)] @@ -2478,6 +2839,14 @@ pub(crate) fn test_sweep_clock_ns() -> u64 { /// Tests: force the auto-sweep rate limiter to consider the interval /// elapsed, so the next signal op sweeps regardless of wall time. +#[cfg(test)] +pub(crate) fn test_touch_sweep_clock() { + if let Ok(r) = region() { + let cell = unsafe { &*(&r.header().sweep_ns as *const u64 as *const AtomicU64) }; + cell.store(monotonic_ns(), Ordering::Release); + } +} + #[cfg(test)] pub(crate) fn test_reset_sweep_clock() { if let Ok(r) = region() { diff --git a/src/engine_sim.rs b/src/engine_sim.rs index 5f4c125..65d0dd1 100644 --- a/src/engine_sim.rs +++ b/src/engine_sim.rs @@ -269,7 +269,9 @@ fn sim_ue45_frame_graph() { // ------------------------------------------------------------------ Unity --- /// Unity job system: per-job events, `Complete()` waits, and the contended -/// graphics-device mutex. +/// graphics-device mutex. Workers wait alertably, as every Wine wait does +/// unless the caller asks otherwise, so the cost of a release with several +/// workers parked shows up in "sem_release". #[test] #[ignore] fn sim_unity_job_system() { @@ -286,13 +288,16 @@ fn sim_unity_job_system() { let next_job = Arc::new(AtomicU32::new(0)); let mut workers = Vec::new(); + let mut alerts = Vec::new(); for _ in 0..WORKERS { let stop = stop.clone(); let events = job_events.clone(); let next_job = next_job.clone(); + let alert = create_event(true, false).unwrap(); + alerts.push(alert); workers.push(thread::spawn(move || { while !stop.load(Ordering::Relaxed) { - match wait_any(&[job_sem], me(), Some(Duration::from_millis(50)), 0) { + match wait_any(&[job_sem], me(), Some(Duration::from_millis(50)), alert) { WaitOutcome::Signaled { .. } => {} WaitOutcome::Timeout => continue, _ => break, @@ -314,11 +319,14 @@ fn sim_unity_job_system() { let mut frames = Frames::new(FPS); let budget = Duration::from_secs_f64(1.0 / FPS as f64); let mut complete = Hist::with_capacity(8192); + let mut release = Hist::with_capacity(8192); for _ in 0..FRAMES { let start = Instant::now(); next_job.store(0, Ordering::Relaxed); for _ in 0..JOBS { + let w = Instant::now(); let _ = sem_release(job_sem, 1); + release.push(w.elapsed()); } // JobHandle.Complete(): one wait per job, in dispatch order. for &event in job_events.iter() { @@ -339,11 +347,12 @@ fn sim_unity_job_system() { let _ = sem_release(job_sem, WORKERS as u32); frames.report("Unity job system", FPS); complete.report("Complete() wait"); + release.report("sem_release"); for w in workers { w.join().unwrap(); } assert_eq!(frames.total as usize, FRAMES); - for e in job_events { + for e in job_events.into_iter().chain(alerts) { let _ = close(e); } let _ = close(job_sem); @@ -743,3 +752,311 @@ fn sim_primitive_costs() { let _ = close(sem); let _ = close(m); } + +// ------------------------------------------------- Unity release cost --- + +/// Thread CPU time (CLOCK_THREAD_CPUTIME_ID) in ns. +fn thread_cpu_ns() -> u64 { + let mut ts = unsafe { std::mem::zeroed::() }; + unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut ts) }; + ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64 +} + +/// (user+sys ns, voluntary switches, involuntary switches) of the calling thread. +fn thread_rusage() -> (u64, u64, u64) { + let mut ru: libc::rusage = unsafe { std::mem::zeroed() }; + unsafe { libc::getrusage(1 /* RUSAGE_THREAD */, &mut ru) }; + let ns = |t: libc::timeval| t.tv_sec as u64 * 1_000_000_000 + t.tv_usec as u64 * 1000; + (ns(ru.ru_utime) + ns(ru.ru_stime), ru.ru_nvcsw as u64, ru.ru_nivcsw as u64) +} + +fn pin_to(cpu: usize) { + unsafe { + let mut set: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_SET(cpu, &mut set); + libc::sched_setaffinity(0, std::mem::size_of::(), &set); + } +} + +thread_local! { + static SIM_FLAG: std::sync::atomic::AtomicI32 = const { std::sync::atomic::AtomicI32::new(0) }; +} + +unsafe extern "C" fn sim_lock_flag() -> *mut i32 { + SIM_FLAG.with(|f| f.as_ptr()) +} + +fn walks_now() -> u64 { + test_counter(8) +} + +fn env_u64(name: &str, default: u64) -> u64 { + std::env::var(name).ok().and_then(|v| v.parse().ok()).unwrap_or(default) +} + +fn spin_us(us: u64) { + if us == 0 { + return; + } + let t = Instant::now(); + let d = Duration::from_micros(us); + while t.elapsed() < d { + std::hint::spin_loop(); + } +} + +/// The Forest's Unity job system as measured in game on the Odin 3: seven +/// workers wait alertably (wait-any with an alert event, as Wine does) on one +/// job semaphore; the main thread releases it one unit at a time, about 300 +/// times per 60 fps frame, in bursts with its own work in between, and waits +/// for each job's completion event (JobHandle.Complete). Each acquired unit is +/// one job of a few microseconds. +/// +/// The judge for the release path is the main thread's CPU per release and +/// its syscalls per release (run under `simpleperf stat --per-thread -e +/// raw_syscalls:sys_enter`; the thread is named `sim-main`), with worker CPU +/// and switches alongside. Knobs (env): SIM_WORKERS 7, SIM_FPS 60, +/// SIM_RELEASES 300 per frame, SIM_BURST 10 releases per burst, SIM_GAP_US +/// main work between two releases of a burst, SIM_MAIN_US main work after a +/// burst before its completion waits, SIM_JOB_US worker work per job, +/// SIM_SECS run length, SIM_PIN 1 = main on the last core, workers on the +/// others. +#[test] +#[ignore] +fn sim_unity_release_cost() { + setup(); + let workers_n = env_u64("SIM_WORKERS", 7) as usize; + let fps = env_u64("SIM_FPS", 60); + let per_frame = env_u64("SIM_RELEASES", 300) as usize; + let burst = env_u64("SIM_BURST", 10).max(1) as usize; + let gap_us = env_u64("SIM_GAP_US", 5); + let main_us = env_u64("SIM_MAIN_US", 150); + let job_us = env_u64("SIM_JOB_US", 60); + let secs = env_u64("SIM_SECS", 8); + let pin = env_u64("SIM_PIN", 1) == 1; + let ncpu = thread::available_parallelism().map(|n| n.get()).unwrap_or(8); + + // SIM_FLAG_HOOK=1: install a host lock-flag callback, as Wine does, so + // lock acquisitions use the flag instead of masking SIGUSR1. + if env_u64("SIM_FLAG_HOOK", 0) == 1 { + set_lock_flag_hook(Some(sim_lock_flag)); + } + let job_sem = create_semaphore(0, 1 << 20).unwrap(); + let done: Arc> = Arc::new((0..burst).map(|_| create_event(false, false).unwrap()).collect()); + let next_job = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + + let mut workers = Vec::new(); + let mut alerts = Vec::new(); + // Threads report over channels and then stay parked until the process + // exits, so simpleperf's per-thread counters are still open at the end. + let (wtx, wrx) = std::sync::mpsc::channel(); + for w in 0..workers_n { + let (stop, done, next_job, wtx) = (stop.clone(), done.clone(), next_job.clone(), wtx.clone()); + let alert = create_event(true, false).unwrap(); + alerts.push(alert); + workers.push( + thread::Builder::new() + .name(format!("sim-w{w}")) + .spawn(move || { + if pin { + pin_to(w % (ncpu - 1)); + } + let r0 = thread_rusage(); + let mut jobs = 0u64; + loop { + match wait_any(&[job_sem], me(), Some(Duration::from_millis(100)), alert) { + WaitOutcome::Signaled { index: 0, .. } => {} + WaitOutcome::Timeout => { + if stop.load(Ordering::Relaxed) { + break; + } + continue; + } + _ => break, + } + if stop.load(Ordering::Relaxed) { + break; + } + let j = next_job.fetch_add(1, Ordering::Relaxed) as usize; + spin_us(job_us); + let _ = event_set(done[j % done.len()]); + jobs += 1; + } + let r1 = thread_rusage(); + let _ = wtx.send((jobs, r1.0 - r0.0, r1.1 - r0.1, r1.2 - r0.2)); + loop { + thread::park(); + } + }) + .unwrap(), + ); + } + // SIM_HOT=1: one SCHED_IDLE spinner per worker core keeps those cores out + // of deep idle, as the game's other threads do, so a woken worker runs + // within a few microseconds (it preempts the spinner) instead of after an + // idle-exit. + let hot = env_u64("SIM_HOT", 0) == 1; + if hot { + for c in 0..ncpu - 1 { + let stop = stop.clone(); + thread::Builder::new() + .name(format!("sim-bg{c}")) + .spawn(move || { + pin_to(c); + let p = libc::sched_param { sched_priority: 0 }; + unsafe { libc::sched_setscheduler(0, 5 /* SCHED_IDLE */, &p) }; + while !stop.load(Ordering::Relaxed) { + std::hint::spin_loop(); + } + }) + .unwrap(); + } + } + thread::sleep(Duration::from_millis(200)); // let every worker park + + let c0 = test_counters(); + let p0 = cpu_time(); + let (mtx, mrx) = std::sync::mpsc::channel(); + let main = { + let done = done.clone(); + thread::Builder::new() + .name("sim-main".into()) + .spawn(move || { + if pin { + pin_to(ncpu - 1); + } + let r0 = thread_rusage(); + let frame = Duration::from_secs_f64(1.0 / fps as f64); + let end = Instant::now() + Duration::from_secs(secs); + let (mut releases, mut rel_cpu, mut rel_wall, mut frames) = (0u64, 0u64, 0u64, 0u64); + // Releases that took a wake walk vs not, and their wall time + // (walks are counted process-wide, but during a release + // section only this thread signals). + let (mut walked, mut walked_ns) = (0u64, 0u64); + while Instant::now() < end { + let fstart = Instant::now(); + let mut left = per_frame; + while left > 0 { + let n = left.min(burst); + left -= n; + // Release section: priced by thread CPU, minus the + // main thread's own spin work in between. + let c = thread_cpu_ns(); + let mut spun = Duration::ZERO; + let mut inside = Duration::ZERO; + for _ in 0..n { + let s = Instant::now(); + spin_us(gap_us); + let w0 = walks_now(); + let t = Instant::now(); + spun += t - s; + let _ = sem_release(job_sem, 1); + let el = t.elapsed(); + inside += el; + if walks_now() != w0 { + walked += 1; + walked_ns += el.as_nanos() as u64; + } + } + rel_cpu += (thread_cpu_ns() - c).saturating_sub(spun.as_nanos() as u64); + rel_wall += inside.as_nanos() as u64; + releases += n as u64; + spin_us(main_us); + for &e in done.iter().take(n) { + let _ = wait_any(&[e], me(), Some(Duration::from_secs(2)), 0); + } + } + frames += 1; + if let Some(rest) = frame.checked_sub(fstart.elapsed()) { + thread::sleep(rest); + } + } + let r1 = thread_rusage(); + println!( + "UNITYSIM main: walked_releases={walked} ({:.3}/release) walk_wall_ns={:.0} nowalk_wall_ns={:.0}", + walked as f64 / releases as f64, + walked_ns as f64 / walked.max(1) as f64, + (rel_wall - walked_ns) as f64 / (releases - walked).max(1) as f64 + ); + let _ = mtx.send((releases, rel_cpu, rel_wall, frames, r1.0 - r0.0, r1.1 - r0.1, r1.2 - r0.2)); + loop { + thread::park(); + } + }) + .unwrap() + }; + let (releases, rel_cpu, rel_wall, frames, main_cpu, main_vcs, main_ivcs) = mrx.recv().unwrap(); + let _ = main; + let c1 = test_counters(); + let p1 = cpu_time(); + stop.store(true, Ordering::Relaxed); + let _ = sem_release(job_sem, workers_n as u32); + let mut w_jobs = 0; + let mut w_cpu = 0; + let mut w_vcs = 0; + let mut w_ivcs = 0; + drop(wtx); + let _ = workers; + for _ in 0..workers_n { + let (j, c, v, i) = wrx.recv().unwrap(); + w_jobs += j; + w_cpu += c; + w_vcs += v; + w_ivcs += i; + } + let d = |name: &str| { + let a = c0.iter().find(|(n, _)| *n == name).unwrap().1; + let b = c1.iter().find(|(n, _)| *n == name).unwrap().1; + (b - a) as f64 + }; + let wall = secs as f64; + let ld = |name: &str| -> f64 { + let i = c0.iter().position(|(n, _)| *n == name).unwrap(); + (c1[i].1 - c0[i].1) as f64 + }; + if ld("wake_lat_cnt") > 0.0 { + println!("UNITYSIM wake_lat_avg_us={:.1}", ld("wake_lat_us_sum") / ld("wake_lat_cnt")); + } + println!( + "UNITYSIM flag_hook={} hot={hot} workers={workers_n} fps={fps} per_frame={per_frame} burst={burst} gap_us={gap_us} main_us={main_us} job_us={job_us} secs={secs} wake_all={} no_sigmask={} debug={}", + std::env::var("SIM_FLAG_HOOK").unwrap_or_default(), + std::env::var("NTSYNC_WAKE_ALL").unwrap_or_default(), + std::env::var("NTSYNC_NO_SIGUSR1_BLOCK").unwrap_or_default(), + std::env::var("NTSYNC_DEBUG").unwrap_or_default(), + ); + println!( + "UNITYSIM frames={frames} fps_got={:.1} releases={releases} rel_cpu_ns={:.0} rel_wall_ns={:.0} main_cpu_pct={:.1} main_vcs_s={:.0} main_ivcs_s={:.0} workers_cpu_pct={:.1} worker_vcs_s={:.0} worker_ivcs_s={:.0} proc_cpu_pct={:.1} jobs={w_jobs}", + frames as f64 / wall, + rel_cpu as f64 / releases as f64, + rel_wall as f64 / releases as f64, + main_cpu as f64 / wall / 1e7, + main_vcs as f64 / wall, + main_ivcs as f64 / wall, + w_cpu as f64 / wall / 1e7, + w_vcs as f64 / wall / workers_n as f64, + w_ivcs as f64 / wall / workers_n as f64, + (p1 - p0).as_secs_f64() / wall * 100.0, + ); + let ops = d("signal_ops"); + println!( + "UNITYSIM walks_per_op={:.3} woken_per_walk={:.3} woken_per_op={:.3} blocked_per_wait={:.3} lock_acq_per_rel={:.3} contended_per_rel={:.3} walks_s={:.0} lock_acq_s={:.0} contended_s={:.0} deferred_s={:.0} forwarded={:.0} zero_wakes_s={:.0} rewalks_s={:.0}", + d("wake_walks") / ops, + d("threads_woken") / d("wake_walks"), + d("threads_woken") / ops, + d("wait_blocked") / d("wait_calls"), + d("lock_acq") / releases as f64, + d("lock_contended") / releases as f64, + d("wake_walks") / wall, + d("lock_acq") / wall, + d("lock_contended") / wall, + d("wakes_deferred") / wall, + d("wakes_forwarded"), + c1.iter().find(|(n, _)| *n == "wakes_unparked").map_or(0.0, |_| d("wakes_unparked")) / wall, + c1.iter().find(|(n, _)| *n == "wake_rewalks").map_or(0.0, |_| d("wake_rewalks")) / wall, + ); + for e in done.iter().copied().chain(alerts) { + let _ = close(e); + } + let _ = close(job_sem); +} diff --git a/src/ffi.rs b/src/ffi.rs index c7c9154..78ce6d6 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -130,6 +130,14 @@ pub unsafe extern "C" fn ntsync_init(path: *const std::ffi::c_char) -> i32 { } } +/// Install the host's per-thread lock-flag callback (NULL removes it). See +/// ntsync_user.h for the protocol its SIGUSR1 handler must follow. Returns 0. +#[no_mangle] +pub extern "C" fn ntsync_set_lock_flag_hook(hook: Option) -> i32 { + core::set_lock_flag_hook(hook); + 0 +} + /// Free all objects whose creator process no longer exists. Returns the /// number of freed objects, or a negative errno. #[no_mangle] diff --git a/src/lib.rs b/src/lib.rs index a4453b2..e5fbfd7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,6 +72,595 @@ mod tests { assert!(close(h)); } + // ---- Budgeted wakes: a signal op wakes only as many wait-any waiters + // as it can satisfy, oldest first; wait-all waiters as before. ---- + + fn wait_until(what: &str, mut cond: impl FnMut() -> bool) { + let start = std::time::Instant::now(); + while !cond() { + assert!(start.elapsed() < Duration::from_secs(10), "timed out waiting for {what}"); + thread::sleep(Duration::from_millis(1)); + } + } + + /// Park `n` single-object waiters on `h`, one at a time so their order + /// on the list is known. Each pushes its id to `order` once it acquires. + fn park_single_waiters( + h: u32, + n: usize, + owner_base: u32, + order: &Arc>>, + ) -> Vec> { + let mut joins = Vec::new(); + for i in 0..n { + let order = order.clone(); + joins.push(thread::spawn(move || { + let out = wait_any(&[h], owner_base + i as u32, None, 0); + assert!(matches!(out, WaitOutcome::Signaled { index: 0, .. })); + order.lock().unwrap().push(i); + })); + wait_until("waiter to park", || test_waiter_counts(h).0 as usize == i + 1); + } + joins + } + + #[test] + fn semaphore_release_wakes_only_released_count() { + setup(); + let h = create_semaphore(0, 10).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let joins = park_single_waiters(h, 4, 9100, &order); + + sem_release(h, 1).unwrap(); + wait_until("one waiter to acquire", || order.lock().unwrap().len() == 1); + thread::sleep(Duration::from_millis(50)); + assert_eq!(order.lock().unwrap().len(), 1, "release(1) satisfied more than one waiter"); + assert_eq!(test_waiter_counts(h), (3, 0), "the other waiters must stay parked"); + + sem_release(h, 2).unwrap(); + wait_until("two more waiters", || order.lock().unwrap().len() == 3); + thread::sleep(Duration::from_millis(50)); + assert_eq!(test_waiter_counts(h), (1, 0)); + + sem_release(h, 1).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(read_sem(h).unwrap().0, 0); + assert!(close(h)); + } + + #[test] + fn single_waiters_are_woken_oldest_first() { + setup(); + let h = create_semaphore(0, 10).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let joins = park_single_waiters(h, 5, 9200, &order); + for k in 1..=5 { + sem_release(h, 1).unwrap(); + wait_until("next waiter", || order.lock().unwrap().len() == k); + } + for j in joins { + j.join().unwrap(); + } + assert_eq!(*order.lock().unwrap(), vec![0, 1, 2, 3, 4]); + assert!(close(h)); + } + + #[test] + fn auto_event_wakes_one_manual_event_wakes_all() { + setup(); + let auto = create_event(false, false).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let joins = park_single_waiters(auto, 3, 9300, &order); + event_set(auto).unwrap(); + wait_until("one auto waiter", || order.lock().unwrap().len() == 1); + thread::sleep(Duration::from_millis(50)); + assert_eq!(order.lock().unwrap().len(), 1); + assert_eq!(test_waiter_counts(auto), (2, 0)); + // A set that lands while the event is still signaled is a no-op, as + // on Windows; keep setting until everyone got through. + let start = std::time::Instant::now(); + while order.lock().unwrap().len() < 3 { + let _ = event_set(auto); + thread::sleep(Duration::from_millis(1)); + assert!(start.elapsed() < Duration::from_secs(10), "auto-event waiter stranded"); + } + for j in joins { + j.join().unwrap(); + } + assert!(close(auto)); + + let manual = create_event(true, false).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let joins = park_single_waiters(manual, 3, 9310, &order); + event_set(manual).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(order.lock().unwrap().len(), 3); + assert!(close(manual)); + } + + #[test] + fn mutex_unlock_hands_over_to_one_waiter() { + setup(); + let m = create_mutex(9400, 1).unwrap(); + let inside = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let done = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let mut joins = Vec::new(); + for i in 0..4u32 { + let (inside, done) = (inside.clone(), done.clone()); + joins.push(thread::spawn(move || { + let me = 9401 + i; + assert!(is_signaled(wait_any(&[m], me, None, 0), 0)); + assert_eq!(inside.fetch_add(1, Ordering::SeqCst), 0, "two owners at once"); + thread::sleep(Duration::from_millis(2)); + inside.fetch_sub(1, Ordering::SeqCst); + mutex_unlock(m, me).unwrap(); + done.fetch_add(1, Ordering::SeqCst); + })); + } + wait_until("all mutex waiters parked", || test_waiter_counts(m).0 == 4); + mutex_unlock(m, 9400).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(done.load(Ordering::SeqCst), 4); + assert!(close(m)); + } + + #[test] + fn multi_object_and_alertable_waiters_are_woken_within_budget() { + setup(); + let sem = create_semaphore(0, 10).unwrap(); + let other = create_event(true, false).unwrap(); + let alert = create_event(true, false).unwrap(); + let got = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let mut joins = Vec::new(); + { + let got = got.clone(); + joins.push(thread::spawn(move || { + assert!(is_signaled(wait_any(&[sem, other], 9500, None, 0), 0)); + got.fetch_add(1, Ordering::SeqCst); + })); + } + wait_until("multi waiter", || test_waiter_counts(sem).1 == 1); + { + let got = got.clone(); + joins.push(thread::spawn(move || { + assert!(is_signaled(wait_any(&[sem], 9501, None, alert), 0)); + got.fetch_add(1, Ordering::SeqCst); + })); + } + wait_until("alertable waiter", || test_waiter_counts(sem).1 == 2); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + joins.extend(park_single_waiters(sem, 2, 9510, &order)); + assert_eq!(test_waiter_counts(sem), (2, 2)); + + // Release one unit: only the oldest waiter (the multi-object one) is + // woken and takes it; the other three stay parked. + sem_release(sem, 1).unwrap(); + wait_until("one acquire", || got.load(Ordering::SeqCst) == 1); + thread::sleep(Duration::from_millis(50)); + assert_eq!(test_waiter_counts(sem), (2, 1), "the other waiters must stay parked"); + assert!(order.lock().unwrap().is_empty()); + sem_release(sem, 3).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(got.load(Ordering::SeqCst) + order.lock().unwrap().len() as u32, 4); + for h in [sem, other, alert] { + assert!(close(h)); + } + } + + /// A wait-all waiter may need more than this object, so it is woken + /// whatever the budget. + #[test] + fn wait_all_waiters_are_always_woken() { + setup(); + let sem = create_semaphore(0, 10).unwrap(); + let gate = create_event(true, true).unwrap(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut joins = park_single_waiters(sem, 1, 9550, &order); + joins.push(thread::spawn(move || { + assert!(matches!(wait_all(&[sem, gate], 9551, None, 0), WaitOutcome::Signaled { .. })); + })); + wait_until("wait-all waiter", || test_waiter_counts(sem) == (1, 1)); + // Two units for two waiters: the single one is the oldest and uses + // the whole budget of the first release if it is the only one woken. + sem_release(sem, 1).unwrap(); + sem_release(sem, 1).unwrap(); + for j in joins { + j.join().unwrap(); + } + assert_eq!(read_sem(sem).unwrap().0, 0); + for h in [sem, gate] { + assert!(close(h)); + } + } + + /// A wait-any waiter woken for a semaphore unit that returns for another + /// reason (its alert fired, or an earlier object in its list was + /// signaled) must pass the wake on, or the unit sits in the semaphore + /// with the next waiter asleep. `alertable` picks the reason. + fn decoy_passes_its_wake_on(alertable: bool, owner_base: u32) { + setup(); + let mut passed_on = 0; + for round in 0..200u32 { + let sem = create_semaphore(0, 10).unwrap(); + let ev = create_event(true, false).unwrap(); + let owner = owner_base + round * 2; + // The decoy parks first, so it is the oldest waiter and the one + // a release of one unit wakes. + let decoy = thread::spawn(move || { + let out = if alertable { + wait_any(&[sem], owner, None, ev) + } else { + wait_any(&[ev, sem], owner, None, 0) + }; + let sem_index = if alertable { 0 } else { 1 }; + assert!(matches!(out, WaitOutcome::Signaled { .. }), "decoy: {out:?}"); + is_signaled(out, sem_index) + }); + wait_until("decoy to park", || test_waiter_counts(sem).1 == 1); + let consumer = thread::spawn(move || { + assert!(is_signaled(wait_any(&[sem], owner + 1, None, 0), 0)); + }); + wait_until("consumer to park", || test_waiter_counts(sem).0 == 1); + + // The release wakes only the decoy. The event lands before the + // decoy gets to run, so it returns on the event and leaves the + // unit behind. + sem_release(sem, 1).unwrap(); + event_set(ev).unwrap(); + if decoy.join().unwrap() { + // The decoy won the race and took the unit itself. + sem_release(sem, 1).unwrap(); + } else { + passed_on += 1; + } + let start = std::time::Instant::now(); + while !consumer.is_finished() { + assert!( + start.elapsed() < Duration::from_secs(10), + "round {round}: consumer left parked, sem count {:?}", + read_sem(sem) + ); + thread::sleep(Duration::from_micros(200)); + } + consumer.join().unwrap(); + assert_eq!(read_sem(sem).unwrap().0, 0); + for h in [sem, ev] { + assert!(close(h)); + } + } + assert!(passed_on > 0, "no round had the decoy return on the event"); + } + + #[test] + fn alerted_waiter_passes_its_wake_on() { + decoy_passes_its_wake_on(true, 9800); + } + + #[test] + fn waiter_satisfied_by_another_object_passes_its_wake_on() { + decoy_passes_its_wake_on(false, 10300); + } + + /// A budgeted wake is charged when its node is detached and issued after + /// the unlock. If the oldest waiter belongs to a process that died, the + /// wake finds nobody parked, and the unit must go to the next waiter + /// instead of staying in the semaphore with that waiter asleep. + #[test] + fn dead_waiter_does_not_strand_a_unit() { + setup(); + let sem = create_semaphore(0, 10).unwrap(); + // Keep the auto-sweep from purging the dead node before the release. + test_touch_sweep_clock(); + let pid = unsafe { libc::fork() }; + if pid == 0 { + let _ = wait_any(&[sem], 11001, None, 0); + unsafe { libc::_exit(0) }; + } + wait_until("child waiter to park", || test_waiter_counts(sem).0 == 1); + unsafe { + libc::kill(pid, libc::SIGKILL); + libc::waitpid(pid, std::ptr::null_mut(), 0); + } + let live = thread::spawn(move || { + assert!(is_signaled(wait_any(&[sem], 11002, Some(Duration::from_secs(10)), 0), 0)); + }); + wait_until("live waiter to park", || test_waiter_counts(sem).0 == 2); + sem_release(sem, 1).unwrap(); + let start = std::time::Instant::now(); + while !live.is_finished() { + assert!(start.elapsed() < Duration::from_secs(5), "unit stranded behind a dead waiter"); + thread::sleep(Duration::from_millis(1)); + } + live.join().unwrap(); + assert_eq!(read_sem(sem).unwrap().0, 0); + assert!(close(sem)); + } + + /// Host cooperation for SIGUSR1 (ntsync_set_lock_flag_hook), modelled on + /// Wine: the handler "suspends" the thread (blocks until resumed) unless + /// the thread's lock flag is set, in which case it marks the signal + /// pending and returns; the library re-raises it after the unlock. + mod usr1 { + use super::super::core::*; + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; + + pub static TARGET: AtomicU32 = AtomicU32::new(0); + pub static FLAG: AtomicI32 = AtomicI32::new(0); + pub static SUSPENDS: AtomicU32 = AtomicU32::new(0); + pub static SUSPENDED: AtomicBool = AtomicBool::new(false); + pub static RESUME: AtomicBool = AtomicBool::new(false); + /// Negative control: the handler suspends regardless of the flag. + pub static IGNORE_FLAG: AtomicBool = AtomicBool::new(false); + + pub unsafe extern "C" fn lock_flag() -> *mut i32 { + if unsafe { libc::gettid() } as u32 == TARGET.load(Ordering::SeqCst) { + FLAG.as_ptr() + } else { + std::ptr::null_mut() + } + } + + pub extern "C" fn handler(_: i32) { + if !IGNORE_FLAG.load(Ordering::SeqCst) && FLAG.load(Ordering::SeqCst) != 0 { + FLAG.fetch_or(LOCK_FLAG_SIGNAL_PENDING, Ordering::SeqCst); + return; + } + SUSPENDS.fetch_add(1, Ordering::SeqCst); + SUSPENDED.store(true, Ordering::SeqCst); + let ts = libc::timespec { tv_sec: 0, tv_nsec: 50_000 }; + while !RESUME.load(Ordering::SeqCst) { + unsafe { libc::nanosleep(&ts, std::ptr::null_mut()) }; + } + RESUME.store(false, Ordering::SeqCst); + SUSPENDED.store(false, Ordering::SeqCst); + } + + pub fn install() { + unsafe { + let mut sa: libc::sigaction = std::mem::zeroed(); + sa.sa_sigaction = handler as extern "C" fn(i32) as usize; + sa.sa_flags = libc::SA_RESTART; + libc::sigemptyset(&mut sa.sa_mask); + libc::sigaction(libc::SIGUSR1, &sa, std::ptr::null_mut()); + } + set_lock_flag_hook(Some(lock_flag)); + } + } + + /// Suspend a thread that is busy taking and dropping the region lock, + /// `rounds` times, and each time check that another thread can still + /// take the lock while it is suspended. Returns (wedged rounds, + /// signals the library deferred). + fn suspend_lock_holder(rounds: u32, ignore_flag: bool) -> (u32, u64) { + setup(); + usr1::install(); + usr1::IGNORE_FLAG.store(ignore_flag, Ordering::SeqCst); + usr1::SUSPENDS.store(0, Ordering::SeqCst); + let stop = Arc::new(AtomicBool::new(false)); + let (tx, rx) = std::sync::mpsc::channel(); + let busy = { + let stop = stop.clone(); + thread::spawn(move || { + let tid = unsafe { libc::gettid() } as u32; + usr1::TARGET.store(tid, Ordering::SeqCst); + tx.send(tid).unwrap(); + while !stop.load(Ordering::Relaxed) { + // create and close each take the region lock. + let e = create_event(false, false).unwrap(); + assert!(close(e)); + } + }) + }; + let tid = rx.recv().unwrap(); + let deferred0 = test_counter(20); + let mut wedged = 0; + for round in 0..rounds { + unsafe { libc::syscall(libc::SYS_tgkill, libc::getpid(), tid as i32, libc::SIGUSR1) }; + let start = std::time::Instant::now(); + while !usr1::SUSPENDED.load(Ordering::SeqCst) { + assert!(start.elapsed() < Duration::from_secs(5), "round {round}: SIGUSR1 lost"); + std::hint::spin_loop(); + } + // Another thread must get the region lock while the target is + // suspended. + let probe = thread::spawn(|| { + let e = create_event(false, false).unwrap(); + assert!(close(e)); + }); + let start = std::time::Instant::now(); + while !probe.is_finished() && start.elapsed() < Duration::from_millis(300) { + thread::sleep(Duration::from_micros(200)); + } + if !probe.is_finished() { + wedged += 1; + } + usr1::RESUME.store(true, Ordering::SeqCst); + while usr1::SUSPENDED.load(Ordering::SeqCst) { + std::hint::spin_loop(); + } + probe.join().unwrap(); + } + stop.store(true, Ordering::Relaxed); + busy.join().unwrap(); + usr1::TARGET.store(0, Ordering::SeqCst); + usr1::IGNORE_FLAG.store(false, Ordering::SeqCst); + set_lock_flag_hook(None); + assert_eq!(usr1::SUSPENDS.load(Ordering::SeqCst), rounds, "every SIGUSR1 suspends exactly once"); + (wedged, test_counter(20) - deferred0) + } + + #[test] + fn sigusr1_in_region_lock_is_deferred_not_lost() { + let (wedged, deferred) = suspend_lock_holder(300, false); + eprintln!("300 suspends: {wedged} wedged, {deferred} deferred inside the lock"); + assert_eq!(wedged, 0, "a thread suspended inside the region lock wedged it"); + assert!(deferred > 0, "no SIGUSR1 landed inside the lock; the test proved nothing"); + } + + /// Negative control for the test above: a handler that ignores the flag + /// suspends the thread inside the lock and wedges it. + #[test] + fn sigusr1_in_region_lock_wedges_without_deferral() { + let (wedged, _) = suspend_lock_holder(300, true); + eprintln!("300 suspends, flag ignored: {wedged} wedged"); + assert!(wedged > 0, "no round wedged; the deferral test cannot tell a fix from luck"); + } + + /// No lost wakeups under contention. Exactly as many units are released + /// as consumers need, so a wake spent on a thread that never takes its + /// unit would leave a consumer parked forever: every consumer must + /// finish within the deadline. Thieves steal units with zero-timeout + /// waits and give them back, so woken waiters regularly lose the race. + #[test] + fn stress_budgeted_wakes_never_strand_a_waiter() { + setup(); + const PER: u32 = 300; + let sem = create_semaphore(0, 100_000).unwrap(); + let other = create_event(true, false).unwrap(); + let alert = create_event(true, false).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let taken = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let mut consumers = Vec::new(); + // 4 single-object consumers, 2 multi-object, 2 alertable (one is both). + for i in 0..7u32 { + let taken = taken.clone(); + consumers.push(thread::spawn(move || { + for _ in 0..PER { + // Consumer 4 returns on `other` and consumers 5 and 6 on + // their alert whenever the toggler fires them; they + // then wait again, having passed their wake on. + loop { + let (out, sem_index) = match i { + 4 => (wait_any(&[other, sem], 9600 + i, None, 0), 1), + 5 => (wait_any(&[sem, other], 9600 + i, None, alert), 0), + 6 => (wait_any(&[sem], 9600 + i, None, alert), 0), + _ => (wait_any(&[sem], 9600 + i, None, 0), 0), + }; + assert!(matches!(out, WaitOutcome::Signaled { .. }), "consumer {i}: {out:?}"); + if is_signaled(out, sem_index) { + break; + } + std::thread::yield_now(); + } + taken.fetch_add(1, Ordering::Relaxed); + } + })); + } + let mut thieves = Vec::new(); + for t in 0..2u32 { + let stop = stop.clone(); + thieves.push(thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + if is_signaled(wait_any(&[sem], 9700 + t, Some(Duration::ZERO), 0), 0) { + std::hint::spin_loop(); + sem_release(sem, 1).unwrap(); + } + std::thread::yield_now(); + } + })); + } + for (t, ev) in [other, alert].into_iter().enumerate() { + let stop = stop.clone(); + thieves.push(thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + event_set(ev).unwrap(); + std::hint::spin_loop(); + event_reset(ev).unwrap(); + thread::sleep(Duration::from_micros(50 + 70 * t as u64)); + } + })); + } + let total = PER * 7; + let mut released = 0; + let mut k = 0u32; + while released < total { + let n = (1 + k % 4).min(total - released); + sem_release(sem, n).unwrap(); + released += n; + k += 1; + if k % 16 == 0 { + thread::sleep(Duration::from_micros(200)); + } + } + let start = std::time::Instant::now(); + while taken.load(Ordering::Relaxed) < total { + assert!( + start.elapsed() < Duration::from_secs(20), + "lost wakeup: {} of {total} units taken, sem count {:?}, waiters {:?}", + taken.load(Ordering::Relaxed), + read_sem(sem), + test_waiter_counts(sem) + ); + thread::sleep(Duration::from_millis(5)); + } + for c in consumers { + c.join().unwrap(); + } + stop.store(true, Ordering::Relaxed); + for t in thieves { + t.join().unwrap(); + } + assert_eq!(read_sem(sem).unwrap().0, 0); + for h in [sem, other, alert] { + assert!(close(h)); + } + } + + /// Mutex and auto-reset event handoffs under contention: mutual + /// exclusion holds and nobody is left parked. + #[test] + fn stress_mutex_and_auto_event_handoff() { + setup(); + let m = create_mutex(0, 0).unwrap(); + let ev = create_event(false, false).unwrap(); + let inside = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let mut joins = Vec::new(); + for i in 0..6u32 { + let inside = inside.clone(); + joins.push(thread::spawn(move || { + let me = 9800 + i; + for _ in 0..500 { + assert!(is_signaled(wait_any(&[m], me, None, 0), 0)); + assert_eq!(inside.fetch_add(1, Ordering::SeqCst), 0, "two mutex owners"); + inside.fetch_sub(1, Ordering::SeqCst); + mutex_unlock(m, me).unwrap(); + } + })); + } + // Auto-event ping-pong: each consumer waits, the producer sets until + // every wait has been satisfied. + let got = Arc::new(std::sync::atomic::AtomicU32::new(0)); + for i in 0..4u32 { + let got = got.clone(); + joins.push(thread::spawn(move || { + for _ in 0..200 { + assert!(is_signaled(wait_any(&[ev], 9900 + i, None, 0), 0)); + got.fetch_add(1, Ordering::SeqCst); + } + })); + } + let start = std::time::Instant::now(); + while got.load(Ordering::SeqCst) < 800 { + let _ = event_set(ev); + std::thread::yield_now(); + assert!(start.elapsed() < Duration::from_secs(20), "auto-event waiter stranded at {}", got.load(Ordering::SeqCst)); + } + for j in joins { + j.join().unwrap(); + } + for h in [m, ev] { + assert!(close(h)); + } + } + #[test] fn handle_zero_is_never_allocated() { setup();