From ff0e303b5b03f4215511b805916fc9abcab9292f Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Sat, 11 Jul 2026 16:56:21 +0200 Subject: [PATCH 1/6] Add profiling_hooks feature exposing jemalloc's experimental sample hooks Signed-off-by: Scott Gerring --- jemalloc-ctl/Cargo.toml | 3 +- jemalloc-ctl/src/macros.rs | 2 + jemalloc-ctl/src/profiling.rs | 291 ++++++++++++++++++++++++++++++++++ jemalloc-sys/Cargo.toml | 1 + jemalloc-sys/README.md | 14 ++ jemalloc-sys/build.rs | 17 ++ 6 files changed, 327 insertions(+), 1 deletion(-) diff --git a/jemalloc-ctl/Cargo.toml b/jemalloc-ctl/Cargo.toml index 1f93c84a4..deea2a2fa 100644 --- a/jemalloc-ctl/Cargo.toml +++ b/jemalloc-ctl/Cargo.toml @@ -38,9 +38,10 @@ default = [] stats = ["tikv-jemalloc-sys/stats"] profiling = ["tikv-jemalloc-sys/profiling"] profiling_libunwind = ["tikv-jemalloc-sys/profiling_libunwind", "profiling"] +profiling_hooks = ["tikv-jemalloc-sys/profiling_hooks", "profiling"] use_std = [ "libc/use_std" ] disable_initial_exec_tls = ["tikv-jemalloc-sys/disable_initial_exec_tls"] [package.metadata.docs.rs] rustdoc-args = [ "--cfg", "jemallocator_docs" ] -features = ["stats", "profiling", "use_std"] +features = ["stats", "profiling", "profiling_hooks", "use_std"] diff --git a/jemalloc-ctl/src/macros.rs b/jemalloc-ctl/src/macros.rs index 94050e354..1eb5b777a 100644 --- a/jemalloc-ctl/src/macros.rs +++ b/jemalloc-ctl/src/macros.rs @@ -130,6 +130,8 @@ macro_rules! make_test { "background_thread" | "max_background_threads" if cfg!(target_os = "macos") => return, + // Skipped: races with the hook test's own `prof_active` writes. + "prof_active" if cfg!(feature = "profiling_hooks") => return, _ => (), } diff --git a/jemalloc-ctl/src/profiling.rs b/jemalloc-ctl/src/profiling.rs index 306ab1a8e..28778a036 100644 --- a/jemalloc-ctl/src/profiling.rs +++ b/jemalloc-ctl/src/profiling.rs @@ -1,6 +1,17 @@ //! `jemalloc`'s run-time configuration for profiling-specific settings. //! //! These settings are controlled by the `MALLOC_CONF` environment variable. +//! +//! This module also exposes on-the-fly control via [`prof_active`] and +//! [`prof_reset`]. +#![cfg_attr( + feature = "profiling_hooks", + doc = " +With the `profiling_hooks` feature, it additionally exposes `jemalloc`'s +experimental `experimental.hooks.prof_sample`/`prof_sample_free`/ +`prof_backtrace` hooks via [`set_prof_sample_hook`], +[`set_prof_sample_free_hook`], and [`set_prof_backtrace_hook`]." +)] option! { lg_prof_interval[ str: b"opt.lg_prof_interval\0", non_str: 2 ] => libc::ssize_t | @@ -153,3 +164,283 @@ option! { /// ``` mib_docs: /// See [`prof_leak`]. } + +option! { + prof_active[ str: b"prof.active\0", non_str: 2 ] => bool | + ops: r,w,u | + docs: + /// On-the-fly activation/deactivation of memory profiling. + /// + /// This is a secondary control mechanism on top of `opt.prof`, and is + /// only effective once `opt.prof` is `true`; when it is `false`, reading + /// [`prof_active`] always returns `false` and writing to it fails with + /// `ENOENT`. `jemalloc` initializes [`prof_active`] to + /// `opt.prof_thread_active_init` (which itself defaults to `true`) as + /// soon as `opt.prof` is `true`, so a build with `opt.prof` enabled + /// samples by default unless [`prof_active`] is set to `false`, e.g. via + /// `prof_active:false` in `MALLOC_CONF`. + /// + /// While inactive, sampling hooks installed via the `profiling_hooks` + /// feature's hook setters remain installed but do not fire, since no + /// allocation is ever selected for sampling. + /// + /// # Examples + /// + /// ``` + /// # #[global_allocator] + /// # static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + /// # + /// # fn main() { + /// use tikv_jemalloc_ctl::profiling; + /// // `false` is always accepted, even if `opt.prof` is disabled at + /// // runtime; writing `true` additionally requires `opt.prof` to be + /// // `true`, else it fails with `ENOENT`. + /// let was_active = profiling::prof_active::write(false).unwrap(); + /// # let _ = was_active; + /// # } + /// ``` + mib_docs: /// See [`prof_active`]. +} + +/// Resets `jemalloc`'s heap profile sample accumulators and, going forward, +/// samples allocations at a rate of one per `2^lg_sample` bytes of +/// allocation activity. +/// +/// Corresponds to `prof.reset`, which is write-only: unlike most keys in +/// this module, there is no matching `read()`/`update()`. +/// +/// # Errors +/// +/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime, e.g. +/// because `MALLOC_CONF`/`JEMALLOC_SYS_WITH_MALLOC_CONF` overrode the +/// `prof:true` this crate's `profiling`/`profiling_hooks` features bake in. +pub fn prof_reset(lg_sample: libc::size_t) -> crate::error::Result<()> { + unsafe { crate::raw::write(b"prof.reset\0", lg_sample) } +} + +#[cfg(feature = "profiling_hooks")] +use libc::{c_uint, c_void}; + +/// Signature of a hook installable via [`set_prof_sample_hook`]. +/// +/// `jemalloc` invokes this hook synchronously, inline on the allocating +/// thread, immediately after it decides to sample an allocation of +/// `usable_size` bytes at `ptr` (the request was for `size` bytes; +/// `usable_size` is jemalloc's usable/rounded-up size). `backtrace` points +/// to `backtrace_length` +/// `void*` frames captured by the installed [`ProfBacktraceHook`] +/// (`backtrace_length` is `0` if [`noop_prof_backtrace_hook`] is installed). +/// +/// # Safety +/// +/// No `jemalloc` mutex is held while this hook runs, but it does run inside +/// `jemalloc`'s `pre_reentrancy`/`post_reentrancy` bracket and may itself +/// allocate/free (including recursively sampling). It must not unwind +/// across the `extern "C"` boundary — a panic here is undefined behavior +/// below Rust 1.81, and this crate is `no_std` by default so `catch_unwind` +/// is unavailable regardless. +#[cfg(feature = "profiling_hooks")] +pub type ProfSampleHook = unsafe extern "C" fn( + ptr: *const c_void, + size: libc::size_t, + backtrace: *mut *mut c_void, + backtrace_length: c_uint, + usable_size: libc::size_t, +); + +/// Signature of a hook installable via [`set_prof_sample_free_hook`]. +/// +/// `jemalloc` invokes this hook synchronously, inline on the freeing +/// thread, just before it frees a previously-sampled allocation of +/// `usable_size` bytes at `ptr`. See [`ProfSampleHook`] for the applicable safety +/// contract. +#[cfg(feature = "profiling_hooks")] +pub type ProfSampleFreeHook = + unsafe extern "C" fn(ptr: *const c_void, usable_size: libc::size_t); + +/// Signature of a hook installable via [`set_prof_backtrace_hook`]. +/// +/// `jemalloc` invokes this hook to capture the stack trace for a sample; it +/// must write at most `max_length` frames into `backtrace` and store the +/// number of frames written through `backtrace_length`. See +/// [`ProfSampleHook`] for the applicable safety contract. +#[cfg(feature = "profiling_hooks")] +pub type ProfBacktraceHook = unsafe extern "C" fn( + backtrace: *mut *mut c_void, + backtrace_length: *mut c_uint, + max_length: c_uint, +); + +/// Installs, replaces, or (with `None`) uninstalls the hook `jemalloc` +/// calls after deciding to sample an allocation, returning the +/// previously-installed hook. +/// +/// Corresponds to `experimental.hooks.prof_sample`. +/// +/// # Errors +/// +/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime — see +/// [`prof_reset`]. Note that `opt.prof` being `true` is sufficient to +/// install a hook; [`prof_active`] need not be `true` (installing while +/// inactive is a no-op until activated). +#[cfg(feature = "profiling_hooks")] +pub fn set_prof_sample_hook( + hook: Option, +) -> crate::error::Result> { + unsafe { crate::raw::update(b"experimental.hooks.prof_sample\0", hook) } +} + +/// Installs, replaces, or (with `None`) uninstalls the hook `jemalloc` +/// calls just before freeing a previously-sampled allocation, returning the +/// previously-installed hook. +/// +/// Corresponds to `experimental.hooks.prof_sample_free`. See +/// [`set_prof_sample_hook`] for the applicable error semantics. +#[cfg(feature = "profiling_hooks")] +pub fn set_prof_sample_free_hook( + hook: Option, +) -> crate::error::Result> { + unsafe { + crate::raw::update(b"experimental.hooks.prof_sample_free\0", hook) + } +} + +/// Installs or replaces the hook `jemalloc` calls to capture a sample's +/// backtrace, returning the previously-installed hook. +/// +/// Corresponds to `experimental.hooks.prof_backtrace`. Unlike +/// [`set_prof_sample_hook`]/[`set_prof_sample_free_hook`], this hook cannot +/// be uninstalled (`jemalloc` rejects a `NULL` new hook with `EINVAL`) — +/// install [`noop_prof_backtrace_hook`] instead of `jemalloc`'s default +/// unwinder if backtraces aren't wanted, and keep the returned previous +/// hook only to log/inspect, not to call directly, since it may itself be a +/// previously-installed [`noop_prof_backtrace_hook`] or `jemalloc`'s +/// internal default depending on prior state. +/// +/// # Errors +/// +/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime — see +/// [`prof_reset`]. +#[cfg(feature = "profiling_hooks")] +pub fn set_prof_backtrace_hook( + hook: ProfBacktraceHook, +) -> crate::error::Result { + unsafe { crate::raw::update(b"experimental.hooks.prof_backtrace\0", hook) } +} + +/// A [`ProfBacktraceHook`] that reports an empty backtrace for every +/// sample. +/// +/// Installing this via [`set_prof_backtrace_hook`] disables `jemalloc`'s +/// own stack unwinding going forward: the per-allocation sampling +/// decision still happens at the configured rate (see [`lg_prof_sample`]) +/// and [`set_prof_sample_hook`]/[`set_prof_sample_free_hook`] hooks still +/// fire, but with `backtrace_length` reported as `0`. Intended for +/// out-of-process samplers (e.g. an eBPF profiler) that capture their own +/// stacks and only need `jemalloc`'s sampling clock, since capturing a +/// backtrace it already unwinds itself is otherwise a per-sample cost +/// (page-aligned promotion, `tcache` bypass, and a `tdata` mutex are still +/// paid regardless of whether a backtrace is captured). +/// +/// # Safety +/// +/// Must only be invoked by `jemalloc` itself as an +/// `experimental.hooks.prof_backtrace` hook, which always passes a non-null +/// `backtrace_length`. +#[cfg(feature = "profiling_hooks")] +pub unsafe extern "C" fn noop_prof_backtrace_hook( + _backtrace: *mut *mut c_void, + backtrace_length: *mut c_uint, + _max_length: c_uint, +) { + *backtrace_length = 0; +} + +// Heap-allocates to force samples, so this needs a real allocator (`use_std`). +#[cfg(all(test, feature = "profiling_hooks", feature = "use_std"))] +mod hook_tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + static SAMPLE_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); + static SAMPLE_FREE_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); + + unsafe extern "C" fn counting_sample_hook( + _ptr: *const c_void, + _size: libc::size_t, + _backtrace: *mut *mut c_void, + _backtrace_length: c_uint, + _usable_size: libc::size_t, + ) { + SAMPLE_HOOK_CALLS.fetch_add(1, Ordering::SeqCst); + } + + unsafe extern "C" fn counting_sample_free_hook( + _ptr: *const c_void, + _usable_size: libc::size_t, + ) { + SAMPLE_FREE_HOOK_CALLS.fetch_add(1, Ordering::SeqCst); + } + + // Exercises the whole hook contract end-to-end against real `jemalloc` + // ctls: activates profiling, resets the sampler to catch every + // allocation, installs counting hooks, allocates/frees, and asserts both + // hooks actually fired before restoring prior state. + #[test] + fn sample_and_sample_free_hooks_fire() { + let was_active = prof_active::read().unwrap(); + let prev_lg_sample = lg_prof_sample::read().unwrap(); + // lg_sample: 0 => average one sample per byte, i.e. every allocation. + // `jemalloc` only recomputes each thread's next sample distance + // (from the new `lg_sample`) once the current, already-primed + // distance (drawn under whatever `lg_sample` was in effect before + // this call, e.g. the crate's default of 512 KiB) has been + // exhausted — so the very next allocation isn't guaranteed to + // sample yet, only allocations after that first one are. + prof_reset(0).unwrap(); + prof_active::write(true).unwrap(); + + let prev_sample = + set_prof_sample_hook(Some(counting_sample_hook)).unwrap(); + let prev_sample_free = + set_prof_sample_free_hook(Some(counting_sample_free_hook)) + .unwrap(); + + // Warm up past any stale pre-reset sample distance: 16 MiB is many + // times the largest plausible leftover distance from a 512 KiB mean. + for _ in 0..16 { + drop(Box::new([0u8; 1024 * 1024])); + } + + let before_sample = SAMPLE_HOOK_CALLS.load(Ordering::SeqCst); + let before_free = SAMPLE_FREE_HOOK_CALLS.load(Ordering::SeqCst); + for _ in 0..16 { + drop(Box::new([0u8; 4096])); + if SAMPLE_HOOK_CALLS.load(Ordering::SeqCst) > before_sample + && SAMPLE_FREE_HOOK_CALLS.load(Ordering::SeqCst) > before_free + { + break; + } + } + + set_prof_sample_hook(prev_sample).unwrap(); + set_prof_sample_free_hook(prev_sample_free).unwrap(); + prof_active::write(was_active).unwrap(); + prof_reset(prev_lg_sample).unwrap(); + + assert!( + SAMPLE_HOOK_CALLS.load(Ordering::SeqCst) > before_sample, + "prof_sample hook did not fire after warm-up with lg_sample=0" + ); + assert!( + SAMPLE_FREE_HOOK_CALLS.load(Ordering::SeqCst) > before_free, + "prof_sample_free hook did not fire after freeing sampled allocations" + ); + } + + #[test] + fn backtrace_hook_can_be_replaced_and_restored() { + let prev = set_prof_backtrace_hook(noop_prof_backtrace_hook).unwrap(); + set_prof_backtrace_hook(prev).unwrap(); + } +} diff --git a/jemalloc-sys/Cargo.toml b/jemalloc-sys/Cargo.toml index 9b3d922f1..53d3314b6 100644 --- a/jemalloc-sys/Cargo.toml +++ b/jemalloc-sys/Cargo.toml @@ -42,6 +42,7 @@ cc = "^1.0.13" default = ["background_threads_runtime_support"] profiling = [] profiling_libunwind = ["profiling"] +profiling_hooks = ["profiling"] debug = [] background_threads_runtime_support = [] background_threads = [ "background_threads_runtime_support" ] diff --git a/jemalloc-sys/README.md b/jemalloc-sys/README.md index bce9ae000..c590a1b68 100644 --- a/jemalloc-sys/README.md +++ b/jemalloc-sys/README.md @@ -54,6 +54,20 @@ This crate provides following cargo feature flags: to be installed. On macOS/iOS, unwind symbols are provided by the system and no extra library is needed. +* `profiling_hooks`: Enables `profiling` automatically and bakes + `prof:true,prof_active:false` into the default `malloc_conf`, so sampling + is installable but inert until a consumer flips `prof.active` at runtime. + Exposes `jemalloc`'s experimental + `experimental.hooks.prof_sample`/`prof_sample_free`/`prof_backtrace` hooks + through `tikv-jemalloc-ctl`'s `profiling` module, letting an external + sampler (e.g. an eBPF profiler) piggyback `jemalloc`'s sampling decision + without its stack walking. Not re-exported by `tikv-jemallocator`. + + Since this crate has `links = "jemalloc"`, there's only one `jemalloc` + build per dependency graph, so enabling this on any dependent applies + `prof:true,prof_active:false` to that shared build for everyone in the + graph too. + * `stats` (configure `jemalloc` with `--enable-stats`): Enable statistics gathering functionality. See the `jemalloc`'s "`opt.stats_print`" option documentation for usage details. diff --git a/jemalloc-sys/build.rs b/jemalloc-sys/build.rs index 845b9d43f..a2a797c85 100644 --- a/jemalloc-sys/build.rs +++ b/jemalloc-sys/build.rs @@ -246,6 +246,23 @@ fn main() { malloc_conf += "background_thread:false"; } + if env::var("CARGO_FEATURE_PROFILING_HOOKS").is_ok() { + info!("CARGO_FEATURE_PROFILING_HOOKS set"); + // Baking bare `prof:true` would also default `prof_active` and + // `prof_thread_active_init` to true. Since `links = "jemalloc"` + // unifies this build across every dependent in the graph, that would + // make the one shared jemalloc actively sample every allocation + // process-wide as soon as any dependent enables this feature, even + // if no hook is ever installed. Keep sampling installable + // (`opt_prof`) but inert until a consumer flips `prof.active` (or + // calls `tikv_jemalloc_ctl::profiling::prof_active::write(true)`) at + // runtime. + if !malloc_conf.is_empty() { + malloc_conf.push(','); + } + malloc_conf.push_str("prof:true,prof_active:false"); + } + if let Ok(malloc_conf_opts) = read_and_watch_env("JEMALLOC_SYS_WITH_MALLOC_CONF") { if !malloc_conf.is_empty() { malloc_conf.push(','); From 3179014d588852dbde740841677d7d847f09cd8b Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 14 Jul 2026 07:38:32 +0200 Subject: [PATCH 2/6] address review bot comment Signed-off-by: Scott Gerring --- jemalloc-ctl/src/profiling.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/jemalloc-ctl/src/profiling.rs b/jemalloc-ctl/src/profiling.rs index 28778a036..6e2f3f16e 100644 --- a/jemalloc-ctl/src/profiling.rs +++ b/jemalloc-ctl/src/profiling.rs @@ -175,11 +175,14 @@ option! { /// only effective once `opt.prof` is `true`; when it is `false`, reading /// [`prof_active`] always returns `false` and writing to it fails with /// `ENOENT`. `jemalloc` initializes [`prof_active`] to - /// `opt.prof_thread_active_init` (which itself defaults to `true`) as - /// soon as `opt.prof` is `true`, so a build with `opt.prof` enabled - /// samples by default unless [`prof_active`] is set to `false`, e.g. via + /// `opt.prof_active` (which itself defaults to `true`) as soon as + /// `opt.prof` is `true`, so a build with `opt.prof` enabled samples by + /// default unless [`prof_active`] is set to `false`, e.g. via /// `prof_active:false` in `MALLOC_CONF`. /// + /// Note: `opt.prof_thread_active_init` is unrelated — it controls the + /// per-thread `thread.prof.active` flag, not this global toggle. + /// /// While inactive, sampling hooks installed via the `profiling_hooks` /// feature's hook setters remain installed but do not fire, since no /// allocation is ever selected for sampling. @@ -213,7 +216,9 @@ option! { /// /// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime, e.g. /// because `MALLOC_CONF`/`JEMALLOC_SYS_WITH_MALLOC_CONF` overrode the -/// `prof:true` this crate's `profiling`/`profiling_hooks` features bake in. +/// `prof:true` that the `profiling_hooks` feature bakes in (the `profiling` +/// feature alone enables `--enable-prof` at build time but does not set +/// `prof:true` in `malloc_conf`). pub fn prof_reset(lg_sample: libc::size_t) -> crate::error::Result<()> { unsafe { crate::raw::write(b"prof.reset\0", lg_sample) } } From b19340abe6f98dce692b13c3856319c6da33634a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 9 Sep 2026 15:51:44 +0200 Subject: [PATCH 3/6] Expose profiling hooks without forcing runtime config Signed-off-by: Scott Gerring --- jemalloc-ctl/Cargo.toml | 3 +- jemalloc-ctl/src/macros.rs | 4 +- jemalloc-ctl/src/profiling.rs | 95 +++++++++++++++++------------------ jemalloc-sys/Cargo.toml | 1 - jemalloc-sys/README.md | 29 +++++------ jemalloc-sys/build.rs | 17 ------- 6 files changed, 65 insertions(+), 84 deletions(-) diff --git a/jemalloc-ctl/Cargo.toml b/jemalloc-ctl/Cargo.toml index deea2a2fa..1f93c84a4 100644 --- a/jemalloc-ctl/Cargo.toml +++ b/jemalloc-ctl/Cargo.toml @@ -38,10 +38,9 @@ default = [] stats = ["tikv-jemalloc-sys/stats"] profiling = ["tikv-jemalloc-sys/profiling"] profiling_libunwind = ["tikv-jemalloc-sys/profiling_libunwind", "profiling"] -profiling_hooks = ["tikv-jemalloc-sys/profiling_hooks", "profiling"] use_std = [ "libc/use_std" ] disable_initial_exec_tls = ["tikv-jemalloc-sys/disable_initial_exec_tls"] [package.metadata.docs.rs] rustdoc-args = [ "--cfg", "jemallocator_docs" ] -features = ["stats", "profiling", "profiling_hooks", "use_std"] +features = ["stats", "profiling", "use_std"] diff --git a/jemalloc-ctl/src/macros.rs b/jemalloc-ctl/src/macros.rs index 1eb5b777a..85b6169f0 100644 --- a/jemalloc-ctl/src/macros.rs +++ b/jemalloc-ctl/src/macros.rs @@ -130,8 +130,8 @@ macro_rules! make_test { "background_thread" | "max_background_threads" if cfg!(target_os = "macos") => return, - // Skipped: races with the hook test's own `prof_active` writes. - "prof_active" if cfg!(feature = "profiling_hooks") => return, + // Requires `opt.prof` and can race with hook tests. + "prof_active" if cfg!(feature = "profiling") => return, _ => (), } diff --git a/jemalloc-ctl/src/profiling.rs b/jemalloc-ctl/src/profiling.rs index 6e2f3f16e..1c878334c 100644 --- a/jemalloc-ctl/src/profiling.rs +++ b/jemalloc-ctl/src/profiling.rs @@ -3,15 +3,10 @@ //! These settings are controlled by the `MALLOC_CONF` environment variable. //! //! This module also exposes on-the-fly control via [`prof_active`] and -//! [`prof_reset`]. -#![cfg_attr( - feature = "profiling_hooks", - doc = " -With the `profiling_hooks` feature, it additionally exposes `jemalloc`'s -experimental `experimental.hooks.prof_sample`/`prof_sample_free`/ -`prof_backtrace` hooks via [`set_prof_sample_hook`], -[`set_prof_sample_free_hook`], and [`set_prof_backtrace_hook`]." -)] +//! [`prof_reset`], along with `jemalloc`'s experimental +//! `experimental.hooks.prof_sample`/`prof_sample_free`/`prof_backtrace` hooks +//! via [`set_prof_sample_hook`], [`set_prof_sample_free_hook`], and +//! [`set_prof_backtrace_hook`]. option! { lg_prof_interval[ str: b"opt.lg_prof_interval\0", non_str: 2 ] => libc::ssize_t | @@ -172,18 +167,19 @@ option! { /// On-the-fly activation/deactivation of memory profiling. /// /// This is a secondary control mechanism on top of `opt.prof`, and is - /// only effective once `opt.prof` is `true`; when it is `false`, reading - /// [`prof_active`] always returns `false` and writing to it fails with - /// `ENOENT`. `jemalloc` initializes [`prof_active`] to + /// only effective once `opt.prof` is `true`. When it is `false`, reading + /// [`prof_active`] returns `false`; writing `false` is accepted as a no-op, + /// while writing `true` fails with `ENOENT`. `jemalloc` initialises + /// [`prof_active`] to /// `opt.prof_active` (which itself defaults to `true`) as soon as - /// `opt.prof` is `true`, so a build with `opt.prof` enabled samples by - /// default unless [`prof_active`] is set to `false`, e.g. via + /// `opt.prof` is `true`, so a configuration with `opt.prof` enabled samples + /// by default unless [`prof_active`] is set to `false`, e.g. via /// `prof_active:false` in `MALLOC_CONF`. /// - /// Note: `opt.prof_thread_active_init` is unrelated — it controls the + /// Note: `opt.prof_thread_active_init` is unrelated. It controls the /// per-thread `thread.prof.active` flag, not this global toggle. /// - /// While inactive, sampling hooks installed via the `profiling_hooks` + /// While inactive, sampling hooks installed via the `profiling` /// feature's hook setters remain installed but do not fire, since no /// allocation is ever selected for sampling. /// @@ -214,16 +210,14 @@ option! { /// /// # Errors /// -/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime, e.g. -/// because `MALLOC_CONF`/`JEMALLOC_SYS_WITH_MALLOC_CONF` overrode the -/// `prof:true` that the `profiling_hooks` feature bakes in (the `profiling` -/// feature alone enables `--enable-prof` at build time but does not set -/// `prof:true` in `malloc_conf`). +/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime. The +/// `profiling` feature enables profiling support but does not set `opt.prof`; +/// configure `prof:true`, for example via `JEMALLOC_SYS_WITH_MALLOC_CONF` at +/// build time. pub fn prof_reset(lg_sample: libc::size_t) -> crate::error::Result<()> { unsafe { crate::raw::write(b"prof.reset\0", lg_sample) } } -#[cfg(feature = "profiling_hooks")] use libc::{c_uint, c_void}; /// Signature of a hook installable via [`set_prof_sample_hook`]. @@ -232,19 +226,16 @@ use libc::{c_uint, c_void}; /// thread, immediately after it decides to sample an allocation of /// `usable_size` bytes at `ptr` (the request was for `size` bytes; /// `usable_size` is jemalloc's usable/rounded-up size). `backtrace` points -/// to `backtrace_length` -/// `void*` frames captured by the installed [`ProfBacktraceHook`] -/// (`backtrace_length` is `0` if [`noop_prof_backtrace_hook`] is installed). +/// to an array of `backtrace_length` `void *` frames captured by the installed +/// [`ProfBacktraceHook`] (`backtrace_length` is `0` if +/// [`noop_prof_backtrace_hook`] is installed). /// /// # Safety /// /// No `jemalloc` mutex is held while this hook runs, but it does run inside -/// `jemalloc`'s `pre_reentrancy`/`post_reentrancy` bracket and may itself -/// allocate/free (including recursively sampling). It must not unwind -/// across the `extern "C"` boundary — a panic here is undefined behavior -/// below Rust 1.81, and this crate is `no_std` by default so `catch_unwind` -/// is unavailable regardless. -#[cfg(feature = "profiling_hooks")] +/// `jemalloc`'s `pre_reentrancy`/`post_reentrancy` bracket. It may itself +/// allocate or free; nested allocator activity is excluded from profiling. +/// It must not unwind across the `extern "C"` boundary. pub type ProfSampleHook = unsafe extern "C" fn( ptr: *const c_void, size: libc::size_t, @@ -259,7 +250,6 @@ pub type ProfSampleHook = unsafe extern "C" fn( /// thread, just before it frees a previously-sampled allocation of /// `usable_size` bytes at `ptr`. See [`ProfSampleHook`] for the applicable safety /// contract. -#[cfg(feature = "profiling_hooks")] pub type ProfSampleFreeHook = unsafe extern "C" fn(ptr: *const c_void, usable_size: libc::size_t); @@ -269,7 +259,6 @@ pub type ProfSampleFreeHook = /// must write at most `max_length` frames into `backtrace` and store the /// number of frames written through `backtrace_length`. See /// [`ProfSampleHook`] for the applicable safety contract. -#[cfg(feature = "profiling_hooks")] pub type ProfBacktraceHook = unsafe extern "C" fn( backtrace: *mut *mut c_void, backtrace_length: *mut c_uint, @@ -284,11 +273,10 @@ pub type ProfBacktraceHook = unsafe extern "C" fn( /// /// # Errors /// -/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime — see +/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime; see /// [`prof_reset`]. Note that `opt.prof` being `true` is sufficient to /// install a hook; [`prof_active`] need not be `true` (installing while /// inactive is a no-op until activated). -#[cfg(feature = "profiling_hooks")] pub fn set_prof_sample_hook( hook: Option, ) -> crate::error::Result> { @@ -301,7 +289,6 @@ pub fn set_prof_sample_hook( /// /// Corresponds to `experimental.hooks.prof_sample_free`. See /// [`set_prof_sample_hook`] for the applicable error semantics. -#[cfg(feature = "profiling_hooks")] pub fn set_prof_sample_free_hook( hook: Option, ) -> crate::error::Result> { @@ -315,18 +302,15 @@ pub fn set_prof_sample_free_hook( /// /// Corresponds to `experimental.hooks.prof_backtrace`. Unlike /// [`set_prof_sample_hook`]/[`set_prof_sample_free_hook`], this hook cannot -/// be uninstalled (`jemalloc` rejects a `NULL` new hook with `EINVAL`) — -/// install [`noop_prof_backtrace_hook`] instead of `jemalloc`'s default -/// unwinder if backtraces aren't wanted, and keep the returned previous -/// hook only to log/inspect, not to call directly, since it may itself be a -/// previously-installed [`noop_prof_backtrace_hook`] or `jemalloc`'s -/// internal default depending on prior state. +/// be uninstalled (`jemalloc` rejects a `NULL` new hook with `EINVAL`). Install +/// [`noop_prof_backtrace_hook`] instead of `jemalloc`'s default unwinder if +/// backtraces aren't wanted. The returned previous hook may be restored later +/// or invoked by the replacement during a valid backtrace-hook call. /// /// # Errors /// -/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime — see +/// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime; see /// [`prof_reset`]. -#[cfg(feature = "profiling_hooks")] pub fn set_prof_backtrace_hook( hook: ProfBacktraceHook, ) -> crate::error::Result { @@ -352,7 +336,6 @@ pub fn set_prof_backtrace_hook( /// Must only be invoked by `jemalloc` itself as an /// `experimental.hooks.prof_backtrace` hook, which always passes a non-null /// `backtrace_length`. -#[cfg(feature = "profiling_hooks")] pub unsafe extern "C" fn noop_prof_backtrace_hook( _backtrace: *mut *mut c_void, backtrace_length: *mut c_uint, @@ -362,11 +345,27 @@ pub unsafe extern "C" fn noop_prof_backtrace_hook( } // Heap-allocates to force samples, so this needs a real allocator (`use_std`). -#[cfg(all(test, feature = "profiling_hooks", feature = "use_std"))] +#[cfg(all(test, feature = "use_std"))] mod hook_tests { use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; + union Conf { + bytes: &'static u8, + c_char: &'static libc::c_char, + } + + // Enable profiling only for this test binary. Normal library builds leave + // the process-wide profiling policy to the final consumer. + #[export_name = "_rjem_malloc_conf"] + pub static TEST_MALLOC_CONF: Option<&'static libc::c_char> = + Some(unsafe { + Conf { + bytes: &b"prof:true,prof_active:false\0"[0], + } + .c_char + }); + static SAMPLE_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); static SAMPLE_FREE_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); @@ -400,8 +399,8 @@ mod hook_tests { // (from the new `lg_sample`) once the current, already-primed // distance (drawn under whatever `lg_sample` was in effect before // this call, e.g. the crate's default of 512 KiB) has been - // exhausted — so the very next allocation isn't guaranteed to - // sample yet, only allocations after that first one are. + // exhausted, so the very next allocation isn't guaranteed to sample + // yet. Only allocations after that first one are. prof_reset(0).unwrap(); prof_active::write(true).unwrap(); diff --git a/jemalloc-sys/Cargo.toml b/jemalloc-sys/Cargo.toml index 53d3314b6..9b3d922f1 100644 --- a/jemalloc-sys/Cargo.toml +++ b/jemalloc-sys/Cargo.toml @@ -42,7 +42,6 @@ cc = "^1.0.13" default = ["background_threads_runtime_support"] profiling = [] profiling_libunwind = ["profiling"] -profiling_hooks = ["profiling"] debug = [] background_threads_runtime_support = [] background_threads = [ "background_threads_runtime_support" ] diff --git a/jemalloc-sys/README.md b/jemalloc-sys/README.md index c590a1b68..f4e6ad075 100644 --- a/jemalloc-sys/README.md +++ b/jemalloc-sys/README.md @@ -45,6 +45,21 @@ This crate provides following cargo feature flags: * `libgcc` (unless --disable-prof-libgcc) * `gcc intrinsics` (unless --disable-prof-gcc) + The matching `profiling` feature in `tikv-jemalloc-ctl` also exposes + `jemalloc`'s experimental `experimental.hooks.prof_sample`/ + `prof_sample_free`/`prof_backtrace` hooks, letting an external sampler (e.g. + an eBPF profiler) piggyback `jemalloc`'s sampling decision without its stack + walking. These hooks are not re-exported by `tikv-jemallocator`. + + The feature compiles profiling support but does not enable profiling. To + enable it, configure `prof:true` before `jemalloc` initialises. This can be + done at process launch with the appropriate `MALLOC_CONF` environment + variable, typically `_RJEM_MALLOC_CONF` for prefixed builds. Use + `prof:true,prof_active:false` to install hooks before enabling sampling + through `prof.active`, or use `prof:true` to begin sampling immediately. + Alternatively, `JEMALLOC_SYS_WITH_MALLOC_CONF` can embed the same + configuration at build time. + * `profiling_libunwind` (configure `jemalloc` with `--enable-prof-libunwind`): Force jemalloc to use `libunwind` for backtracing during heap profiling instead of the default gcc-based unwinding, which has a @@ -54,20 +69,6 @@ This crate provides following cargo feature flags: to be installed. On macOS/iOS, unwind symbols are provided by the system and no extra library is needed. -* `profiling_hooks`: Enables `profiling` automatically and bakes - `prof:true,prof_active:false` into the default `malloc_conf`, so sampling - is installable but inert until a consumer flips `prof.active` at runtime. - Exposes `jemalloc`'s experimental - `experimental.hooks.prof_sample`/`prof_sample_free`/`prof_backtrace` hooks - through `tikv-jemalloc-ctl`'s `profiling` module, letting an external - sampler (e.g. an eBPF profiler) piggyback `jemalloc`'s sampling decision - without its stack walking. Not re-exported by `tikv-jemallocator`. - - Since this crate has `links = "jemalloc"`, there's only one `jemalloc` - build per dependency graph, so enabling this on any dependent applies - `prof:true,prof_active:false` to that shared build for everyone in the - graph too. - * `stats` (configure `jemalloc` with `--enable-stats`): Enable statistics gathering functionality. See the `jemalloc`'s "`opt.stats_print`" option documentation for usage details. diff --git a/jemalloc-sys/build.rs b/jemalloc-sys/build.rs index a2a797c85..845b9d43f 100644 --- a/jemalloc-sys/build.rs +++ b/jemalloc-sys/build.rs @@ -246,23 +246,6 @@ fn main() { malloc_conf += "background_thread:false"; } - if env::var("CARGO_FEATURE_PROFILING_HOOKS").is_ok() { - info!("CARGO_FEATURE_PROFILING_HOOKS set"); - // Baking bare `prof:true` would also default `prof_active` and - // `prof_thread_active_init` to true. Since `links = "jemalloc"` - // unifies this build across every dependent in the graph, that would - // make the one shared jemalloc actively sample every allocation - // process-wide as soon as any dependent enables this feature, even - // if no hook is ever installed. Keep sampling installable - // (`opt_prof`) but inert until a consumer flips `prof.active` (or - // calls `tikv_jemalloc_ctl::profiling::prof_active::write(true)`) at - // runtime. - if !malloc_conf.is_empty() { - malloc_conf.push(','); - } - malloc_conf.push_str("prof:true,prof_active:false"); - } - if let Ok(malloc_conf_opts) = read_and_watch_env("JEMALLOC_SYS_WITH_MALLOC_CONF") { if !malloc_conf.is_empty() { malloc_conf.push(','); From da68df265d37a6ee3cbcff93f52a35e33c64ee0e Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 17 Sep 2026 10:32:53 +0200 Subject: [PATCH 4/6] Address more review feedback Signed-off-by: Scott Gerring --- ci/run.sh | 13 +++++++++---- jemalloc-ctl/src/profiling.rs | 33 +++++++++++---------------------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index 56ef68600..621995cdf 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -59,10 +59,15 @@ case "${TARGET}" in cargo test --target "${TARGET}" \ --manifest-path jemalloc-ctl/Cargo.toml \ --no-default-features - # FIXME: cross fails to pass features to jemalloc-ctl - # ${CARGO_CMD} test --target "${TARGET}" \ - # --manifest-path jemalloc-ctl \ - # --no-default-features --features use_std + ( + # This malloc_conf is for the Rust hook tests and breaks jemalloc's C tests. + unset JEMALLOC_SYS_RUN_JEMALLOC_TESTS + JEMALLOC_SYS_WITH_MALLOC_CONF=prof:true,prof_active:false \ + cargo test --target "${TARGET}" \ + --manifest-path jemalloc-ctl/Cargo.toml \ + --no-default-features \ + --features 'profiling use_std' + ) ;; esac diff --git a/jemalloc-ctl/src/profiling.rs b/jemalloc-ctl/src/profiling.rs index 1c878334c..5a4fe769a 100644 --- a/jemalloc-ctl/src/profiling.rs +++ b/jemalloc-ctl/src/profiling.rs @@ -298,14 +298,15 @@ pub fn set_prof_sample_free_hook( } /// Installs or replaces the hook `jemalloc` calls to capture a sample's -/// backtrace, returning the previously-installed hook. +/// backtrace, returning the previously-installed hook, if any. /// /// Corresponds to `experimental.hooks.prof_backtrace`. Unlike /// [`set_prof_sample_hook`]/[`set_prof_sample_free_hook`], this hook cannot /// be uninstalled (`jemalloc` rejects a `NULL` new hook with `EINVAL`). Install /// [`noop_prof_backtrace_hook`] instead of `jemalloc`'s default unwinder if -/// backtraces aren't wanted. The returned previous hook may be restored later -/// or invoked by the replacement during a valid backtrace-hook call. +/// backtraces aren't wanted. If present, the returned previous hook may be +/// restored later or invoked by the replacement during a valid backtrace-hook +/// call. /// /// # Errors /// @@ -313,8 +314,10 @@ pub fn set_prof_sample_free_hook( /// [`prof_reset`]. pub fn set_prof_backtrace_hook( hook: ProfBacktraceHook, -) -> crate::error::Result { - unsafe { crate::raw::update(b"experimental.hooks.prof_backtrace\0", hook) } +) -> crate::error::Result> { + unsafe { + crate::raw::update(b"experimental.hooks.prof_backtrace\0", Some(hook)) + } } /// A [`ProfBacktraceHook`] that reports an empty backtrace for every @@ -350,22 +353,6 @@ mod hook_tests { use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; - union Conf { - bytes: &'static u8, - c_char: &'static libc::c_char, - } - - // Enable profiling only for this test binary. Normal library builds leave - // the process-wide profiling policy to the final consumer. - #[export_name = "_rjem_malloc_conf"] - pub static TEST_MALLOC_CONF: Option<&'static libc::c_char> = - Some(unsafe { - Conf { - bytes: &b"prof:true,prof_active:false\0"[0], - } - .c_char - }); - static SAMPLE_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); static SAMPLE_FREE_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); @@ -444,7 +431,9 @@ mod hook_tests { #[test] fn backtrace_hook_can_be_replaced_and_restored() { - let prev = set_prof_backtrace_hook(noop_prof_backtrace_hook).unwrap(); + let prev = set_prof_backtrace_hook(noop_prof_backtrace_hook) + .unwrap() + .expect("jemalloc had no previous backtrace hook"); set_prof_backtrace_hook(prev).unwrap(); } } From 8b8a06cfe65319c0dcc34b1d02eed5a96c9050bc Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 21 Sep 2026 08:08:09 +0200 Subject: [PATCH 5/6] address review feedback Signed-off-by: Scott Gerring --- jemalloc-ctl/src/profiling.rs | 57 ++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/jemalloc-ctl/src/profiling.rs b/jemalloc-ctl/src/profiling.rs index 5a4fe769a..dada7fdbc 100644 --- a/jemalloc-ctl/src/profiling.rs +++ b/jemalloc-ctl/src/profiling.rs @@ -7,6 +7,11 @@ //! `experimental.hooks.prof_sample`/`prof_sample_free`/`prof_backtrace` hooks //! via [`set_prof_sample_hook`], [`set_prof_sample_free_hook`], and //! [`set_prof_backtrace_hook`]. +//! +//! # Experimental hook API +//! +//! `jemalloc` considers these hook mallctls experimental. Their names and +//! callback ABIs may change between `jemalloc` versions without notice. option! { lg_prof_interval[ str: b"opt.lg_prof_interval\0", non_str: 2 ] => libc::ssize_t | @@ -194,8 +199,8 @@ option! { /// // `false` is always accepted, even if `opt.prof` is disabled at /// // runtime; writing `true` additionally requires `opt.prof` to be /// // `true`, else it fails with `ENOENT`. - /// let was_active = profiling::prof_active::write(false).unwrap(); - /// # let _ = was_active; + /// let was_active = profiling::prof_active::update(false).unwrap(); + /// profiling::prof_active::write(was_active).unwrap(); /// # } /// ``` mib_docs: /// See [`prof_active`]. @@ -277,7 +282,13 @@ pub type ProfBacktraceHook = unsafe extern "C" fn( /// [`prof_reset`]. Note that `opt.prof` being `true` is sufficient to /// install a hook; [`prof_active`] need not be `true` (installing while /// inactive is a no-op until activated). -pub fn set_prof_sample_hook( +/// +/// # Safety +/// +/// The caller must ensure the linked `jemalloc` uses the [`ProfSampleHook`] ABI +/// documented here and that `hook`, if present, upholds that type's safety +/// contract for every invocation while installed. +pub unsafe fn set_prof_sample_hook( hook: Option, ) -> crate::error::Result> { unsafe { crate::raw::update(b"experimental.hooks.prof_sample\0", hook) } @@ -289,7 +300,13 @@ pub fn set_prof_sample_hook( /// /// Corresponds to `experimental.hooks.prof_sample_free`. See /// [`set_prof_sample_hook`] for the applicable error semantics. -pub fn set_prof_sample_free_hook( +/// +/// # Safety +/// +/// The caller must ensure the linked `jemalloc` uses the +/// [`ProfSampleFreeHook`] ABI documented here and that `hook`, if present, +/// upholds that type's safety contract for every invocation while installed. +pub unsafe fn set_prof_sample_free_hook( hook: Option, ) -> crate::error::Result> { unsafe { @@ -312,7 +329,13 @@ pub fn set_prof_sample_free_hook( /// /// Returns an error (`ENOENT`) if `opt.prof` is `false` at runtime; see /// [`prof_reset`]. -pub fn set_prof_backtrace_hook( +/// +/// # Safety +/// +/// The caller must ensure the linked `jemalloc` uses the [`ProfBacktraceHook`] +/// ABI documented here and that `hook` upholds that type's safety contract for +/// every invocation while installed. +pub unsafe fn set_prof_backtrace_hook( hook: ProfBacktraceHook, ) -> crate::error::Result> { unsafe { @@ -380,7 +403,8 @@ mod hook_tests { #[test] fn sample_and_sample_free_hooks_fire() { let was_active = prof_active::read().unwrap(); - let prev_lg_sample = lg_prof_sample::read().unwrap(); + let prev_lg_sample: libc::size_t = + unsafe { crate::raw::read(b"prof.lg_sample\0") }.unwrap(); // lg_sample: 0 => average one sample per byte, i.e. every allocation. // `jemalloc` only recomputes each thread's next sample distance // (from the new `lg_sample`) once the current, already-primed @@ -392,10 +416,12 @@ mod hook_tests { prof_active::write(true).unwrap(); let prev_sample = - set_prof_sample_hook(Some(counting_sample_hook)).unwrap(); - let prev_sample_free = - set_prof_sample_free_hook(Some(counting_sample_free_hook)) + unsafe { set_prof_sample_hook(Some(counting_sample_hook)) } .unwrap(); + let prev_sample_free = unsafe { + set_prof_sample_free_hook(Some(counting_sample_free_hook)) + } + .unwrap(); // Warm up past any stale pre-reset sample distance: 16 MiB is many // times the largest plausible leftover distance from a 512 KiB mean. @@ -414,8 +440,8 @@ mod hook_tests { } } - set_prof_sample_hook(prev_sample).unwrap(); - set_prof_sample_free_hook(prev_sample_free).unwrap(); + unsafe { set_prof_sample_hook(prev_sample) }.unwrap(); + unsafe { set_prof_sample_free_hook(prev_sample_free) }.unwrap(); prof_active::write(was_active).unwrap(); prof_reset(prev_lg_sample).unwrap(); @@ -431,9 +457,10 @@ mod hook_tests { #[test] fn backtrace_hook_can_be_replaced_and_restored() { - let prev = set_prof_backtrace_hook(noop_prof_backtrace_hook) - .unwrap() - .expect("jemalloc had no previous backtrace hook"); - set_prof_backtrace_hook(prev).unwrap(); + let prev = + unsafe { set_prof_backtrace_hook(noop_prof_backtrace_hook) } + .unwrap() + .expect("jemalloc had no previous backtrace hook"); + unsafe { set_prof_backtrace_hook(prev) }.unwrap(); } } From 3718fd25c477b26fb77f305dddda03cf37f7a425 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 21 Sep 2026 08:26:25 +0200 Subject: [PATCH 6/6] more cleanup Signed-off-by: Scott Gerring --- CHANGELOG.md | 9 +++ jemalloc-ctl/src/profiling.rs | 133 +++++++++++++++++++++------------- jemalloc-sys/README.md | 11 ++- 3 files changed, 98 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b185c866c..53f13b88e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# Unreleased + +- jemalloc-ctl: expose `prof.active`, `prof.lg_sample`, and `prof.reset` via + `profiling::{prof_active, lg_sample, prof_reset}` +- jemalloc-ctl: expose jemalloc's experimental sample hooks under the + `profiling` feature (`set_prof_sample_hook`, `set_prof_sample_free_hook`, + `set_prof_backtrace_hook`, `noop_prof_backtrace_hook`, and the `Prof*Hook` + types) + # 0.7.0 - 2026-05-25 - Reverse order of MAKEFLAGS priority (#152) diff --git a/jemalloc-ctl/src/profiling.rs b/jemalloc-ctl/src/profiling.rs index dada7fdbc..a77e52e45 100644 --- a/jemalloc-ctl/src/profiling.rs +++ b/jemalloc-ctl/src/profiling.rs @@ -11,7 +11,11 @@ //! # Experimental hook API //! //! `jemalloc` considers these hook mallctls experimental. Their names and -//! callback ABIs may change between `jemalloc` versions without notice. +//! callback ABIs may change between versions without notice. Hooks are +//! process-wide, may run concurrently, and replacement does not wait for +//! in-flight calls. + +use libc::{c_uint, c_void}; option! { lg_prof_interval[ str: b"opt.lg_prof_interval\0", non_str: 2 ] => libc::ssize_t | @@ -181,12 +185,13 @@ option! { /// by default unless [`prof_active`] is set to `false`, e.g. via /// `prof_active:false` in `MALLOC_CONF`. /// - /// Note: `opt.prof_thread_active_init` is unrelated. It controls the - /// per-thread `thread.prof.active` flag, not this global toggle. + /// `thread.prof.active` is a separate per-thread gate, initialised from + /// `opt.prof_thread_active_init`; both it and this global control must be + /// active for a thread to sample. /// - /// While inactive, sampling hooks installed via the `profiling` - /// feature's hook setters remain installed but do not fire, since no - /// allocation is ever selected for sampling. + /// While this control is inactive, no new allocation samples are selected, + /// so [`ProfSampleHook`] does not fire. [`ProfSampleFreeHook`] can still + /// fire for allocations sampled before deactivation. /// /// # Examples /// @@ -206,9 +211,20 @@ option! { mib_docs: /// See [`prof_active`]. } +option! { + lg_sample[ str: b"prof.lg_sample\0", non_str: 2 ] => libc::size_t | + ops: r | + docs: + /// Current log base 2 of the mean number of bytes between samples. + /// + /// Initialised from [`lg_prof_sample`] and updated by [`prof_reset`]. + mib_docs: /// See [`lg_sample`]. +} + /// Resets `jemalloc`'s heap profile sample accumulators and, going forward, -/// samples allocations at a rate of one per `2^lg_sample` bytes of -/// allocation activity. +/// draws sample intervals from a geometric distribution with a mean of +/// `2^new_lg_sample` bytes of allocation activity (values `>= 64` are clamped +/// to `63`). Sampling still requires `prof.active` and `thread.prof.active`. /// /// Corresponds to `prof.reset`, which is write-only: unlike most keys in /// this module, there is no matching `read()`/`update()`. @@ -219,12 +235,10 @@ option! { /// `profiling` feature enables profiling support but does not set `opt.prof`; /// configure `prof:true`, for example via `JEMALLOC_SYS_WITH_MALLOC_CONF` at /// build time. -pub fn prof_reset(lg_sample: libc::size_t) -> crate::error::Result<()> { - unsafe { crate::raw::write(b"prof.reset\0", lg_sample) } +pub fn prof_reset(new_lg_sample: libc::size_t) -> crate::error::Result<()> { + unsafe { crate::raw::write(b"prof.reset\0", new_lg_sample) } } -use libc::{c_uint, c_void}; - /// Signature of a hook installable via [`set_prof_sample_hook`]. /// /// `jemalloc` invokes this hook synchronously, inline on the allocating @@ -237,10 +251,11 @@ use libc::{c_uint, c_void}; /// /// # Safety /// -/// No `jemalloc` mutex is held while this hook runs, but it does run inside -/// `jemalloc`'s `pre_reentrancy`/`post_reentrancy` bracket. It may itself -/// allocate or free; nested allocator activity is excluded from profiling. -/// It must not unwind across the `extern "C"` boundary. +/// Arguments are valid only during the call. The hook must not deallocate +/// `ptr`, retain any argument, or mutate `backtrace`. No `jemalloc` mutex is +/// held, and the hook may allocate or free other allocations; nested allocator +/// activity is excluded from profiling. Hooks may run concurrently and must +/// not unwind across the `extern "C"` boundary. pub type ProfSampleHook = unsafe extern "C" fn( ptr: *const c_void, size: libc::size_t, @@ -253,17 +268,27 @@ pub type ProfSampleHook = unsafe extern "C" fn( /// /// `jemalloc` invokes this hook synchronously, inline on the freeing /// thread, just before it frees a previously-sampled allocation of -/// `usable_size` bytes at `ptr`. See [`ProfSampleHook`] for the applicable safety -/// contract. +/// `usable_size` bytes at `ptr`. +/// +/// # Safety +/// +/// `ptr` is valid only during the call and must not be retained or deallocated. +/// The hook may allocate or free other allocations; nested allocator activity +/// is excluded from profiling. Hooks may run concurrently and must not unwind +/// across the `extern "C"` boundary. pub type ProfSampleFreeHook = unsafe extern "C" fn(ptr: *const c_void, usable_size: libc::size_t); /// Signature of a hook installable via [`set_prof_backtrace_hook`]. /// -/// `jemalloc` invokes this hook to capture the stack trace for a sample; it -/// must write at most `max_length` frames into `backtrace` and store the -/// number of frames written through `backtrace_length`. See -/// [`ProfSampleHook`] for the applicable safety contract. +/// `jemalloc` invokes this hook to capture the stack trace for a sample. +/// +/// # Safety +/// +/// The pointers are valid only during the call and must not be retained. The +/// hook must write at most `max_length` frames into `backtrace`, store that +/// count through `backtrace_length`, support concurrent calls, and not unwind +/// across the `extern "C"` boundary. pub type ProfBacktraceHook = unsafe extern "C" fn( backtrace: *mut *mut c_void, backtrace_length: *mut c_uint, @@ -285,9 +310,9 @@ pub type ProfBacktraceHook = unsafe extern "C" fn( /// /// # Safety /// -/// The caller must ensure the linked `jemalloc` uses the [`ProfSampleHook`] ABI -/// documented here and that `hook`, if present, upholds that type's safety -/// contract for every invocation while installed. +/// The caller must ensure the linked `jemalloc` uses the documented +/// [`ProfSampleHook`] ABI and that `hook`, if present, upholds its contract. +/// Replaced hooks may still be in flight, so their state must remain valid. pub unsafe fn set_prof_sample_hook( hook: Option, ) -> crate::error::Result> { @@ -303,9 +328,9 @@ pub unsafe fn set_prof_sample_hook( /// /// # Safety /// -/// The caller must ensure the linked `jemalloc` uses the -/// [`ProfSampleFreeHook`] ABI documented here and that `hook`, if present, -/// upholds that type's safety contract for every invocation while installed. +/// The caller must ensure the linked `jemalloc` uses the documented +/// [`ProfSampleFreeHook`] ABI and that `hook`, if present, upholds its contract. +/// Replaced hooks may still be in flight, so their state must remain valid. pub unsafe fn set_prof_sample_free_hook( hook: Option, ) -> crate::error::Result> { @@ -332,9 +357,9 @@ pub unsafe fn set_prof_sample_free_hook( /// /// # Safety /// -/// The caller must ensure the linked `jemalloc` uses the [`ProfBacktraceHook`] -/// ABI documented here and that `hook` upholds that type's safety contract for -/// every invocation while installed. +/// The caller must ensure the linked `jemalloc` uses the documented +/// [`ProfBacktraceHook`] ABI and that `hook` upholds its contract. Replaced +/// hooks may still be in flight, so their state must remain valid. pub unsafe fn set_prof_backtrace_hook( hook: ProfBacktraceHook, ) -> crate::error::Result> { @@ -348,7 +373,7 @@ pub unsafe fn set_prof_backtrace_hook( /// /// Installing this via [`set_prof_backtrace_hook`] disables `jemalloc`'s /// own stack unwinding going forward: the per-allocation sampling -/// decision still happens at the configured rate (see [`lg_prof_sample`]) +/// decision still happens at the configured rate (see [`lg_sample`]) /// and [`set_prof_sample_hook`]/[`set_prof_sample_free_hook`] hooks still /// fire, but with `backtrace_length` reported as `0`. Intended for /// out-of-process samplers (e.g. an eBPF profiler) that capture their own @@ -378,14 +403,17 @@ mod hook_tests { static SAMPLE_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); static SAMPLE_FREE_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); + static SAMPLE_BACKTRACE_LENGTH: AtomicUsize = AtomicUsize::new(usize::MAX); unsafe extern "C" fn counting_sample_hook( _ptr: *const c_void, _size: libc::size_t, _backtrace: *mut *mut c_void, - _backtrace_length: c_uint, + backtrace_length: c_uint, _usable_size: libc::size_t, ) { + SAMPLE_BACKTRACE_LENGTH + .store(backtrace_length as usize, Ordering::SeqCst); SAMPLE_HOOK_CALLS.fetch_add(1, Ordering::SeqCst); } @@ -402,9 +430,14 @@ mod hook_tests { // hooks actually fired before restoring prior state. #[test] fn sample_and_sample_free_hooks_fire() { - let was_active = prof_active::read().unwrap(); - let prev_lg_sample: libc::size_t = - unsafe { crate::raw::read(b"prof.lg_sample\0") }.unwrap(); + // Requires `opt.prof`; see the `profiling` feature docs. The CI job + // for this test bakes `prof:true` via `JEMALLOC_SYS_WITH_MALLOC_CONF`. + if !prof::read().unwrap() { + return; + } + let was_active = prof_active::update(false).unwrap(); + let prev_lg_sample = lg_sample::read().unwrap(); + SAMPLE_BACKTRACE_LENGTH.store(usize::MAX, Ordering::SeqCst); // lg_sample: 0 => average one sample per byte, i.e. every allocation. // `jemalloc` only recomputes each thread's next sample distance // (from the new `lg_sample`) once the current, already-primed @@ -413,8 +446,10 @@ mod hook_tests { // exhausted, so the very next allocation isn't guaranteed to sample // yet. Only allocations after that first one are. prof_reset(0).unwrap(); - prof_active::write(true).unwrap(); - + let prev_backtrace = + unsafe { set_prof_backtrace_hook(noop_prof_backtrace_hook) } + .unwrap() + .expect("jemalloc had no previous backtrace hook"); let prev_sample = unsafe { set_prof_sample_hook(Some(counting_sample_hook)) } .unwrap(); @@ -422,10 +457,12 @@ mod hook_tests { set_prof_sample_free_hook(Some(counting_sample_free_hook)) } .unwrap(); + prof_active::write(true).unwrap(); - // Warm up past any stale pre-reset sample distance: 16 MiB is many - // times the largest plausible leftover distance from a 512 KiB mean. - for _ in 0..16 { + // Warm up past any sample distance primed under the previous + // `lg_sample`. The default's geometric interval can reach ~18 MiB, so + // burn well past that before checking for samples. + for _ in 0..32 { drop(Box::new([0u8; 1024 * 1024])); } @@ -440,10 +477,12 @@ mod hook_tests { } } + prof_active::write(false).unwrap(); unsafe { set_prof_sample_hook(prev_sample) }.unwrap(); unsafe { set_prof_sample_free_hook(prev_sample_free) }.unwrap(); - prof_active::write(was_active).unwrap(); + unsafe { set_prof_backtrace_hook(prev_backtrace) }.unwrap(); prof_reset(prev_lg_sample).unwrap(); + prof_active::write(was_active).unwrap(); assert!( SAMPLE_HOOK_CALLS.load(Ordering::SeqCst) > before_sample, @@ -453,14 +492,6 @@ mod hook_tests { SAMPLE_FREE_HOOK_CALLS.load(Ordering::SeqCst) > before_free, "prof_sample_free hook did not fire after freeing sampled allocations" ); - } - - #[test] - fn backtrace_hook_can_be_replaced_and_restored() { - let prev = - unsafe { set_prof_backtrace_hook(noop_prof_backtrace_hook) } - .unwrap() - .expect("jemalloc had no previous backtrace hook"); - unsafe { set_prof_backtrace_hook(prev) }.unwrap(); + assert_eq!(SAMPLE_BACKTRACE_LENGTH.load(Ordering::SeqCst), 0); } } diff --git a/jemalloc-sys/README.md b/jemalloc-sys/README.md index f4e6ad075..e67a4e930 100644 --- a/jemalloc-sys/README.md +++ b/jemalloc-sys/README.md @@ -48,16 +48,19 @@ This crate provides following cargo feature flags: The matching `profiling` feature in `tikv-jemalloc-ctl` also exposes `jemalloc`'s experimental `experimental.hooks.prof_sample`/ `prof_sample_free`/`prof_backtrace` hooks, letting an external sampler (e.g. - an eBPF profiler) piggyback `jemalloc`'s sampling decision without its stack - walking. These hooks are not re-exported by `tikv-jemallocator`. + an eBPF profiler) piggyback `jemalloc`'s sampling decision. To avoid + `jemalloc`'s own stack walk, install `noop_prof_backtrace_hook` through + `set_prof_backtrace_hook`. These hooks are not re-exported by + `tikv-jemallocator`. The feature compiles profiling support but does not enable profiling. To enable it, configure `prof:true` before `jemalloc` initialises. This can be done at process launch with the appropriate `MALLOC_CONF` environment variable, typically `_RJEM_MALLOC_CONF` for prefixed builds. Use `prof:true,prof_active:false` to install hooks before enabling sampling - through `prof.active`, or use `prof:true` to begin sampling immediately. - Alternatively, `JEMALLOC_SYS_WITH_MALLOC_CONF` can embed the same + through `prof.active`, or use `prof:true` to begin sampling immediately with + the default per-thread settings. Alternatively, + `JEMALLOC_SYS_WITH_MALLOC_CONF` can embed the same configuration at build time. * `profiling_libunwind` (configure `jemalloc` with `--enable-prof-libunwind`):