From b16f260a25e3b7ec2793e1875156577f4e13f675 Mon Sep 17 00:00:00 2001 From: Nadav Amit Date: Fri, 4 Sep 2026 11:37:31 +0300 Subject: [PATCH 1/2] mm/page_cache_ext: hold registry lock across sample snip+score+putback The snip-then-score-unlocked optimization in __bpf_cache_ext_list_sample() races with valid_folios_del(): the cache_ext_list_node wrapper is kfreed after that path drops the registry write_lock, so nodes sitting in the per-CPU sample_folios_arr[] can be freed while score_fn() dereferences them. The LIST_POISON check in __putback_list_nodes() catches only some of these, and itself reads freed memory. Hold the registry write_lock across snip + score + putback. score_fn is non-sleepable, so calling it under rwlock_t is safe. Note: this serializes sampling against all list updates and measurably slows sampling-based policies under memory pressure (we measured ~30% throughput loss for the LHD policy on YCSB-C/LevelDB with a 4 GiB cgroup, on a 6.18 port of this code); a perf-preserving fix would refcount the node wrapper. Correctness first. Co-Authored-By: Claude Fable 5 --- mm/page_cache_ext_ds.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/mm/page_cache_ext_ds.c b/mm/page_cache_ext_ds.c index 5e50b8cdf..cdfe3529a 100644 --- a/mm/page_cache_ext_ds.c +++ b/mm/page_cache_ext_ds.c @@ -476,10 +476,21 @@ int __bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, pr_err("cache_ext: list is NULL\n"); return -1; } + /* + * Hold the registry write_lock across snip + score + putback. + * valid_folios_del() kfrees the cache_ext_list_node wrapper after + * dropping the same registry write_lock, so releasing the lock + * during scoring exposes the per-CPU sample_folios_arr[] to a + * use-after-free: a folio removed from the page cache while its + * node sits in the sample array is freed underneath score_fn() + * (the LIST_POISON check in __putback_list_nodes() detects only + * some of these, and itself reads freed memory). Serializing the + * whole sample is the simple fix; a perf-preserving alternative + * would refcount the wrapper. + */ write_lock(®istry->lock); - // Optimization: Snip the front of the list and select the pages without - // holding the lock. + // Phase 1: snip the front of the list. for (int i = 0; i < num_folios_to_sample; i++) { if (list_empty(&list_ptr->head)) { pr_warn("cache_ext: ran out of folios to sample\n"); @@ -497,9 +508,9 @@ int __bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, list_del_init(&node->node); } - write_unlock(®istry->lock); - - // 1. For every n elements, evict the one with the min score + // Phase 2: score under write_lock. The score_fn callback is + // non-sleepable, so calling it under rwlock_t is safe. + // For every n elements, evict the one with the min score. ctx->nr_folios_to_evict = 0; int sample_folios_idx = 0; for (int i = 0; i < ctx->request_nr_folios_to_evict; i++) { @@ -528,8 +539,8 @@ int __bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, ctx->nr_folios_to_evict++; } - // 2. Put everything to the back of the list. - write_lock(®istry->lock); + // Phase 3: put everything to the back of the list, still under + // the write_lock taken before Phase 1. __putback_list_nodes(list_ptr, sample_folios_arr, sample_folios_size); write_unlock(®istry->lock); From c34e3505566d1713e559fc53eed9c4657fda1141 Mon Sep 17 00:00:00 2001 From: Nadav Amit Date: Fri, 4 Sep 2026 11:37:31 +0300 Subject: [PATCH 2/2] mm/page_cache_ext: unlock iterate_extended with the condition it locked with cache_ext_list_iterate_extended() chooses read_lock vs write_lock by comparing the iterate modes against CACHE_EXT_ITERATE_SKIP, but releases by comparing against CACHE_EXT_CONTINUE_ITER -- an iterate *return code* from a different enum. This works today only because both constants happen to be 0; reordering either enum would silently pair read_lock with write_unlock and corrupt the rwlock. Use the same condition on both sides. Co-Authored-By: Claude Fable 5 --- mm/page_cache_ext_ds.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mm/page_cache_ext_ds.c b/mm/page_cache_ext_ds.c index cdfe3529a..7ea4d87bd 100644 --- a/mm/page_cache_ext_ds.c +++ b/mm/page_cache_ext_ds.c @@ -347,7 +347,14 @@ int cache_ext_list_iterate_extended(struct mem_cgroup *memcg, } } - if (opts->continue_mode == CACHE_EXT_CONTINUE_ITER && opts->evict_mode == CACHE_EXT_CONTINUE_ITER) + /* + * Must mirror the lock-acquisition condition above. Comparing + * against CACHE_EXT_CONTINUE_ITER (an iterate *return code*) only + * worked because it happens to equal CACHE_EXT_ITERATE_SKIP (0); + * reordering either enum would silently turn this into a + * read_lock/write_unlock mismatch and corrupt the rwlock. + */ + if (opts->continue_mode == CACHE_EXT_ITERATE_SKIP && opts->evict_mode == CACHE_EXT_ITERATE_SKIP) read_unlock(®istry->lock); else write_unlock(®istry->lock);