diff --git a/mm/page_cache_ext_ds.c b/mm/page_cache_ext_ds.c index 5e50b8cdf..7ea4d87bd 100644 --- a/mm/page_cache_ext_ds.c +++ b/mm/page_cache_ext_ds.c @@ -347,7 +347,14 @@ int cache_ext_list_iterate_extended(struct mem_cgroup *memcg, } } - if (opts->continue_mode == CACHE_EXT_CONTINUE_ITER && opts->evict_mode == CACHE_EXT_CONTINUE_ITER) + /* + * Must mirror the lock-acquisition condition above. Comparing + * against CACHE_EXT_CONTINUE_ITER (an iterate *return code*) only + * worked because it happens to equal CACHE_EXT_ITERATE_SKIP (0); + * reordering either enum would silently turn this into a + * read_lock/write_unlock mismatch and corrupt the rwlock. + */ + if (opts->continue_mode == CACHE_EXT_ITERATE_SKIP && opts->evict_mode == CACHE_EXT_ITERATE_SKIP) read_unlock(®istry->lock); else write_unlock(®istry->lock); @@ -476,10 +483,21 @@ int __bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, pr_err("cache_ext: list is NULL\n"); return -1; } + /* + * Hold the registry write_lock across snip + score + putback. + * valid_folios_del() kfrees the cache_ext_list_node wrapper after + * dropping the same registry write_lock, so releasing the lock + * during scoring exposes the per-CPU sample_folios_arr[] to a + * use-after-free: a folio removed from the page cache while its + * node sits in the sample array is freed underneath score_fn() + * (the LIST_POISON check in __putback_list_nodes() detects only + * some of these, and itself reads freed memory). Serializing the + * whole sample is the simple fix; a perf-preserving alternative + * would refcount the wrapper. + */ write_lock(®istry->lock); - // Optimization: Snip the front of the list and select the pages without - // holding the lock. + // Phase 1: snip the front of the list. for (int i = 0; i < num_folios_to_sample; i++) { if (list_empty(&list_ptr->head)) { pr_warn("cache_ext: ran out of folios to sample\n"); @@ -497,9 +515,9 @@ int __bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, list_del_init(&node->node); } - write_unlock(®istry->lock); - - // 1. For every n elements, evict the one with the min score + // Phase 2: score under write_lock. The score_fn callback is + // non-sleepable, so calling it under rwlock_t is safe. + // For every n elements, evict the one with the min score. ctx->nr_folios_to_evict = 0; int sample_folios_idx = 0; for (int i = 0; i < ctx->request_nr_folios_to_evict; i++) { @@ -528,8 +546,8 @@ int __bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, ctx->nr_folios_to_evict++; } - // 2. Put everything to the back of the list. - write_lock(®istry->lock); + // Phase 3: put everything to the back of the list, still under + // the write_lock taken before Phase 1. __putback_list_nodes(list_ptr, sample_folios_arr, sample_folios_size); write_unlock(®istry->lock);