tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

slab: remove the do_slab_free() fastpath

We have removed cpu slab usage from allocation paths. Now remove
do_slab_free() which was freeing objects to the cpu slab when
the object belonged to it. Instead call __slab_free() directly,
which was previously the fallback.

This simplifies kfree_nolock() - when freeing to percpu sheaf
fails, we can call defer_free() directly.

Also remove functions that became unused.

Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Reviewed-by: Hao Li <hao.li@linux.dev>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

Vlastimil Babka bdc9282f e323b52c

+13 -136
+13 -136
mm/slub.c
··· 3678 3678 return cpu; 3679 3679 } 3680 3680 3681 - static inline void note_cmpxchg_failure(const char *n, 3682 - const struct kmem_cache *s, unsigned long tid) 3683 - { 3684 - #ifdef SLUB_DEBUG_CMPXCHG 3685 - unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); 3686 - 3687 - pr_info("%s %s: cmpxchg redo ", n, s->name); 3688 - 3689 - if (IS_ENABLED(CONFIG_PREEMPTION) && 3690 - tid_to_cpu(tid) != tid_to_cpu(actual_tid)) { 3691 - pr_warn("due to cpu change %d -> %d\n", 3692 - tid_to_cpu(tid), tid_to_cpu(actual_tid)); 3693 - } else if (tid_to_event(tid) != tid_to_event(actual_tid)) { 3694 - pr_warn("due to cpu running other code. Event %ld->%ld\n", 3695 - tid_to_event(tid), tid_to_event(actual_tid)); 3696 - } else { 3697 - pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", 3698 - actual_tid, tid, next_tid(tid)); 3699 - } 3700 - #endif 3701 - stat(s, CMPXCHG_DOUBLE_CPU_FAIL); 3702 - } 3703 - 3704 3681 static void init_kmem_cache_cpus(struct kmem_cache *s) 3705 3682 { 3706 3683 #ifdef CONFIG_PREEMPT_RT ··· 4214 4237 return gfp_pfmemalloc_allowed(gfpflags); 4215 4238 4216 4239 return true; 4217 - } 4218 - 4219 - static inline bool 4220 - __update_cpu_freelist_fast(struct kmem_cache *s, 4221 - void *freelist_old, void *freelist_new, 4222 - unsigned long tid) 4223 - { 4224 - struct freelist_tid old = { .freelist = freelist_old, .tid = tid }; 4225 - struct freelist_tid new = { .freelist = freelist_new, .tid = next_tid(tid) }; 4226 - 4227 - return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid, 4228 - &old.freelist_tid, new.freelist_tid); 4229 4240 } 4230 4241 4231 4242 /* ··· 6153 6188 irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); 6154 6189 } 6155 6190 6156 - /* 6157 - * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 6158 - * can perform fastpath freeing without additional function calls. 6159 - * 6160 - * The fastpath is only possible if we are freeing to the current cpu slab 6161 - * of this processor. This typically the case if we have just allocated 6162 - * the item before. 6163 - * 6164 - * If fastpath is not possible then fall back to __slab_free where we deal 6165 - * with all sorts of special processing. 6166 - * 6167 - * Bulk free of a freelist with several objects (all pointing to the 6168 - * same slab) possible by specifying head and tail ptr, plus objects 6169 - * count (cnt). Bulk free indicated by tail pointer being set. 6170 - */ 6171 - static __always_inline void do_slab_free(struct kmem_cache *s, 6172 - struct slab *slab, void *head, void *tail, 6173 - int cnt, unsigned long addr) 6174 - { 6175 - /* cnt == 0 signals that it's called from kfree_nolock() */ 6176 - bool allow_spin = cnt; 6177 - struct kmem_cache_cpu *c; 6178 - unsigned long tid; 6179 - void **freelist; 6180 - 6181 - redo: 6182 - /* 6183 - * Determine the currently cpus per cpu slab. 6184 - * The cpu may change afterward. However that does not matter since 6185 - * data is retrieved via this pointer. If we are on the same cpu 6186 - * during the cmpxchg then the free will succeed. 6187 - */ 6188 - c = raw_cpu_ptr(s->cpu_slab); 6189 - tid = READ_ONCE(c->tid); 6190 - 6191 - /* Same with comment on barrier() in __slab_alloc_node() */ 6192 - barrier(); 6193 - 6194 - if (unlikely(slab != c->slab)) { 6195 - if (unlikely(!allow_spin)) { 6196 - /* 6197 - * __slab_free() can locklessly cmpxchg16 into a slab, 6198 - * but then it might need to take spin_lock 6199 - * for further processing. 6200 - * Avoid the complexity and simply add to a deferred list. 6201 - */ 6202 - defer_free(s, head); 6203 - } else { 6204 - __slab_free(s, slab, head, tail, cnt, addr); 6205 - } 6206 - return; 6207 - } 6208 - 6209 - if (unlikely(!allow_spin)) { 6210 - if ((in_nmi() || !USE_LOCKLESS_FAST_PATH()) && 6211 - local_lock_is_locked(&s->cpu_slab->lock)) { 6212 - defer_free(s, head); 6213 - return; 6214 - } 6215 - cnt = 1; /* restore cnt. kfree_nolock() frees one object at a time */ 6216 - } 6217 - 6218 - if (USE_LOCKLESS_FAST_PATH()) { 6219 - freelist = READ_ONCE(c->freelist); 6220 - 6221 - set_freepointer(s, tail, freelist); 6222 - 6223 - if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) { 6224 - note_cmpxchg_failure("slab_free", s, tid); 6225 - goto redo; 6226 - } 6227 - } else { 6228 - __maybe_unused unsigned long flags = 0; 6229 - 6230 - /* Update the free list under the local lock */ 6231 - local_lock_cpu_slab(s, flags); 6232 - c = this_cpu_ptr(s->cpu_slab); 6233 - if (unlikely(slab != c->slab)) { 6234 - local_unlock_cpu_slab(s, flags); 6235 - goto redo; 6236 - } 6237 - tid = c->tid; 6238 - freelist = c->freelist; 6239 - 6240 - set_freepointer(s, tail, freelist); 6241 - c->freelist = head; 6242 - c->tid = next_tid(tid); 6243 - 6244 - local_unlock_cpu_slab(s, flags); 6245 - } 6246 - stat_add(s, FREE_FASTPATH, cnt); 6247 - } 6248 - 6249 6191 static __fastpath_inline 6250 6192 void slab_free(struct kmem_cache *s, struct slab *slab, void *object, 6251 6193 unsigned long addr) ··· 6169 6297 return; 6170 6298 } 6171 6299 6172 - do_slab_free(s, slab, object, object, 1, addr); 6300 + __slab_free(s, slab, object, object, 1, addr); 6173 6301 } 6174 6302 6175 6303 #ifdef CONFIG_MEMCG ··· 6178 6306 void memcg_alloc_abort_single(struct kmem_cache *s, void *object) 6179 6307 { 6180 6308 if (likely(slab_free_hook(s, object, slab_want_init_on_free(s), false))) 6181 - do_slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_); 6309 + __slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_); 6182 6310 } 6183 6311 #endif 6184 6312 ··· 6193 6321 * to remove objects, whose reuse must be delayed. 6194 6322 */ 6195 6323 if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt))) 6196 - do_slab_free(s, slab, head, tail, cnt, addr); 6324 + __slab_free(s, slab, head, tail, cnt, addr); 6197 6325 } 6198 6326 6199 6327 #ifdef CONFIG_SLUB_RCU_DEBUG ··· 6219 6347 6220 6348 /* resume freeing */ 6221 6349 if (slab_free_hook(s, object, slab_want_init_on_free(s), true)) 6222 - do_slab_free(s, slab, object, object, 1, _THIS_IP_); 6350 + __slab_free(s, slab, object, object, 1, _THIS_IP_); 6223 6351 } 6224 6352 #endif /* CONFIG_SLUB_RCU_DEBUG */ 6225 6353 6226 6354 #ifdef CONFIG_KASAN_GENERIC 6227 6355 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr) 6228 6356 { 6229 - do_slab_free(cache, virt_to_slab(x), x, x, 1, addr); 6357 + __slab_free(cache, virt_to_slab(x), x, x, 1, addr); 6230 6358 } 6231 6359 #endif 6232 6360 ··· 6442 6570 return; 6443 6571 } 6444 6572 6445 - do_slab_free(s, slab, x, x, 0, _RET_IP_); 6573 + /* 6574 + * __slab_free() can locklessly cmpxchg16 into a slab, but then it might 6575 + * need to take spin_lock for further processing. 6576 + * Avoid the complexity and simply add to a deferred list. 6577 + */ 6578 + defer_free(s, x); 6446 6579 } 6447 6580 EXPORT_SYMBOL_GPL(kfree_nolock); 6448 6581 ··· 6873 6996 if (kfence_free(df.freelist)) 6874 6997 continue; 6875 6998 6876 - do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, 6999 + __slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, 6877 7000 _RET_IP_); 6878 7001 } while (likely(size)); 6879 7002 } ··· 6959 7082 cnt++; 6960 7083 object = get_freepointer(s, object); 6961 7084 } while (object); 6962 - do_slab_free(s, slab, head, tail, cnt, _RET_IP_); 7085 + __slab_free(s, slab, head, tail, cnt, _RET_IP_); 6963 7086 } 6964 7087 6965 7088 if (refilled >= max)