tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

slub: remove CONFIG_SLUB_TINY specific code paths

CONFIG_SLUB_TINY minimizes the SLUB's memory overhead in multiple ways,
mainly by avoiding percpu caching of slabs and objects. It also reduces
code size by replacing some code paths with simplified ones through
ifdefs, but the benefits of that are smaller and would complicate the
upcoming changes.

Thus remove these code paths and associated ifdefs and simplify the code
base.

Link: https://patch.msgid.link/20251105-sheaves-cleanups-v1-4-b8218e1ac7ef@suse.cz
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

Vlastimil Babka 31e0886f 1ce20c28

+4 -105
-2
mm/slab.h
··· 236 236 * Slab cache management. 237 237 */ 238 238 struct kmem_cache { 239 - #ifndef CONFIG_SLUB_TINY 240 239 struct kmem_cache_cpu __percpu *cpu_slab; 241 240 struct lock_class_key lock_key; 242 - #endif 243 241 struct slub_percpu_sheaves __percpu *cpu_sheaves; 244 242 /* Used for retrieving partial slabs, etc. */ 245 243 slab_flags_t flags;
+4 -103
mm/slub.c
··· 410 410 NR_SLUB_STAT_ITEMS 411 411 }; 412 412 413 - #ifndef CONFIG_SLUB_TINY 414 413 /* 415 414 * When changing the layout, make sure freelist and tid are still compatible 416 415 * with this_cpu_cmpxchg_double() alignment requirements. ··· 431 432 unsigned int stat[NR_SLUB_STAT_ITEMS]; 432 433 #endif 433 434 }; 434 - #endif /* CONFIG_SLUB_TINY */ 435 435 436 436 static inline void stat(const struct kmem_cache *s, enum stat_item si) 437 437 { ··· 595 597 return freelist_ptr_decode(s, p, ptr_addr); 596 598 } 597 599 598 - #ifndef CONFIG_SLUB_TINY 599 600 static void prefetch_freepointer(const struct kmem_cache *s, void *object) 600 601 { 601 602 prefetchw(object + s->offset); 602 603 } 603 - #endif 604 604 605 605 /* 606 606 * When running under KMSAN, get_freepointer_safe() may return an uninitialized ··· 710 714 return s->cpu_partial_slabs; 711 715 } 712 716 #else 717 + #ifdef SLAB_SUPPORTS_SYSFS 713 718 static inline void 714 719 slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) 715 720 { 716 721 } 722 + #endif 717 723 718 724 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) 719 725 { ··· 2024 2026 int objects) {} 2025 2027 static inline void dec_slabs_node(struct kmem_cache *s, int node, 2026 2028 int objects) {} 2027 - #ifndef CONFIG_SLUB_TINY 2028 2029 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 2029 2030 void **freelist, void *nextfree) 2030 2031 { 2031 2032 return false; 2032 2033 } 2033 - #endif 2034 2034 #endif /* CONFIG_SLUB_DEBUG */ 2035 2035 2036 2036 #ifdef CONFIG_SLAB_OBJ_EXT ··· 3619 3623 return get_any_partial(s, pc); 3620 3624 } 3621 3625 3622 - #ifndef CONFIG_SLUB_TINY 3623 - 3624 3626 #ifdef CONFIG_PREEMPTION 3625 3627 /* 3626 3628 * Calculate the next globally unique transaction for disambiguation ··· 4018 4024 return c->slab || slub_percpu_partial(c); 4019 4025 } 4020 4026 4021 - #else /* CONFIG_SLUB_TINY */ 4022 - static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { } 4023 - static inline bool has_cpu_slab(int cpu, struct kmem_cache *s) { return false; } 4024 - static inline void flush_this_cpu_slab(struct kmem_cache *s) { } 4025 - #endif /* CONFIG_SLUB_TINY */ 4026 - 4027 4027 static bool has_pcs_used(int cpu, struct kmem_cache *s) 4028 4028 { 4029 4029 struct slub_percpu_sheaves *pcs; ··· 4358 4370 return true; 4359 4371 } 4360 4372 4361 - #ifndef CONFIG_SLUB_TINY 4362 4373 static inline bool 4363 4374 __update_cpu_freelist_fast(struct kmem_cache *s, 4364 4375 void *freelist_old, void *freelist_new, ··· 4621 4634 pc.orig_size = orig_size; 4622 4635 slab = get_partial(s, node, &pc); 4623 4636 if (slab) { 4624 - if (kmem_cache_debug(s)) { 4637 + if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4625 4638 freelist = pc.object; 4626 4639 /* 4627 4640 * For debug caches here we had to go through ··· 4659 4672 4660 4673 stat(s, ALLOC_SLAB); 4661 4674 4662 - if (kmem_cache_debug(s)) { 4675 + if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4663 4676 freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); 4664 4677 4665 4678 if (unlikely(!freelist)) { ··· 4871 4884 4872 4885 return object; 4873 4886 } 4874 - #else /* CONFIG_SLUB_TINY */ 4875 - static void *__slab_alloc_node(struct kmem_cache *s, 4876 - gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 4877 - { 4878 - struct partial_context pc; 4879 - struct slab *slab; 4880 - void *object; 4881 - 4882 - pc.flags = gfpflags; 4883 - pc.orig_size = orig_size; 4884 - slab = get_partial(s, node, &pc); 4885 - 4886 - if (slab) 4887 - return pc.object; 4888 - 4889 - slab = new_slab(s, gfpflags, node); 4890 - if (unlikely(!slab)) { 4891 - slab_out_of_memory(s, gfpflags, node); 4892 - return NULL; 4893 - } 4894 - 4895 - object = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); 4896 - 4897 - return object; 4898 - } 4899 - #endif /* CONFIG_SLUB_TINY */ 4900 4887 4901 4888 /* 4902 4889 * If the object has been wiped upon free, make sure it's fully initialized by ··· 5721 5760 * it did local_lock_irqsave(&s->cpu_slab->lock, flags). 5722 5761 * In this case fast path with __update_cpu_freelist_fast() is not safe. 5723 5762 */ 5724 - #ifndef CONFIG_SLUB_TINY 5725 5763 if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock)) 5726 - #endif 5727 5764 ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size); 5728 5765 5729 5766 if (PTR_ERR(ret) == -EBUSY) { ··· 6512 6553 llist_for_each_safe(pos, t, llnode) { 6513 6554 struct slab *slab = container_of(pos, struct slab, llnode); 6514 6555 6515 - #ifdef CONFIG_SLUB_TINY 6516 - free_slab(slab->slab_cache, slab); 6517 - #else 6518 6556 if (slab->frozen) 6519 6557 deactivate_slab(slab->slab_cache, slab, slab->flush_freelist); 6520 6558 else 6521 6559 free_slab(slab->slab_cache, slab); 6522 - #endif 6523 6560 } 6524 6561 } 6525 6562 ··· 6551 6596 irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); 6552 6597 } 6553 6598 6554 - #ifndef CONFIG_SLUB_TINY 6555 6599 /* 6556 6600 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 6557 6601 * can perform fastpath freeing without additional function calls. ··· 6643 6689 } 6644 6690 stat_add(s, FREE_FASTPATH, cnt); 6645 6691 } 6646 - #else /* CONFIG_SLUB_TINY */ 6647 - static void do_slab_free(struct kmem_cache *s, 6648 - struct slab *slab, void *head, void *tail, 6649 - int cnt, unsigned long addr) 6650 - { 6651 - __slab_free(s, slab, head, tail, cnt, addr); 6652 - } 6653 - #endif /* CONFIG_SLUB_TINY */ 6654 6692 6655 6693 static __fastpath_inline 6656 6694 void slab_free(struct kmem_cache *s, struct slab *slab, void *object, ··· 6920 6974 * since kasan quarantine takes locks and not supported from NMI. 6921 6975 */ 6922 6976 kasan_slab_free(s, x, false, false, /* skip quarantine */true); 6923 - #ifndef CONFIG_SLUB_TINY 6924 6977 do_slab_free(s, slab, x, x, 0, _RET_IP_); 6925 - #else 6926 - defer_free(s, x); 6927 - #endif 6928 6978 } 6929 6979 EXPORT_SYMBOL_GPL(kfree_nolock); 6930 6980 ··· 7370 7428 } 7371 7429 EXPORT_SYMBOL(kmem_cache_free_bulk); 7372 7430 7373 - #ifndef CONFIG_SLUB_TINY 7374 7431 static inline 7375 7432 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 7376 7433 void **p) ··· 7434 7493 return 0; 7435 7494 7436 7495 } 7437 - #else /* CONFIG_SLUB_TINY */ 7438 - static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, 7439 - size_t size, void **p) 7440 - { 7441 - int i; 7442 - 7443 - for (i = 0; i < size; i++) { 7444 - void *object = kfence_alloc(s, s->object_size, flags); 7445 - 7446 - if (unlikely(object)) { 7447 - p[i] = object; 7448 - continue; 7449 - } 7450 - 7451 - p[i] = __slab_alloc_node(s, flags, NUMA_NO_NODE, 7452 - _RET_IP_, s->object_size); 7453 - if (unlikely(!p[i])) 7454 - goto error; 7455 - 7456 - maybe_wipe_obj_freeptr(s, p[i]); 7457 - } 7458 - 7459 - return i; 7460 - 7461 - error: 7462 - __kmem_cache_free_bulk(s, i, p); 7463 - return 0; 7464 - } 7465 - #endif /* CONFIG_SLUB_TINY */ 7466 7496 7467 7497 /* Note that interrupts must be enabled when calling this function. */ 7468 7498 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, ··· 7652 7740 barn_init(barn); 7653 7741 } 7654 7742 7655 - #ifndef CONFIG_SLUB_TINY 7656 7743 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 7657 7744 { 7658 7745 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < ··· 7672 7761 7673 7762 return 1; 7674 7763 } 7675 - #else 7676 - static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 7677 - { 7678 - return 1; 7679 - } 7680 - #endif /* CONFIG_SLUB_TINY */ 7681 7764 7682 7765 static int init_percpu_sheaves(struct kmem_cache *s) 7683 7766 { ··· 7761 7856 cache_random_seq_destroy(s); 7762 7857 if (s->cpu_sheaves) 7763 7858 pcs_destroy(s); 7764 - #ifndef CONFIG_SLUB_TINY 7765 7859 #ifdef CONFIG_PREEMPT_RT 7766 7860 if (s->cpu_slab) 7767 7861 lockdep_unregister_key(&s->lock_key); 7768 7862 #endif 7769 7863 free_percpu(s->cpu_slab); 7770 - #endif 7771 7864 free_kmem_cache_nodes(s); 7772 7865 } 7773 7866 ··· 8508 8605 8509 8606 void __init kmem_cache_init_late(void) 8510 8607 { 8511 - #ifndef CONFIG_SLUB_TINY 8512 8608 flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); 8513 8609 WARN_ON(!flushwq); 8514 - #endif 8515 8610 } 8516 8611 8517 8612 struct kmem_cache *