Merge branch 'slab/for-6.19/sheaves_cleanups' into slab/for-next

-6

include/linux/gfp_types.h

··· 55 55 #ifdef CONFIG_LOCKDEP 56 56 ___GFP_NOLOCKDEP_BIT, 57 57 #endif 58 - #ifdef CONFIG_SLAB_OBJ_EXT 59 58 ___GFP_NO_OBJ_EXT_BIT, 60 - #endif 61 59 ___GFP_LAST_BIT 62 60 }; 63 61 ··· 96 98 #else 97 99 #define ___GFP_NOLOCKDEP 0 98 100 #endif 99 - #ifdef CONFIG_SLAB_OBJ_EXT 100 101 #define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) 101 - #else 102 - #define ___GFP_NO_OBJ_EXT 0 103 - #endif 104 102 105 103 /* 106 104 * Physical address zone modifiers (see linux/mmzone.h - low four bits)

-2

mm/slab.h

··· 236 236 * Slab cache management. 237 237 */ 238 238 struct kmem_cache { 239 - #ifndef CONFIG_SLUB_TINY 240 239 struct kmem_cache_cpu __percpu *cpu_slab; 241 240 struct lock_class_key lock_key; 242 - #endif 243 241 struct slub_percpu_sheaves __percpu *cpu_sheaves; 244 242 /* Used for retrieving partial slabs, etc. */ 245 243 slab_flags_t flags;

+173 -158

mm/slub.c

··· 410 410 NR_SLUB_STAT_ITEMS 411 411 }; 412 412 413 - #ifndef CONFIG_SLUB_TINY 414 413 /* 415 414 * When changing the layout, make sure freelist and tid are still compatible 416 415 * with this_cpu_cmpxchg_double() alignment requirements. ··· 431 432 unsigned int stat[NR_SLUB_STAT_ITEMS]; 432 433 #endif 433 434 }; 434 - #endif /* CONFIG_SLUB_TINY */ 435 435 436 436 static inline void stat(const struct kmem_cache *s, enum stat_item si) 437 437 { ··· 467 469 struct rcu_head rcu_head; 468 470 struct list_head barn_list; 469 471 /* only used for prefilled sheafs */ 470 - unsigned int capacity; 472 + struct { 473 + unsigned int capacity; 474 + bool pfmemalloc; 475 + }; 471 476 }; 472 477 struct kmem_cache *cache; 473 478 unsigned int size; ··· 595 594 return freelist_ptr_decode(s, p, ptr_addr); 596 595 } 597 596 598 - #ifndef CONFIG_SLUB_TINY 599 597 static void prefetch_freepointer(const struct kmem_cache *s, void *object) 600 598 { 601 599 prefetchw(object + s->offset); 602 600 } 603 - #endif 604 601 605 602 /* 606 603 * When running under KMSAN, get_freepointer_safe() may return an uninitialized ··· 710 711 return s->cpu_partial_slabs; 711 712 } 712 713 #else 714 + #ifdef SLAB_SUPPORTS_SYSFS 713 715 static inline void 714 716 slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) 715 717 { 716 718 } 719 + #endif 717 720 718 721 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) 719 722 { ··· 2028 2027 int objects) {} 2029 2028 static inline void dec_slabs_node(struct kmem_cache *s, int node, 2030 2029 int objects) {} 2031 - #ifndef CONFIG_SLUB_TINY 2032 2030 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 2033 2031 void **freelist, void *nextfree) 2034 2032 { 2035 2033 return false; 2036 2034 } 2037 - #endif 2038 2035 #endif /* CONFIG_SLUB_DEBUG */ 2036 + 2037 + /* 2038 + * The allocated objcg pointers array is not accounted directly. 2039 + * Moreover, it should not come from DMA buffer and is not readily 2040 + * reclaimable. So those GFP bits should be masked off. 2041 + */ 2042 + #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ 2043 + __GFP_ACCOUNT | __GFP_NOFAIL) 2039 2044 2040 2045 #ifdef CONFIG_SLAB_OBJ_EXT 2041 2046 ··· 2092 2085 struct slabobj_ext *vec, unsigned int objects) {} 2093 2086 2094 2087 #endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ 2095 - 2096 - /* 2097 - * The allocated objcg pointers array is not accounted directly. 2098 - * Moreover, it should not come from DMA buffer and is not readily 2099 - * reclaimable. So those GFP bits should be masked off. 2100 - */ 2101 - #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ 2102 - __GFP_ACCOUNT | __GFP_NOFAIL) 2103 2088 2104 2089 static inline void init_slab_obj_exts(struct slab *slab) 2105 2090 { ··· 2600 2601 2601 2602 static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp) 2602 2603 { 2603 - struct slab_sheaf *sheaf = kzalloc(struct_size(sheaf, objects, 2604 - s->sheaf_capacity), gfp); 2604 + struct slab_sheaf *sheaf; 2605 + size_t sheaf_size; 2606 + 2607 + if (gfp & __GFP_NO_OBJ_EXT) 2608 + return NULL; 2609 + 2610 + gfp &= ~OBJCGS_CLEAR_MASK; 2611 + 2612 + /* 2613 + * Prevent recursion to the same cache, or a deep stack of kmallocs of 2614 + * varying sizes (sheaf capacity might differ for each kmalloc size 2615 + * bucket) 2616 + */ 2617 + if (s->flags & SLAB_KMALLOC) 2618 + gfp |= __GFP_NO_OBJ_EXT; 2619 + 2620 + sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity); 2621 + sheaf = kzalloc(sheaf_size, gfp); 2605 2622 2606 2623 if (unlikely(!sheaf)) 2607 2624 return NULL; ··· 2670 2655 if (!sheaf) 2671 2656 return NULL; 2672 2657 2673 - if (refill_sheaf(s, sheaf, gfp)) { 2658 + if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC)) { 2674 2659 free_empty_sheaf(s, sheaf); 2675 2660 return NULL; 2676 2661 } ··· 2748 2733 sheaf->size = 0; 2749 2734 } 2750 2735 2751 - static void __rcu_free_sheaf_prepare(struct kmem_cache *s, 2736 + static bool __rcu_free_sheaf_prepare(struct kmem_cache *s, 2752 2737 struct slab_sheaf *sheaf) 2753 2738 { 2754 2739 bool init = slab_want_init_on_free(s); 2755 2740 void **p = &sheaf->objects[0]; 2756 2741 unsigned int i = 0; 2742 + bool pfmemalloc = false; 2757 2743 2758 2744 while (i < sheaf->size) { 2759 2745 struct slab *slab = virt_to_slab(p[i]); ··· 2767 2751 continue; 2768 2752 } 2769 2753 2754 + if (slab_test_pfmemalloc(slab)) 2755 + pfmemalloc = true; 2756 + 2770 2757 i++; 2771 2758 } 2759 + 2760 + return pfmemalloc; 2772 2761 } 2773 2762 2774 2763 static void rcu_free_sheaf_nobarn(struct rcu_head *head) ··· 3036 3015 3037 3016 static void barn_shrink(struct kmem_cache *s, struct node_barn *barn) 3038 3017 { 3039 - struct list_head empty_list; 3040 - struct list_head full_list; 3018 + LIST_HEAD(empty_list); 3019 + LIST_HEAD(full_list); 3041 3020 struct slab_sheaf *sheaf, *sheaf2; 3042 3021 unsigned long flags; 3043 - 3044 - INIT_LIST_HEAD(&empty_list); 3045 - INIT_LIST_HEAD(&full_list); 3046 3022 3047 3023 spin_lock_irqsave(&barn->lock, flags); 3048 3024 ··· 3636 3618 return get_any_partial(s, pc); 3637 3619 } 3638 3620 3639 - #ifndef CONFIG_SLUB_TINY 3640 - 3641 3621 #ifdef CONFIG_PREEMPTION 3642 3622 /* 3643 3623 * Calculate the next globally unique transaction for disambiguation ··· 4035 4019 return c->slab || slub_percpu_partial(c); 4036 4020 } 4037 4021 4038 - #else /* CONFIG_SLUB_TINY */ 4039 - static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { } 4040 - static inline bool has_cpu_slab(int cpu, struct kmem_cache *s) { return false; } 4041 - static inline void flush_this_cpu_slab(struct kmem_cache *s) { } 4042 - #endif /* CONFIG_SLUB_TINY */ 4043 - 4044 4022 static bool has_pcs_used(int cpu, struct kmem_cache *s) 4045 4023 { 4046 4024 struct slub_percpu_sheaves *pcs; ··· 4375 4365 return true; 4376 4366 } 4377 4367 4378 - #ifndef CONFIG_SLUB_TINY 4379 4368 static inline bool 4380 4369 __update_cpu_freelist_fast(struct kmem_cache *s, 4381 4370 void *freelist_old, void *freelist_new, ··· 4638 4629 pc.orig_size = orig_size; 4639 4630 slab = get_partial(s, node, &pc); 4640 4631 if (slab) { 4641 - if (kmem_cache_debug(s)) { 4632 + if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4642 4633 freelist = pc.object; 4643 4634 /* 4644 4635 * For debug caches here we had to go through ··· 4676 4667 4677 4668 stat(s, ALLOC_SLAB); 4678 4669 4679 - if (kmem_cache_debug(s)) { 4670 + if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4680 4671 freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); 4681 4672 4682 - if (unlikely(!freelist)) 4673 + if (unlikely(!freelist)) { 4674 + /* This could cause an endless loop. Fail instead. */ 4675 + if (!allow_spin) 4676 + return NULL; 4683 4677 goto new_objects; 4678 + } 4684 4679 4685 4680 if (s->flags & SLAB_STORE_USER) 4686 4681 set_track(s, freelist, TRACK_ALLOC, addr, ··· 4888 4875 4889 4876 return object; 4890 4877 } 4891 - #else /* CONFIG_SLUB_TINY */ 4892 - static void *__slab_alloc_node(struct kmem_cache *s, 4893 - gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 4894 - { 4895 - struct partial_context pc; 4896 - struct slab *slab; 4897 - void *object; 4898 - 4899 - pc.flags = gfpflags; 4900 - pc.orig_size = orig_size; 4901 - slab = get_partial(s, node, &pc); 4902 - 4903 - if (slab) 4904 - return pc.object; 4905 - 4906 - slab = new_slab(s, gfpflags, node); 4907 - if (unlikely(!slab)) { 4908 - slab_out_of_memory(s, gfpflags, node); 4909 - return NULL; 4910 - } 4911 - 4912 - object = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); 4913 - 4914 - return object; 4915 - } 4916 - #endif /* CONFIG_SLUB_TINY */ 4917 4878 4918 4879 /* 4919 4880 * If the object has been wiped upon free, make sure it's fully initialized by ··· 5028 5041 return NULL; 5029 5042 5030 5043 if (empty) { 5031 - if (!refill_sheaf(s, empty, gfp)) { 5044 + if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC)) { 5032 5045 full = empty; 5033 5046 } else { 5034 5047 /* ··· 5328 5341 } 5329 5342 EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); 5330 5343 5344 + static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s, 5345 + struct slab_sheaf *sheaf, gfp_t gfp) 5346 + { 5347 + int ret = 0; 5348 + 5349 + ret = refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC); 5350 + 5351 + if (likely(!ret || !gfp_pfmemalloc_allowed(gfp))) 5352 + return ret; 5353 + 5354 + /* 5355 + * if we are allowed to, refill sheaf with pfmemalloc but then remember 5356 + * it for when it's returned 5357 + */ 5358 + ret = refill_sheaf(s, sheaf, gfp); 5359 + sheaf->pfmemalloc = true; 5360 + 5361 + return ret; 5362 + } 5363 + 5331 5364 /* 5332 5365 * returns a sheaf that has at least the requested size 5333 5366 * when prefilling is needed, do so with given gfp flags ··· 5382 5375 sheaf->cache = s; 5383 5376 sheaf->capacity = size; 5384 5377 5378 + /* 5379 + * we do not need to care about pfmemalloc here because oversize 5380 + * sheaves area always flushed and freed when returned 5381 + */ 5385 5382 if (!__kmem_cache_alloc_bulk(s, gfp, size, 5386 5383 &sheaf->objects[0])) { 5387 5384 kfree(sheaf); ··· 5422 5411 if (!sheaf) 5423 5412 sheaf = alloc_empty_sheaf(s, gfp); 5424 5413 5425 - if (sheaf && sheaf->size < size) { 5426 - if (refill_sheaf(s, sheaf, gfp)) { 5414 + if (sheaf) { 5415 + sheaf->capacity = s->sheaf_capacity; 5416 + sheaf->pfmemalloc = false; 5417 + 5418 + if (sheaf->size < size && 5419 + __prefill_sheaf_pfmemalloc(s, sheaf, gfp)) { 5427 5420 sheaf_flush_unused(s, sheaf); 5428 5421 free_empty_sheaf(s, sheaf); 5429 5422 sheaf = NULL; 5430 5423 } 5431 5424 } 5432 - 5433 - if (sheaf) 5434 - sheaf->capacity = s->sheaf_capacity; 5435 5425 5436 5426 return sheaf; 5437 5427 } ··· 5453 5441 struct slub_percpu_sheaves *pcs; 5454 5442 struct node_barn *barn; 5455 5443 5456 - if (unlikely(sheaf->capacity != s->sheaf_capacity)) { 5444 + if (unlikely((sheaf->capacity != s->sheaf_capacity) 5445 + || sheaf->pfmemalloc)) { 5457 5446 sheaf_flush_unused(s, sheaf); 5458 5447 kfree(sheaf); 5459 5448 return; ··· 5520 5507 5521 5508 if (likely(sheaf->capacity >= size)) { 5522 5509 if (likely(sheaf->capacity == s->sheaf_capacity)) 5523 - return refill_sheaf(s, sheaf, gfp); 5510 + return __prefill_sheaf_pfmemalloc(s, sheaf, gfp); 5524 5511 5525 5512 if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size, 5526 5513 &sheaf->objects[sheaf->size])) { ··· 5553 5540 * 5554 5541 * The gfp parameter is meant only to specify __GFP_ZERO or __GFP_ACCOUNT 5555 5542 * memcg charging is forced over limit if necessary, to avoid failure. 5543 + * 5544 + * It is possible that the allocation comes from kfence and then the sheaf 5545 + * size is not decreased. 5556 5546 */ 5557 5547 void * 5558 5548 kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp, ··· 5567 5551 if (sheaf->size == 0) 5568 5552 goto out; 5569 5553 5570 - ret = sheaf->objects[--sheaf->size]; 5554 + ret = kfence_alloc(s, s->object_size, gfp); 5555 + 5556 + if (likely(!ret)) 5557 + ret = sheaf->objects[--sheaf->size]; 5571 5558 5572 5559 init = slab_want_init_on_alloc(gfp, s); 5573 5560 ··· 5738 5719 * it did local_lock_irqsave(&s->cpu_slab->lock, flags). 5739 5720 * In this case fast path with __update_cpu_freelist_fast() is not safe. 5740 5721 */ 5741 - #ifndef CONFIG_SLUB_TINY 5742 5722 if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock)) 5743 - #endif 5744 5723 ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size); 5745 5724 5746 5725 if (PTR_ERR(ret) == -EBUSY) { ··· 5876 5859 unsigned long addr) 5877 5860 5878 5861 { 5879 - void *prior; 5880 - int was_frozen; 5862 + void *old_head; 5863 + bool was_frozen, was_full; 5881 5864 struct slab new; 5882 5865 unsigned long counters; 5883 5866 struct kmem_cache_node *n = NULL; ··· 5891 5874 return; 5892 5875 } 5893 5876 5877 + /* 5878 + * It is enough to test IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) below 5879 + * instead of kmem_cache_has_cpu_partial(s), because kmem_cache_debug(s) 5880 + * is the only other reason it can be false, and it is already handled 5881 + * above. 5882 + */ 5883 + 5894 5884 do { 5895 5885 if (unlikely(n)) { 5896 5886 spin_unlock_irqrestore(&n->list_lock, flags); 5897 5887 n = NULL; 5898 5888 } 5899 - prior = slab->freelist; 5889 + old_head = slab->freelist; 5900 5890 counters = slab->counters; 5901 - set_freepointer(s, tail, prior); 5891 + set_freepointer(s, tail, old_head); 5902 5892 new.counters = counters; 5903 - was_frozen = new.frozen; 5893 + was_frozen = !!new.frozen; 5894 + was_full = (old_head == NULL); 5904 5895 new.inuse -= cnt; 5905 - if ((!new.inuse || !prior) && !was_frozen) { 5906 - /* Needs to be taken off a list */ 5907 - if (!kmem_cache_has_cpu_partial(s) || prior) { 5896 + /* 5897 + * Might need to be taken off (due to becoming empty) or added 5898 + * to (due to not being full anymore) the partial list. 5899 + * Unless it's frozen. 5900 + */ 5901 + if ((!new.inuse || was_full) && !was_frozen) { 5902 + /* 5903 + * If slab becomes non-full and we have cpu partial 5904 + * lists, we put it there unconditionally to avoid 5905 + * taking the list_lock. Otherwise we need it. 5906 + */ 5907 + if (!(IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full)) { 5908 5908 5909 5909 n = get_node(s, slab_nid(slab)); 5910 5910 /* ··· 5939 5905 } 5940 5906 5941 5907 } while (!slab_update_freelist(s, slab, 5942 - prior, counters, 5908 + old_head, counters, 5943 5909 head, new.counters, 5944 5910 "__slab_free")); 5945 5911 ··· 5951 5917 * activity can be necessary. 5952 5918 */ 5953 5919 stat(s, FREE_FROZEN); 5954 - } else if (kmem_cache_has_cpu_partial(s) && !prior) { 5920 + } else if (IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full) { 5955 5921 /* 5956 5922 * If we started with a full slab then put it onto the 5957 5923 * per cpu partial list. ··· 5960 5926 stat(s, CPU_PARTIAL_FREE); 5961 5927 } 5962 5928 5929 + /* 5930 + * In other cases we didn't take the list_lock because the slab 5931 + * was already on the partial list and will remain there. 5932 + */ 5933 + 5963 5934 return; 5964 5935 } 5965 5936 ··· 5972 5933 * This slab was partially empty but not on the per-node partial list, 5973 5934 * in which case we shouldn't manipulate its list, just return. 5974 5935 */ 5975 - if (prior && !on_node_partial) { 5936 + if (!was_full && !on_node_partial) { 5976 5937 spin_unlock_irqrestore(&n->list_lock, flags); 5977 5938 return; 5978 5939 } 5979 5940 5941 + /* 5942 + * If slab became empty, should we add/keep it on the partial list or we 5943 + * have enough? 5944 + */ 5980 5945 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) 5981 5946 goto slab_empty; 5982 5947 5983 5948 /* 5984 5949 * Objects left in the slab. If it was not on the partial list before 5985 - * then add it. 5950 + * then add it. This can only happen when cache has no per cpu partial 5951 + * list otherwise we would have put it there. 5986 5952 */ 5987 - if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { 5953 + if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) { 5988 5954 add_partial(n, slab, DEACTIVATE_TO_TAIL); 5989 5955 stat(s, FREE_ADD_PARTIAL); 5990 5956 } ··· 5997 5953 return; 5998 5954 5999 5955 slab_empty: 6000 - if (prior) { 6001 - /* 6002 - * Slab on the partial list. 6003 - */ 5956 + /* 5957 + * The slab could have a single object and thus go from full to empty in 5958 + * a single free, but more likely it was on the partial list. Remove it. 5959 + */ 5960 + if (likely(!was_full)) { 6004 5961 remove_partial(n, slab); 6005 5962 stat(s, FREE_REMOVE_PARTIAL); 6006 5963 } ··· 6226 6181 * handles it fine. The only downside is that sheaf will serve fewer 6227 6182 * allocations when reused. It only happens due to debugging, which is a 6228 6183 * performance hit anyway. 6184 + * 6185 + * If it returns true, there was at least one object from pfmemalloc 6186 + * slab so simply flush everything. 6229 6187 */ 6230 - __rcu_free_sheaf_prepare(s, sheaf); 6188 + if (__rcu_free_sheaf_prepare(s, sheaf)) 6189 + goto flush; 6231 6190 6232 6191 n = get_node(s, sheaf->node); 6233 6192 if (!n) ··· 6386 6337 continue; 6387 6338 } 6388 6339 6389 - if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != node)) { 6340 + if (unlikely((IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != node) 6341 + || slab_test_pfmemalloc(slab))) { 6390 6342 remote_objects[remote_nr] = p[i]; 6391 6343 p[i] = p[--size]; 6392 6344 if (++remote_nr >= PCS_BATCH_MAX) ··· 6529 6479 llist_for_each_safe(pos, t, llnode) { 6530 6480 struct slab *slab = container_of(pos, struct slab, llnode); 6531 6481 6532 - #ifdef CONFIG_SLUB_TINY 6533 - free_slab(slab->slab_cache, slab); 6534 - #else 6535 6482 if (slab->frozen) 6536 6483 deactivate_slab(slab->slab_cache, slab, slab->flush_freelist); 6537 6484 else 6538 6485 free_slab(slab->slab_cache, slab); 6539 - #endif 6540 6486 } 6541 6487 } 6542 6488 ··· 6568 6522 irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); 6569 6523 } 6570 6524 6571 - #ifndef CONFIG_SLUB_TINY 6572 6525 /* 6573 6526 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 6574 6527 * can perform fastpath freeing without additional function calls. ··· 6660 6615 } 6661 6616 stat_add(s, FREE_FASTPATH, cnt); 6662 6617 } 6663 - #else /* CONFIG_SLUB_TINY */ 6664 - static void do_slab_free(struct kmem_cache *s, 6665 - struct slab *slab, void *head, void *tail, 6666 - int cnt, unsigned long addr) 6667 - { 6668 - __slab_free(s, slab, head, tail, cnt, addr); 6669 - } 6670 - #endif /* CONFIG_SLUB_TINY */ 6671 6618 6672 6619 static __fastpath_inline 6673 6620 void slab_free(struct kmem_cache *s, struct slab *slab, void *object, ··· 6672 6635 return; 6673 6636 6674 6637 if (s->cpu_sheaves && likely(!IS_ENABLED(CONFIG_NUMA) || 6675 - slab_nid(slab) == numa_mem_id())) { 6638 + slab_nid(slab) == numa_mem_id()) 6639 + && likely(!slab_test_pfmemalloc(slab))) { 6676 6640 if (likely(free_to_pcs(s, object))) 6677 6641 return; 6678 6642 } ··· 6937 6899 * since kasan quarantine takes locks and not supported from NMI. 6938 6900 */ 6939 6901 kasan_slab_free(s, x, false, false, /* skip quarantine */true); 6940 - #ifndef CONFIG_SLUB_TINY 6941 6902 do_slab_free(s, slab, x, x, 0, _RET_IP_); 6942 - #else 6943 - defer_free(s, x); 6944 - #endif 6945 6903 } 6946 6904 EXPORT_SYMBOL_GPL(kfree_nolock); 6947 6905 ··· 7387 7353 } 7388 7354 EXPORT_SYMBOL(kmem_cache_free_bulk); 7389 7355 7390 - #ifndef CONFIG_SLUB_TINY 7391 7356 static inline 7392 7357 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 7393 7358 void **p) ··· 7404 7371 local_lock_irqsave(&s->cpu_slab->lock, irqflags); 7405 7372 7406 7373 for (i = 0; i < size; i++) { 7407 - void *object = kfence_alloc(s, s->object_size, flags); 7374 + void *object = c->freelist; 7408 7375 7409 - if (unlikely(object)) { 7410 - p[i] = object; 7411 - continue; 7412 - } 7413 - 7414 - object = c->freelist; 7415 7376 if (unlikely(!object)) { 7416 7377 /* 7417 7378 * We may have removed an object from c->freelist using ··· 7451 7424 return 0; 7452 7425 7453 7426 } 7454 - #else /* CONFIG_SLUB_TINY */ 7455 - static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, 7456 - size_t size, void **p) 7457 - { 7458 - int i; 7459 - 7460 - for (i = 0; i < size; i++) { 7461 - void *object = kfence_alloc(s, s->object_size, flags); 7462 - 7463 - if (unlikely(object)) { 7464 - p[i] = object; 7465 - continue; 7466 - } 7467 - 7468 - p[i] = __slab_alloc_node(s, flags, NUMA_NO_NODE, 7469 - _RET_IP_, s->object_size); 7470 - if (unlikely(!p[i])) 7471 - goto error; 7472 - 7473 - maybe_wipe_obj_freeptr(s, p[i]); 7474 - } 7475 - 7476 - return i; 7477 - 7478 - error: 7479 - __kmem_cache_free_bulk(s, i, p); 7480 - return 0; 7481 - } 7482 - #endif /* CONFIG_SLUB_TINY */ 7483 7427 7484 7428 /* Note that interrupts must be enabled when calling this function. */ 7485 7429 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, 7486 7430 void **p) 7487 7431 { 7488 7432 unsigned int i = 0; 7433 + void *kfence_obj; 7489 7434 7490 7435 if (!size) 7491 7436 return 0; ··· 7465 7466 s = slab_pre_alloc_hook(s, flags); 7466 7467 if (unlikely(!s)) 7467 7468 return 0; 7469 + 7470 + /* 7471 + * to make things simpler, only assume at most once kfence allocated 7472 + * object per bulk allocation and choose its index randomly 7473 + */ 7474 + kfence_obj = kfence_alloc(s, s->object_size, flags); 7475 + 7476 + if (unlikely(kfence_obj)) { 7477 + if (unlikely(size == 1)) { 7478 + p[0] = kfence_obj; 7479 + goto out; 7480 + } 7481 + size--; 7482 + } 7468 7483 7469 7484 if (s->cpu_sheaves) 7470 7485 i = alloc_from_pcs_bulk(s, size, p); ··· 7491 7478 if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) { 7492 7479 if (i > 0) 7493 7480 __kmem_cache_free_bulk(s, i, p); 7481 + if (kfence_obj) 7482 + __kfence_free(kfence_obj); 7494 7483 return 0; 7495 7484 } 7496 7485 } 7497 7486 7487 + if (unlikely(kfence_obj)) { 7488 + int idx = get_random_u32_below(size + 1); 7489 + 7490 + if (idx != size) 7491 + p[size] = p[idx]; 7492 + p[idx] = kfence_obj; 7493 + 7494 + size++; 7495 + } 7496 + 7497 + out: 7498 7498 /* 7499 7499 * memcg and kmem_cache debug support and memory initialization. 7500 7500 * Done outside of the IRQ disabled fastpath loop. ··· 7669 7643 barn_init(barn); 7670 7644 } 7671 7645 7672 - #ifndef CONFIG_SLUB_TINY 7673 7646 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 7674 7647 { 7675 7648 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < ··· 7689 7664 7690 7665 return 1; 7691 7666 } 7692 - #else 7693 - static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 7694 - { 7695 - return 1; 7696 - } 7697 - #endif /* CONFIG_SLUB_TINY */ 7698 7667 7699 7668 static int init_percpu_sheaves(struct kmem_cache *s) 7700 7669 { ··· 7778 7759 cache_random_seq_destroy(s); 7779 7760 if (s->cpu_sheaves) 7780 7761 pcs_destroy(s); 7781 - #ifndef CONFIG_SLUB_TINY 7782 7762 #ifdef CONFIG_PREEMPT_RT 7783 7763 if (s->cpu_slab) 7784 7764 lockdep_unregister_key(&s->lock_key); 7785 7765 #endif 7786 7766 free_percpu(s->cpu_slab); 7787 - #endif 7788 7767 free_kmem_cache_nodes(s); 7789 7768 } 7790 7769 ··· 8536 8519 8537 8520 void __init kmem_cache_init_late(void) 8538 8521 { 8539 - #ifndef CONFIG_SLUB_TINY 8540 8522 flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); 8541 8523 WARN_ON(!flushwq); 8542 - #endif 8543 8524 } 8544 8525 8545 8526 struct kmem_cache *

Configure Feed

Configure Feed