slab: make percpu sheaves compatible with kmalloc_nolock()/kfree_nolock()

+60 -22

1 changed file

expand all

slub.c

+60 -22

mm/slub.c

··· 2889 2889 s->cpu_sheaves = NULL; 2890 2890 } 2891 2891 2892 - static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn) 2892 + static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn, 2893 + bool allow_spin) 2893 2894 { 2894 2895 struct slab_sheaf *empty = NULL; 2895 2896 unsigned long flags; ··· 2898 2897 if (!data_race(barn->nr_empty)) 2899 2898 return NULL; 2900 2899 2901 - spin_lock_irqsave(&barn->lock, flags); 2900 + if (likely(allow_spin)) 2901 + spin_lock_irqsave(&barn->lock, flags); 2902 + else if (!spin_trylock_irqsave(&barn->lock, flags)) 2903 + return NULL; 2902 2904 2903 2905 if (likely(barn->nr_empty)) { 2904 2906 empty = list_first_entry(&barn->sheaves_empty, ··· 2978 2974 * change. 2979 2975 */ 2980 2976 static struct slab_sheaf * 2981 - barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty) 2977 + barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty, 2978 + bool allow_spin) 2982 2979 { 2983 2980 struct slab_sheaf *full = NULL; 2984 2981 unsigned long flags; ··· 2987 2982 if (!data_race(barn->nr_full)) 2988 2983 return NULL; 2989 2984 2990 - spin_lock_irqsave(&barn->lock, flags); 2985 + if (likely(allow_spin)) 2986 + spin_lock_irqsave(&barn->lock, flags); 2987 + else if (!spin_trylock_irqsave(&barn->lock, flags)) 2988 + return NULL; 2991 2989 2992 2990 if (likely(barn->nr_full)) { 2993 2991 full = list_first_entry(&barn->sheaves_full, struct slab_sheaf, ··· 3011 3003 * barn. But if there are too many full sheaves, reject this with -E2BIG. 3012 3004 */ 3013 3005 static struct slab_sheaf * 3014 - barn_replace_full_sheaf(struct node_barn *barn, struct slab_sheaf *full) 3006 + barn_replace_full_sheaf(struct node_barn *barn, struct slab_sheaf *full, 3007 + bool allow_spin) 3015 3008 { 3016 3009 struct slab_sheaf *empty; 3017 3010 unsigned long flags; ··· 3023 3014 if (!data_race(barn->nr_empty)) 3024 3015 return ERR_PTR(-ENOMEM); 3025 3016 3026 - spin_lock_irqsave(&barn->lock, flags); 3017 + if (likely(allow_spin)) 3018 + spin_lock_irqsave(&barn->lock, flags); 3019 + else if (!spin_trylock_irqsave(&barn->lock, flags)) 3020 + return ERR_PTR(-EBUSY); 3027 3021 3028 3022 if (likely(barn->nr_empty)) { 3029 3023 empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf, ··· 5020 5008 return NULL; 5021 5009 } 5022 5010 5023 - full = barn_replace_empty_sheaf(barn, pcs->main); 5011 + full = barn_replace_empty_sheaf(barn, pcs->main, 5012 + gfpflags_allow_spinning(gfp)); 5024 5013 5025 5014 if (full) { 5026 5015 stat(s, BARN_GET); ··· 5038 5025 empty = pcs->spare; 5039 5026 pcs->spare = NULL; 5040 5027 } else { 5041 - empty = barn_get_empty_sheaf(barn); 5028 + empty = barn_get_empty_sheaf(barn, true); 5042 5029 } 5043 5030 } 5044 5031 ··· 5178 5165 } 5179 5166 5180 5167 static __fastpath_inline 5181 - unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p) 5168 + unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, gfp_t gfp, size_t size, 5169 + void **p) 5182 5170 { 5183 5171 struct slub_percpu_sheaves *pcs; 5184 5172 struct slab_sheaf *main; ··· 5213 5199 return allocated; 5214 5200 } 5215 5201 5216 - full = barn_replace_empty_sheaf(barn, pcs->main); 5202 + full = barn_replace_empty_sheaf(barn, pcs->main, 5203 + gfpflags_allow_spinning(gfp)); 5217 5204 5218 5205 if (full) { 5219 5206 stat(s, BARN_GET); ··· 5715 5700 gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags; 5716 5701 struct kmem_cache *s; 5717 5702 bool can_retry = true; 5718 - void *ret = ERR_PTR(-EBUSY); 5703 + void *ret; 5719 5704 5720 5705 VM_WARN_ON_ONCE(gfp_flags & ~(__GFP_ACCOUNT | __GFP_ZERO | 5721 5706 __GFP_NO_OBJ_EXT)); ··· 5745 5730 * spin_trylock_irqsave(&n->list_lock, ...) 5746 5731 */ 5747 5732 return NULL; 5733 + 5734 + ret = alloc_from_pcs(s, alloc_gfp, node); 5735 + if (ret) 5736 + goto success; 5737 + 5738 + ret = ERR_PTR(-EBUSY); 5748 5739 5749 5740 /* 5750 5741 * Do not call slab_alloc_node(), since trylock mode isn't ··· 5788 5767 ret = NULL; 5789 5768 } 5790 5769 5770 + success: 5791 5771 maybe_wipe_obj_freeptr(s, ret); 5792 5772 slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret, 5793 5773 slab_want_init_on_alloc(alloc_gfp, s), size); ··· 6109 6087 * unlocked. 6110 6088 */ 6111 6089 static struct slub_percpu_sheaves * 6112 - __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs) 6090 + __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 6091 + bool allow_spin) 6113 6092 { 6114 6093 struct slab_sheaf *empty; 6115 6094 struct node_barn *barn; ··· 6134 6111 put_fail = false; 6135 6112 6136 6113 if (!pcs->spare) { 6137 - empty = barn_get_empty_sheaf(barn); 6114 + empty = barn_get_empty_sheaf(barn, allow_spin); 6138 6115 if (empty) { 6139 6116 pcs->spare = pcs->main; 6140 6117 pcs->main = empty; ··· 6148 6125 return pcs; 6149 6126 } 6150 6127 6151 - empty = barn_replace_full_sheaf(barn, pcs->main); 6128 + empty = barn_replace_full_sheaf(barn, pcs->main, allow_spin); 6152 6129 6153 6130 if (!IS_ERR(empty)) { 6154 6131 stat(s, BARN_PUT); ··· 6156 6133 return pcs; 6157 6134 } 6158 6135 6159 - if (PTR_ERR(empty) == -E2BIG) { 6136 + /* sheaf_flush_unused() doesn't support !allow_spin */ 6137 + if (PTR_ERR(empty) == -E2BIG && allow_spin) { 6160 6138 /* Since we got here, spare exists and is full */ 6161 6139 struct slab_sheaf *to_flush = pcs->spare; 6162 6140 ··· 6181 6157 6182 6158 alloc_empty: 6183 6159 local_unlock(&s->cpu_sheaves->lock); 6160 + 6161 + /* 6162 + * alloc_empty_sheaf() doesn't support !allow_spin and it's 6163 + * easier to fall back to freeing directly without sheaves 6164 + * than add the support (and to sheaf_flush_unused() above) 6165 + */ 6166 + if (!allow_spin) 6167 + return NULL; 6184 6168 6185 6169 empty = alloc_empty_sheaf(s, GFP_NOWAIT); 6186 6170 if (empty) ··· 6232 6200 * The object is expected to have passed slab_free_hook() already. 6233 6201 */ 6234 6202 static __fastpath_inline 6235 - bool free_to_pcs(struct kmem_cache *s, void *object) 6203 + bool free_to_pcs(struct kmem_cache *s, void *object, bool allow_spin) 6236 6204 { 6237 6205 struct slub_percpu_sheaves *pcs; 6238 6206 ··· 6243 6211 6244 6212 if (unlikely(pcs->main->size == s->sheaf_capacity)) { 6245 6213 6246 - pcs = __pcs_replace_full_main(s, pcs); 6214 + pcs = __pcs_replace_full_main(s, pcs, allow_spin); 6247 6215 if (unlikely(!pcs)) 6248 6216 return false; 6249 6217 } ··· 6368 6336 goto fail; 6369 6337 } 6370 6338 6371 - empty = barn_get_empty_sheaf(barn); 6339 + empty = barn_get_empty_sheaf(barn, true); 6372 6340 6373 6341 if (empty) { 6374 6342 pcs->rcu_free = empty; ··· 6488 6456 goto no_empty; 6489 6457 6490 6458 if (!pcs->spare) { 6491 - empty = barn_get_empty_sheaf(barn); 6459 + empty = barn_get_empty_sheaf(barn, true); 6492 6460 if (!empty) 6493 6461 goto no_empty; 6494 6462 ··· 6502 6470 goto do_free; 6503 6471 } 6504 6472 6505 - empty = barn_replace_full_sheaf(barn, pcs->main); 6473 + empty = barn_replace_full_sheaf(barn, pcs->main, true); 6506 6474 if (IS_ERR(empty)) { 6507 6475 stat(s, BARN_PUT_FAIL); 6508 6476 goto no_empty; ··· 6754 6722 6755 6723 if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id()) 6756 6724 && likely(!slab_test_pfmemalloc(slab))) { 6757 - if (likely(free_to_pcs(s, object))) 6725 + if (likely(free_to_pcs(s, object, true))) 6758 6726 return; 6759 6727 } 6760 6728 ··· 7025 6993 * since kasan quarantine takes locks and not supported from NMI. 7026 6994 */ 7027 6995 kasan_slab_free(s, x, false, false, /* skip quarantine */true); 6996 + 6997 + if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id())) { 6998 + if (likely(free_to_pcs(s, x, false))) 6999 + return; 7000 + } 7001 + 7028 7002 do_slab_free(s, slab, x, x, 0, _RET_IP_); 7029 7003 } 7030 7004 EXPORT_SYMBOL_GPL(kfree_nolock); ··· 7583 7545 size--; 7584 7546 } 7585 7547 7586 - i = alloc_from_pcs_bulk(s, size, p); 7548 + i = alloc_from_pcs_bulk(s, flags, size, p); 7587 7549 7588 7550 if (i < size) { 7589 7551 /*

Configure Feed

Configure Feed