tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm/page_alloc: simplify and cleanup pcp locking

The pcp locking relies on pcp_spin_trylock() which has to be used together
with pcp_trylock_prepare()/pcp_trylock_finish() to work properly on !SMP
!RT configs. This is tedious and error-prone.

We can remove pcp_spin_lock() and underlying pcpu_spin_lock() because we
don't use it. Afterwards pcp_spin_unlock() is only used together with
pcp_spin_trylock(). Therefore we can add the UP_flags parameter to them
both and handle pcp_trylock_prepare()/finish() within.

Additionally for the configs where pcp_trylock_prepare()/finish() are
no-op (SMP || RT) make them pass &UP_flags to a no-op inline function.
This ensures typechecking and makes the local variable "used" so we can
remove the __maybe_unused attributes.

In my compile testing, bloat-o-meter reported no change on SMP config, so
the compiler is capable of optimizing away the no-ops same as before, and
we have simplified the code using pcp_spin_trylock().

Link: https://lkml.kernel.org/r/20251015-b4-pcp-lock-cleanup-v2-1-740d999595d5@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: Joshua Hahn <joshua.hahnjy@gmail.com>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Vlastimil Babka and committed by
Andrew Morton 0f21b911 91e69129

+40 -59
+40 -59
mm/page_alloc.c
··· 99 99 /* 100 100 * On SMP, spin_trylock is sufficient protection. 101 101 * On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP. 102 + * Pass flags to a no-op inline function to typecheck and silence the unused 103 + * variable warning. 102 104 */ 103 - #define pcp_trylock_prepare(flags) do { } while (0) 104 - #define pcp_trylock_finish(flag) do { } while (0) 105 + static inline void __pcp_trylock_noop(unsigned long *flags) { } 106 + #define pcp_trylock_prepare(flags) __pcp_trylock_noop(&(flags)) 107 + #define pcp_trylock_finish(flags) __pcp_trylock_noop(&(flags)) 105 108 #else 106 109 107 110 /* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */ ··· 132 129 * Generic helper to lookup and a per-cpu variable with an embedded spinlock. 133 130 * Return value should be used with equivalent unlock helper. 134 131 */ 135 - #define pcpu_spin_lock(type, member, ptr) \ 136 - ({ \ 137 - type *_ret; \ 138 - pcpu_task_pin(); \ 139 - _ret = this_cpu_ptr(ptr); \ 140 - spin_lock(&_ret->member); \ 141 - _ret; \ 142 - }) 143 - 144 132 #define pcpu_spin_trylock(type, member, ptr) \ 145 133 ({ \ 146 134 type *_ret; \ ··· 151 157 }) 152 158 153 159 /* struct per_cpu_pages specific helpers. */ 154 - #define pcp_spin_lock(ptr) \ 155 - pcpu_spin_lock(struct per_cpu_pages, lock, ptr) 160 + #define pcp_spin_trylock(ptr, UP_flags) \ 161 + ({ \ 162 + struct per_cpu_pages *__ret; \ 163 + pcp_trylock_prepare(UP_flags); \ 164 + __ret = pcpu_spin_trylock(struct per_cpu_pages, lock, ptr); \ 165 + if (!__ret) \ 166 + pcp_trylock_finish(UP_flags); \ 167 + __ret; \ 168 + }) 156 169 157 - #define pcp_spin_trylock(ptr) \ 158 - pcpu_spin_trylock(struct per_cpu_pages, lock, ptr) 159 - 160 - #define pcp_spin_unlock(ptr) \ 161 - pcpu_spin_unlock(lock, ptr) 170 + #define pcp_spin_unlock(ptr, UP_flags) \ 171 + ({ \ 172 + pcpu_spin_unlock(lock, ptr); \ 173 + pcp_trylock_finish(UP_flags); \ 174 + }) 162 175 163 176 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 164 177 DEFINE_PER_CPU(int, numa_node); ··· 2888 2887 if (to_free == 0 || pcp->count == 0) 2889 2888 break; 2890 2889 2891 - pcp_spin_unlock(pcp); 2892 - pcp_trylock_finish(*UP_flags); 2890 + pcp_spin_unlock(pcp, *UP_flags); 2893 2891 2894 - pcp_trylock_prepare(*UP_flags); 2895 - pcp = pcp_spin_trylock(zone->per_cpu_pageset); 2892 + pcp = pcp_spin_trylock(zone->per_cpu_pageset, *UP_flags); 2896 2893 if (!pcp) { 2897 - pcp_trylock_finish(*UP_flags); 2898 2894 ret = false; 2899 2895 break; 2900 2896 } ··· 2902 2904 * returned in an unlocked state. 2903 2905 */ 2904 2906 if (smp_processor_id() != cpu) { 2905 - pcp_spin_unlock(pcp); 2906 - pcp_trylock_finish(*UP_flags); 2907 + pcp_spin_unlock(pcp, *UP_flags); 2907 2908 ret = false; 2908 2909 break; 2909 2910 } ··· 2934 2937 static void __free_frozen_pages(struct page *page, unsigned int order, 2935 2938 fpi_t fpi_flags) 2936 2939 { 2937 - unsigned long __maybe_unused UP_flags; 2940 + unsigned long UP_flags; 2938 2941 struct per_cpu_pages *pcp; 2939 2942 struct zone *zone; 2940 2943 unsigned long pfn = page_to_pfn(page); ··· 2970 2973 add_page_to_zone_llist(zone, page, order); 2971 2974 return; 2972 2975 } 2973 - pcp_trylock_prepare(UP_flags); 2974 - pcp = pcp_spin_trylock(zone->per_cpu_pageset); 2976 + pcp = pcp_spin_trylock(zone->per_cpu_pageset, UP_flags); 2975 2977 if (pcp) { 2976 2978 if (!free_frozen_page_commit(zone, pcp, page, migratetype, 2977 2979 order, fpi_flags, &UP_flags)) 2978 2980 return; 2979 - pcp_spin_unlock(pcp); 2981 + pcp_spin_unlock(pcp, UP_flags); 2980 2982 } else { 2981 2983 free_one_page(zone, page, pfn, order, fpi_flags); 2982 2984 } 2983 - pcp_trylock_finish(UP_flags); 2984 2985 } 2985 2986 2986 2987 void free_frozen_pages(struct page *page, unsigned int order) ··· 2991 2996 */ 2992 2997 void free_unref_folios(struct folio_batch *folios) 2993 2998 { 2994 - unsigned long __maybe_unused UP_flags; 2999 + unsigned long UP_flags; 2995 3000 struct per_cpu_pages *pcp = NULL; 2996 3001 struct zone *locked_zone = NULL; 2997 3002 int i, j; ··· 3034 3039 if (zone != locked_zone || 3035 3040 is_migrate_isolate(migratetype)) { 3036 3041 if (pcp) { 3037 - pcp_spin_unlock(pcp); 3038 - pcp_trylock_finish(UP_flags); 3042 + pcp_spin_unlock(pcp, UP_flags); 3039 3043 locked_zone = NULL; 3040 3044 pcp = NULL; 3041 3045 } ··· 3053 3059 * trylock is necessary as folios may be getting freed 3054 3060 * from IRQ or SoftIRQ context after an IO completion. 3055 3061 */ 3056 - pcp_trylock_prepare(UP_flags); 3057 - pcp = pcp_spin_trylock(zone->per_cpu_pageset); 3062 + pcp = pcp_spin_trylock(zone->per_cpu_pageset, UP_flags); 3058 3063 if (unlikely(!pcp)) { 3059 - pcp_trylock_finish(UP_flags); 3060 3064 free_one_page(zone, &folio->page, pfn, 3061 3065 order, FPI_NONE); 3062 3066 continue; ··· 3077 3085 } 3078 3086 } 3079 3087 3080 - if (pcp) { 3081 - pcp_spin_unlock(pcp); 3082 - pcp_trylock_finish(UP_flags); 3083 - } 3088 + if (pcp) 3089 + pcp_spin_unlock(pcp, UP_flags); 3084 3090 folio_batch_reinit(folios); 3085 3091 } 3086 3092 ··· 3329 3339 struct per_cpu_pages *pcp; 3330 3340 struct list_head *list; 3331 3341 struct page *page; 3332 - unsigned long __maybe_unused UP_flags; 3342 + unsigned long UP_flags; 3333 3343 3334 3344 /* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */ 3335 - pcp_trylock_prepare(UP_flags); 3336 - pcp = pcp_spin_trylock(zone->per_cpu_pageset); 3337 - if (!pcp) { 3338 - pcp_trylock_finish(UP_flags); 3345 + pcp = pcp_spin_trylock(zone->per_cpu_pageset, UP_flags); 3346 + if (!pcp) 3339 3347 return NULL; 3340 - } 3341 3348 3342 3349 /* 3343 3350 * On allocation, reduce the number of pages that are batch freed. ··· 3344 3357 pcp->free_count >>= 1; 3345 3358 list = &pcp->lists[order_to_pindex(migratetype, order)]; 3346 3359 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); 3347 - pcp_spin_unlock(pcp); 3348 - pcp_trylock_finish(UP_flags); 3360 + pcp_spin_unlock(pcp, UP_flags); 3349 3361 if (page) { 3350 3362 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3351 3363 zone_statistics(preferred_zone, zone, 1); ··· 5031 5045 struct page **page_array) 5032 5046 { 5033 5047 struct page *page; 5034 - unsigned long __maybe_unused UP_flags; 5048 + unsigned long UP_flags; 5035 5049 struct zone *zone; 5036 5050 struct zoneref *z; 5037 5051 struct per_cpu_pages *pcp; ··· 5125 5139 goto failed; 5126 5140 5127 5141 /* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */ 5128 - pcp_trylock_prepare(UP_flags); 5129 - pcp = pcp_spin_trylock(zone->per_cpu_pageset); 5142 + pcp = pcp_spin_trylock(zone->per_cpu_pageset, UP_flags); 5130 5143 if (!pcp) 5131 - goto failed_irq; 5144 + goto failed; 5132 5145 5133 5146 /* Attempt the batch allocation */ 5134 5147 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; ··· 5144 5159 if (unlikely(!page)) { 5145 5160 /* Try and allocate at least one page */ 5146 5161 if (!nr_account) { 5147 - pcp_spin_unlock(pcp); 5148 - goto failed_irq; 5162 + pcp_spin_unlock(pcp, UP_flags); 5163 + goto failed; 5149 5164 } 5150 5165 break; 5151 5166 } ··· 5156 5171 page_array[nr_populated++] = page; 5157 5172 } 5158 5173 5159 - pcp_spin_unlock(pcp); 5160 - pcp_trylock_finish(UP_flags); 5174 + pcp_spin_unlock(pcp, UP_flags); 5161 5175 5162 5176 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); 5163 5177 zone_statistics(zonelist_zone(ac.preferred_zoneref), zone, nr_account); 5164 5178 5165 5179 out: 5166 5180 return nr_populated; 5167 - 5168 - failed_irq: 5169 - pcp_trylock_finish(UP_flags); 5170 5181 5171 5182 failed: 5172 5183 page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask);