tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm/page_alloc: remove IRQ saving/restoring from pcp locking

Effectively revert commit 038a102535eb ("mm/page_alloc: prevent pcp
corruption with SMP=n"). The original problem is now avoided by
pcp_spin_trylock() always failing on CONFIG_SMP=n, so we do not need to
disable IRQs anymore.

It's not a complete revert, because keeping the pcp_spin_(un)lock()
wrappers is useful. Rename them from _maybe_irqsave/restore to _nopin.
The difference from pcp_spin_trylock()/pcp_spin_unlock() is that the
_nopin variants don't perform pcpu_task_pin/unpin().

Link: https://lkml.kernel.org/r/20260227-b4-pcp-locking-cleanup-v1-2-f7e22e603447@kernel.org
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: David Hildenbrand (Arm) <david@kernel.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Vlastimil Babka and committed by
Andrew Morton 0a2c52a9 a373f371

+16 -30
+16 -30
mm/page_alloc.c
··· 154 154 BUG_ON(1) 155 155 #endif 156 156 157 - #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) 158 - static inline void __flags_noop(unsigned long *flags) { } 159 - #define pcp_spin_lock_maybe_irqsave(ptr, flags) \ 160 - ({ \ 161 - __flags_noop(&(flags)); \ 162 - spin_lock(&(ptr)->lock); \ 163 - }) 164 - #define pcp_spin_unlock_maybe_irqrestore(ptr, flags) \ 165 - ({ \ 166 - spin_unlock(&(ptr)->lock); \ 167 - __flags_noop(&(flags)); \ 168 - }) 169 - #else 170 - #define pcp_spin_lock_maybe_irqsave(ptr, flags) \ 171 - spin_lock_irqsave(&(ptr)->lock, flags) 172 - #define pcp_spin_unlock_maybe_irqrestore(ptr, flags) \ 173 - spin_unlock_irqrestore(&(ptr)->lock, flags) 174 - #endif 157 + /* 158 + * In some cases we do not need to pin the task to the CPU because we are 159 + * already given a specific cpu's pcp pointer. 160 + */ 161 + #define pcp_spin_lock_nopin(ptr) \ 162 + spin_lock(&(ptr)->lock) 163 + #define pcp_spin_unlock_nopin(ptr) \ 164 + spin_unlock(&(ptr)->lock) 175 165 176 166 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 177 167 DEFINE_PER_CPU(int, numa_node); ··· 2558 2568 bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp) 2559 2569 { 2560 2570 int high_min, to_drain, to_drain_batched, batch; 2561 - unsigned long UP_flags; 2562 2571 bool todo = false; 2563 2572 2564 2573 high_min = READ_ONCE(pcp->high_min); ··· 2577 2588 to_drain = pcp->count - pcp->high; 2578 2589 while (to_drain > 0) { 2579 2590 to_drain_batched = min(to_drain, batch); 2580 - pcp_spin_lock_maybe_irqsave(pcp, UP_flags); 2591 + pcp_spin_lock_nopin(pcp); 2581 2592 free_pcppages_bulk(zone, to_drain_batched, pcp, 0); 2582 - pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); 2593 + pcp_spin_unlock_nopin(pcp); 2583 2594 todo = true; 2584 2595 2585 2596 to_drain -= to_drain_batched; ··· 2596 2607 */ 2597 2608 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2598 2609 { 2599 - unsigned long UP_flags; 2600 2610 int to_drain, batch; 2601 2611 2602 2612 batch = READ_ONCE(pcp->batch); 2603 2613 to_drain = min(pcp->count, batch); 2604 2614 if (to_drain > 0) { 2605 - pcp_spin_lock_maybe_irqsave(pcp, UP_flags); 2615 + pcp_spin_lock_nopin(pcp); 2606 2616 free_pcppages_bulk(zone, to_drain, pcp, 0); 2607 - pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); 2617 + pcp_spin_unlock_nopin(pcp); 2608 2618 } 2609 2619 } 2610 2620 #endif ··· 2614 2626 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2615 2627 { 2616 2628 struct per_cpu_pages *pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 2617 - unsigned long UP_flags; 2618 2629 int count; 2619 2630 2620 2631 do { 2621 - pcp_spin_lock_maybe_irqsave(pcp, UP_flags); 2632 + pcp_spin_lock_nopin(pcp); 2622 2633 count = pcp->count; 2623 2634 if (count) { 2624 2635 int to_drain = min(count, ··· 2626 2639 free_pcppages_bulk(zone, to_drain, pcp, 0); 2627 2640 count -= to_drain; 2628 2641 } 2629 - pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); 2642 + pcp_spin_unlock_nopin(pcp); 2630 2643 } while (count); 2631 2644 } 2632 2645 ··· 6110 6123 { 6111 6124 struct per_cpu_pages *pcp; 6112 6125 struct cpu_cacheinfo *cci; 6113 - unsigned long UP_flags; 6114 6126 6115 6127 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 6116 6128 cci = get_cpu_cacheinfo(cpu); ··· 6120 6134 * This can reduce zone lock contention without hurting 6121 6135 * cache-hot pages sharing. 6122 6136 */ 6123 - pcp_spin_lock_maybe_irqsave(pcp, UP_flags); 6137 + pcp_spin_lock_nopin(pcp); 6124 6138 if ((cci->per_cpu_data_slice_size >> PAGE_SHIFT) > 3 * pcp->batch) 6125 6139 pcp->flags |= PCPF_FREE_HIGH_BATCH; 6126 6140 else 6127 6141 pcp->flags &= ~PCPF_FREE_HIGH_BATCH; 6128 - pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); 6142 + pcp_spin_unlock_nopin(pcp); 6129 6143 } 6130 6144 6131 6145 void setup_pcp_cacheinfo(unsigned int cpu)