mm: do not open-code lruvec lock

Now we have lruvec_unlock(), lruvec_unlock_irq() and
lruvec_unlock_irqrestore(), but no the paired lruvec_lock(),
lruvec_lock_irq() and lruvec_lock_irqsave().

There is currently no use case for lruvec_lock_irqsave(), so only
introduce lruvec_lock_irq(), and change all open-code places to use this
helper function. This looks cleaner and prepares for reparenting LRU
pages, preventing user from missing RCU lock calls due to open-code lruvec
lock.

Link: https://lore.kernel.org/2d0bafe7564e17ece46dfd58197af22ce57017dc.1772711148.git.zhengqi.arch@bytedance.com
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Acked-by: Muchun Song <muchun.song@linux.dev>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Cc: Allen Pais <apais@linux.microsoft.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Chen Ridong <chenridong@huawei.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Hamza Mahfooz <hamzamahfooz@linux.microsoft.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Imran Khan <imran.f.khan@oracle.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kamalesh Babulal <kamalesh.babulal@oracle.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <Liam.Howlett@oracle.com>
Cc: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Michal Koutný <mkoutny@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <songmuchun@bytedance.com>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Usama Arif <usamaarif642@gmail.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Wei Xu <weixugc@google.com>
Cc: Yosry Ahmed <yosry@kernel.org>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Qi Zheng and committed by

Andrew Morton 2 months ago d14f8785 50738297

+24 -19

2 changed files

expand all

include

linux

memcontrol.h

vmscan.c

include/linux/memcontrol.h

··· 1498 1498 return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec)); 1499 1499 } 1500 1500 1501 + static inline void lruvec_lock_irq(struct lruvec *lruvec) 1502 + { 1503 + spin_lock_irq(&lruvec->lru_lock); 1504 + } 1505 + 1501 1506 static inline void lruvec_unlock(struct lruvec *lruvec) 1502 1507 { 1503 1508 spin_unlock(&lruvec->lru_lock);

+19 -19

mm/vmscan.c

··· 1998 1998 1999 1999 lru_add_drain(); 2000 2000 2001 - spin_lock_irq(&lruvec->lru_lock); 2001 + lruvec_lock_irq(lruvec); 2002 2002 2003 2003 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, 2004 2004 &nr_scanned, sc, lru); ··· 2008 2008 mod_lruvec_state(lruvec, item, nr_scanned); 2009 2009 mod_lruvec_state(lruvec, PGSCAN_ANON + file, nr_scanned); 2010 2010 2011 - spin_unlock_irq(&lruvec->lru_lock); 2011 + lruvec_unlock_irq(lruvec); 2012 2012 2013 2013 if (nr_taken == 0) 2014 2014 return 0; ··· 2025 2025 mod_lruvec_state(lruvec, item, nr_reclaimed); 2026 2026 mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed); 2027 2027 2028 - spin_lock_irq(&lruvec->lru_lock); 2028 + lruvec_lock_irq(lruvec); 2029 2029 lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout, 2030 2030 nr_scanned - nr_reclaimed); 2031 2031 ··· 2104 2104 2105 2105 lru_add_drain(); 2106 2106 2107 - spin_lock_irq(&lruvec->lru_lock); 2107 + lruvec_lock_irq(lruvec); 2108 2108 2109 2109 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, 2110 2110 &nr_scanned, sc, lru); ··· 2113 2113 2114 2114 mod_lruvec_state(lruvec, PGREFILL, nr_scanned); 2115 2115 2116 - spin_unlock_irq(&lruvec->lru_lock); 2116 + lruvec_unlock_irq(lruvec); 2117 2117 2118 2118 while (!list_empty(&l_hold)) { 2119 2119 struct folio *folio; ··· 2169 2169 count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); 2170 2170 mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2171 2171 2172 - spin_lock_irq(&lruvec->lru_lock); 2172 + lruvec_lock_irq(lruvec); 2173 2173 lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated); 2174 2174 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, 2175 2175 nr_deactivate, nr_rotated, sc->priority, file); ··· 3803 3803 } 3804 3804 3805 3805 if (walk->batched) { 3806 - spin_lock_irq(&lruvec->lru_lock); 3806 + lruvec_lock_irq(lruvec); 3807 3807 reset_batch_size(walk); 3808 - spin_unlock_irq(&lruvec->lru_lock); 3808 + lruvec_unlock_irq(lruvec); 3809 3809 } 3810 3810 3811 3811 cond_resched(); ··· 3965 3965 if (seq < READ_ONCE(lrugen->max_seq)) 3966 3966 return false; 3967 3967 3968 - spin_lock_irq(&lruvec->lru_lock); 3968 + lruvec_lock_irq(lruvec); 3969 3969 3970 3970 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 3971 3971 ··· 3980 3980 if (inc_min_seq(lruvec, type, swappiness)) 3981 3981 continue; 3982 3982 3983 - spin_unlock_irq(&lruvec->lru_lock); 3983 + lruvec_unlock_irq(lruvec); 3984 3984 cond_resched(); 3985 3985 goto restart; 3986 3986 } ··· 4015 4015 /* make sure preceding modifications appear */ 4016 4016 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); 4017 4017 unlock: 4018 - spin_unlock_irq(&lruvec->lru_lock); 4018 + lruvec_unlock_irq(lruvec); 4019 4019 4020 4020 return success; 4021 4021 } ··· 4715 4715 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4716 4716 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4717 4717 4718 - spin_lock_irq(&lruvec->lru_lock); 4718 + lruvec_lock_irq(lruvec); 4719 4719 4720 4720 scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness, &type, &list); 4721 4721 ··· 4724 4724 if (evictable_min_seq(lrugen->min_seq, swappiness) + MIN_NR_GENS > lrugen->max_seq) 4725 4725 scanned = 0; 4726 4726 4727 - spin_unlock_irq(&lruvec->lru_lock); 4727 + lruvec_unlock_irq(lruvec); 4728 4728 4729 4729 if (list_empty(&list)) 4730 4730 return scanned; ··· 4762 4762 walk = current->reclaim_state->mm_walk; 4763 4763 if (walk && walk->batched) { 4764 4764 walk->lruvec = lruvec; 4765 - spin_lock_irq(&lruvec->lru_lock); 4765 + lruvec_lock_irq(lruvec); 4766 4766 reset_batch_size(walk); 4767 - spin_unlock_irq(&lruvec->lru_lock); 4767 + lruvec_unlock_irq(lruvec); 4768 4768 } 4769 4769 4770 4770 mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), ··· 5202 5202 for_each_node(nid) { 5203 5203 struct lruvec *lruvec = get_lruvec(memcg, nid); 5204 5204 5205 - spin_lock_irq(&lruvec->lru_lock); 5205 + lruvec_lock_irq(lruvec); 5206 5206 5207 5207 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 5208 5208 VM_WARN_ON_ONCE(!state_is_valid(lruvec)); ··· 5210 5210 lruvec->lrugen.enabled = enabled; 5211 5211 5212 5212 while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { 5213 - spin_unlock_irq(&lruvec->lru_lock); 5213 + lruvec_unlock_irq(lruvec); 5214 5214 cond_resched(); 5215 - spin_lock_irq(&lruvec->lru_lock); 5215 + lruvec_lock_irq(lruvec); 5216 5216 } 5217 5217 5218 - spin_unlock_irq(&lruvec->lru_lock); 5218 + lruvec_unlock_irq(lruvec); 5219 5219 } 5220 5220 5221 5221 cond_resched();

Configure Feed

Configure Feed