mm: optimize lru_note_cost() by adding lru_note_cost_unlock_irq()

Dropping a lock, just to demand it again for an afterthought, cannot be
good if contended: convert lru_note_cost() to lru_note_cost_unlock_irq().

[hughd@google.com: delete unneeded comment]
Link: https://lkml.kernel.org/r/dbf9352a-1ed9-a021-c0c7-9309ac73e174@google.com
Link: https://lkml.kernel.org/r/21100102-51b6-79d5-03db-1bb7f97fa94c@google.com
Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Roman Gushchin <roman.gushchin@linux.dev>
Tested-by: Roman Gushchin <roman.gushchin@linux.dev>
Reviewed-by: Shakeel Butt <shakeel.butt@linux.dev>
Cc: David Hildenbrand <david@redhat.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Michal Hocko <mhocko@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Hugh Dickins and committed by

Andrew Morton 10 months ago 3865301d 526660b9

+25 -21

3 changed files

expand all

include

linux

swap.h

swap.c

vmscan.c

+3 -2

include/linux/swap.h

··· 376 376 377 377 378 378 /* linux/mm/swap.c */ 379 - void lru_note_cost(struct lruvec *lruvec, bool file, 380 - unsigned int nr_io, unsigned int nr_rotated); 379 + void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, 380 + unsigned int nr_io, unsigned int nr_rotated) 381 + __releases(lruvec->lru_lock); 381 382 void lru_note_cost_refault(struct folio *); 382 383 void folio_add_lru(struct folio *); 383 384 void folio_add_lru_vma(struct folio *, struct vm_area_struct *);

+19 -14

mm/swap.c

··· 237 237 folio_batch_add_and_move(folio, lru_move_tail, true); 238 238 } 239 239 240 - void lru_note_cost(struct lruvec *lruvec, bool file, 241 - unsigned int nr_io, unsigned int nr_rotated) 240 + void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, 241 + unsigned int nr_io, unsigned int nr_rotated) 242 + __releases(lruvec->lru_lock) 242 243 { 243 244 unsigned long cost; 244 245 ··· 251 250 * different between them, adjust scan balance for CPU work. 252 251 */ 253 252 cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated; 253 + if (!cost) { 254 + spin_unlock_irq(&lruvec->lru_lock); 255 + return; 256 + } 254 257 255 - do { 258 + for (;;) { 256 259 unsigned long lrusize; 257 260 258 - /* 259 - * Hold lruvec->lru_lock is safe here, since 260 - * 1) The pinned lruvec in reclaim, or 261 - * 2) From a pre-LRU page during refault (which also holds the 262 - * rcu lock, so would be safe even if the page was on the LRU 263 - * and could move simultaneously to a new lruvec). 264 - */ 265 - spin_lock_irq(&lruvec->lru_lock); 266 261 /* Record cost event */ 267 262 if (file) 268 263 lruvec->file_cost += cost; ··· 282 285 lruvec->file_cost /= 2; 283 286 lruvec->anon_cost /= 2; 284 287 } 288 + 285 289 spin_unlock_irq(&lruvec->lru_lock); 286 - } while ((lruvec = parent_lruvec(lruvec))); 290 + lruvec = parent_lruvec(lruvec); 291 + if (!lruvec) 292 + break; 293 + spin_lock_irq(&lruvec->lru_lock); 294 + } 287 295 } 288 296 289 297 void lru_note_cost_refault(struct folio *folio) 290 298 { 291 - lru_note_cost(folio_lruvec(folio), folio_is_file_lru(folio), 292 - folio_nr_pages(folio), 0); 299 + struct lruvec *lruvec; 300 + 301 + lruvec = folio_lruvec_lock_irq(folio); 302 + lru_note_cost_unlock_irq(lruvec, folio_is_file_lru(folio), 303 + folio_nr_pages(folio), 0); 293 304 } 294 305 295 306 static void lru_activate(struct lruvec *lruvec, struct folio *folio)

+3 -5

mm/vmscan.c

··· 2053 2053 __count_vm_events(item, nr_reclaimed); 2054 2054 count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); 2055 2055 __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); 2056 - spin_unlock_irq(&lruvec->lru_lock); 2057 2056 2058 - lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed); 2057 + lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout, 2058 + nr_scanned - nr_reclaimed); 2059 2059 2060 2060 /* 2061 2061 * If dirty folios are scanned that are not queued for IO, it ··· 2201 2201 count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); 2202 2202 2203 2203 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2204 - spin_unlock_irq(&lruvec->lru_lock); 2205 2204 2206 - if (nr_rotated) 2207 - lru_note_cost(lruvec, file, 0, nr_rotated); 2205 + lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated); 2208 2206 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, 2209 2207 nr_deactivate, nr_rotated, sc->priority, file); 2210 2208 }

Configure Feed

Configure Feed