mm: revert "mm/gup: clear the LRU flag of a page before adding to LRU batch"

This reverts commit 33dfe9204f29: now that
collect_longterm_unpinnable_folios() is checking ref_count instead of lru,
and mlock/munlock do not participate in the revised LRU flag clearing,
those changes are misleading, and enlarge the window during which
mlock/munlock may miss an mlock_count update.

It is possible (I'd hesitate to claim probable) that the greater
likelihood of missed mlock_count updates would explain the "Realtime
threads delayed due to kcompactd0" observed on 6.12 in the Link below. If
that is the case, this reversion will help; but a complete solution needs
also a further patch, beyond the scope of this series.

Included some 80-column cleanup around folio_batch_add_and_move().

The role of folio_test_clear_lru() (before taking per-memcg lru_lock) is
questionable since 6.13 removed mem_cgroup_move_account() etc; but perhaps
there are still some races which need it - not examined here.

Link: https://lore.kernel.org/linux-mm/DU0PR01MB10385345F7153F334100981888259A@DU0PR01MB10385.eurprd01.prod.exchangelabs.com/
Link: https://lkml.kernel.org/r/05905d7b-ed14-68b1-79d8-bdec30367eba@google.com
Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: David Hildenbrand <david@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@kernel.org>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Chris Li <chrisl@kernel.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Keir Fraser <keirf@google.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Li Zhe <lizhe.67@bytedance.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Peter Xu <peterx@redhat.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Shivank Garg <shivankg@amd.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wei Xu <weixugc@google.com>
Cc: Will Deacon <will@kernel.org>
Cc: yangge <yangge1116@126.com>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Hugh Dickins and committed by

Andrew Morton 9 months ago afb99e9f a09a8a1f

+26 -24

1 changed file

expand all

swap.c

+26 -24

mm/swap.c

··· 164 164 for (i = 0; i < folio_batch_count(fbatch); i++) { 165 165 struct folio *folio = fbatch->folios[i]; 166 166 167 + /* block memcg migration while the folio moves between lru */ 168 + if (move_fn != lru_add && !folio_test_clear_lru(folio)) 169 + continue; 170 + 167 171 folio_lruvec_relock_irqsave(folio, &lruvec, &flags); 168 172 move_fn(lruvec, folio); 169 173 ··· 180 176 } 181 177 182 178 static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch, 183 - struct folio *folio, move_fn_t move_fn, 184 - bool on_lru, bool disable_irq) 179 + struct folio *folio, move_fn_t move_fn, bool disable_irq) 185 180 { 186 181 unsigned long flags; 187 - 188 - if (on_lru && !folio_test_clear_lru(folio)) 189 - return; 190 182 191 183 folio_get(folio); 192 184 ··· 191 191 else 192 192 local_lock(&cpu_fbatches.lock); 193 193 194 - if (!folio_batch_add(this_cpu_ptr(fbatch), folio) || folio_test_large(folio) || 195 - lru_cache_disabled()) 194 + if (!folio_batch_add(this_cpu_ptr(fbatch), folio) || 195 + folio_test_large(folio) || lru_cache_disabled()) 196 196 folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn); 197 197 198 198 if (disable_irq) ··· 201 201 local_unlock(&cpu_fbatches.lock); 202 202 } 203 203 204 - #define folio_batch_add_and_move(folio, op, on_lru) \ 205 - __folio_batch_add_and_move( \ 206 - &cpu_fbatches.op, \ 207 - folio, \ 208 - op, \ 209 - on_lru, \ 210 - offsetof(struct cpu_fbatches, op) >= offsetof(struct cpu_fbatches, lock_irq) \ 204 + #define folio_batch_add_and_move(folio, op) \ 205 + __folio_batch_add_and_move( \ 206 + &cpu_fbatches.op, \ 207 + folio, \ 208 + op, \ 209 + offsetof(struct cpu_fbatches, op) >= \ 210 + offsetof(struct cpu_fbatches, lock_irq) \ 211 211 ) 212 212 213 213 static void lru_move_tail(struct lruvec *lruvec, struct folio *folio) ··· 231 231 void folio_rotate_reclaimable(struct folio *folio) 232 232 { 233 233 if (folio_test_locked(folio) || folio_test_dirty(folio) || 234 - folio_test_unevictable(folio)) 234 + folio_test_unevictable(folio) || !folio_test_lru(folio)) 235 235 return; 236 236 237 - folio_batch_add_and_move(folio, lru_move_tail, true); 237 + folio_batch_add_and_move(folio, lru_move_tail); 238 238 } 239 239 240 240 void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, ··· 328 328 329 329 void folio_activate(struct folio *folio) 330 330 { 331 - if (folio_test_active(folio) || folio_test_unevictable(folio)) 331 + if (folio_test_active(folio) || folio_test_unevictable(folio) || 332 + !folio_test_lru(folio)) 332 333 return; 333 334 334 - folio_batch_add_and_move(folio, lru_activate, true); 335 + folio_batch_add_and_move(folio, lru_activate); 335 336 } 336 337 337 338 #else ··· 508 507 lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) 509 508 folio_set_active(folio); 510 509 511 - folio_batch_add_and_move(folio, lru_add, false); 510 + folio_batch_add_and_move(folio, lru_add); 512 511 } 513 512 EXPORT_SYMBOL(folio_add_lru); 514 513 ··· 686 685 void deactivate_file_folio(struct folio *folio) 687 686 { 688 687 /* Deactivating an unevictable folio will not accelerate reclaim */ 689 - if (folio_test_unevictable(folio)) 688 + if (folio_test_unevictable(folio) || !folio_test_lru(folio)) 690 689 return; 691 690 692 691 if (lru_gen_enabled() && lru_gen_clear_refs(folio)) 693 692 return; 694 693 695 - folio_batch_add_and_move(folio, lru_deactivate_file, true); 694 + folio_batch_add_and_move(folio, lru_deactivate_file); 696 695 } 697 696 698 697 /* ··· 705 704 */ 706 705 void folio_deactivate(struct folio *folio) 707 706 { 708 - if (folio_test_unevictable(folio)) 707 + if (folio_test_unevictable(folio) || !folio_test_lru(folio)) 709 708 return; 710 709 711 710 if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio)) 712 711 return; 713 712 714 - folio_batch_add_and_move(folio, lru_deactivate, true); 713 + folio_batch_add_and_move(folio, lru_deactivate); 715 714 } 716 715 717 716 /** ··· 724 723 void folio_mark_lazyfree(struct folio *folio) 725 724 { 726 725 if (!folio_test_anon(folio) || !folio_test_swapbacked(folio) || 726 + !folio_test_lru(folio) || 727 727 folio_test_swapcache(folio) || folio_test_unevictable(folio)) 728 728 return; 729 729 730 - folio_batch_add_and_move(folio, lru_lazyfree, true); 730 + folio_batch_add_and_move(folio, lru_lazyfree); 731 731 } 732 732 733 733 void lru_add_drain(void)

Configure Feed

Configure Feed