tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm, swap: use unified helper for swap cache look up

The swap cache lookup helper swap_cache_get_folio currently does readahead
updates as well, so callers that are not doing swapin from any VMA or
mapping are forced to reuse filemap helpers instead, and have to access
the swap cache space directly.

So decouple readahead update with swap cache lookup. Move the readahead
update part into a standalone helper. Let the caller call the readahead
update helper if they do readahead. And convert all swap cache lookups to
use swap_cache_get_folio.

After this commit, there are only three special cases for accessing swap
cache space now: huge memory splitting, migration, and shmem replacing,
because they need to lock the XArray. The following commits will wrap
their accesses to the swap cache too, with special helpers.

And worth noting, currently dropbehind is not supported for anon folio,
and we will never see a dropbehind folio in swap cache. The unified
helper can be updated later to handle that.

While at it, add proper kernedoc for touched helpers.

No functional change.

Link: https://lkml.kernel.org/r/20250916160100.31545-3-ryncsn@gmail.com
Signed-off-by: Kairui Song <kasong@tencent.com>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Reviewed-by: Barry Song <baohua@kernel.org>
Acked-by: David Hildenbrand <david@redhat.com>
Acked-by: Chris Li <chrisl@kernel.org>
Acked-by: Nhat Pham <nphamcs@gmail.com>
Suggested-by: Chris Li <chrisl@kernel.org>
Cc: Baoquan He <bhe@redhat.com>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: kernel test robot <oliver.sang@intel.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Yosry Ahmed <yosryahmed@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: SeongJae Park <sj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Kairui Song and committed by
Andrew Morton f2812461 87cc5157

+79 -68
+4 -2
mm/memory.c
··· 4660 4660 if (unlikely(!si)) 4661 4661 goto out; 4662 4662 4663 - folio = swap_cache_get_folio(entry, vma, vmf->address); 4664 - if (folio) 4663 + folio = swap_cache_get_folio(entry); 4664 + if (folio) { 4665 + swap_update_readahead(folio, vma, vmf->address); 4665 4666 page = folio_file_page(folio, swp_offset(entry)); 4667 + } 4666 4668 swapcache = folio; 4667 4669 4668 4670 if (!folio) {
+1 -2
mm/mincore.c
··· 76 76 if (!si) 77 77 return 0; 78 78 } 79 - folio = filemap_get_entry(swap_address_space(entry), 80 - swap_cache_index(entry)); 79 + folio = swap_cache_get_folio(entry); 81 80 if (shmem) 82 81 put_swap_device(si); 83 82 /* The swap cache space contains either folio, shadow or NULL */
+3 -1
mm/shmem.c
··· 2317 2317 } 2318 2318 2319 2319 /* Look it up and read it in.. */ 2320 - folio = swap_cache_get_folio(swap, NULL, 0); 2320 + folio = swap_cache_get_folio(swap); 2321 2321 if (!folio) { 2322 2322 if (data_race(si->flags & SWP_SYNCHRONOUS_IO)) { 2323 2323 /* Direct swapin skipping swap cache & readahead */ ··· 2342 2342 count_vm_event(PGMAJFAULT); 2343 2343 count_memcg_event_mm(fault_mm, PGMAJFAULT); 2344 2344 } 2345 + } else { 2346 + swap_update_readahead(folio, NULL, 0); 2345 2347 } 2346 2348 2347 2349 if (order > folio_order(folio)) {
+9 -4
mm/swap.h
··· 62 62 void clear_shadow_from_swap_cache(int type, unsigned long begin, 63 63 unsigned long end); 64 64 void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); 65 - struct folio *swap_cache_get_folio(swp_entry_t entry, 66 - struct vm_area_struct *vma, unsigned long addr); 65 + struct folio *swap_cache_get_folio(swp_entry_t entry); 67 66 struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 68 67 struct vm_area_struct *vma, unsigned long addr, 69 68 struct swap_iocb **plug); ··· 73 74 struct mempolicy *mpol, pgoff_t ilx); 74 75 struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag, 75 76 struct vm_fault *vmf); 77 + void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, 78 + unsigned long addr); 76 79 77 80 static inline unsigned int folio_swap_flags(struct folio *folio) 78 81 { ··· 160 159 return NULL; 161 160 } 162 161 162 + static inline void swap_update_readahead(struct folio *folio, 163 + struct vm_area_struct *vma, unsigned long addr) 164 + { 165 + } 166 + 163 167 static inline int swap_writeout(struct folio *folio, 164 168 struct swap_iocb **swap_plug) 165 169 { ··· 175 169 { 176 170 } 177 171 178 - static inline struct folio *swap_cache_get_folio(swp_entry_t entry, 179 - struct vm_area_struct *vma, unsigned long addr) 172 + static inline struct folio *swap_cache_get_folio(swp_entry_t entry) 180 173 { 181 174 return NULL; 182 175 }
+55 -50
mm/swap_state.c
··· 69 69 printk("Total swap = %lukB\n", K(total_swap_pages)); 70 70 } 71 71 72 + /** 73 + * swap_cache_get_folio - Looks up a folio in the swap cache. 74 + * @entry: swap entry used for the lookup. 75 + * 76 + * A found folio will be returned unlocked and with its refcount increased. 77 + * 78 + * Context: Caller must ensure @entry is valid and protect the swap device 79 + * with reference count or locks. 80 + * Return: Returns the found folio on success, NULL otherwise. The caller 81 + * must lock and check if the folio still matches the swap entry before 82 + * use. 83 + */ 84 + struct folio *swap_cache_get_folio(swp_entry_t entry) 85 + { 86 + struct folio *folio = filemap_get_folio(swap_address_space(entry), 87 + swap_cache_index(entry)); 88 + if (IS_ERR(folio)) 89 + return NULL; 90 + return folio; 91 + } 92 + 72 93 void *get_shadow_from_swap_cache(swp_entry_t entry) 73 94 { 74 95 struct address_space *address_space = swap_address_space(entry); ··· 293 272 return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap); 294 273 } 295 274 296 - /* 297 - * Lookup a swap entry in the swap cache. A found folio will be returned 298 - * unlocked and with its refcount incremented - we rely on the kernel 299 - * lock getting page table operations atomic even if we drop the folio 300 - * lock before returning. 301 - * 302 - * Caller must lock the swap device or hold a reference to keep it valid. 275 + /** 276 + * swap_update_readahead - Update the readahead statistics of VMA or globally. 277 + * @folio: the swap cache folio that just got hit. 278 + * @vma: the VMA that should be updated, could be NULL for global update. 279 + * @addr: the addr that triggered the swapin, ignored if @vma is NULL. 303 280 */ 304 - struct folio *swap_cache_get_folio(swp_entry_t entry, 305 - struct vm_area_struct *vma, unsigned long addr) 281 + void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, 282 + unsigned long addr) 306 283 { 307 - struct folio *folio; 284 + bool readahead, vma_ra = swap_use_vma_readahead(); 308 285 309 - folio = filemap_get_folio(swap_address_space(entry), swap_cache_index(entry)); 310 - if (!IS_ERR(folio)) { 311 - bool vma_ra = swap_use_vma_readahead(); 312 - bool readahead; 286 + /* 287 + * At the moment, we don't support PG_readahead for anon THP 288 + * so let's bail out rather than confusing the readahead stat. 289 + */ 290 + if (unlikely(folio_test_large(folio))) 291 + return; 313 292 314 - /* 315 - * At the moment, we don't support PG_readahead for anon THP 316 - * so let's bail out rather than confusing the readahead stat. 317 - */ 318 - if (unlikely(folio_test_large(folio))) 319 - return folio; 293 + readahead = folio_test_clear_readahead(folio); 294 + if (vma && vma_ra) { 295 + unsigned long ra_val; 296 + int win, hits; 320 297 321 - readahead = folio_test_clear_readahead(folio); 322 - if (vma && vma_ra) { 323 - unsigned long ra_val; 324 - int win, hits; 325 - 326 - ra_val = GET_SWAP_RA_VAL(vma); 327 - win = SWAP_RA_WIN(ra_val); 328 - hits = SWAP_RA_HITS(ra_val); 329 - if (readahead) 330 - hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX); 331 - atomic_long_set(&vma->swap_readahead_info, 332 - SWAP_RA_VAL(addr, win, hits)); 333 - } 334 - 335 - if (readahead) { 336 - count_vm_event(SWAP_RA_HIT); 337 - if (!vma || !vma_ra) 338 - atomic_inc(&swapin_readahead_hits); 339 - } 340 - } else { 341 - folio = NULL; 298 + ra_val = GET_SWAP_RA_VAL(vma); 299 + win = SWAP_RA_WIN(ra_val); 300 + hits = SWAP_RA_HITS(ra_val); 301 + if (readahead) 302 + hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX); 303 + atomic_long_set(&vma->swap_readahead_info, 304 + SWAP_RA_VAL(addr, win, hits)); 342 305 } 343 306 344 - return folio; 307 + if (readahead) { 308 + count_vm_event(SWAP_RA_HIT); 309 + if (!vma || !vma_ra) 310 + atomic_inc(&swapin_readahead_hits); 311 + } 345 312 } 346 313 347 314 struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, ··· 345 336 *new_page_allocated = false; 346 337 for (;;) { 347 338 int err; 348 - /* 349 - * First check the swap cache. Since this is normally 350 - * called after swap_cache_get_folio() failed, re-calling 351 - * that would confuse statistics. 352 - */ 353 - folio = filemap_get_folio(swap_address_space(entry), 354 - swap_cache_index(entry)); 355 - if (!IS_ERR(folio)) 339 + 340 + /* Check the swap cache in case the folio is already there */ 341 + folio = swap_cache_get_folio(entry); 342 + if (folio) 356 343 goto got_folio; 357 344 358 345 /*
+5 -6
mm/swapfile.c
··· 213 213 unsigned long offset, unsigned long flags) 214 214 { 215 215 swp_entry_t entry = swp_entry(si->type, offset); 216 - struct address_space *address_space = swap_address_space(entry); 217 216 struct swap_cluster_info *ci; 218 217 struct folio *folio; 219 218 int ret, nr_pages; 220 219 bool need_reclaim; 221 220 222 221 again: 223 - folio = filemap_get_folio(address_space, swap_cache_index(entry)); 224 - if (IS_ERR(folio)) 222 + folio = swap_cache_get_folio(entry); 223 + if (!folio) 225 224 return 0; 226 225 227 226 nr_pages = folio_nr_pages(folio); ··· 2130 2131 pte_unmap(pte); 2131 2132 pte = NULL; 2132 2133 2133 - folio = swap_cache_get_folio(entry, vma, addr); 2134 + folio = swap_cache_get_folio(entry); 2134 2135 if (!folio) { 2135 2136 struct vm_fault vmf = { 2136 2137 .vma = vma, ··· 2356 2357 (i = find_next_to_unuse(si, i)) != 0) { 2357 2358 2358 2359 entry = swp_entry(type, i); 2359 - folio = filemap_get_folio(swap_address_space(entry), swap_cache_index(entry)); 2360 - if (IS_ERR(folio)) 2360 + folio = swap_cache_get_folio(entry); 2361 + if (!folio) 2361 2362 continue; 2362 2363 2363 2364 /*
+2 -3
mm/userfaultfd.c
··· 1489 1489 * separately to allow proper handling. 1490 1490 */ 1491 1491 if (!src_folio) 1492 - folio = filemap_get_folio(swap_address_space(entry), 1493 - swap_cache_index(entry)); 1494 - if (!IS_ERR_OR_NULL(folio)) { 1492 + folio = swap_cache_get_folio(entry); 1493 + if (folio) { 1495 1494 if (folio_test_large(folio)) { 1496 1495 ret = -EBUSY; 1497 1496 folio_put(folio);