mm, swap: cleanup swap cache API and add kerneldoc

+1 -1

mm/filemap.c

··· 4525 4525 * invalidation, so there might not be 4526 4526 * a shadow in the swapcache (yet). 4527 4527 */ 4528 - shadow = get_shadow_from_swap_cache(swp); 4528 + shadow = swap_cache_get_shadow(swp); 4529 4529 if (!shadow) 4530 4530 goto resched; 4531 4531 }

+1 -1

mm/memory-failure.c

··· 1127 1127 struct folio *folio = page_folio(p); 1128 1128 int ret; 1129 1129 1130 - delete_from_swap_cache(folio); 1130 + swap_cache_del_folio(folio); 1131 1131 1132 1132 ret = delete_from_lru_cache(folio) ? MF_FAILED : MF_RECOVERED; 1133 1133 folio_unlock(folio);

+1 -1

mm/memory.c

··· 4699 4699 4700 4700 memcg1_swapin(entry, nr_pages); 4701 4701 4702 - shadow = get_shadow_from_swap_cache(entry); 4702 + shadow = swap_cache_get_shadow(entry); 4703 4703 if (shadow) 4704 4704 workingset_refault(folio, shadow); 4705 4705

+5 -5

mm/shmem.c

··· 1661 1661 } 1662 1662 1663 1663 /* 1664 - * The delete_from_swap_cache() below could be left for 1664 + * The swap_cache_del_folio() below could be left for 1665 1665 * shrink_folio_list()'s folio_free_swap() to dispose of; 1666 1666 * but I'm a little nervous about letting this folio out of 1667 1667 * shmem_writeout() in a hybrid half-tmpfs-half-swap state 1668 1668 * e.g. folio_mapping(folio) might give an unexpected answer. 1669 1669 */ 1670 - delete_from_swap_cache(folio); 1670 + swap_cache_del_folio(folio); 1671 1671 goto redirty; 1672 1672 } 1673 1673 if (nr_pages > 1) ··· 2045 2045 new->swap = entry; 2046 2046 2047 2047 memcg1_swapin(entry, nr_pages); 2048 - shadow = get_shadow_from_swap_cache(entry); 2048 + shadow = swap_cache_get_shadow(entry); 2049 2049 if (shadow) 2050 2050 workingset_refault(new, shadow); 2051 2051 folio_add_lru(new); ··· 2183 2183 nr_pages = folio_nr_pages(folio); 2184 2184 folio_wait_writeback(folio); 2185 2185 if (!skip_swapcache) 2186 - delete_from_swap_cache(folio); 2186 + swap_cache_del_folio(folio); 2187 2187 /* 2188 2188 * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks 2189 2189 * won't be 0 when inode is released and thus trigger WARN_ON(i_blocks) ··· 2422 2422 folio->swap.val = 0; 2423 2423 swapcache_clear(si, swap, nr_pages); 2424 2424 } else { 2425 - delete_from_swap_cache(folio); 2425 + swap_cache_del_folio(folio); 2426 2426 } 2427 2427 folio_mark_dirty(folio); 2428 2428 swap_free_nr(swap, nr_pages);

+28 -22

mm/swap.h

··· 167 167 return folio_entry.val == round_down(entry.val, nr_pages); 168 168 } 169 169 170 - void show_swap_cache_info(void); 171 - void *get_shadow_from_swap_cache(swp_entry_t entry); 172 - int add_to_swap_cache(struct folio *folio, swp_entry_t entry, 173 - gfp_t gfp, void **shadowp); 174 - void __delete_from_swap_cache(struct folio *folio, 175 - swp_entry_t entry, void *shadow); 176 - void delete_from_swap_cache(struct folio *folio); 177 - void clear_shadow_from_swap_cache(int type, unsigned long begin, 178 - unsigned long end); 179 - void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); 170 + /* 171 + * All swap cache helpers below require the caller to ensure the swap entries 172 + * used are valid and stablize the device by any of the following ways: 173 + * - Hold a reference by get_swap_device(): this ensures a single entry is 174 + * valid and increases the swap device's refcount. 175 + * - Locking a folio in the swap cache: this ensures the folio's swap entries 176 + * are valid and pinned, also implies reference to the device. 177 + * - Locking anything referencing the swap entry: e.g. PTL that protects 178 + * swap entries in the page table, similar to locking swap cache folio. 179 + * - See the comment of get_swap_device() for more complex usage. 180 + */ 180 181 struct folio *swap_cache_get_folio(swp_entry_t entry); 182 + void *swap_cache_get_shadow(swp_entry_t entry); 183 + int swap_cache_add_folio(struct folio *folio, swp_entry_t entry, 184 + gfp_t gfp, void **shadow); 185 + void swap_cache_del_folio(struct folio *folio); 186 + void __swap_cache_del_folio(struct folio *folio, 187 + swp_entry_t entry, void *shadow); 188 + void swap_cache_clear_shadow(int type, unsigned long begin, 189 + unsigned long end); 190 + 191 + void show_swap_cache_info(void); 192 + void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); 181 193 struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 182 194 struct vm_area_struct *vma, unsigned long addr, 183 195 struct swap_iocb **plug); ··· 317 305 return NULL; 318 306 } 319 307 320 - static inline void *get_shadow_from_swap_cache(swp_entry_t entry) 308 + static inline void *swap_cache_get_shadow(swp_entry_t entry) 321 309 { 322 310 return NULL; 323 311 } 324 312 325 - static inline int add_to_swap_cache(struct folio *folio, swp_entry_t entry, 326 - gfp_t gfp_mask, void **shadowp) 313 + static inline int swap_cache_add_folio(swp_entry_t entry, struct folio *folio, 314 + gfp_t gfp, void **shadow) 327 315 { 328 - return -1; 316 + return -EINVAL; 329 317 } 330 318 331 - static inline void __delete_from_swap_cache(struct folio *folio, 332 - swp_entry_t entry, void *shadow) 333 - { 334 - } 335 - 336 - static inline void delete_from_swap_cache(struct folio *folio) 319 + static inline void swap_cache_del_folio(struct folio *folio) 337 320 { 338 321 } 339 322 340 - static inline void clear_shadow_from_swap_cache(int type, unsigned long begin, 341 - unsigned long end) 323 + static inline void __swap_cache_del_folio(struct folio *folio, swp_entry_t entry, void *shadow) 342 324 { 343 325 } 344 326

+62 -24

mm/swap_state.c

··· 78 78 * Context: Caller must ensure @entry is valid and protect the swap device 79 79 * with reference count or locks. 80 80 * Return: Returns the found folio on success, NULL otherwise. The caller 81 - * must lock and check if the folio still matches the swap entry before 82 - * use (e.g. with folio_matches_swap_entry). 81 + * must lock nd check if the folio still matches the swap entry before 82 + * use (e.g., folio_matches_swap_entry). 83 83 */ 84 84 struct folio *swap_cache_get_folio(swp_entry_t entry) 85 85 { ··· 90 90 return folio; 91 91 } 92 92 93 - void *get_shadow_from_swap_cache(swp_entry_t entry) 93 + /** 94 + * swap_cache_get_shadow - Looks up a shadow in the swap cache. 95 + * @entry: swap entry used for the lookup. 96 + * 97 + * Context: Caller must ensure @entry is valid and protect the swap device 98 + * with reference count or locks. 99 + * Return: Returns either NULL or an XA_VALUE (shadow). 100 + */ 101 + void *swap_cache_get_shadow(swp_entry_t entry) 94 102 { 95 103 struct address_space *address_space = swap_address_space(entry); 96 104 pgoff_t idx = swap_cache_index(entry); ··· 110 102 return NULL; 111 103 } 112 104 113 - /* 114 - * add_to_swap_cache resembles filemap_add_folio on swapper_space, 115 - * but sets SwapCache flag and 'swap' instead of mapping and index. 105 + /** 106 + * swap_cache_add_folio - Add a folio into the swap cache. 107 + * @folio: The folio to be added. 108 + * @entry: The swap entry corresponding to the folio. 109 + * @gfp: gfp_mask for XArray node allocation. 110 + * @shadowp: If a shadow is found, return the shadow. 111 + * 112 + * Context: Caller must ensure @entry is valid and protect the swap device 113 + * with reference count or locks. 114 + * The caller also needs to mark the corresponding swap_map slots with 115 + * SWAP_HAS_CACHE to avoid race or conflict. 116 + * Return: Returns 0 on success, error code otherwise. 116 117 */ 117 - int add_to_swap_cache(struct folio *folio, swp_entry_t entry, 118 - gfp_t gfp, void **shadowp) 118 + int swap_cache_add_folio(struct folio *folio, swp_entry_t entry, 119 + gfp_t gfp, void **shadowp) 119 120 { 120 121 struct address_space *address_space = swap_address_space(entry); 121 122 pgoff_t idx = swap_cache_index(entry); ··· 172 155 return xas_error(&xas); 173 156 } 174 157 175 - /* 176 - * This must be called only on folios that have 177 - * been verified to be in the swap cache. 158 + /** 159 + * __swap_cache_del_folio - Removes a folio from the swap cache. 160 + * @folio: The folio. 161 + * @entry: The first swap entry that the folio corresponds to. 162 + * @shadow: shadow value to be filled in the swap cache. 163 + * 164 + * Removes a folio from the swap cache and fills a shadow in place. 165 + * This won't put the folio's refcount. The caller has to do that. 166 + * 167 + * Context: Caller must hold the xa_lock, ensure the folio is 168 + * locked and in the swap cache, using the index of @entry. 178 169 */ 179 - void __delete_from_swap_cache(struct folio *folio, 180 - swp_entry_t entry, void *shadow) 170 + void __swap_cache_del_folio(struct folio *folio, 171 + swp_entry_t entry, void *shadow) 181 172 { 182 173 struct address_space *address_space = swap_address_space(entry); 183 174 int i; ··· 211 186 __lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr); 212 187 } 213 188 214 - /* 215 - * This must be called only on folios that have 216 - * been verified to be in the swap cache and locked. 217 - * It will never put the folio into the free list, 218 - * the caller has a reference on the folio. 189 + /** 190 + * swap_cache_del_folio - Removes a folio from the swap cache. 191 + * @folio: The folio. 192 + * 193 + * Same as __swap_cache_del_folio, but handles lock and refcount. The 194 + * caller must ensure the folio is either clean or has a swap count 195 + * equal to zero, or it may cause data loss. 196 + * 197 + * Context: Caller must ensure the folio is locked and in the swap cache. 219 198 */ 220 - void delete_from_swap_cache(struct folio *folio) 199 + void swap_cache_del_folio(struct folio *folio) 221 200 { 222 201 swp_entry_t entry = folio->swap; 223 202 struct address_space *address_space = swap_address_space(entry); 224 203 225 204 xa_lock_irq(&address_space->i_pages); 226 - __delete_from_swap_cache(folio, entry, NULL); 205 + __swap_cache_del_folio(folio, entry, NULL); 227 206 xa_unlock_irq(&address_space->i_pages); 228 207 229 208 put_swap_folio(folio, entry); 230 209 folio_ref_sub(folio, folio_nr_pages(folio)); 231 210 } 232 211 233 - void clear_shadow_from_swap_cache(int type, unsigned long begin, 234 - unsigned long end) 212 + /** 213 + * swap_cache_clear_shadow - Clears a set of shadows in the swap cache. 214 + * @type: Indicates the swap device. 215 + * @begin: Beginning offset of the range. 216 + * @end: Ending offset of the range. 217 + * 218 + * Context: Caller must ensure the range is valid and hold a reference to 219 + * the swap device. 220 + */ 221 + void swap_cache_clear_shadow(int type, unsigned long begin, 222 + unsigned long end) 235 223 { 236 224 unsigned long curr = begin; 237 225 void *old; ··· 431 393 goto put_and_return; 432 394 433 395 /* 434 - * We might race against __delete_from_swap_cache(), and 396 + * We might race against __swap_cache_del_folio(), and 435 397 * stumble across a swap_map entry whose SWAP_HAS_CACHE 436 398 * has not yet been cleared. Or race against another 437 399 * __read_swap_cache_async(), which has set SWAP_HAS_CACHE ··· 450 412 goto fail_unlock; 451 413 452 414 /* May fail (-ENOMEM) if XArray node allocation failed. */ 453 - if (add_to_swap_cache(new_folio, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow)) 415 + if (swap_cache_add_folio(new_folio, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow)) 454 416 goto fail_unlock; 455 417 456 418 memcg1_swapin(entry, 1);

+4 -4

mm/swapfile.c

··· 267 267 if (!need_reclaim) 268 268 goto out_unlock; 269 269 270 - delete_from_swap_cache(folio); 270 + swap_cache_del_folio(folio); 271 271 folio_set_dirty(folio); 272 272 ret = nr_pages; 273 273 out_unlock: ··· 1124 1124 swap_slot_free_notify(si->bdev, offset); 1125 1125 offset++; 1126 1126 } 1127 - clear_shadow_from_swap_cache(si->type, begin, end); 1127 + swap_cache_clear_shadow(si->type, begin, end); 1128 1128 1129 1129 /* 1130 1130 * Make sure that try_to_unuse() observes si->inuse_pages reaching 0 ··· 1289 1289 * TODO: this could cause a theoretical memory reclaim 1290 1290 * deadlock in the swap out path. 1291 1291 */ 1292 - if (add_to_swap_cache(folio, entry, gfp | __GFP_NOMEMALLOC, NULL)) 1292 + if (swap_cache_add_folio(folio, entry, gfp | __GFP_NOMEMALLOC, NULL)) 1293 1293 goto out_free; 1294 1294 1295 1295 return 0; ··· 1759 1759 if (folio_swapped(folio)) 1760 1760 return false; 1761 1761 1762 - delete_from_swap_cache(folio); 1762 + swap_cache_del_folio(folio); 1763 1763 folio_set_dirty(folio); 1764 1764 return true; 1765 1765 }

+1 -1

mm/vmscan.c

··· 776 776 777 777 if (reclaimed && !mapping_exiting(mapping)) 778 778 shadow = workingset_eviction(folio, target_memcg); 779 - __delete_from_swap_cache(folio, swap, shadow); 779 + __swap_cache_del_folio(folio, swap, shadow); 780 780 memcg1_swapout(folio, swap); 781 781 xa_unlock_irq(&mapping->i_pages); 782 782 put_swap_folio(folio, swap);

+1 -1

mm/zswap.c

··· 1069 1069 1070 1070 out: 1071 1071 if (ret && ret != -EEXIST) { 1072 - delete_from_swap_cache(folio); 1072 + swap_cache_del_folio(folio); 1073 1073 folio_unlock(folio); 1074 1074 } 1075 1075 folio_put(folio);

Configure Feed

Configure Feed