mm, swap: rename and move some swap cluster definition and helpers

-34

include/linux/swap.h

··· 236 236 #define SWAP_CONT_MAX 0x7f /* Max count */ 237 237 238 238 /* 239 - * We use this to track usage of a cluster. A cluster is a block of swap disk 240 - * space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All 241 - * free clusters are organized into a list. We fetch an entry from the list to 242 - * get a free cluster. 243 - * 244 - * The flags field determines if a cluster is free. This is 245 - * protected by cluster lock. 246 - */ 247 - struct swap_cluster_info { 248 - spinlock_t lock; /* 249 - * Protect swap_cluster_info fields 250 - * other than list, and swap_info_struct->swap_map 251 - * elements corresponding to the swap cluster. 252 - */ 253 - u16 count; 254 - u8 flags; 255 - u8 order; 256 - struct list_head list; 257 - }; 258 - 259 - /* All on-list cluster must have a non-zero flag. */ 260 - enum swap_cluster_flags { 261 - CLUSTER_FLAG_NONE = 0, /* For temporary off-list cluster */ 262 - CLUSTER_FLAG_FREE, 263 - CLUSTER_FLAG_NONFULL, 264 - CLUSTER_FLAG_FRAG, 265 - /* Clusters with flags above are allocatable */ 266 - CLUSTER_FLAG_USABLE = CLUSTER_FLAG_FRAG, 267 - CLUSTER_FLAG_FULL, 268 - CLUSTER_FLAG_DISCARD, 269 - CLUSTER_FLAG_MAX, 270 - }; 271 - 272 - /* 273 239 * The first page in the swap file is the swap header, which is always marked 274 240 * bad to prevent it from being allocated as an entry. This also prevents the 275 241 * cluster to which it belongs being marked free. Therefore 0 is safe to use as

+70

mm/swap.h

··· 7 7 8 8 extern int page_cluster; 9 9 10 + #ifdef CONFIG_THP_SWAP 11 + #define SWAPFILE_CLUSTER HPAGE_PMD_NR 12 + #define swap_entry_order(order) (order) 13 + #else 14 + #define SWAPFILE_CLUSTER 256 15 + #define swap_entry_order(order) 0 16 + #endif 17 + 18 + /* 19 + * We use this to track usage of a cluster. A cluster is a block of swap disk 20 + * space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All 21 + * free clusters are organized into a list. We fetch an entry from the list to 22 + * get a free cluster. 23 + * 24 + * The flags field determines if a cluster is free. This is 25 + * protected by cluster lock. 26 + */ 27 + struct swap_cluster_info { 28 + spinlock_t lock; /* 29 + * Protect swap_cluster_info fields 30 + * other than list, and swap_info_struct->swap_map 31 + * elements corresponding to the swap cluster. 32 + */ 33 + u16 count; 34 + u8 flags; 35 + u8 order; 36 + struct list_head list; 37 + }; 38 + 39 + /* All on-list cluster must have a non-zero flag. */ 40 + enum swap_cluster_flags { 41 + CLUSTER_FLAG_NONE = 0, /* For temporary off-list cluster */ 42 + CLUSTER_FLAG_FREE, 43 + CLUSTER_FLAG_NONFULL, 44 + CLUSTER_FLAG_FRAG, 45 + /* Clusters with flags above are allocatable */ 46 + CLUSTER_FLAG_USABLE = CLUSTER_FLAG_FRAG, 47 + CLUSTER_FLAG_FULL, 48 + CLUSTER_FLAG_DISCARD, 49 + CLUSTER_FLAG_MAX, 50 + }; 51 + 10 52 #ifdef CONFIG_SWAP 11 53 #include <linux/swapops.h> /* for swp_offset */ 12 54 #include <linux/blk_types.h> /* for bio_end_io_t */ 55 + 56 + static inline struct swap_cluster_info *swp_offset_cluster( 57 + struct swap_info_struct *si, pgoff_t offset) 58 + { 59 + return &si->cluster_info[offset / SWAPFILE_CLUSTER]; 60 + } 61 + 62 + /** 63 + * swap_cluster_lock - Lock and return the swap cluster of given offset. 64 + * @si: swap device the cluster belongs to. 65 + * @offset: the swap entry offset, pointing to a valid slot. 66 + * 67 + * Context: The caller must ensure the offset is in the valid range and 68 + * protect the swap device with reference count or locks. 69 + */ 70 + static inline struct swap_cluster_info *swap_cluster_lock( 71 + struct swap_info_struct *si, unsigned long offset) 72 + { 73 + struct swap_cluster_info *ci = swp_offset_cluster(si, offset); 74 + 75 + spin_lock(&ci->lock); 76 + return ci; 77 + } 78 + 79 + static inline void swap_cluster_unlock(struct swap_cluster_info *ci) 80 + { 81 + spin_unlock(&ci->lock); 82 + } 13 83 14 84 /* linux/mm/page_io.c */ 15 85 int sio_pool_init(void);

+29 -68

mm/swapfile.c

··· 58 58 static void swap_range_alloc(struct swap_info_struct *si, 59 59 unsigned int nr_entries); 60 60 static bool folio_swapcache_freeable(struct folio *folio); 61 - static struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, 62 - unsigned long offset); 63 - static inline void unlock_cluster(struct swap_cluster_info *ci); 64 61 65 62 static DEFINE_SPINLOCK(swap_lock); 66 63 static unsigned int nr_swapfiles; ··· 255 258 * swap_map is HAS_CACHE only, which means the slots have no page table 256 259 * reference or pending writeback, and can't be allocated to others. 257 260 */ 258 - ci = lock_cluster(si, offset); 261 + ci = swap_cluster_lock(si, offset); 259 262 need_reclaim = swap_only_has_cache(si, offset, nr_pages); 260 - unlock_cluster(ci); 263 + swap_cluster_unlock(ci); 261 264 if (!need_reclaim) 262 265 goto out_unlock; 263 266 ··· 382 385 } 383 386 } 384 387 385 - #ifdef CONFIG_THP_SWAP 386 - #define SWAPFILE_CLUSTER HPAGE_PMD_NR 387 - 388 - #define swap_entry_order(order) (order) 389 - #else 390 - #define SWAPFILE_CLUSTER 256 391 - 392 - /* 393 - * Define swap_entry_order() as constant to let compiler to optimize 394 - * out some code if !CONFIG_THP_SWAP 395 - */ 396 - #define swap_entry_order(order) 0 397 - #endif 398 388 #define LATENCY_LIMIT 256 399 389 400 390 static inline bool cluster_is_empty(struct swap_cluster_info *info) ··· 409 425 return ci - si->cluster_info; 410 426 } 411 427 412 - static inline struct swap_cluster_info *offset_to_cluster(struct swap_info_struct *si, 413 - unsigned long offset) 414 - { 415 - return &si->cluster_info[offset / SWAPFILE_CLUSTER]; 416 - } 417 - 418 428 static inline unsigned int cluster_offset(struct swap_info_struct *si, 419 429 struct swap_cluster_info *ci) 420 430 { 421 431 return cluster_index(si, ci) * SWAPFILE_CLUSTER; 422 - } 423 - 424 - static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, 425 - unsigned long offset) 426 - { 427 - struct swap_cluster_info *ci; 428 - 429 - ci = offset_to_cluster(si, offset); 430 - spin_lock(&ci->lock); 431 - 432 - return ci; 433 - } 434 - 435 - static inline void unlock_cluster(struct swap_cluster_info *ci) 436 - { 437 - spin_unlock(&ci->lock); 438 432 } 439 433 440 434 static void move_cluster(struct swap_info_struct *si, ··· 770 808 } 771 809 out: 772 810 relocate_cluster(si, ci); 773 - unlock_cluster(ci); 811 + swap_cluster_unlock(ci); 774 812 if (si->flags & SWP_SOLIDSTATE) { 775 813 this_cpu_write(percpu_swap_cluster.offset[order], next); 776 814 this_cpu_write(percpu_swap_cluster.si[order], si); ··· 837 875 if (ci->flags == CLUSTER_FLAG_NONE) 838 876 relocate_cluster(si, ci); 839 877 840 - unlock_cluster(ci); 878 + swap_cluster_unlock(ci); 841 879 if (to_scan <= 0) 842 880 break; 843 881 } ··· 876 914 if (offset == SWAP_ENTRY_INVALID) 877 915 goto new_cluster; 878 916 879 - ci = lock_cluster(si, offset); 917 + ci = swap_cluster_lock(si, offset); 880 918 /* Cluster could have been used by another order */ 881 919 if (cluster_is_usable(ci, order)) { 882 920 if (cluster_is_empty(ci)) ··· 884 922 found = alloc_swap_scan_cluster(si, ci, offset, 885 923 order, usage); 886 924 } else { 887 - unlock_cluster(ci); 925 + swap_cluster_unlock(ci); 888 926 } 889 927 if (found) 890 928 goto done; ··· 1165 1203 if (!si || !offset || !get_swap_device_info(si)) 1166 1204 return false; 1167 1205 1168 - ci = lock_cluster(si, offset); 1206 + ci = swap_cluster_lock(si, offset); 1169 1207 if (cluster_is_usable(ci, order)) { 1170 1208 if (cluster_is_empty(ci)) 1171 1209 offset = cluster_offset(si, ci); ··· 1173 1211 if (found) 1174 1212 *entry = swp_entry(si->type, found); 1175 1213 } else { 1176 - unlock_cluster(ci); 1214 + swap_cluster_unlock(ci); 1177 1215 } 1178 1216 1179 1217 put_swap_device(si); ··· 1441 1479 unsigned long offset = swp_offset(entry); 1442 1480 struct swap_cluster_info *ci; 1443 1481 1444 - ci = lock_cluster(si, offset); 1445 - if (swap_only_has_cache(si, offset, nr)) 1482 + ci = swap_cluster_lock(si, offset); 1483 + if (swap_only_has_cache(si, offset, nr)) { 1446 1484 swap_entries_free(si, ci, entry, nr); 1447 - else { 1485 + } else { 1448 1486 for (int i = 0; i < nr; i++, entry.val++) 1449 1487 swap_entry_put_locked(si, ci, entry, SWAP_HAS_CACHE); 1450 1488 } 1451 - unlock_cluster(ci); 1489 + swap_cluster_unlock(ci); 1452 1490 } 1453 1491 1454 1492 static bool swap_entries_put_map(struct swap_info_struct *si, ··· 1466 1504 if (count != 1 && count != SWAP_MAP_SHMEM) 1467 1505 goto fallback; 1468 1506 1469 - ci = lock_cluster(si, offset); 1507 + ci = swap_cluster_lock(si, offset); 1470 1508 if (!swap_is_last_map(si, offset, nr, &has_cache)) { 1471 1509 goto locked_fallback; 1472 1510 } ··· 1475 1513 else 1476 1514 for (i = 0; i < nr; i++) 1477 1515 WRITE_ONCE(si->swap_map[offset + i], SWAP_HAS_CACHE); 1478 - unlock_cluster(ci); 1516 + swap_cluster_unlock(ci); 1479 1517 1480 1518 return has_cache; 1481 1519 1482 1520 fallback: 1483 - ci = lock_cluster(si, offset); 1521 + ci = swap_cluster_lock(si, offset); 1484 1522 locked_fallback: 1485 1523 for (i = 0; i < nr; i++, entry.val++) { 1486 1524 count = swap_entry_put_locked(si, ci, entry, 1); 1487 1525 if (count == SWAP_HAS_CACHE) 1488 1526 has_cache = true; 1489 1527 } 1490 - unlock_cluster(ci); 1528 + swap_cluster_unlock(ci); 1491 1529 return has_cache; 1492 - 1493 1530 } 1494 1531 1495 1532 /* ··· 1538 1577 unsigned char *map_end = map + nr_pages; 1539 1578 1540 1579 /* It should never free entries across different clusters */ 1541 - VM_BUG_ON(ci != offset_to_cluster(si, offset + nr_pages - 1)); 1580 + VM_BUG_ON(ci != swp_offset_cluster(si, offset + nr_pages - 1)); 1542 1581 VM_BUG_ON(cluster_is_empty(ci)); 1543 1582 VM_BUG_ON(ci->count < nr_pages); 1544 1583 ··· 1613 1652 struct swap_cluster_info *ci; 1614 1653 int count; 1615 1654 1616 - ci = lock_cluster(si, offset); 1655 + ci = swap_cluster_lock(si, offset); 1617 1656 count = swap_count(si->swap_map[offset]); 1618 - unlock_cluster(ci); 1657 + swap_cluster_unlock(ci); 1619 1658 return !!count; 1620 1659 } 1621 1660 ··· 1638 1677 1639 1678 offset = swp_offset(entry); 1640 1679 1641 - ci = lock_cluster(si, offset); 1680 + ci = swap_cluster_lock(si, offset); 1642 1681 1643 1682 count = swap_count(si->swap_map[offset]); 1644 1683 if (!(count & COUNT_CONTINUED)) ··· 1661 1700 n *= (SWAP_CONT_MAX + 1); 1662 1701 } while (tmp_count & COUNT_CONTINUED); 1663 1702 out: 1664 - unlock_cluster(ci); 1703 + swap_cluster_unlock(ci); 1665 1704 return count; 1666 1705 } 1667 1706 ··· 1676 1715 int i; 1677 1716 bool ret = false; 1678 1717 1679 - ci = lock_cluster(si, offset); 1718 + ci = swap_cluster_lock(si, offset); 1680 1719 if (nr_pages == 1) { 1681 1720 if (swap_count(map[roffset])) 1682 1721 ret = true; ··· 1689 1728 } 1690 1729 } 1691 1730 unlock_out: 1692 - unlock_cluster(ci); 1731 + swap_cluster_unlock(ci); 1693 1732 return ret; 1694 1733 } 1695 1734 ··· 2623 2662 BUG_ON(si->flags & SWP_WRITEOK); 2624 2663 2625 2664 for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) { 2626 - ci = lock_cluster(si, offset); 2627 - unlock_cluster(ci); 2665 + ci = swap_cluster_lock(si, offset); 2666 + swap_cluster_unlock(ci); 2628 2667 } 2629 2668 } 2630 2669 ··· 3540 3579 offset = swp_offset(entry); 3541 3580 VM_WARN_ON(nr > SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER); 3542 3581 VM_WARN_ON(usage == 1 && nr > 1); 3543 - ci = lock_cluster(si, offset); 3582 + ci = swap_cluster_lock(si, offset); 3544 3583 3545 3584 err = 0; 3546 3585 for (i = 0; i < nr; i++) { ··· 3595 3634 } 3596 3635 3597 3636 unlock_out: 3598 - unlock_cluster(ci); 3637 + swap_cluster_unlock(ci); 3599 3638 return err; 3600 3639 } 3601 3640 ··· 3694 3733 3695 3734 offset = swp_offset(entry); 3696 3735 3697 - ci = lock_cluster(si, offset); 3736 + ci = swap_cluster_lock(si, offset); 3698 3737 3699 3738 count = swap_count(si->swap_map[offset]); 3700 3739 ··· 3754 3793 out_unlock_cont: 3755 3794 spin_unlock(&si->cont_lock); 3756 3795 out: 3757 - unlock_cluster(ci); 3796 + swap_cluster_unlock(ci); 3758 3797 put_swap_device(si); 3759 3798 outer: 3760 3799 if (page)

Configure Feed

Configure Feed