mm: swap: allow storage of all mTHP orders

Multi-size THP enables performance improvements by allocating large,
pte-mapped folios for anonymous memory. However I've observed that on an
arm64 system running a parallel workload (e.g. kernel compilation) across
many cores, under high memory pressure, the speed regresses. This is due
to bottlenecking on the increased number of TLBIs added due to all the
extra folio splitting when the large folios are swapped out.

Therefore, solve this regression by adding support for swapping out mTHP
without needing to split the folio, just like is already done for
PMD-sized THP. This change only applies when CONFIG_THP_SWAP is enabled,
and when the swap backing store is a non-rotating block device. These are
the same constraints as for the existing PMD-sized THP swap-out support.

Note that no attempt is made to swap-in (m)THP here - this is still done
page-by-page, like for PMD-sized THP. But swapping-out mTHP is a
prerequisite for swapping-in mTHP.

The main change here is to improve the swap entry allocator so that it can
allocate any power-of-2 number of contiguous entries between [1, (1 <<
PMD_ORDER)]. This is done by allocating a cluster for each distinct order
and allocating sequentially from it until the cluster is full. This
ensures that we don't need to search the map and we get no fragmentation
due to alignment padding for different orders in the cluster. If there is
no current cluster for a given order, we attempt to allocate a free
cluster from the list. If there are no free clusters, we fail the
allocation and the caller can fall back to splitting the folio and
allocates individual entries (as per existing PMD-sized THP fallback).

The per-order current clusters are maintained per-cpu using the existing
infrastructure. This is done to avoid interleving pages from different
tasks, which would prevent IO being batched. This is already done for the
order-0 allocations so we follow the same pattern.

As is done for order-0 per-cpu clusters, the scanner now can steal order-0
entries from any per-cpu-per-order reserved cluster. This ensures that
when the swap file is getting full, space doesn't get tied up in the
per-cpu reserves.

This change only modifies swap to be able to accept any order mTHP. It
doesn't change the callers to elide doing the actual split. That will be
done in separate changes.

Link: https://lkml.kernel.org/r/20240408183946.2991168-6-ryan.roberts@arm.com
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: "Huang, Ying" <ying.huang@intel.com>
Cc: Barry Song <21cnbao@gmail.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: Chris Li <chrisl@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Gao Xiang <xiang@kernel.org>
Cc: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Lance Yang <ioworker0@gmail.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Ryan Roberts and committed by

Andrew Morton 2 years ago 845982eb 9faaa0f8

+98 -72

2 changed files

expand all

include

linux

swap.h

swapfile.c

+7 -1

include/linux/swap.h

··· 268 268 */ 269 269 #define SWAP_NEXT_INVALID 0 270 270 271 + #ifdef CONFIG_THP_SWAP 272 + #define SWAP_NR_ORDERS (PMD_ORDER + 1) 273 + #else 274 + #define SWAP_NR_ORDERS 1 275 + #endif 276 + 271 277 /* 272 278 * We assign a cluster to each CPU, so each CPU can allocate swap entry from 273 279 * its own cluster and swapout sequentially. The purpose is to optimize swapout 274 280 * throughput. 275 281 */ 276 282 struct percpu_cluster { 277 - unsigned int next; /* Likely next allocation offset */ 283 + unsigned int next[SWAP_NR_ORDERS]; /* Likely next allocation offset */ 278 284 }; 279 285 280 286 struct swap_cluster_list {

+91 -71

mm/swapfile.c

··· 551 551 552 552 /* 553 553 * The cluster corresponding to page_nr will be used. The cluster will be 554 - * removed from free cluster list and its usage counter will be increased. 554 + * removed from free cluster list and its usage counter will be increased by 555 + * count. 555 556 */ 556 - static void inc_cluster_info_page(struct swap_info_struct *p, 557 - struct swap_cluster_info *cluster_info, unsigned long page_nr) 557 + static void add_cluster_info_page(struct swap_info_struct *p, 558 + struct swap_cluster_info *cluster_info, unsigned long page_nr, 559 + unsigned long count) 558 560 { 559 561 unsigned long idx = page_nr / SWAPFILE_CLUSTER; 560 562 ··· 565 563 if (cluster_is_free(&cluster_info[idx])) 566 564 alloc_cluster(p, idx); 567 565 568 - VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER); 566 + VM_BUG_ON(cluster_count(&cluster_info[idx]) + count > SWAPFILE_CLUSTER); 569 567 cluster_set_count(&cluster_info[idx], 570 - cluster_count(&cluster_info[idx]) + 1); 568 + cluster_count(&cluster_info[idx]) + count); 569 + } 570 + 571 + /* 572 + * The cluster corresponding to page_nr will be used. The cluster will be 573 + * removed from free cluster list and its usage counter will be increased by 1. 574 + */ 575 + static void inc_cluster_info_page(struct swap_info_struct *p, 576 + struct swap_cluster_info *cluster_info, unsigned long page_nr) 577 + { 578 + add_cluster_info_page(p, cluster_info, page_nr, 1); 571 579 } 572 580 573 581 /* ··· 607 595 */ 608 596 static bool 609 597 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, 610 - unsigned long offset) 598 + unsigned long offset, int order) 611 599 { 612 600 struct percpu_cluster *percpu_cluster; 613 601 bool conflict; ··· 621 609 return false; 622 610 623 611 percpu_cluster = this_cpu_ptr(si->percpu_cluster); 624 - percpu_cluster->next = SWAP_NEXT_INVALID; 612 + percpu_cluster->next[order] = SWAP_NEXT_INVALID; 613 + return true; 614 + } 615 + 616 + static inline bool swap_range_empty(char *swap_map, unsigned int start, 617 + unsigned int nr_pages) 618 + { 619 + unsigned int i; 620 + 621 + for (i = 0; i < nr_pages; i++) { 622 + if (swap_map[start + i]) 623 + return false; 624 + } 625 + 625 626 return true; 626 627 } 627 628 628 629 /* 629 - * Try to get a swap entry from current cpu's swap entry pool (a cluster). This 630 - * might involve allocating a new cluster for current CPU too. 630 + * Try to get swap entries with specified order from current cpu's swap entry 631 + * pool (a cluster). This might involve allocating a new cluster for current CPU 632 + * too. 631 633 */ 632 634 static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, 633 - unsigned long *offset, unsigned long *scan_base) 635 + unsigned long *offset, unsigned long *scan_base, int order) 634 636 { 637 + unsigned int nr_pages = 1 << order; 635 638 struct percpu_cluster *cluster; 636 639 struct swap_cluster_info *ci; 637 640 unsigned int tmp, max; 638 641 639 642 new_cluster: 640 643 cluster = this_cpu_ptr(si->percpu_cluster); 641 - tmp = cluster->next; 644 + tmp = cluster->next[order]; 642 645 if (tmp == SWAP_NEXT_INVALID) { 643 646 if (!cluster_list_empty(&si->free_clusters)) { 644 647 tmp = cluster_next(&si->free_clusters.head) * ··· 674 647 675 648 /* 676 649 * Other CPUs can use our cluster if they can't find a free cluster, 677 - * check if there is still free entry in the cluster 650 + * check if there is still free entry in the cluster, maintaining 651 + * natural alignment. 678 652 */ 679 653 max = min_t(unsigned long, si->max, ALIGN(tmp + 1, SWAPFILE_CLUSTER)); 680 654 if (tmp < max) { 681 655 ci = lock_cluster(si, tmp); 682 656 while (tmp < max) { 683 - if (!si->swap_map[tmp]) 657 + if (swap_range_empty(si->swap_map, tmp, nr_pages)) 684 658 break; 685 - tmp++; 659 + tmp += nr_pages; 686 660 } 687 661 unlock_cluster(ci); 688 662 } 689 663 if (tmp >= max) { 690 - cluster->next = SWAP_NEXT_INVALID; 664 + cluster->next[order] = SWAP_NEXT_INVALID; 691 665 goto new_cluster; 692 666 } 693 667 *offset = tmp; 694 668 *scan_base = tmp; 695 - tmp += 1; 696 - cluster->next = tmp < max ? tmp : SWAP_NEXT_INVALID; 669 + tmp += nr_pages; 670 + cluster->next[order] = tmp < max ? tmp : SWAP_NEXT_INVALID; 697 671 return true; 698 672 } 699 673 ··· 824 796 825 797 static int scan_swap_map_slots(struct swap_info_struct *si, 826 798 unsigned char usage, int nr, 827 - swp_entry_t slots[]) 799 + swp_entry_t slots[], int order) 828 800 { 829 801 struct swap_cluster_info *ci; 830 802 unsigned long offset; 831 803 unsigned long scan_base; 832 804 unsigned long last_in_cluster = 0; 833 805 int latency_ration = LATENCY_LIMIT; 806 + unsigned int nr_pages = 1 << order; 834 807 int n_ret = 0; 835 808 bool scanned_many = false; 836 809 ··· 845 816 * But we do now try to find an empty cluster. -Andrea 846 817 * And we let swap pages go all over an SSD partition. Hugh 847 818 */ 819 + 820 + if (order > 0) { 821 + /* 822 + * Should not even be attempting large allocations when huge 823 + * page swap is disabled. Warn and fail the allocation. 824 + */ 825 + if (!IS_ENABLED(CONFIG_THP_SWAP) || 826 + nr_pages > SWAPFILE_CLUSTER) { 827 + VM_WARN_ON_ONCE(1); 828 + return 0; 829 + } 830 + 831 + /* 832 + * Swapfile is not block device or not using clusters so unable 833 + * to allocate large entries. 834 + */ 835 + if (!(si->flags & SWP_BLKDEV) || !si->cluster_info) 836 + return 0; 837 + } 848 838 849 839 si->flags += SWP_SCANNING; 850 840 /* ··· 879 831 880 832 /* SSD algorithm */ 881 833 if (si->cluster_info) { 882 - if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 834 + if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base, order)) { 835 + if (order > 0) 836 + goto no_page; 883 837 goto scan; 838 + } 884 839 } else if (unlikely(!si->cluster_nr--)) { 885 840 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { 886 841 si->cluster_nr = SWAPFILE_CLUSTER - 1; ··· 925 874 926 875 checks: 927 876 if (si->cluster_info) { 928 - while (scan_swap_map_ssd_cluster_conflict(si, offset)) { 877 + while (scan_swap_map_ssd_cluster_conflict(si, offset, order)) { 929 878 /* take a break if we already got some slots */ 930 879 if (n_ret) 931 880 goto done; 932 881 if (!scan_swap_map_try_ssd_cluster(si, &offset, 933 - &scan_base)) 882 + &scan_base, order)) { 883 + if (order > 0) 884 + goto no_page; 934 885 goto scan; 886 + } 935 887 } 936 888 } 937 889 if (!(si->flags & SWP_WRITEOK)) ··· 965 911 else 966 912 goto done; 967 913 } 968 - WRITE_ONCE(si->swap_map[offset], usage); 969 - inc_cluster_info_page(si, si->cluster_info, offset); 914 + memset(si->swap_map + offset, usage, nr_pages); 915 + add_cluster_info_page(si, si->cluster_info, offset, nr_pages); 970 916 unlock_cluster(ci); 971 917 972 - swap_range_alloc(si, offset, 1); 918 + swap_range_alloc(si, offset, nr_pages); 973 919 slots[n_ret++] = swp_entry(si->type, offset); 974 920 975 921 /* got enough slots or reach max slots? */ ··· 990 936 991 937 /* try to get more slots in cluster */ 992 938 if (si->cluster_info) { 993 - if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 939 + if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base, order)) 994 940 goto checks; 941 + if (order > 0) 942 + goto done; 995 943 } else if (si->cluster_nr && !si->swap_map[++offset]) { 996 944 /* non-ssd case, still more slots in cluster? */ 997 945 --si->cluster_nr; ··· 1020 964 } 1021 965 1022 966 done: 1023 - set_cluster_next(si, offset + 1); 967 + if (order == 0) 968 + set_cluster_next(si, offset + 1); 1024 969 si->flags -= SWP_SCANNING; 1025 970 return n_ret; 1026 971 1027 972 scan: 973 + VM_WARN_ON(order > 0); 1028 974 spin_unlock(&si->lock); 1029 975 while (++offset <= READ_ONCE(si->highest_bit)) { 1030 976 if (unlikely(--latency_ration < 0)) { ··· 1055 997 return n_ret; 1056 998 } 1057 999 1058 - static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) 1059 - { 1060 - unsigned long idx; 1061 - struct swap_cluster_info *ci; 1062 - unsigned long offset; 1063 - 1064 - /* 1065 - * Should not even be attempting cluster allocations when huge 1066 - * page swap is disabled. Warn and fail the allocation. 1067 - */ 1068 - if (!IS_ENABLED(CONFIG_THP_SWAP)) { 1069 - VM_WARN_ON_ONCE(1); 1070 - return 0; 1071 - } 1072 - 1073 - if (cluster_list_empty(&si->free_clusters)) 1074 - return 0; 1075 - 1076 - idx = cluster_list_first(&si->free_clusters); 1077 - offset = idx * SWAPFILE_CLUSTER; 1078 - ci = lock_cluster(si, offset); 1079 - alloc_cluster(si, idx); 1080 - cluster_set_count(ci, SWAPFILE_CLUSTER); 1081 - 1082 - memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER); 1083 - unlock_cluster(ci); 1084 - swap_range_alloc(si, offset, SWAPFILE_CLUSTER); 1085 - *slot = swp_entry(si->type, offset); 1086 - 1087 - return 1; 1088 - } 1089 - 1090 1000 static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) 1091 1001 { 1092 1002 unsigned long offset = idx * SWAPFILE_CLUSTER; ··· 1076 1050 long avail_pgs; 1077 1051 int n_ret = 0; 1078 1052 int node; 1079 - 1080 - /* Only single cluster request supported */ 1081 - WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER); 1082 1053 1083 1054 spin_lock(&swap_avail_lock); 1084 1055 ··· 1112 1089 spin_unlock(&si->lock); 1113 1090 goto nextsi; 1114 1091 } 1115 - if (size == SWAPFILE_CLUSTER) { 1116 - if (si->flags & SWP_BLKDEV) 1117 - n_ret = swap_alloc_cluster(si, swp_entries); 1118 - } else 1119 - n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, 1120 - n_goal, swp_entries); 1092 + n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, 1093 + n_goal, swp_entries, order); 1121 1094 spin_unlock(&si->lock); 1122 - if (n_ret || size == SWAPFILE_CLUSTER) 1095 + if (n_ret || size > 1) 1123 1096 goto check_out; 1124 1097 cond_resched(); 1125 1098 ··· 1692 1673 1693 1674 /* This is called for allocating swap entry, not cache */ 1694 1675 spin_lock(&si->lock); 1695 - if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry)) 1676 + if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry, 0)) 1696 1677 atomic_long_dec(&nr_swap_pages); 1697 1678 spin_unlock(&si->lock); 1698 1679 fail: ··· 3146 3127 p->flags |= SWP_SYNCHRONOUS_IO; 3147 3128 3148 3129 if (p->bdev && bdev_nonrot(p->bdev)) { 3149 - int cpu; 3130 + int cpu, i; 3150 3131 unsigned long ci, nr_cluster; 3151 3132 3152 3133 p->flags |= SWP_SOLIDSTATE; ··· 3184 3165 struct percpu_cluster *cluster; 3185 3166 3186 3167 cluster = per_cpu_ptr(p->percpu_cluster, cpu); 3187 - cluster->next = SWAP_NEXT_INVALID; 3168 + for (i = 0; i < SWAP_NR_ORDERS; i++) 3169 + cluster->next[i] = SWAP_NEXT_INVALID; 3188 3170 } 3189 3171 } else { 3190 3172 atomic_inc(&nr_rotate_swap);

Configure Feed

Configure Feed