mm, swap: simplify percpu cluster updating

Instead of using a returning argument, we can simply store the next
cluster offset to the fixed percpu location, which reduce the stack usage
and simplify the function:

Object size:
./scripts/bloat-o-meter mm/swapfile.o mm/swapfile.o.new
add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-271 (-271)
Function old new delta
get_swap_pages 2847 2733 -114
alloc_swap_scan_cluster 894 737 -157
Total: Before=30833, After=30562, chg -0.88%

Stack usage:
Before:
swapfile.c:1190:5:get_swap_pages 240 static

After:
swapfile.c:1185:5:get_swap_pages 216 static

Link: https://lkml.kernel.org/r/20250113175732.48099-11-ryncsn@gmail.com
Signed-off-by: Kairui Song <kasong@tencent.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: Chis Li <chrisl@kernel.org>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickens <hughd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kalesh Singh <kaleshsingh@google.com>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Kairui Song and committed by

Andrew Morton 1 year ago e3ae2dec 3b644773

+31 -39

2 changed files

expand all

include

linux

swap.h

swapfile.c

+2 -2

include/linux/swap.h

··· 274 274 * The first page in the swap file is the swap header, which is always marked 275 275 * bad to prevent it from being allocated as an entry. This also prevents the 276 276 * cluster to which it belongs being marked free. Therefore 0 is safe to use as 277 - * a sentinel to indicate next is not valid in percpu_cluster. 277 + * a sentinel to indicate an entry is not valid. 278 278 */ 279 - #define SWAP_NEXT_INVALID 0 279 + #define SWAP_ENTRY_INVALID 0 280 280 281 281 #ifdef CONFIG_THP_SWAP 282 282 #define SWAP_NR_ORDERS (PMD_ORDER + 1)

+29 -37

mm/swapfile.c

··· 765 765 return true; 766 766 } 767 767 768 - static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, unsigned long offset, 769 - unsigned int *foundp, unsigned int order, 768 + /* Try use a new cluster for current CPU and allocate from it. */ 769 + static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, 770 + struct swap_cluster_info *ci, 771 + unsigned long offset, 772 + unsigned int order, 770 773 unsigned char usage) 771 774 { 772 - unsigned long start = offset & ~(SWAPFILE_CLUSTER - 1); 775 + unsigned int next = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; 776 + unsigned long start = ALIGN_DOWN(offset, SWAPFILE_CLUSTER); 773 777 unsigned long end = min(start + SWAPFILE_CLUSTER, si->max); 774 778 unsigned int nr_pages = 1 << order; 775 779 bool need_reclaim, ret; 776 - struct swap_cluster_info *ci; 777 780 778 - ci = &si->cluster_info[offset / SWAPFILE_CLUSTER]; 779 781 lockdep_assert_held(&ci->lock); 780 782 781 - if (end < nr_pages || ci->count + nr_pages > SWAPFILE_CLUSTER) { 782 - offset = SWAP_NEXT_INVALID; 783 + if (end < nr_pages || ci->count + nr_pages > SWAPFILE_CLUSTER) 783 784 goto out; 784 - } 785 785 786 786 for (end -= nr_pages; offset <= end; offset += nr_pages) { 787 787 need_reclaim = false; ··· 795 795 * cluster has no flag set, and change of list 796 796 * won't cause fragmentation. 797 797 */ 798 - if (!cluster_is_usable(ci, order)) { 799 - offset = SWAP_NEXT_INVALID; 798 + if (!cluster_is_usable(ci, order)) 800 799 goto out; 801 - } 802 800 if (cluster_is_empty(ci)) 803 801 offset = start; 804 802 /* Reclaim failed but cluster is usable, try next */ 805 803 if (!ret) 806 804 continue; 807 805 } 808 - if (!cluster_alloc_range(si, ci, offset, usage, order)) { 809 - offset = SWAP_NEXT_INVALID; 810 - goto out; 811 - } 812 - *foundp = offset; 813 - if (ci->count == SWAPFILE_CLUSTER) { 814 - offset = SWAP_NEXT_INVALID; 815 - goto out; 816 - } 806 + if (!cluster_alloc_range(si, ci, offset, usage, order)) 807 + break; 808 + found = offset; 817 809 offset += nr_pages; 810 + if (ci->count < SWAPFILE_CLUSTER && offset <= end) 811 + next = offset; 818 812 break; 819 813 } 820 - if (offset > end) 821 - offset = SWAP_NEXT_INVALID; 822 814 out: 823 815 relocate_cluster(si, ci); 824 816 unlock_cluster(ci); 825 - return offset; 817 + __this_cpu_write(si->percpu_cluster->next[order], next); 818 + return found; 826 819 } 827 820 828 821 /* Return true if reclaimed a whole cluster */ ··· 884 891 if (cluster_is_usable(ci, order)) { 885 892 if (cluster_is_empty(ci)) 886 893 offset = cluster_offset(si, ci); 887 - offset = alloc_swap_scan_cluster(si, offset, &found, 888 - order, usage); 894 + found = alloc_swap_scan_cluster(si, ci, offset, 895 + order, usage); 889 896 } else { 890 897 unlock_cluster(ci); 891 898 } ··· 896 903 new_cluster: 897 904 ci = isolate_lock_cluster(si, &si->free_clusters); 898 905 if (ci) { 899 - offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci), 900 - &found, order, usage); 906 + found = alloc_swap_scan_cluster(si, ci, cluster_offset(si, ci), 907 + order, usage); 901 908 if (found) 902 909 goto done; 903 910 } ··· 910 917 unsigned int frags = 0, frags_existing; 911 918 912 919 while ((ci = isolate_lock_cluster(si, &si->nonfull_clusters[order]))) { 913 - offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci), 914 - &found, order, usage); 920 + found = alloc_swap_scan_cluster(si, ci, cluster_offset(si, ci), 921 + order, usage); 915 922 if (found) 916 923 goto done; 917 924 /* Clusters failed to allocate are moved to frag_clusters */ ··· 928 935 * per-CPU usage, but they could contain newly released 929 936 * reclaimable (eg. lazy-freed swap cache) slots. 930 937 */ 931 - offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci), 932 - &found, order, usage); 938 + found = alloc_swap_scan_cluster(si, ci, cluster_offset(si, ci), 939 + order, usage); 933 940 if (found) 934 941 goto done; 935 942 frags++; ··· 955 962 */ 956 963 while ((ci = isolate_lock_cluster(si, &si->frag_clusters[o]))) { 957 964 atomic_long_dec(&si->frag_cluster_nr[o]); 958 - offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci), 959 - &found, order, usage); 965 + found = alloc_swap_scan_cluster(si, ci, cluster_offset(si, ci), 966 + 0, usage); 960 967 if (found) 961 968 goto done; 962 969 } 963 970 964 971 while ((ci = isolate_lock_cluster(si, &si->nonfull_clusters[o]))) { 965 - offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci), 966 - &found, order, usage); 972 + found = alloc_swap_scan_cluster(si, ci, cluster_offset(si, ci), 973 + 0, usage); 967 974 if (found) 968 975 goto done; 969 976 } 970 977 } 971 978 done: 972 - __this_cpu_write(si->percpu_cluster->next[order], offset); 973 979 local_unlock(&si->percpu_cluster->lock); 974 980 975 981 return found; ··· 3192 3200 3193 3201 cluster = per_cpu_ptr(si->percpu_cluster, cpu); 3194 3202 for (i = 0; i < SWAP_NR_ORDERS; i++) 3195 - cluster->next[i] = SWAP_NEXT_INVALID; 3203 + cluster->next[i] = SWAP_ENTRY_INVALID; 3196 3204 local_lock_init(&cluster->lock); 3197 3205 } 3198 3206

Configure Feed

Configure Feed