Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm, swap: clean up plist removal and adding
When the swap device is full (inuse_pages == pages), it should be removed
from the allocation available plist. If any slot is freed, the swap
device should be added back to the plist. Additionally, during swapon or
swapoff, the swap device is forcefully added or removed.
Currently, the condition (inuse_pages == pages) is checked after every
counter update, then remove or add the device accordingly. This is
serialized by si->lock.
This commit decouples it from the protection of si->lock and reworked
plist removal and adding, making it possible to get rid of the hard
dependency on si->lock in allocation path in later commits.
To achieve this, simply using another lock is not an optimal approach, as
the overhead is observable for a hot counter, and may cause complex
locking issues. Thus, this commit manages to make it a lock-free atomic
operation, by embedding the plist state into the second highest bit of the
atomic counter.
Simply making the counter an atomic will not work, if the update and plist
status check are not performed atomically, we may miss an addition or
removal. With the embedded info we can update the counter and check the
plist status with single atomic operations, and avoid any extra overheads:
If the counter is full (inuse_pages == pages) and the off-list bit is
unset, we attempt to remove it from the plist. If the counter is not full
(inuse_pages != pages) and the off-list bit is set, we attempt to add it
to the plist. Removing, adding and bit update is serialized with a lock,
which is a cold path. Ordinary counter updates will be lock-free.
Link: https://lkml.kernel.org/r/20250113175732.48099-7-ryncsn@gmail.com
Signed-off-by: Kairui Song <kasong@tencent.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: Chis Li <chrisl@kernel.org>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickens <hughd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kalesh Singh <kaleshsingh@google.com>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Kairui Song
and committed by
Andrew Morton
b228386c
27701521
+139
-51
+1
-1
include/linux/swap.h
+1
-1
include/linux/swap.h
···
306
306
/* list of cluster that are fragmented or contented */
307
307
unsigned int frag_cluster_nr[SWAP_NR_ORDERS];
308
308
unsigned int pages; /* total of usable pages of swap */
309
-
unsigned int inuse_pages; /* number of those currently in use */
309
+
atomic_long_t inuse_pages; /* number of those currently in use */
310
310
struct percpu_cluster __percpu *percpu_cluster; /* per cpu's swap location */
311
311
struct rb_root swap_extent_root;/* root of the swap extent rbtree */
312
312
struct block_device *bdev; /* swap device or bdev of swap file */
+138
-50
mm/swapfile.c
+138
-50
mm/swapfile.c
···
128
128
return ent & ~SWAP_HAS_CACHE; /* may include COUNT_CONTINUED flag */
129
129
}
130
130
131
+
/*
132
+
* Use the second highest bit of inuse_pages counter as the indicator
133
+
* if one swap device is on the available plist, so the atomic can
134
+
* still be updated arithmetically while having special data embedded.
135
+
*
136
+
* inuse_pages counter is the only thing indicating if a device should
137
+
* be on avail_lists or not (except swapon / swapoff). By embedding the
138
+
* off-list bit in the atomic counter, updates no longer need any lock
139
+
* to check the list status.
140
+
*
141
+
* This bit will be set if the device is not on the plist and not
142
+
* usable, will be cleared if the device is on the plist.
143
+
*/
144
+
#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_t) - 2))
145
+
#define SWAP_USAGE_COUNTER_MASK (~SWAP_USAGE_OFFLIST_BIT)
146
+
static long swap_usage_in_pages(struct swap_info_struct *si)
147
+
{
148
+
return atomic_long_read(&si->inuse_pages) & SWAP_USAGE_COUNTER_MASK;
149
+
}
150
+
131
151
/* Reclaim the swap entry anyway if possible */
132
152
#define TTRS_ANYWAY 0x1
133
153
/*
···
737
717
int nr_reclaim;
738
718
739
719
if (force)
740
-
to_scan = si->inuse_pages / SWAPFILE_CLUSTER;
720
+
to_scan = swap_usage_in_pages(si) / SWAPFILE_CLUSTER;
741
721
742
722
while (!list_empty(&si->full_clusters)) {
743
723
ci = list_first_entry(&si->full_clusters, struct swap_cluster_info, list);
···
892
872
return found;
893
873
}
894
874
895
-
static void __del_from_avail_list(struct swap_info_struct *si)
875
+
/* SWAP_USAGE_OFFLIST_BIT can only be set by this helper. */
876
+
static void del_from_avail_list(struct swap_info_struct *si, bool swapoff)
896
877
{
897
878
int nid;
879
+
unsigned long pages;
898
880
899
-
assert_spin_locked(&si->lock);
881
+
spin_lock(&swap_avail_lock);
882
+
883
+
if (swapoff) {
884
+
/*
885
+
* Forcefully remove it. Clear the SWP_WRITEOK flags for
886
+
* swapoff here so it's synchronized by both si->lock and
887
+
* swap_avail_lock, to ensure the result can be seen by
888
+
* add_to_avail_list.
889
+
*/
890
+
lockdep_assert_held(&si->lock);
891
+
si->flags &= ~SWP_WRITEOK;
892
+
atomic_long_or(SWAP_USAGE_OFFLIST_BIT, &si->inuse_pages);
893
+
} else {
894
+
/*
895
+
* If not called by swapoff, take it off-list only if it's
896
+
* full and SWAP_USAGE_OFFLIST_BIT is not set (strictly
897
+
* si->inuse_pages == pages), any concurrent slot freeing,
898
+
* or device already removed from plist by someone else
899
+
* will make this return false.
900
+
*/
901
+
pages = si->pages;
902
+
if (!atomic_long_try_cmpxchg(&si->inuse_pages, &pages,
903
+
pages | SWAP_USAGE_OFFLIST_BIT))
904
+
goto skip;
905
+
}
906
+
900
907
for_each_node(nid)
901
908
plist_del(&si->avail_lists[nid], &swap_avail_heads[nid]);
909
+
910
+
skip:
911
+
spin_unlock(&swap_avail_lock);
902
912
}
903
913
904
-
static void del_from_avail_list(struct swap_info_struct *si)
914
+
/* SWAP_USAGE_OFFLIST_BIT can only be cleared by this helper. */
915
+
static void add_to_avail_list(struct swap_info_struct *si, bool swapon)
905
916
{
917
+
int nid;
918
+
long val;
919
+
unsigned long pages;
920
+
906
921
spin_lock(&swap_avail_lock);
907
-
__del_from_avail_list(si);
922
+
923
+
/* Corresponding to SWP_WRITEOK clearing in del_from_avail_list */
924
+
if (swapon) {
925
+
lockdep_assert_held(&si->lock);
926
+
si->flags |= SWP_WRITEOK;
927
+
} else {
928
+
if (!(READ_ONCE(si->flags) & SWP_WRITEOK))
929
+
goto skip;
930
+
}
931
+
932
+
if (!(atomic_long_read(&si->inuse_pages) & SWAP_USAGE_OFFLIST_BIT))
933
+
goto skip;
934
+
935
+
val = atomic_long_fetch_and_relaxed(~SWAP_USAGE_OFFLIST_BIT, &si->inuse_pages);
936
+
937
+
/*
938
+
* When device is full and device is on the plist, only one updater will
939
+
* see (inuse_pages == si->pages) and will call del_from_avail_list. If
940
+
* that updater happen to be here, just skip adding.
941
+
*/
942
+
pages = si->pages;
943
+
if (val == pages) {
944
+
/* Just like the cmpxchg in del_from_avail_list */
945
+
if (atomic_long_try_cmpxchg(&si->inuse_pages, &pages,
946
+
pages | SWAP_USAGE_OFFLIST_BIT))
947
+
goto skip;
948
+
}
949
+
950
+
for_each_node(nid)
951
+
plist_add(&si->avail_lists[nid], &swap_avail_heads[nid]);
952
+
953
+
skip:
908
954
spin_unlock(&swap_avail_lock);
955
+
}
956
+
957
+
/*
958
+
* swap_usage_add / swap_usage_sub of each slot are serialized by ci->lock
959
+
* within each cluster, so the total contribution to the global counter should
960
+
* always be positive and cannot exceed the total number of usable slots.
961
+
*/
962
+
static bool swap_usage_add(struct swap_info_struct *si, unsigned int nr_entries)
963
+
{
964
+
long val = atomic_long_add_return_relaxed(nr_entries, &si->inuse_pages);
965
+
966
+
/*
967
+
* If device is full, and SWAP_USAGE_OFFLIST_BIT is not set,
968
+
* remove it from the plist.
969
+
*/
970
+
if (unlikely(val == si->pages)) {
971
+
del_from_avail_list(si, false);
972
+
return true;
973
+
}
974
+
975
+
return false;
976
+
}
977
+
978
+
static void swap_usage_sub(struct swap_info_struct *si, unsigned int nr_entries)
979
+
{
980
+
long val = atomic_long_sub_return_relaxed(nr_entries, &si->inuse_pages);
981
+
982
+
/*
983
+
* If device is not full, and SWAP_USAGE_OFFLIST_BIT is set,
984
+
* remove it from the plist.
985
+
*/
986
+
if (unlikely(val & SWAP_USAGE_OFFLIST_BIT))
987
+
add_to_avail_list(si, false);
909
988
}
910
989
911
990
static void swap_range_alloc(struct swap_info_struct *si,
912
991
unsigned int nr_entries)
913
992
{
914
-
WRITE_ONCE(si->inuse_pages, si->inuse_pages + nr_entries);
915
-
if (si->inuse_pages == si->pages) {
916
-
del_from_avail_list(si);
917
-
993
+
if (swap_usage_add(si, nr_entries)) {
918
994
if (vm_swap_full())
919
995
schedule_work(&si->reclaim_work);
920
996
}
921
-
}
922
-
923
-
static void add_to_avail_list(struct swap_info_struct *si)
924
-
{
925
-
int nid;
926
-
927
-
spin_lock(&swap_avail_lock);
928
-
for_each_node(nid)
929
-
plist_add(&si->avail_lists[nid], &swap_avail_heads[nid]);
930
-
spin_unlock(&swap_avail_lock);
931
997
}
932
998
933
999
static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
···
1031
925
for (i = 0; i < nr_entries; i++)
1032
926
clear_bit(offset + i, si->zeromap);
1033
927
1034
-
if (si->inuse_pages == si->pages)
1035
-
add_to_avail_list(si);
1036
928
if (si->flags & SWP_BLKDEV)
1037
929
swap_slot_free_notify =
1038
930
si->bdev->bd_disk->fops->swap_slot_free_notify;
···
1050
946
*/
1051
947
smp_wmb();
1052
948
atomic_long_add(nr_entries, &nr_swap_pages);
1053
-
WRITE_ONCE(si->inuse_pages, si->inuse_pages - nr_entries);
949
+
swap_usage_sub(si, nr_entries);
1054
950
}
1055
951
1056
952
static int cluster_alloc_swap(struct swap_info_struct *si,
···
1140
1036
plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]);
1141
1037
spin_unlock(&swap_avail_lock);
1142
1038
spin_lock(&si->lock);
1143
-
if ((si->inuse_pages == si->pages) || !(si->flags & SWP_WRITEOK)) {
1144
-
spin_lock(&swap_avail_lock);
1145
-
if (plist_node_empty(&si->avail_lists[node])) {
1146
-
spin_unlock(&si->lock);
1147
-
goto nextsi;
1148
-
}
1149
-
WARN(!(si->flags & SWP_WRITEOK),
1150
-
"swap_info %d in list but !SWP_WRITEOK\n",
1151
-
si->type);
1152
-
__del_from_avail_list(si);
1153
-
spin_unlock(&si->lock);
1154
-
goto nextsi;
1155
-
}
1156
1039
n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE,
1157
1040
n_goal, swp_entries, order);
1158
1041
spin_unlock(&si->lock);
···
1148
1057
cond_resched();
1149
1058
1150
1059
spin_lock(&swap_avail_lock);
1151
-
nextsi:
1152
1060
/*
1153
1061
* if we got here, it's likely that si was almost full before,
1154
1062
* and since scan_swap_map_slots() can drop the si->lock,
···
1879
1789
if (sis->flags & SWP_WRITEOK) {
1880
1790
n = sis->pages;
1881
1791
if (free)
1882
-
n -= sis->inuse_pages;
1792
+
n -= swap_usage_in_pages(sis);
1883
1793
}
1884
1794
spin_unlock(&sis->lock);
1885
1795
}
···
2214
2124
swp_entry_t entry;
2215
2125
unsigned int i;
2216
2126
2217
-
if (!READ_ONCE(si->inuse_pages))
2127
+
if (!swap_usage_in_pages(si))
2218
2128
goto success;
2219
2129
2220
2130
retry:
···
2227
2137
2228
2138
spin_lock(&mmlist_lock);
2229
2139
p = &init_mm.mmlist;
2230
-
while (READ_ONCE(si->inuse_pages) &&
2140
+
while (swap_usage_in_pages(si) &&
2231
2141
!signal_pending(current) &&
2232
2142
(p = p->next) != &init_mm.mmlist) {
2233
2143
···
2255
2165
mmput(prev_mm);
2256
2166
2257
2167
i = 0;
2258
-
while (READ_ONCE(si->inuse_pages) &&
2168
+
while (swap_usage_in_pages(si) &&
2259
2169
!signal_pending(current) &&
2260
2170
(i = find_next_to_unuse(si, i)) != 0) {
2261
2171
···
2290
2200
* folio_alloc_swap(), temporarily hiding that swap. It's easy
2291
2201
* and robust (though cpu-intensive) just to keep retrying.
2292
2202
*/
2293
-
if (READ_ONCE(si->inuse_pages)) {
2203
+
if (swap_usage_in_pages(si)) {
2294
2204
if (!signal_pending(current))
2295
2205
goto retry;
2296
2206
return -EINTR;
···
2317
2227
unsigned int type;
2318
2228
2319
2229
for (type = 0; type < nr_swapfiles; type++)
2320
-
if (swap_info[type]->inuse_pages)
2230
+
if (swap_usage_in_pages(swap_info[type]))
2321
2231
return;
2322
2232
spin_lock(&mmlist_lock);
2323
2233
list_for_each_safe(p, next, &init_mm.mmlist)
···
2496
2406
2497
2407
static void _enable_swap_info(struct swap_info_struct *si)
2498
2408
{
2499
-
si->flags |= SWP_WRITEOK;
2500
2409
atomic_long_add(si->pages, &nr_swap_pages);
2501
2410
total_swap_pages += si->pages;
2502
2411
···
2512
2423
*/
2513
2424
plist_add(&si->list, &swap_active_head);
2514
2425
2515
-
/* add to available list if swap device is not full */
2516
-
if (si->inuse_pages < si->pages)
2517
-
add_to_avail_list(si);
2426
+
/* Add back to available list */
2427
+
add_to_avail_list(si, true);
2518
2428
}
2519
2429
2520
2430
static void enable_swap_info(struct swap_info_struct *si, int prio,
···
2611
2523
goto out_dput;
2612
2524
}
2613
2525
spin_lock(&p->lock);
2614
-
del_from_avail_list(p);
2526
+
del_from_avail_list(p, true);
2615
2527
if (p->prio < 0) {
2616
2528
struct swap_info_struct *si = p;
2617
2529
int nid;
···
2629
2541
plist_del(&p->list, &swap_active_head);
2630
2542
atomic_long_sub(p->pages, &nr_swap_pages);
2631
2543
total_swap_pages -= p->pages;
2632
-
p->flags &= ~SWP_WRITEOK;
2633
2544
spin_unlock(&p->lock);
2634
2545
spin_unlock(&swap_lock);
2635
2546
···
2808
2721
}
2809
2722
2810
2723
bytes = K(si->pages);
2811
-
inuse = K(READ_ONCE(si->inuse_pages));
2724
+
inuse = K(swap_usage_in_pages(si));
2812
2725
2813
2726
file = si->swap_file;
2814
2727
len = seq_file_path(swap, file, " \t\n\\");
···
2925
2838
}
2926
2839
spin_lock_init(&p->lock);
2927
2840
spin_lock_init(&p->cont_lock);
2841
+
atomic_long_set(&p->inuse_pages, SWAP_USAGE_OFFLIST_BIT);
2928
2842
init_completion(&p->comp);
2929
2843
2930
2844
return p;
···
3423
3335
struct swap_info_struct *si = swap_info[type];
3424
3336
3425
3337
if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
3426
-
nr_to_be_unused += READ_ONCE(si->inuse_pages);
3338
+
nr_to_be_unused += swap_usage_in_pages(si);
3427
3339
}
3428
3340
val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
3429
3341
val->totalswap = total_swap_pages + nr_to_be_unused;