Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm, swap: tidy up swap device and cluster info helpers
swp_swap_info is the most commonly used helper for retrieving swap info.
It has an internal check that may lead to a NULL return value, but almost
none of its caller checks the return value, making the internal check
pointless. In fact, most of these callers already ensured the entry is
valid and never expect a NULL value.
Tidy this up and improve the function names. If the caller can make sure
the swap entry/type is valid and the device is pinned, use the new
introduced __swap_entry_to_info/__swap_type_to_info instead. They have
more debug sanity checks and lower overhead as they are inlined.
Callers that may expect a NULL value should use
swap_entry_to_info/swap_type_to_info instead.
No feature change. The rearranged codes should have had no effect, or
they should have been hitting NULL de-ref bugs already. Only some new
sanity checks are added so potential issues may show up in debug build.
The new helpers will be frequently used with swap table later when working
with swap cache folios. A locked swap cache folio ensures the entries are
valid and stable so these helpers are very helpful.
Link: https://lkml.kernel.org/r/20250916160100.31545-8-ryncsn@gmail.com
Signed-off-by: Kairui Song <kasong@tencent.com>
Acked-by: Chris Li <chrisl@kernel.org>
Reviewed-by: Barry Song <baohua@kernel.org>
Acked-by: David Hildenbrand <david@redhat.com>
Suggested-by: Chris Li <chrisl@kernel.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: kernel test robot <oliver.sang@intel.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: SeongJae Park <sj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Kairui Song
and committed by
Andrew Morton
0fcf8ef4
4522aed4
+60
-37
-6
include/linux/swap.h
-6
include/linux/swap.h
···
479
479
extern int __swap_count(swp_entry_t entry);
480
480
extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry);
481
481
extern int swp_swapcount(swp_entry_t entry);
482
-
struct swap_info_struct *swp_swap_info(swp_entry_t entry);
483
482
struct backing_dev_info;
484
483
extern int init_swap_address_space(unsigned int type, unsigned long nr_pages);
485
484
extern void exit_swap_address_space(unsigned int type);
···
491
492
}
492
493
493
494
#else /* CONFIG_SWAP */
494
-
static inline struct swap_info_struct *swp_swap_info(swp_entry_t entry)
495
-
{
496
-
return NULL;
497
-
}
498
-
499
495
static inline struct swap_info_struct *get_swap_device(swp_entry_t entry)
500
496
{
501
497
return NULL;
+6
-6
mm/page_io.c
+6
-6
mm/page_io.c
···
204
204
static void swap_zeromap_folio_set(struct folio *folio)
205
205
{
206
206
struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio);
207
-
struct swap_info_struct *sis = swp_swap_info(folio->swap);
207
+
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
208
208
int nr_pages = folio_nr_pages(folio);
209
209
swp_entry_t entry;
210
210
unsigned int i;
···
223
223
224
224
static void swap_zeromap_folio_clear(struct folio *folio)
225
225
{
226
-
struct swap_info_struct *sis = swp_swap_info(folio->swap);
226
+
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
227
227
swp_entry_t entry;
228
228
unsigned int i;
229
229
···
374
374
static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
375
375
{
376
376
struct swap_iocb *sio = swap_plug ? *swap_plug : NULL;
377
-
struct swap_info_struct *sis = swp_swap_info(folio->swap);
377
+
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
378
378
struct file *swap_file = sis->swap_file;
379
379
loff_t pos = swap_dev_pos(folio->swap);
380
380
···
446
446
447
447
void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)
448
448
{
449
-
struct swap_info_struct *sis = swp_swap_info(folio->swap);
449
+
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
450
450
451
451
VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
452
452
/*
···
537
537
538
538
static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)
539
539
{
540
-
struct swap_info_struct *sis = swp_swap_info(folio->swap);
540
+
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
541
541
struct swap_iocb *sio = NULL;
542
542
loff_t pos = swap_dev_pos(folio->swap);
543
543
···
608
608
609
609
void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
610
610
{
611
-
struct swap_info_struct *sis = swp_swap_info(folio->swap);
611
+
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
612
612
bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO;
613
613
bool workingset = folio_test_workingset(folio);
614
614
unsigned long pflags;
+33
-5
mm/swap.h
+33
-5
mm/swap.h
···
15
15
#define swap_entry_order(order) 0
16
16
#endif
17
17
18
+
extern struct swap_info_struct *swap_info[];
19
+
18
20
/*
19
21
* We use this to track usage of a cluster. A cluster is a block of swap disk
20
22
* space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All
···
55
53
#include <linux/swapops.h> /* for swp_offset */
56
54
#include <linux/blk_types.h> /* for bio_end_io_t */
57
55
58
-
static inline struct swap_cluster_info *swp_offset_cluster(
56
+
/*
57
+
* Callers of all helpers below must ensure the entry, type, or offset is
58
+
* valid, and protect the swap device with reference count or locks.
59
+
*/
60
+
static inline struct swap_info_struct *__swap_type_to_info(int type)
61
+
{
62
+
struct swap_info_struct *si;
63
+
64
+
si = READ_ONCE(swap_info[type]); /* rcu_dereference() */
65
+
VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
66
+
return si;
67
+
}
68
+
69
+
static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry)
70
+
{
71
+
return __swap_type_to_info(swp_type(entry));
72
+
}
73
+
74
+
static inline struct swap_cluster_info *__swap_offset_to_cluster(
59
75
struct swap_info_struct *si, pgoff_t offset)
60
76
{
77
+
VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
78
+
VM_WARN_ON_ONCE(offset >= si->max);
61
79
return &si->cluster_info[offset / SWAPFILE_CLUSTER];
62
80
}
63
81
···
92
70
static inline struct swap_cluster_info *swap_cluster_lock(
93
71
struct swap_info_struct *si, unsigned long offset)
94
72
{
95
-
struct swap_cluster_info *ci = swp_offset_cluster(si, offset);
73
+
struct swap_cluster_info *ci = __swap_offset_to_cluster(si, offset);
96
74
75
+
VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
97
76
spin_lock(&ci->lock);
98
77
return ci;
99
78
}
···
193
170
194
171
static inline unsigned int folio_swap_flags(struct folio *folio)
195
172
{
196
-
return swp_swap_info(folio->swap)->flags;
173
+
return __swap_entry_to_info(folio->swap)->flags;
197
174
}
198
175
199
176
/*
···
204
181
static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
205
182
bool *is_zeromap)
206
183
{
207
-
struct swap_info_struct *sis = swp_swap_info(entry);
184
+
struct swap_info_struct *sis = __swap_entry_to_info(entry);
208
185
unsigned long start = swp_offset(entry);
209
186
unsigned long end = start + max_nr;
210
187
bool first_bit;
···
223
200
224
201
static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
225
202
{
226
-
struct swap_info_struct *si = swp_swap_info(entry);
203
+
struct swap_info_struct *si = __swap_entry_to_info(entry);
227
204
pgoff_t offset = swp_offset(entry);
228
205
int i;
229
206
···
242
219
243
220
#else /* CONFIG_SWAP */
244
221
struct swap_iocb;
222
+
static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry)
223
+
{
224
+
return NULL;
225
+
}
226
+
245
227
static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
246
228
{
247
229
}
+2
-2
mm/swap_state.c
+2
-2
mm/swap_state.c
···
336
336
struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated,
337
337
bool skip_if_exists)
338
338
{
339
-
struct swap_info_struct *si = swp_swap_info(entry);
339
+
struct swap_info_struct *si = __swap_entry_to_info(entry);
340
340
struct folio *folio;
341
341
struct folio *new_folio = NULL;
342
342
struct folio *result = NULL;
···
560
560
unsigned long offset = entry_offset;
561
561
unsigned long start_offset, end_offset;
562
562
unsigned long mask;
563
-
struct swap_info_struct *si = swp_swap_info(entry);
563
+
struct swap_info_struct *si = __swap_entry_to_info(entry);
564
564
struct blk_plug plug;
565
565
struct swap_iocb *splug = NULL;
566
566
bool page_allocated;
+19
-18
mm/swapfile.c
+19
-18
mm/swapfile.c
···
102
102
static struct plist_head *swap_avail_heads;
103
103
static DEFINE_SPINLOCK(swap_avail_lock);
104
104
105
-
static struct swap_info_struct *swap_info[MAX_SWAPFILES];
105
+
struct swap_info_struct *swap_info[MAX_SWAPFILES];
106
106
107
107
static DEFINE_MUTEX(swapon_mutex);
108
108
···
124
124
.lock = INIT_LOCAL_LOCK(),
125
125
};
126
126
127
-
static struct swap_info_struct *swap_type_to_swap_info(int type)
127
+
/* May return NULL on invalid type, caller must check for NULL return */
128
+
static struct swap_info_struct *swap_type_to_info(int type)
128
129
{
129
130
if (type >= MAX_SWAPFILES)
130
131
return NULL;
131
-
132
132
return READ_ONCE(swap_info[type]); /* rcu_dereference() */
133
+
}
134
+
135
+
/* May return NULL on invalid entry, caller must check for NULL return */
136
+
static struct swap_info_struct *swap_entry_to_info(swp_entry_t entry)
137
+
{
138
+
return swap_type_to_info(swp_type(entry));
133
139
}
134
140
135
141
static inline unsigned char swap_count(unsigned char ent)
···
348
342
349
343
sector_t swap_folio_sector(struct folio *folio)
350
344
{
351
-
struct swap_info_struct *sis = swp_swap_info(folio->swap);
345
+
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
352
346
struct swap_extent *se;
353
347
sector_t sector;
354
348
pgoff_t offset;
···
1306
1300
1307
1301
if (!entry.val)
1308
1302
goto out;
1309
-
si = swp_swap_info(entry);
1303
+
si = swap_entry_to_info(entry);
1310
1304
if (!si)
1311
1305
goto bad_nofile;
1312
1306
if (data_race(!(si->flags & SWP_USED)))
···
1421
1415
1422
1416
if (!entry.val)
1423
1417
goto out;
1424
-
si = swp_swap_info(entry);
1418
+
si = swap_entry_to_info(entry);
1425
1419
if (!si)
1426
1420
goto bad_nofile;
1427
1421
if (!get_swap_device_info(si))
···
1544
1538
unsigned char *map_end = map + nr_pages;
1545
1539
1546
1540
/* It should never free entries across different clusters */
1547
-
VM_BUG_ON(ci != swp_offset_cluster(si, offset + nr_pages - 1));
1541
+
VM_BUG_ON(ci != __swap_offset_to_cluster(si, offset + nr_pages - 1));
1548
1542
VM_BUG_ON(cluster_is_empty(ci));
1549
1543
VM_BUG_ON(ci->count < nr_pages);
1550
1544
···
1602
1596
1603
1597
int __swap_count(swp_entry_t entry)
1604
1598
{
1605
-
struct swap_info_struct *si = swp_swap_info(entry);
1599
+
struct swap_info_struct *si = __swap_entry_to_info(entry);
1606
1600
pgoff_t offset = swp_offset(entry);
1607
1601
1608
1602
return swap_count(si->swap_map[offset]);
···
1833
1827
1834
1828
swp_entry_t get_swap_page_of_type(int type)
1835
1829
{
1836
-
struct swap_info_struct *si = swap_type_to_swap_info(type);
1830
+
struct swap_info_struct *si = swap_type_to_info(type);
1837
1831
unsigned long offset;
1838
1832
swp_entry_t entry = {0};
1839
1833
···
1914
1908
*/
1915
1909
sector_t swapdev_block(int type, pgoff_t offset)
1916
1910
{
1917
-
struct swap_info_struct *si = swap_type_to_swap_info(type);
1911
+
struct swap_info_struct *si = swap_type_to_info(type);
1918
1912
struct swap_extent *se;
1919
1913
1920
1914
if (!si || !(si->flags & SWP_WRITEOK))
···
2843
2837
if (!l)
2844
2838
return SEQ_START_TOKEN;
2845
2839
2846
-
for (type = 0; (si = swap_type_to_swap_info(type)); type++) {
2840
+
for (type = 0; (si = swap_type_to_info(type)); type++) {
2847
2841
if (!(si->flags & SWP_USED) || !si->swap_map)
2848
2842
continue;
2849
2843
if (!--l)
···
2864
2858
type = si->type + 1;
2865
2859
2866
2860
++(*pos);
2867
-
for (; (si = swap_type_to_swap_info(type)); type++) {
2861
+
for (; (si = swap_type_to_info(type)); type++) {
2868
2862
if (!(si->flags & SWP_USED) || !si->swap_map)
2869
2863
continue;
2870
2864
return si;
···
3537
3531
unsigned char has_cache;
3538
3532
int err, i;
3539
3533
3540
-
si = swp_swap_info(entry);
3534
+
si = swap_entry_to_info(entry);
3541
3535
if (WARN_ON_ONCE(!si)) {
3542
3536
pr_err("%s%08lx\n", Bad_file, entry.val);
3543
3537
return -EINVAL;
···
3650
3644
void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr)
3651
3645
{
3652
3646
swap_entries_put_cache(si, entry, nr);
3653
-
}
3654
-
3655
-
struct swap_info_struct *swp_swap_info(swp_entry_t entry)
3656
-
{
3657
-
return swap_type_to_swap_info(swp_type(entry));
3658
3647
}
3659
3648
3660
3649
/*