mm, swap: hold a reference during scan and cleanup flag usage

The flag SWP_SCANNING was used as an indicator of whether a device is
being scanned for allocation, and prevents swapoff. Combined with
SWP_WRITEOK, they work as a set of barriers for a clean swapoff:

1. Swapoff clears SWP_WRITEOK, allocation requests will see
~SWP_WRITEOK and abort as it's serialized by si->lock.
2. Swapoff unuses all allocated entries.
3. Swapoff waits for SWP_SCANNING flag to be cleared, so ongoing
allocations will stop, preventing UAF.
4. Now swapoff can free everything safely.

This will make the allocation path have a hard dependency on si->lock.
Allocation always have to acquire si->lock first for setting SWP_SCANNING
and checking SWP_WRITEOK.

This commit removes this flag, and just uses the existing per-CPU refcount
instead to prevent UAF in step 3, which serves well for such usage without
dependency on si->lock, and scales very well too. Just hold a reference
during the whole scan and allocation process. Swapoff will kill and wait
for the counter.

And for preventing any allocation from happening after step 1 so the unuse
in step 2 can ensure all slots are free, swapoff will acquire the ci->lock
of each cluster one by one to ensure all allocations see ~SWP_WRITEOK and
abort.

This way these dependences on si->lock are gone. And worth noting we
can't kill the refcount as the first step for swapoff as the unuse process
have to acquire the refcount.

Link: https://lkml.kernel.org/r/20250113175732.48099-8-ryncsn@gmail.com
Signed-off-by: Kairui Song <kasong@tencent.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: Chis Li <chrisl@kernel.org>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickens <hughd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kalesh Singh <kaleshsingh@google.com>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Kairui Song and committed by

Andrew Morton 1 year ago 9a0ddeb7 b228386c

+57 -34

2 changed files

expand all

include

linux

swap.h

swapfile.c

-1

include/linux/swap.h

··· 219 219 SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */ 220 220 SWP_SYNCHRONOUS_IO = (1 << 12), /* synchronous IO is efficient */ 221 221 /* add others here before... */ 222 - SWP_SCANNING = (1 << 14), /* refcount in scan_swap_map */ 223 222 }; 224 223 225 224 #define SWAP_CLUSTER_MAX 32UL

+57 -33

mm/swapfile.c

··· 658 658 { 659 659 unsigned int nr_pages = 1 << order; 660 660 661 + lockdep_assert_held(&ci->lock); 662 + 661 663 if (!(si->flags & SWP_WRITEOK)) 662 664 return false; 663 665 ··· 1061 1059 { 1062 1060 int n_ret = 0; 1063 1061 1064 - si->flags += SWP_SCANNING; 1065 - 1066 1062 while (n_ret < nr) { 1067 1063 unsigned long offset = cluster_alloc_swap_entry(si, order, usage); 1068 1064 ··· 1068 1068 break; 1069 1069 slots[n_ret++] = swp_entry(si->type, offset); 1070 1070 } 1071 - 1072 - si->flags -= SWP_SCANNING; 1073 1071 1074 1072 return n_ret; 1075 1073 } ··· 1110 1112 return cluster_alloc_swap(si, usage, nr, slots, order); 1111 1113 } 1112 1114 1115 + static bool get_swap_device_info(struct swap_info_struct *si) 1116 + { 1117 + if (!percpu_ref_tryget_live(&si->users)) 1118 + return false; 1119 + /* 1120 + * Guarantee the si->users are checked before accessing other 1121 + * fields of swap_info_struct, and si->flags (SWP_WRITEOK) is 1122 + * up to dated. 1123 + * 1124 + * Paired with the spin_unlock() after setup_swap_info() in 1125 + * enable_swap_info(), and smp_wmb() in swapoff. 1126 + */ 1127 + smp_rmb(); 1128 + return true; 1129 + } 1130 + 1113 1131 int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_order) 1114 1132 { 1115 1133 int order = swap_entry_order(entry_order); ··· 1153 1139 /* requeue si to after same-priority siblings */ 1154 1140 plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]); 1155 1141 spin_unlock(&swap_avail_lock); 1156 - spin_lock(&si->lock); 1157 - n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, 1158 - n_goal, swp_entries, order); 1159 - spin_unlock(&si->lock); 1160 - if (n_ret || size > 1) 1161 - goto check_out; 1162 - cond_resched(); 1142 + if (get_swap_device_info(si)) { 1143 + spin_lock(&si->lock); 1144 + n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, 1145 + n_goal, swp_entries, order); 1146 + spin_unlock(&si->lock); 1147 + put_swap_device(si); 1148 + if (n_ret || size > 1) 1149 + goto check_out; 1150 + cond_resched(); 1151 + } 1163 1152 1164 1153 spin_lock(&swap_avail_lock); 1165 1154 /* ··· 1313 1296 si = swp_swap_info(entry); 1314 1297 if (!si) 1315 1298 goto bad_nofile; 1316 - if (!percpu_ref_tryget_live(&si->users)) 1299 + if (!get_swap_device_info(si)) 1317 1300 goto out; 1318 - /* 1319 - * Guarantee the si->users are checked before accessing other 1320 - * fields of swap_info_struct. 1321 - * 1322 - * Paired with the spin_unlock() after setup_swap_info() in 1323 - * enable_swap_info(). 1324 - */ 1325 - smp_rmb(); 1326 1301 offset = swp_offset(entry); 1327 1302 if (offset >= si->max) 1328 1303 goto put_out; ··· 1794 1785 goto fail; 1795 1786 1796 1787 /* This is called for allocating swap entry, not cache */ 1797 - spin_lock(&si->lock); 1798 - if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry, 0)) 1799 - atomic_long_dec(&nr_swap_pages); 1800 - spin_unlock(&si->lock); 1788 + if (get_swap_device_info(si)) { 1789 + spin_lock(&si->lock); 1790 + if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry, 0)) 1791 + atomic_long_dec(&nr_swap_pages); 1792 + spin_unlock(&si->lock); 1793 + put_swap_device(si); 1794 + } 1801 1795 fail: 1802 1796 return entry; 1803 1797 } ··· 2574 2562 return ret; 2575 2563 } 2576 2564 2565 + /* 2566 + * Called after clearing SWP_WRITEOK, ensures cluster_alloc_range 2567 + * see the updated flags, so there will be no more allocations. 2568 + */ 2569 + static void wait_for_allocation(struct swap_info_struct *si) 2570 + { 2571 + unsigned long offset; 2572 + unsigned long end = ALIGN(si->max, SWAPFILE_CLUSTER); 2573 + struct swap_cluster_info *ci; 2574 + 2575 + BUG_ON(si->flags & SWP_WRITEOK); 2576 + 2577 + for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) { 2578 + ci = lock_cluster(si, offset); 2579 + unlock_cluster(ci); 2580 + offset += SWAPFILE_CLUSTER; 2581 + } 2582 + } 2583 + 2577 2584 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) 2578 2585 { 2579 2586 struct swap_info_struct *p = NULL; ··· 2663 2632 spin_unlock(&p->lock); 2664 2633 spin_unlock(&swap_lock); 2665 2634 2635 + wait_for_allocation(p); 2636 + 2666 2637 disable_swap_slots_cache_lock(); 2667 2638 2668 2639 set_current_oom_origin(); ··· 2706 2673 spin_lock(&swap_lock); 2707 2674 spin_lock(&p->lock); 2708 2675 drain_mmlist(); 2709 - 2710 - /* wait for anyone still in scan_swap_map_slots */ 2711 - while (p->flags >= SWP_SCANNING) { 2712 - spin_unlock(&p->lock); 2713 - spin_unlock(&swap_lock); 2714 - schedule_timeout_uninterruptible(1); 2715 - spin_lock(&swap_lock); 2716 - spin_lock(&p->lock); 2717 - } 2718 2676 2719 2677 swap_file = p->swap_file; 2720 2678 p->swap_file = NULL;

Configure Feed

Configure Feed