tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'locking-core-2025-12-01' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Ingo Molnar:
"Mutexes:

- Redo __mutex_init() to reduce generated code size (Sebastian
Andrzej Siewior)

Seqlocks:

- Introduce scoped_seqlock_read() (Peter Zijlstra)

- Change thread_group_cputime() to use scoped_seqlock_read() (Oleg
Nesterov)

- Change do_task_stat() to use scoped_seqlock_read() (Oleg Nesterov)

- Change do_io_accounting() to use scoped_seqlock_read() (Oleg
Nesterov)

- Fix the incorrect documentation of read_seqbegin_or_lock() /
need_seqretry() (Oleg Nesterov)

- Allow KASAN to fail optimizing (Peter Zijlstra)

Local lock updates:

- Fix all kernel-doc warnings (Randy Dunlap)

- Add the <linux/local_lock*.h> headers to MAINTAINERS (Sebastian
Andrzej Siewior)

- Reduce the risk of shadowing via s/l/__l/ and s/tl/__tl/ (Vincent
Mailhol)

Lock debugging:

- spinlock/debug: Fix data-race in do_raw_write_lock (Alexander
Sverdlin)

Atomic primitives infrastructure:

- atomic: Skip alignment check for try_cmpxchg() old arg (Arnd
Bergmann)

Rust runtime integration:

- sync: atomic: Enable generated Atomic<T> usage (Boqun Feng)

- sync: atomic: Implement Debug for Atomic<Debug> (Boqun Feng)

- debugfs: Remove Rust native atomics and replace them with Linux
versions (Boqun Feng)

- debugfs: Implement Reader for Mutex<T> only when T is Unpin (Boqun
Feng)

- lock: guard: Add T: Unpin bound to DerefMut (Daniel Almeida)

- lock: Pin the inner data (Daniel Almeida)

- lock: Add a Pin<&mut T> accessor (Daniel Almeida)"

* tag 'locking-core-2025-12-01' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
locking/local_lock: Fix all kernel-doc warnings
locking/local_lock: s/l/__l/ and s/tl/__tl/ to reduce the risk of shadowing
locking/local_lock: Add the <linux/local_lock*.h> headers to MAINTAINERS
locking/mutex: Redo __mutex_init() to reduce generated code size
rust: debugfs: Replace the usage of Rust native atomics
rust: sync: atomic: Implement Debug for Atomic<Debug>
rust: sync: atomic: Make Atomic*Ops pub(crate)
seqlock: Allow KASAN to fail optimizing
rust: debugfs: Implement Reader for Mutex<T> only when T is Unpin
seqlock: Change do_io_accounting() to use scoped_seqlock_read()
seqlock: Change do_task_stat() to use scoped_seqlock_read()
seqlock: Change thread_group_cputime() to use scoped_seqlock_read()
seqlock: Introduce scoped_seqlock_read()
documentation: seqlock: fix the wrong documentation of read_seqbegin_or_lock/need_seqretry
atomic: Skip alignment check for try_cmpxchg() old arg
rust: lock: Add a Pin<&mut T> accessor
rust: lock: Pin the inner data
rust: lock: guard: Add T: Unpin bound to DerefMut
locking/spinlock/debug: Fix data-race in do_raw_write_lock

Linus Torvalds b53440f8 1b5dd298

+339 -172
+5 -4
Documentation/locking/seqlock.rst
··· 220 220 according to a passed marker. This is used to avoid lockless readers 221 221 starvation (too much retry loops) in case of a sharp spike in write 222 222 activity. First, a lockless read is tried (even marker passed). If 223 - that trial fails (odd sequence counter is returned, which is used as 224 - the next iteration marker), the lockless read is transformed to a 225 - full locking read and no retry loop is necessary:: 223 + that trial fails (sequence counter doesn't match), make the marker 224 + odd for the next iteration, the lockless read is transformed to a 225 + full locking read and no retry loop is necessary, for example:: 226 226 227 227 /* marker; even initialization */ 228 - int seq = 0; 228 + int seq = 1; 229 229 do { 230 + seq++; /* 2 on the 1st/lockless path, otherwise odd */ 230 231 read_seqbegin_or_lock(&foo_seqlock, &seq); 231 232 232 233 /* ... [[read-side critical section]] ... */
+1
MAINTAINERS
··· 14536 14536 T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking/core 14537 14537 F: Documentation/locking/ 14538 14538 F: arch/*/include/asm/spinlock*.h 14539 + F: include/linux/local_lock*.h 14539 14540 F: include/linux/lockdep*.h 14540 14541 F: include/linux/mutex*.h 14541 14542 F: include/linux/rwlock*.h
+2 -7
fs/proc/array.c
··· 481 481 unsigned long flags; 482 482 int exit_code = task->exit_code; 483 483 struct signal_struct *sig = task->signal; 484 - unsigned int seq = 1; 485 484 486 485 state = *get_task_state(task); 487 486 vsize = eip = esp = 0; ··· 537 538 if (permitted && (!whole || num_threads < 2)) 538 539 wchan = !task_is_running(task); 539 540 540 - do { 541 - seq++; /* 2 on the 1st/lockless path, otherwise odd */ 542 - flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq); 543 - 541 + scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) { 544 542 cmin_flt = sig->cmin_flt; 545 543 cmaj_flt = sig->cmaj_flt; 546 544 cutime = sig->cutime; ··· 559 563 } 560 564 rcu_read_unlock(); 561 565 } 562 - } while (need_seqretry(&sig->stats_lock, seq)); 563 - done_seqretry_irqrestore(&sig->stats_lock, seq, flags); 566 + } 564 567 565 568 if (whole) { 566 569 thread_group_cputime_adjusted(task, &utime, &stime);
+3 -10
fs/proc/base.c
··· 3043 3043 if (whole) { 3044 3044 struct signal_struct *sig = task->signal; 3045 3045 struct task_struct *t; 3046 - unsigned int seq = 1; 3047 - unsigned long flags; 3048 3046 3049 - rcu_read_lock(); 3050 - do { 3051 - seq++; /* 2 on the 1st/lockless path, otherwise odd */ 3052 - flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq); 3053 - 3047 + guard(rcu)(); 3048 + scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) { 3054 3049 acct = sig->ioac; 3055 3050 __for_each_thread(sig, t) 3056 3051 task_io_accounting_add(&acct, &t->ioac); 3057 3052 3058 - } while (need_seqretry(&sig->stats_lock, seq)); 3059 - done_seqretry_irqrestore(&sig->stats_lock, seq, flags); 3060 - rcu_read_unlock(); 3053 + } 3061 3054 } else { 3062 3055 acct = task->ioac; 3063 3056 }
+13 -13
include/linux/atomic/atomic-instrumented.h
··· 1276 1276 { 1277 1277 kcsan_mb(); 1278 1278 instrument_atomic_read_write(v, sizeof(*v)); 1279 - instrument_atomic_read_write(old, sizeof(*old)); 1279 + instrument_read_write(old, sizeof(*old)); 1280 1280 return raw_atomic_try_cmpxchg(v, old, new); 1281 1281 } 1282 1282 ··· 1298 1298 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1299 1299 { 1300 1300 instrument_atomic_read_write(v, sizeof(*v)); 1301 - instrument_atomic_read_write(old, sizeof(*old)); 1301 + instrument_read_write(old, sizeof(*old)); 1302 1302 return raw_atomic_try_cmpxchg_acquire(v, old, new); 1303 1303 } 1304 1304 ··· 1321 1321 { 1322 1322 kcsan_release(); 1323 1323 instrument_atomic_read_write(v, sizeof(*v)); 1324 - instrument_atomic_read_write(old, sizeof(*old)); 1324 + instrument_read_write(old, sizeof(*old)); 1325 1325 return raw_atomic_try_cmpxchg_release(v, old, new); 1326 1326 } 1327 1327 ··· 1343 1343 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 1344 1344 { 1345 1345 instrument_atomic_read_write(v, sizeof(*v)); 1346 - instrument_atomic_read_write(old, sizeof(*old)); 1346 + instrument_read_write(old, sizeof(*old)); 1347 1347 return raw_atomic_try_cmpxchg_relaxed(v, old, new); 1348 1348 } 1349 1349 ··· 2854 2854 { 2855 2855 kcsan_mb(); 2856 2856 instrument_atomic_read_write(v, sizeof(*v)); 2857 - instrument_atomic_read_write(old, sizeof(*old)); 2857 + instrument_read_write(old, sizeof(*old)); 2858 2858 return raw_atomic64_try_cmpxchg(v, old, new); 2859 2859 } 2860 2860 ··· 2876 2876 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2877 2877 { 2878 2878 instrument_atomic_read_write(v, sizeof(*v)); 2879 - instrument_atomic_read_write(old, sizeof(*old)); 2879 + instrument_read_write(old, sizeof(*old)); 2880 2880 return raw_atomic64_try_cmpxchg_acquire(v, old, new); 2881 2881 } 2882 2882 ··· 2899 2899 { 2900 2900 kcsan_release(); 2901 2901 instrument_atomic_read_write(v, sizeof(*v)); 2902 - instrument_atomic_read_write(old, sizeof(*old)); 2902 + instrument_read_write(old, sizeof(*old)); 2903 2903 return raw_atomic64_try_cmpxchg_release(v, old, new); 2904 2904 } 2905 2905 ··· 2921 2921 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 2922 2922 { 2923 2923 instrument_atomic_read_write(v, sizeof(*v)); 2924 - instrument_atomic_read_write(old, sizeof(*old)); 2924 + instrument_read_write(old, sizeof(*old)); 2925 2925 return raw_atomic64_try_cmpxchg_relaxed(v, old, new); 2926 2926 } 2927 2927 ··· 4432 4432 { 4433 4433 kcsan_mb(); 4434 4434 instrument_atomic_read_write(v, sizeof(*v)); 4435 - instrument_atomic_read_write(old, sizeof(*old)); 4435 + instrument_read_write(old, sizeof(*old)); 4436 4436 return raw_atomic_long_try_cmpxchg(v, old, new); 4437 4437 } 4438 4438 ··· 4454 4454 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) 4455 4455 { 4456 4456 instrument_atomic_read_write(v, sizeof(*v)); 4457 - instrument_atomic_read_write(old, sizeof(*old)); 4457 + instrument_read_write(old, sizeof(*old)); 4458 4458 return raw_atomic_long_try_cmpxchg_acquire(v, old, new); 4459 4459 } 4460 4460 ··· 4477 4477 { 4478 4478 kcsan_release(); 4479 4479 instrument_atomic_read_write(v, sizeof(*v)); 4480 - instrument_atomic_read_write(old, sizeof(*old)); 4480 + instrument_read_write(old, sizeof(*old)); 4481 4481 return raw_atomic_long_try_cmpxchg_release(v, old, new); 4482 4482 } 4483 4483 ··· 4499 4499 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) 4500 4500 { 4501 4501 instrument_atomic_read_write(v, sizeof(*v)); 4502 - instrument_atomic_read_write(old, sizeof(*old)); 4502 + instrument_read_write(old, sizeof(*old)); 4503 4503 return raw_atomic_long_try_cmpxchg_relaxed(v, old, new); 4504 4504 } 4505 4505 ··· 5050 5050 5051 5051 5052 5052 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ 5053 - // 8829b337928e9508259079d32581775ececd415b 5053 + // f618ac667f868941a84ce0ab2242f1786e049ed4
+3 -1
include/linux/local_lock.h
··· 6 6 7 7 /** 8 8 * local_lock_init - Runtime initialize a lock instance 9 + * @lock: The lock variable 9 10 */ 10 11 #define local_lock_init(lock) __local_lock_init(lock) 11 12 ··· 53 52 __local_unlock_irqrestore(this_cpu_ptr(lock), flags) 54 53 55 54 /** 56 - * local_lock_init - Runtime initialize a lock instance 55 + * local_trylock_init - Runtime initialize a lock instance 56 + * @lock: The lock variable 57 57 */ 58 58 #define local_trylock_init(lock) __local_trylock_init(lock) 59 59
+31 -31
include/linux/local_lock_internal.h
··· 99 99 100 100 #define __local_lock_acquire(lock) \ 101 101 do { \ 102 - local_trylock_t *tl; \ 103 - local_lock_t *l; \ 102 + local_trylock_t *__tl; \ 103 + local_lock_t *__l; \ 104 104 \ 105 - l = (local_lock_t *)(lock); \ 106 - tl = (local_trylock_t *)l; \ 105 + __l = (local_lock_t *)(lock); \ 106 + __tl = (local_trylock_t *)__l; \ 107 107 _Generic((lock), \ 108 108 local_trylock_t *: ({ \ 109 - lockdep_assert(tl->acquired == 0); \ 110 - WRITE_ONCE(tl->acquired, 1); \ 109 + lockdep_assert(__tl->acquired == 0); \ 110 + WRITE_ONCE(__tl->acquired, 1); \ 111 111 }), \ 112 112 local_lock_t *: (void)0); \ 113 - local_lock_acquire(l); \ 113 + local_lock_acquire(__l); \ 114 114 } while (0) 115 115 116 116 #define __local_lock(lock) \ ··· 133 133 134 134 #define __local_trylock(lock) \ 135 135 ({ \ 136 - local_trylock_t *tl; \ 136 + local_trylock_t *__tl; \ 137 137 \ 138 138 preempt_disable(); \ 139 - tl = (lock); \ 140 - if (READ_ONCE(tl->acquired)) { \ 139 + __tl = (lock); \ 140 + if (READ_ONCE(__tl->acquired)) { \ 141 141 preempt_enable(); \ 142 - tl = NULL; \ 142 + __tl = NULL; \ 143 143 } else { \ 144 - WRITE_ONCE(tl->acquired, 1); \ 144 + WRITE_ONCE(__tl->acquired, 1); \ 145 145 local_trylock_acquire( \ 146 - (local_lock_t *)tl); \ 146 + (local_lock_t *)__tl); \ 147 147 } \ 148 - !!tl; \ 148 + !!__tl; \ 149 149 }) 150 150 151 151 #define __local_trylock_irqsave(lock, flags) \ 152 152 ({ \ 153 - local_trylock_t *tl; \ 153 + local_trylock_t *__tl; \ 154 154 \ 155 155 local_irq_save(flags); \ 156 - tl = (lock); \ 157 - if (READ_ONCE(tl->acquired)) { \ 156 + __tl = (lock); \ 157 + if (READ_ONCE(__tl->acquired)) { \ 158 158 local_irq_restore(flags); \ 159 - tl = NULL; \ 159 + __tl = NULL; \ 160 160 } else { \ 161 - WRITE_ONCE(tl->acquired, 1); \ 161 + WRITE_ONCE(__tl->acquired, 1); \ 162 162 local_trylock_acquire( \ 163 - (local_lock_t *)tl); \ 163 + (local_lock_t *)__tl); \ 164 164 } \ 165 - !!tl; \ 165 + !!__tl; \ 166 166 }) 167 167 168 168 /* preemption or migration must be disabled before calling __local_lock_is_locked */ ··· 170 170 171 171 #define __local_lock_release(lock) \ 172 172 do { \ 173 - local_trylock_t *tl; \ 174 - local_lock_t *l; \ 173 + local_trylock_t *__tl; \ 174 + local_lock_t *__l; \ 175 175 \ 176 - l = (local_lock_t *)(lock); \ 177 - tl = (local_trylock_t *)l; \ 178 - local_lock_release(l); \ 176 + __l = (local_lock_t *)(lock); \ 177 + __tl = (local_trylock_t *)__l; \ 178 + local_lock_release(__l); \ 179 179 _Generic((lock), \ 180 180 local_trylock_t *: ({ \ 181 - lockdep_assert(tl->acquired == 1); \ 182 - WRITE_ONCE(tl->acquired, 0); \ 181 + lockdep_assert(__tl->acquired == 1); \ 182 + WRITE_ONCE(__tl->acquired, 0); \ 183 183 }), \ 184 184 local_lock_t *: (void)0); \ 185 185 } while (0) ··· 223 223 #define INIT_LOCAL_LOCK(lockname) __LOCAL_SPIN_LOCK_UNLOCKED((lockname)) 224 224 #define INIT_LOCAL_TRYLOCK(lockname) __LOCAL_SPIN_LOCK_UNLOCKED((lockname)) 225 225 226 - #define __local_lock_init(l) \ 226 + #define __local_lock_init(__l) \ 227 227 do { \ 228 - local_spin_lock_init((l)); \ 228 + local_spin_lock_init((__l)); \ 229 229 } while (0) 230 230 231 - #define __local_trylock_init(l) __local_lock_init(l) 231 + #define __local_trylock_init(__l) __local_lock_init(__l) 232 232 233 233 #define __local_lock(__lock) \ 234 234 do { \
+35 -10
include/linux/mutex.h
··· 86 86 #define DEFINE_MUTEX(mutexname) \ 87 87 struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) 88 88 89 - extern void __mutex_init(struct mutex *lock, const char *name, 90 - struct lock_class_key *key); 89 + #ifdef CONFIG_DEBUG_LOCK_ALLOC 90 + void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key); 91 + 92 + static inline void __mutex_init(struct mutex *lock, const char *name, 93 + struct lock_class_key *key) 94 + { 95 + mutex_init_lockep(lock, name, key); 96 + } 97 + #else 98 + extern void mutex_init_generic(struct mutex *lock); 99 + 100 + static inline void __mutex_init(struct mutex *lock, const char *name, 101 + struct lock_class_key *key) 102 + { 103 + mutex_init_generic(lock); 104 + } 105 + #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ 91 106 92 107 /** 93 108 * mutex_is_locked - is the mutex locked ··· 126 111 #define DEFINE_MUTEX(mutexname) \ 127 112 struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) 128 113 129 - extern void __mutex_rt_init(struct mutex *lock, const char *name, 130 - struct lock_class_key *key); 131 - 132 114 #define mutex_is_locked(l) rt_mutex_base_is_locked(&(l)->rtmutex) 133 115 134 - #define __mutex_init(mutex, name, key) \ 135 - do { \ 136 - rt_mutex_base_init(&(mutex)->rtmutex); \ 137 - __mutex_rt_init((mutex), name, key); \ 138 - } while (0) 116 + #ifdef CONFIG_DEBUG_LOCK_ALLOC 117 + extern void mutex_rt_init_lockdep(struct mutex *mutex, const char *name, 118 + struct lock_class_key *key); 139 119 120 + static inline void __mutex_init(struct mutex *lock, const char *name, 121 + struct lock_class_key *key) 122 + { 123 + mutex_rt_init_lockdep(lock, name, key); 124 + } 125 + 126 + #else 127 + extern void mutex_rt_init_generic(struct mutex *mutex); 128 + 129 + static inline void __mutex_init(struct mutex *lock, const char *name, 130 + struct lock_class_key *key) 131 + { 132 + mutex_rt_init_generic(lock); 133 + } 134 + #endif /* !CONFIG_LOCKDEP */ 140 135 #endif /* CONFIG_PREEMPT_RT */ 141 136 142 137 #ifdef CONFIG_DEBUG_MUTEXES
+114
include/linux/seqlock.h
··· 1209 1209 if (seq & 1) 1210 1210 read_sequnlock_excl_irqrestore(lock, flags); 1211 1211 } 1212 + 1213 + enum ss_state { 1214 + ss_done = 0, 1215 + ss_lock, 1216 + ss_lock_irqsave, 1217 + ss_lockless, 1218 + }; 1219 + 1220 + struct ss_tmp { 1221 + enum ss_state state; 1222 + unsigned long data; 1223 + spinlock_t *lock; 1224 + spinlock_t *lock_irqsave; 1225 + }; 1226 + 1227 + static inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) 1228 + { 1229 + if (sst->lock) 1230 + spin_unlock(sst->lock); 1231 + if (sst->lock_irqsave) 1232 + spin_unlock_irqrestore(sst->lock_irqsave, sst->data); 1233 + } 1234 + 1235 + extern void __scoped_seqlock_invalid_target(void); 1236 + 1237 + #if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || defined(CONFIG_KASAN) 1238 + /* 1239 + * For some reason some GCC-8 architectures (nios2, alpha) have trouble 1240 + * determining that the ss_done state is impossible in __scoped_seqlock_next() 1241 + * below. 1242 + * 1243 + * Similarly KASAN is known to confuse compilers enough to break this. But we 1244 + * don't care about code quality for KASAN builds anyway. 1245 + */ 1246 + static inline void __scoped_seqlock_bug(void) { } 1247 + #else 1248 + /* 1249 + * Canary for compiler optimization -- if the compiler doesn't realize this is 1250 + * an impossible state, it very likely generates sub-optimal code here. 1251 + */ 1252 + extern void __scoped_seqlock_bug(void); 1253 + #endif 1254 + 1255 + static inline void 1256 + __scoped_seqlock_next(struct ss_tmp *sst, seqlock_t *lock, enum ss_state target) 1257 + { 1258 + switch (sst->state) { 1259 + case ss_done: 1260 + __scoped_seqlock_bug(); 1261 + return; 1262 + 1263 + case ss_lock: 1264 + case ss_lock_irqsave: 1265 + sst->state = ss_done; 1266 + return; 1267 + 1268 + case ss_lockless: 1269 + if (!read_seqretry(lock, sst->data)) { 1270 + sst->state = ss_done; 1271 + return; 1272 + } 1273 + break; 1274 + } 1275 + 1276 + switch (target) { 1277 + case ss_done: 1278 + __scoped_seqlock_invalid_target(); 1279 + return; 1280 + 1281 + case ss_lock: 1282 + sst->lock = &lock->lock; 1283 + spin_lock(sst->lock); 1284 + sst->state = ss_lock; 1285 + return; 1286 + 1287 + case ss_lock_irqsave: 1288 + sst->lock_irqsave = &lock->lock; 1289 + spin_lock_irqsave(sst->lock_irqsave, sst->data); 1290 + sst->state = ss_lock_irqsave; 1291 + return; 1292 + 1293 + case ss_lockless: 1294 + sst->data = read_seqbegin(lock); 1295 + return; 1296 + } 1297 + } 1298 + 1299 + #define __scoped_seqlock_read(_seqlock, _target, _s) \ 1300 + for (struct ss_tmp _s __cleanup(__scoped_seqlock_cleanup) = \ 1301 + { .state = ss_lockless, .data = read_seqbegin(_seqlock) }; \ 1302 + _s.state != ss_done; \ 1303 + __scoped_seqlock_next(&_s, _seqlock, _target)) 1304 + 1305 + /** 1306 + * scoped_seqlock_read (lock, ss_state) - execute the read side critical 1307 + * section without manual sequence 1308 + * counter handling or calls to other 1309 + * helpers 1310 + * @lock: pointer to seqlock_t protecting the data 1311 + * @ss_state: one of {ss_lock, ss_lock_irqsave, ss_lockless} indicating 1312 + * the type of critical read section 1313 + * 1314 + * Example: 1315 + * 1316 + * scoped_seqlock_read (&lock, ss_lock) { 1317 + * // read-side critical section 1318 + * } 1319 + * 1320 + * Starts with a lockess pass first. If it fails, restarts the critical 1321 + * section with the lock held. 1322 + */ 1323 + #define scoped_seqlock_read(_seqlock, _target) \ 1324 + __scoped_seqlock_read(_seqlock, _target, __UNIQUE_ID(seqlock)) 1325 + 1212 1326 #endif /* __LINUX_SEQLOCK_H */
+1 -9
kernel/locking/mutex-debug.c
··· 78 78 } 79 79 } 80 80 81 - void debug_mutex_init(struct mutex *lock, const char *name, 82 - struct lock_class_key *key) 81 + void debug_mutex_init(struct mutex *lock) 83 82 { 84 - #ifdef CONFIG_DEBUG_LOCK_ALLOC 85 - /* 86 - * Make sure we are not reinitializing a held lock: 87 - */ 88 - debug_check_no_locks_freed((void *)lock, sizeof(*lock)); 89 - lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP); 90 - #endif 91 83 lock->magic = lock; 92 84 } 93 85
+22 -6
kernel/locking/mutex.c
··· 43 43 # define MUTEX_WARN_ON(cond) 44 44 #endif 45 45 46 - void 47 - __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key) 46 + static void __mutex_init_generic(struct mutex *lock) 48 47 { 49 48 atomic_long_set(&lock->owner, 0); 50 49 raw_spin_lock_init(&lock->wait_lock); ··· 51 52 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER 52 53 osq_lock_init(&lock->osq); 53 54 #endif 54 - 55 - debug_mutex_init(lock, name, key); 55 + debug_mutex_init(lock); 56 56 } 57 - EXPORT_SYMBOL(__mutex_init); 58 57 59 58 static inline struct task_struct *__owner_task(unsigned long owner) 60 59 { ··· 139 142 * There is nothing that would stop spreading the lockdep annotations outwards 140 143 * except more code. 141 144 */ 145 + void mutex_init_generic(struct mutex *lock) 146 + { 147 + __mutex_init_generic(lock); 148 + } 149 + EXPORT_SYMBOL(mutex_init_generic); 142 150 143 151 /* 144 152 * Optimistic trylock that only works in the uncontended case. Make sure to ··· 168 166 169 167 return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL); 170 168 } 171 - #endif 169 + 170 + #else /* !CONFIG_DEBUG_LOCK_ALLOC */ 171 + 172 + void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key) 173 + { 174 + __mutex_init_generic(lock); 175 + 176 + /* 177 + * Make sure we are not reinitializing a held lock: 178 + */ 179 + debug_check_no_locks_freed((void *)lock, sizeof(*lock)); 180 + lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP); 181 + } 182 + EXPORT_SYMBOL(mutex_init_lockep); 183 + #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ 172 184 173 185 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag) 174 186 {
+2 -3
kernel/locking/mutex.h
··· 59 59 extern void debug_mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter, 60 60 struct task_struct *task); 61 61 extern void debug_mutex_unlock(struct mutex *lock); 62 - extern void debug_mutex_init(struct mutex *lock, const char *name, 63 - struct lock_class_key *key); 62 + extern void debug_mutex_init(struct mutex *lock); 64 63 #else /* CONFIG_DEBUG_MUTEXES */ 65 64 # define debug_mutex_lock_common(lock, waiter) do { } while (0) 66 65 # define debug_mutex_wake_waiter(lock, waiter) do { } while (0) ··· 67 68 # define debug_mutex_add_waiter(lock, waiter, ti) do { } while (0) 68 69 # define debug_mutex_remove_waiter(lock, waiter, ti) do { } while (0) 69 70 # define debug_mutex_unlock(lock) do { } while (0) 70 - # define debug_mutex_init(lock, name, key) do { } while (0) 71 + # define debug_mutex_init(lock) do { } while (0) 71 72 #endif /* !CONFIG_DEBUG_MUTEXES */ 72 73 #endif /* CONFIG_PREEMPT_RT */
+15 -4
kernel/locking/rtmutex_api.c
··· 515 515 516 516 #ifdef CONFIG_PREEMPT_RT 517 517 /* Mutexes */ 518 - void __mutex_rt_init(struct mutex *mutex, const char *name, 519 - struct lock_class_key *key) 518 + static void __mutex_rt_init_generic(struct mutex *mutex) 520 519 { 520 + rt_mutex_base_init(&mutex->rtmutex); 521 521 debug_check_no_locks_freed((void *)mutex, sizeof(*mutex)); 522 - lockdep_init_map_wait(&mutex->dep_map, name, key, 0, LD_WAIT_SLEEP); 523 522 } 524 - EXPORT_SYMBOL(__mutex_rt_init); 525 523 526 524 static __always_inline int __mutex_lock_common(struct mutex *lock, 527 525 unsigned int state, ··· 540 542 } 541 543 542 544 #ifdef CONFIG_DEBUG_LOCK_ALLOC 545 + void mutex_rt_init_lockdep(struct mutex *mutex, const char *name, struct lock_class_key *key) 546 + { 547 + __mutex_rt_init_generic(mutex); 548 + lockdep_init_map_wait(&mutex->dep_map, name, key, 0, LD_WAIT_SLEEP); 549 + } 550 + EXPORT_SYMBOL(mutex_rt_init_lockdep); 551 + 543 552 void __sched mutex_lock_nested(struct mutex *lock, unsigned int subclass) 544 553 { 545 554 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); ··· 602 597 } 603 598 EXPORT_SYMBOL_GPL(_mutex_trylock_nest_lock); 604 599 #else /* CONFIG_DEBUG_LOCK_ALLOC */ 600 + 601 + void mutex_rt_init_generic(struct mutex *mutex) 602 + { 603 + __mutex_rt_init_generic(mutex); 604 + } 605 + EXPORT_SYMBOL(mutex_rt_init_generic); 605 606 606 607 void __sched mutex_lock(struct mutex *lock) 607 608 {
+2 -2
kernel/locking/spinlock_debug.c
··· 184 184 static inline void debug_write_lock_before(rwlock_t *lock) 185 185 { 186 186 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic"); 187 - RWLOCK_BUG_ON(lock->owner == current, lock, "recursion"); 188 - RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(), 187 + RWLOCK_BUG_ON(READ_ONCE(lock->owner) == current, lock, "recursion"); 188 + RWLOCK_BUG_ON(READ_ONCE(lock->owner_cpu) == raw_smp_processor_id(), 189 189 lock, "cpu recursion"); 190 190 } 191 191
+5 -15
kernel/sched/cputime.c
··· 313 313 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times) 314 314 { 315 315 struct signal_struct *sig = tsk->signal; 316 - u64 utime, stime; 317 316 struct task_struct *t; 318 - unsigned int seq, nextseq; 319 - unsigned long flags; 317 + u64 utime, stime; 320 318 321 319 /* 322 320 * Update current task runtime to account pending time since last ··· 327 329 if (same_thread_group(current, tsk)) 328 330 (void) task_sched_runtime(current); 329 331 330 - rcu_read_lock(); 331 - /* Attempt a lockless read on the first round. */ 332 - nextseq = 0; 333 - do { 334 - seq = nextseq; 335 - flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq); 332 + guard(rcu)(); 333 + scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) { 336 334 times->utime = sig->utime; 337 335 times->stime = sig->stime; 338 336 times->sum_exec_runtime = sig->sum_sched_runtime; 339 337 340 - for_each_thread(tsk, t) { 338 + __for_each_thread(sig, t) { 341 339 task_cputime(t, &utime, &stime); 342 340 times->utime += utime; 343 341 times->stime += stime; 344 342 times->sum_exec_runtime += read_sum_exec_runtime(t); 345 343 } 346 - /* If lockless access failed, take the lock. */ 347 - nextseq = 1; 348 - } while (need_seqretry(&sig->stats_lock, seq)); 349 - done_seqretry_irqrestore(&sig->stats_lock, seq, flags); 350 - rcu_read_unlock(); 344 + } 351 345 } 352 346 353 347 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
+18 -37
rust/kernel/debugfs/traits.rs
··· 4 4 //! Traits for rendering or updating values exported to DebugFS. 5 5 6 6 use crate::prelude::*; 7 + use crate::sync::atomic::{Atomic, AtomicBasicOps, AtomicType, Relaxed}; 7 8 use crate::sync::Mutex; 8 9 use crate::uaccess::UserSliceReader; 9 10 use core::fmt::{self, Debug, Formatter}; 10 11 use core::str::FromStr; 11 - use core::sync::atomic::{ 12 - AtomicI16, AtomicI32, AtomicI64, AtomicI8, AtomicIsize, AtomicU16, AtomicU32, AtomicU64, 13 - AtomicU8, AtomicUsize, Ordering, 14 - }; 15 12 16 13 /// A trait for types that can be written into a string. 17 14 /// ··· 47 50 fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result; 48 51 } 49 52 50 - impl<T: FromStr> Reader for Mutex<T> { 53 + impl<T: FromStr + Unpin> Reader for Mutex<T> { 51 54 fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { 52 55 let mut buf = [0u8; 128]; 53 56 if reader.len() > buf.len() { ··· 63 66 } 64 67 } 65 68 66 - macro_rules! impl_reader_for_atomic { 67 - ($(($atomic_type:ty, $int_type:ty)),*) => { 68 - $( 69 - impl Reader for $atomic_type { 70 - fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { 71 - let mut buf = [0u8; 21]; // Enough for a 64-bit number. 72 - if reader.len() > buf.len() { 73 - return Err(EINVAL); 74 - } 75 - let n = reader.len(); 76 - reader.read_slice(&mut buf[..n])?; 69 + impl<T: AtomicType + FromStr> Reader for Atomic<T> 70 + where 71 + T::Repr: AtomicBasicOps, 72 + { 73 + fn read_from_slice(&self, reader: &mut UserSliceReader) -> Result { 74 + let mut buf = [0u8; 21]; // Enough for a 64-bit number. 75 + if reader.len() > buf.len() { 76 + return Err(EINVAL); 77 + } 78 + let n = reader.len(); 79 + reader.read_slice(&mut buf[..n])?; 77 80 78 - let s = core::str::from_utf8(&buf[..n]).map_err(|_| EINVAL)?; 79 - let val = s.trim().parse::<$int_type>().map_err(|_| EINVAL)?; 80 - self.store(val, Ordering::Relaxed); 81 - Ok(()) 82 - } 83 - } 84 - )* 85 - }; 81 + let s = core::str::from_utf8(&buf[..n]).map_err(|_| EINVAL)?; 82 + let val = s.trim().parse::<T>().map_err(|_| EINVAL)?; 83 + self.store(val, Relaxed); 84 + Ok(()) 85 + } 86 86 } 87 - 88 - impl_reader_for_atomic!( 89 - (AtomicI16, i16), 90 - (AtomicI32, i32), 91 - (AtomicI64, i64), 92 - (AtomicI8, i8), 93 - (AtomicIsize, isize), 94 - (AtomicU16, u16), 95 - (AtomicU32, u32), 96 - (AtomicU64, u64), 97 - (AtomicU8, u8), 98 - (AtomicUsize, usize) 99 - );
+11 -1
rust/kernel/sync/atomic.rs
··· 22 22 23 23 pub use internal::AtomicImpl; 24 24 pub use ordering::{Acquire, Full, Relaxed, Release}; 25 + pub(crate) use internal::{AtomicArithmeticOps, AtomicBasicOps, AtomicExchangeOps}; 25 26 26 27 use crate::build_error; 27 - use internal::{AtomicArithmeticOps, AtomicBasicOps, AtomicExchangeOps, AtomicRepr}; 28 + use internal::AtomicRepr; 28 29 use ordering::OrderingType; 29 30 30 31 /// A memory location which can be safely modified from multiple execution contexts. ··· 304 303 OrderingType::Release => T::Repr::atomic_set_release(&self.0, v), 305 304 _ => build_error!("Wrong ordering"), 306 305 } 306 + } 307 + } 308 + 309 + impl<T: AtomicType + core::fmt::Debug> core::fmt::Debug for Atomic<T> 310 + where 311 + T::Repr: AtomicBasicOps, 312 + { 313 + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 314 + core::fmt::Debug::fmt(&self.load(Relaxed), f) 307 315 } 308 316 } 309 317
+37 -4
rust/kernel/sync/lock.rs
··· 11 11 types::{NotThreadSafe, Opaque, ScopeGuard}, 12 12 }; 13 13 use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; 14 - use pin_init::{pin_data, pin_init, PinInit}; 14 + use pin_init::{pin_data, pin_init, PinInit, Wrapper}; 15 15 16 16 pub mod mutex; 17 17 pub mod spinlock; ··· 115 115 _pin: PhantomPinned, 116 116 117 117 /// The data protected by the lock. 118 + #[pin] 118 119 pub(crate) data: UnsafeCell<T>, 119 120 } 120 121 ··· 128 127 129 128 impl<T, B: Backend> Lock<T, B> { 130 129 /// Constructs a new lock initialiser. 131 - pub fn new(t: T, name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> { 130 + pub fn new( 131 + t: impl PinInit<T>, 132 + name: &'static CStr, 133 + key: Pin<&'static LockClassKey>, 134 + ) -> impl PinInit<Self> { 132 135 pin_init!(Self { 133 - data: UnsafeCell::new(t), 136 + data <- UnsafeCell::pin_init(t), 134 137 _pin: PhantomPinned, 135 138 // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have 136 139 // static lifetimes so they live indefinitely. ··· 245 240 246 241 cb() 247 242 } 243 + 244 + /// Returns a pinned mutable reference to the protected data. 245 + /// 246 + /// The guard implements [`DerefMut`] when `T: Unpin`, so for [`Unpin`] 247 + /// types [`DerefMut`] should be used instead of this function. 248 + /// 249 + /// [`DerefMut`]: core::ops::DerefMut 250 + /// [`Unpin`]: core::marker::Unpin 251 + /// 252 + /// # Examples 253 + /// 254 + /// ``` 255 + /// # use kernel::sync::{Mutex, MutexGuard}; 256 + /// # use core::{pin::Pin, marker::PhantomPinned}; 257 + /// struct Data(PhantomPinned); 258 + /// 259 + /// fn example(mutex: &Mutex<Data>) { 260 + /// let mut data: MutexGuard<'_, Data> = mutex.lock(); 261 + /// let mut data: Pin<&mut Data> = data.as_mut(); 262 + /// } 263 + /// ``` 264 + pub fn as_mut(&mut self) -> Pin<&mut T> { 265 + // SAFETY: `self.lock.data` is structurally pinned. 266 + unsafe { Pin::new_unchecked(&mut *self.lock.data.get()) } 267 + } 248 268 } 249 269 250 270 impl<T: ?Sized, B: Backend> core::ops::Deref for Guard<'_, T, B> { ··· 281 251 } 282 252 } 283 253 284 - impl<T: ?Sized, B: Backend> core::ops::DerefMut for Guard<'_, T, B> { 254 + impl<T: ?Sized, B: Backend> core::ops::DerefMut for Guard<'_, T, B> 255 + where 256 + T: Unpin, 257 + { 285 258 fn deref_mut(&mut self) -> &mut Self::Target { 286 259 // SAFETY: The caller owns the lock, so it is safe to deref the protected data. 287 260 unsafe { &mut *self.lock.data.get() }
+4 -1
rust/kernel/sync/lock/global.rs
··· 106 106 } 107 107 } 108 108 109 - impl<B: GlobalLockBackend> core::ops::DerefMut for GlobalGuard<B> { 109 + impl<B: GlobalLockBackend> core::ops::DerefMut for GlobalGuard<B> 110 + where 111 + B::Item: Unpin, 112 + { 110 113 fn deref_mut(&mut self) -> &mut Self::Target { 111 114 &mut self.inner 112 115 }
+5 -7
samples/rust/rust_debugfs.rs
··· 32 32 //! ``` 33 33 34 34 use core::str::FromStr; 35 - use core::sync::atomic::AtomicUsize; 36 - use core::sync::atomic::Ordering; 37 35 use kernel::c_str; 38 36 use kernel::debugfs::{Dir, File}; 39 37 use kernel::new_mutex; 40 38 use kernel::prelude::*; 39 + use kernel::sync::atomic::{Atomic, Relaxed}; 41 40 use kernel::sync::Mutex; 42 - 43 41 use kernel::{acpi, device::Core, of, platform, str::CString, types::ARef}; 44 42 45 43 kernel::module_platform_driver! { ··· 57 59 #[pin] 58 60 _compatible: File<CString>, 59 61 #[pin] 60 - counter: File<AtomicUsize>, 62 + counter: File<Atomic<usize>>, 61 63 #[pin] 62 64 inner: File<Mutex<Inner>>, 63 65 } ··· 107 109 ) -> Result<Pin<KBox<Self>>> { 108 110 let result = KBox::try_pin_init(RustDebugFs::new(pdev), GFP_KERNEL)?; 109 111 // We can still mutate fields through the files which are atomic or mutexed: 110 - result.counter.store(91, Ordering::Relaxed); 112 + result.counter.store(91, Relaxed); 111 113 { 112 114 let mut guard = result.inner.lock(); 113 115 guard.x = guard.y; ··· 118 120 } 119 121 120 122 impl RustDebugFs { 121 - fn build_counter(dir: &Dir) -> impl PinInit<File<AtomicUsize>> + '_ { 122 - dir.read_write_file(c_str!("counter"), AtomicUsize::new(0)) 123 + fn build_counter(dir: &Dir) -> impl PinInit<File<Atomic<usize>>> + '_ { 124 + dir.read_write_file(c_str!("counter"), Atomic::<usize>::new(0)) 123 125 } 124 126 125 127 fn build_inner(dir: &Dir) -> impl PinInit<File<Mutex<Inner>>> + '_ {
+3 -3
samples/rust/rust_debugfs_scoped.rs
··· 6 6 //! `Scope::dir` to create a variety of files without the need to separately 7 7 //! track them all. 8 8 9 - use core::sync::atomic::AtomicUsize; 10 9 use kernel::debugfs::{Dir, Scope}; 11 10 use kernel::prelude::*; 11 + use kernel::sync::atomic::Atomic; 12 12 use kernel::sync::Mutex; 13 13 use kernel::{c_str, new_mutex, str::CString}; 14 14 ··· 62 62 let file_name = CString::try_from_fmt(fmt!("{name_str}"))?; 63 63 for sub in items { 64 64 nums.push( 65 - AtomicUsize::new(sub.parse().map_err(|_| EINVAL)?), 65 + Atomic::<usize>::new(sub.parse().map_err(|_| EINVAL)?), 66 66 GFP_KERNEL, 67 67 )?; 68 68 } ··· 109 109 110 110 struct DeviceData { 111 111 name: CString, 112 - nums: KVec<AtomicUsize>, 112 + nums: KVec<Atomic<usize>>, 113 113 } 114 114 115 115 fn init_control(base_dir: &Dir, dyn_dirs: Dir) -> impl PinInit<Scope<ModuleData>> + '_ {
+7 -4
scripts/atomic/gen-atomic-instrumented.sh
··· 12 12 local arg="$1"; shift 13 13 local type="${arg%%:*}" 14 14 local name="$(gen_param_name "${arg}")" 15 - local rw="write" 15 + local rw="atomic_write" 16 16 17 17 case "${type#c}" in 18 18 i) return;; ··· 20 20 21 21 if [ ${type#c} != ${type} ]; then 22 22 # We don't write to constant parameters. 23 - rw="read" 23 + rw="atomic_read" 24 + elif [ "${type}" = "p" ] ; then 25 + # The "old" argument in try_cmpxchg() gets accessed non-atomically 26 + rw="read_write" 24 27 elif [ "${meta}" != "s" ]; then 25 28 # An atomic RMW: if this parameter is not a constant, and this atomic is 26 29 # not just a 's'tore, this parameter is both read from and written to. 27 - rw="read_write" 30 + rw="atomic_read_write" 28 31 fi 29 32 30 - printf "\tinstrument_atomic_${rw}(${name}, sizeof(*${name}));\n" 33 + printf "\tinstrument_${rw}(${name}, sizeof(*${name}));\n" 31 34 } 32 35 33 36 #gen_params_checks(meta, arg...)