tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'locking-core-2024-11-18' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Ingo Molnar:
"Lockdep:
- Enable PROVE_RAW_LOCK_NESTING with PROVE_LOCKING (Sebastian Andrzej
Siewior)
- Add lockdep_cleanup_dead_cpu() (David Woodhouse)

futexes:
- Use atomic64_inc_return() in get_inode_sequence_number() (Uros
Bizjak)
- Use atomic64_try_cmpxchg_relaxed() in get_inode_sequence_number()
(Uros Bizjak)

RT locking:
- Add sparse annotation PREEMPT_RT's locking (Sebastian Andrzej
Siewior)

spinlocks:
- Use atomic_try_cmpxchg_release() in osq_unlock() (Uros Bizjak)

atomics:
- x86: Use ALT_OUTPUT_SP() for __alternative_atomic64() (Uros Bizjak)
- x86: Use ALT_OUTPUT_SP() for __arch_{,try_}cmpxchg64_emu() (Uros
Bizjak)

KCSAN, seqlocks:
- Support seqcount_latch_t (Marco Elver)

<linux/cleanup.h>:
- Add if_not_guard() conditional guard helper (David Lechner)
- Adjust scoped_guard() macros to avoid potential warning (Przemek
Kitszel)
- Remove address space of returned pointer (Uros Bizjak)

WW mutexes:
- locking/ww_mutex: Adjust to lockdep nest_lock requirements (Thomas
Hellström)

Rust integration:
- Fix raw_spin_lock initialization on PREEMPT_RT (Eder Zulian)

Misc cleanups & fixes:
- lockdep: Fix wait-type check related warnings (Ahmed Ehab)
- lockdep: Use info level for initial info messages (Jiri Slaby)
- spinlocks: Make __raw_* lock ops static (Geert Uytterhoeven)
- pvqspinlock: Convert fields of 'enum vcpu_state' to uppercase
(Qiuxu Zhuo)
- iio: magnetometer: Fix if () scoped_guard() formatting (Stephen
Rothwell)
- rtmutex: Fix misleading comment (Peter Zijlstra)
- percpu-rw-semaphores: Fix grammar in percpu-rw-semaphore.rst (Xiu
Jianfeng)"

* tag 'locking-core-2024-11-18' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (29 commits)
locking/Documentation: Fix grammar in percpu-rw-semaphore.rst
iio: magnetometer: fix if () scoped_guard() formatting
rust: helpers: Avoid raw_spin_lock initialization for PREEMPT_RT
kcsan, seqlock: Fix incorrect assumption in read_seqbegin()
seqlock, treewide: Switch to non-raw seqcount_latch interface
kcsan, seqlock: Support seqcount_latch_t
time/sched_clock: Broaden sched_clock()'s instrumentation coverage
time/sched_clock: Swap update_clock_read_data() latch writes
locking/atomic/x86: Use ALT_OUTPUT_SP() for __arch_{,try_}cmpxchg64_emu()
locking/atomic/x86: Use ALT_OUTPUT_SP() for __alternative_atomic64()
cleanup: Add conditional guard helper
cleanup: Adjust scoped_guard() macros to avoid potential warning
locking/osq_lock: Use atomic_try_cmpxchg_release() in osq_unlock()
cleanup: Remove address space of returned pointer
locking/rtmutex: Fix misleading comment
locking/rt: Annotate unlock followed by lock for sparse.
locking/rt: Add sparse annotation for RCU.
locking/rt: Remove one __cond_lock() in RT's spin_trylock_irqsave()
locking/rt: Add sparse annotation PREEMPT_RT's sleeping locks.
locking/pvqspinlock: Convert fields of 'enum vcpu_state' to uppercase
...

Linus Torvalds 364eeb79 d8d78a90

+354 -160
+2 -2
Documentation/locking/percpu-rw-semaphore.rst
··· 16 16 hundreds of milliseconds. 17 17 18 18 The lock is declared with "struct percpu_rw_semaphore" type. 19 - The lock is initialized percpu_init_rwsem, it returns 0 on success and 20 - -ENOMEM on allocation failure. 19 + The lock is initialized with percpu_init_rwsem, it returns 0 on success 20 + and -ENOMEM on allocation failure. 21 21 The lock must be freed with percpu_free_rwsem to avoid memory leak. 22 22 23 23 The lock is locked for read with percpu_down_read, percpu_up_read and
+1 -1
Documentation/locking/seqlock.rst
··· 153 153 from interruption by readers. This is typically the case when the read 154 154 side can be invoked from NMI handlers. 155 155 156 - Check `raw_write_seqcount_latch()` for more information. 156 + Check `write_seqcount_latch()` for more information. 157 157 158 158 159 159 .. _seqlock_t:
+2 -1
arch/x86/include/asm/atomic64_32.h
··· 51 51 #ifdef CONFIG_X86_CMPXCHG64 52 52 #define __alternative_atomic64(f, g, out, in...) \ 53 53 asm volatile("call %c[func]" \ 54 - : out : [func] "i" (atomic64_##g##_cx8), ## in) 54 + : ALT_OUTPUT_SP(out) \ 55 + : [func] "i" (atomic64_##g##_cx8), ## in) 55 56 56 57 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8) 57 58 #else
+3 -3
arch/x86/include/asm/cmpxchg_32.h
··· 94 94 asm volatile(ALTERNATIVE(_lock_loc \ 95 95 "call cmpxchg8b_emu", \ 96 96 _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \ 97 - : "+a" (o.low), "+d" (o.high) \ 97 + : ALT_OUTPUT_SP("+a" (o.low), "+d" (o.high)) \ 98 98 : "b" (n.low), "c" (n.high), [ptr] "S" (_ptr) \ 99 99 : "memory"); \ 100 100 \ ··· 123 123 "call cmpxchg8b_emu", \ 124 124 _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \ 125 125 CC_SET(e) \ 126 - : CC_OUT(e) (ret), \ 127 - "+a" (o.low), "+d" (o.high) \ 126 + : ALT_OUTPUT_SP(CC_OUT(e) (ret), \ 127 + "+a" (o.low), "+d" (o.high)) \ 128 128 : "b" (n.low), "c" (n.high), [ptr] "S" (_ptr) \ 129 129 : "memory"); \ 130 130 \
+3 -2
arch/x86/kernel/tsc.c
··· 174 174 175 175 c2n = per_cpu_ptr(&cyc2ns, cpu); 176 176 177 - raw_write_seqcount_latch(&c2n->seq); 177 + write_seqcount_latch_begin(&c2n->seq); 178 178 c2n->data[0] = data; 179 - raw_write_seqcount_latch(&c2n->seq); 179 + write_seqcount_latch(&c2n->seq); 180 180 c2n->data[1] = data; 181 + write_seqcount_latch_end(&c2n->seq); 181 182 } 182 183 183 184 static void set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now)
+2 -1
drivers/iio/magnetometer/af8133j.c
··· 312 312 * When suspended, just store the new range to data->range to be 313 313 * applied later during power up. 314 314 */ 315 - if (!pm_runtime_status_suspended(dev)) 315 + if (!pm_runtime_status_suspended(dev)) { 316 316 scoped_guard(mutex, &data->mutex) 317 317 ret = regmap_write(data->regmap, 318 318 AF8133J_REG_RANGE, range); 319 + } 319 320 320 321 pm_runtime_enable(dev); 321 322
+57 -10
include/linux/cleanup.h
··· 273 273 * an anonymous instance of the (guard) class, not recommended for 274 274 * conditional locks. 275 275 * 276 + * if_not_guard(name, args...) { <error handling> }: 277 + * convenience macro for conditional guards that calls the statement that 278 + * follows only if the lock was not acquired (typically an error return). 279 + * 280 + * Only for conditional locks. 281 + * 276 282 * scoped_guard (name, args...) { }: 277 283 * similar to CLASS(name, scope)(args), except the variable (with the 278 284 * explicit name 'scope') is declard in a for-loop such that its scope is ··· 291 285 * similar to scoped_guard(), except it does fail when the lock 292 286 * acquire fails. 293 287 * 288 + * Only for conditional locks. 294 289 */ 295 290 291 + #define __DEFINE_CLASS_IS_CONDITIONAL(_name, _is_cond) \ 292 + static __maybe_unused const bool class_##_name##_is_conditional = _is_cond 293 + 296 294 #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ 295 + __DEFINE_CLASS_IS_CONDITIONAL(_name, false); \ 297 296 DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \ 298 297 static inline void * class_##_name##_lock_ptr(class_##_name##_t *_T) \ 299 - { return *_T; } 298 + { return (void *)(__force unsigned long)*_T; } 300 299 301 300 #define DEFINE_GUARD_COND(_name, _ext, _condlock) \ 301 + __DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \ 302 302 EXTEND_CLASS(_name, _ext, \ 303 303 ({ void *_t = _T; if (_T && !(_condlock)) _t = NULL; _t; }), \ 304 304 class_##_name##_t _T) \ ··· 315 303 CLASS(_name, __UNIQUE_ID(guard)) 316 304 317 305 #define __guard_ptr(_name) class_##_name##_lock_ptr 306 + #define __is_cond_ptr(_name) class_##_name##_is_conditional 318 307 319 - #define scoped_guard(_name, args...) \ 320 - for (CLASS(_name, scope)(args), \ 321 - *done = NULL; __guard_ptr(_name)(&scope) && !done; done = (void *)1) 308 + /* 309 + * Helper macro for scoped_guard(). 310 + * 311 + * Note that the "!__is_cond_ptr(_name)" part of the condition ensures that 312 + * compiler would be sure that for the unconditional locks the body of the 313 + * loop (caller-provided code glued to the else clause) could not be skipped. 314 + * It is needed because the other part - "__guard_ptr(_name)(&scope)" - is too 315 + * hard to deduce (even if could be proven true for unconditional locks). 316 + */ 317 + #define __scoped_guard(_name, _label, args...) \ 318 + for (CLASS(_name, scope)(args); \ 319 + __guard_ptr(_name)(&scope) || !__is_cond_ptr(_name); \ 320 + ({ goto _label; })) \ 321 + if (0) { \ 322 + _label: \ 323 + break; \ 324 + } else 322 325 323 - #define scoped_cond_guard(_name, _fail, args...) \ 324 - for (CLASS(_name, scope)(args), \ 325 - *done = NULL; !done; done = (void *)1) \ 326 - if (!__guard_ptr(_name)(&scope)) _fail; \ 327 - else 326 + #define scoped_guard(_name, args...) \ 327 + __scoped_guard(_name, __UNIQUE_ID(label), args) 328 + 329 + #define __scoped_cond_guard(_name, _fail, _label, args...) \ 330 + for (CLASS(_name, scope)(args); true; ({ goto _label; })) \ 331 + if (!__guard_ptr(_name)(&scope)) { \ 332 + BUILD_BUG_ON(!__is_cond_ptr(_name)); \ 333 + _fail; \ 334 + _label: \ 335 + break; \ 336 + } else 337 + 338 + #define scoped_cond_guard(_name, _fail, args...) \ 339 + __scoped_cond_guard(_name, _fail, __UNIQUE_ID(label), args) 340 + 341 + #define __if_not_guard(_name, _id, args...) \ 342 + BUILD_BUG_ON(!__is_cond_ptr(_name)); \ 343 + CLASS(_name, _id)(args); \ 344 + if (!__guard_ptr(_name)(&_id)) 345 + 346 + #define if_not_guard(_name, args...) \ 347 + __if_not_guard(_name, __UNIQUE_ID(guard), args) 328 348 329 349 /* 330 350 * Additional helper macros for generating lock guards with types, either for ··· 391 347 \ 392 348 static inline void *class_##_name##_lock_ptr(class_##_name##_t *_T) \ 393 349 { \ 394 - return _T->lock; \ 350 + return (void *)(__force unsigned long)_T->lock; \ 395 351 } 396 352 397 353 ··· 413 369 } 414 370 415 371 #define DEFINE_LOCK_GUARD_1(_name, _type, _lock, _unlock, ...) \ 372 + __DEFINE_CLASS_IS_CONDITIONAL(_name, false); \ 416 373 __DEFINE_UNLOCK_GUARD(_name, _type, _unlock, __VA_ARGS__) \ 417 374 __DEFINE_LOCK_GUARD_1(_name, _type, _lock) 418 375 419 376 #define DEFINE_LOCK_GUARD_0(_name, _lock, _unlock, ...) \ 377 + __DEFINE_CLASS_IS_CONDITIONAL(_name, false); \ 420 378 __DEFINE_UNLOCK_GUARD(_name, void, _unlock, __VA_ARGS__) \ 421 379 __DEFINE_LOCK_GUARD_0(_name, _lock) 422 380 423 381 #define DEFINE_LOCK_GUARD_1_COND(_name, _ext, _condlock) \ 382 + __DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \ 424 383 EXTEND_CLASS(_name, _ext, \ 425 384 ({ class_##_name##_t _t = { .lock = l }, *_T = &_t;\ 426 385 if (_T->lock && !(_condlock)) _T->lock = NULL; \
+6
include/linux/irqflags.h
··· 18 18 #include <asm/irqflags.h> 19 19 #include <asm/percpu.h> 20 20 21 + struct task_struct; 22 + 21 23 /* Currently lockdep_softirqs_on/off is used only by lockdep */ 22 24 #ifdef CONFIG_PROVE_LOCKING 23 25 extern void lockdep_softirqs_on(unsigned long ip); ··· 27 25 extern void lockdep_hardirqs_on_prepare(void); 28 26 extern void lockdep_hardirqs_on(unsigned long ip); 29 27 extern void lockdep_hardirqs_off(unsigned long ip); 28 + extern void lockdep_cleanup_dead_cpu(unsigned int cpu, 29 + struct task_struct *idle); 30 30 #else 31 31 static inline void lockdep_softirqs_on(unsigned long ip) { } 32 32 static inline void lockdep_softirqs_off(unsigned long ip) { } 33 33 static inline void lockdep_hardirqs_on_prepare(void) { } 34 34 static inline void lockdep_hardirqs_on(unsigned long ip) { } 35 35 static inline void lockdep_hardirqs_off(unsigned long ip) { } 36 + static inline void lockdep_cleanup_dead_cpu(unsigned int cpu, 37 + struct task_struct *idle) {} 36 38 #endif 37 39 38 40 #ifdef CONFIG_TRACE_IRQFLAGS
+1 -1
include/linux/lockdep.h
··· 173 173 (lock)->dep_map.lock_type) 174 174 175 175 #define lockdep_set_subclass(lock, sub) \ 176 - lockdep_init_map_type(&(lock)->dep_map, #lock, (lock)->dep_map.key, sub,\ 176 + lockdep_init_map_type(&(lock)->dep_map, (lock)->dep_map.name, (lock)->dep_map.key, sub,\ 177 177 (lock)->dep_map.wait_type_inner, \ 178 178 (lock)->dep_map.wait_type_outer, \ 179 179 (lock)->dep_map.lock_type)
+11 -9
include/linux/rbtree_latch.h
··· 14 14 * 15 15 * If we need to allow unconditional lookups (say as required for NMI context 16 16 * usage) we need a more complex setup; this data structure provides this by 17 - * employing the latch technique -- see @raw_write_seqcount_latch -- to 17 + * employing the latch technique -- see @write_seqcount_latch_begin -- to 18 18 * implement a latched RB-tree which does allow for unconditional lookups by 19 19 * virtue of always having (at least) one stable copy of the tree. 20 20 * ··· 132 132 * @ops: operators defining the node order 133 133 * 134 134 * It inserts @node into @root in an ordered fashion such that we can always 135 - * observe one complete tree. See the comment for raw_write_seqcount_latch(). 135 + * observe one complete tree. See the comment for write_seqcount_latch_begin(). 136 136 * 137 137 * The inserts use rcu_assign_pointer() to publish the element such that the 138 138 * tree structure is stored before we can observe the new @node. ··· 145 145 struct latch_tree_root *root, 146 146 const struct latch_tree_ops *ops) 147 147 { 148 - raw_write_seqcount_latch(&root->seq); 148 + write_seqcount_latch_begin(&root->seq); 149 149 __lt_insert(node, root, 0, ops->less); 150 - raw_write_seqcount_latch(&root->seq); 150 + write_seqcount_latch(&root->seq); 151 151 __lt_insert(node, root, 1, ops->less); 152 + write_seqcount_latch_end(&root->seq); 152 153 } 153 154 154 155 /** ··· 160 159 * 161 160 * Removes @node from the trees @root in an ordered fashion such that we can 162 161 * always observe one complete tree. See the comment for 163 - * raw_write_seqcount_latch(). 162 + * write_seqcount_latch_begin(). 164 163 * 165 164 * It is assumed that @node will observe one RCU quiescent state before being 166 165 * reused of freed. ··· 173 172 struct latch_tree_root *root, 174 173 const struct latch_tree_ops *ops) 175 174 { 176 - raw_write_seqcount_latch(&root->seq); 175 + write_seqcount_latch_begin(&root->seq); 177 176 __lt_erase(node, root, 0); 178 - raw_write_seqcount_latch(&root->seq); 177 + write_seqcount_latch(&root->seq); 179 178 __lt_erase(node, root, 1); 179 + write_seqcount_latch_end(&root->seq); 180 180 } 181 181 182 182 /** ··· 206 204 unsigned int seq; 207 205 208 206 do { 209 - seq = raw_read_seqcount_latch(&root->seq); 207 + seq = read_seqcount_latch(&root->seq); 210 208 node = __lt_find(key, root, seq & 1, ops->comp); 211 - } while (raw_read_seqcount_latch_retry(&root->seq, seq)); 209 + } while (read_seqcount_latch_retry(&root->seq, seq)); 212 210 213 211 return node; 214 212 }
+5 -5
include/linux/rwlock_rt.h
··· 24 24 __rt_rwlock_init(rwl, #rwl, &__key); \ 25 25 } while (0) 26 26 27 - extern void rt_read_lock(rwlock_t *rwlock); 27 + extern void rt_read_lock(rwlock_t *rwlock) __acquires(rwlock); 28 28 extern int rt_read_trylock(rwlock_t *rwlock); 29 - extern void rt_read_unlock(rwlock_t *rwlock); 30 - extern void rt_write_lock(rwlock_t *rwlock); 31 - extern void rt_write_lock_nested(rwlock_t *rwlock, int subclass); 29 + extern void rt_read_unlock(rwlock_t *rwlock) __releases(rwlock); 30 + extern void rt_write_lock(rwlock_t *rwlock) __acquires(rwlock); 31 + extern void rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(rwlock); 32 32 extern int rt_write_trylock(rwlock_t *rwlock); 33 - extern void rt_write_unlock(rwlock_t *rwlock); 33 + extern void rt_write_unlock(rwlock_t *rwlock) __releases(rwlock); 34 34 35 35 static __always_inline void read_lock(rwlock_t *rwlock) 36 36 {
+72 -26
include/linux/seqlock.h
··· 622 622 } 623 623 624 624 /** 625 + * read_seqcount_latch() - pick even/odd latch data copy 626 + * @s: Pointer to seqcount_latch_t 627 + * 628 + * See write_seqcount_latch() for details and a full reader/writer usage 629 + * example. 630 + * 631 + * Return: sequence counter raw value. Use the lowest bit as an index for 632 + * picking which data copy to read. The full counter must then be checked 633 + * with read_seqcount_latch_retry(). 634 + */ 635 + static __always_inline unsigned read_seqcount_latch(const seqcount_latch_t *s) 636 + { 637 + kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); 638 + return raw_read_seqcount_latch(s); 639 + } 640 + 641 + /** 625 642 * raw_read_seqcount_latch_retry() - end a seqcount_latch_t read section 626 643 * @s: Pointer to seqcount_latch_t 627 644 * @start: count, from raw_read_seqcount_latch() ··· 653 636 } 654 637 655 638 /** 639 + * read_seqcount_latch_retry() - end a seqcount_latch_t read section 640 + * @s: Pointer to seqcount_latch_t 641 + * @start: count, from read_seqcount_latch() 642 + * 643 + * Return: true if a read section retry is required, else false 644 + */ 645 + static __always_inline int 646 + read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start) 647 + { 648 + kcsan_atomic_next(0); 649 + return raw_read_seqcount_latch_retry(s, start); 650 + } 651 + 652 + /** 656 653 * raw_write_seqcount_latch() - redirect latch readers to even/odd copy 654 + * @s: Pointer to seqcount_latch_t 655 + */ 656 + static __always_inline void raw_write_seqcount_latch(seqcount_latch_t *s) 657 + { 658 + smp_wmb(); /* prior stores before incrementing "sequence" */ 659 + s->seqcount.sequence++; 660 + smp_wmb(); /* increment "sequence" before following stores */ 661 + } 662 + 663 + /** 664 + * write_seqcount_latch_begin() - redirect latch readers to odd copy 657 665 * @s: Pointer to seqcount_latch_t 658 666 * 659 667 * The latch technique is a multiversion concurrency control method that allows ··· 707 665 * 708 666 * void latch_modify(struct latch_struct *latch, ...) 709 667 * { 710 - * smp_wmb(); // Ensure that the last data[1] update is visible 711 - * latch->seq.sequence++; 712 - * smp_wmb(); // Ensure that the seqcount update is visible 713 - * 668 + * write_seqcount_latch_begin(&latch->seq); 714 669 * modify(latch->data[0], ...); 715 - * 716 - * smp_wmb(); // Ensure that the data[0] update is visible 717 - * latch->seq.sequence++; 718 - * smp_wmb(); // Ensure that the seqcount update is visible 719 - * 670 + * write_seqcount_latch(&latch->seq); 720 671 * modify(latch->data[1], ...); 672 + * write_seqcount_latch_end(&latch->seq); 721 673 * } 722 674 * 723 675 * The query will have a form like:: ··· 722 686 * unsigned seq, idx; 723 687 * 724 688 * do { 725 - * seq = raw_read_seqcount_latch(&latch->seq); 689 + * seq = read_seqcount_latch(&latch->seq); 726 690 * 727 691 * idx = seq & 0x01; 728 692 * entry = data_query(latch->data[idx], ...); 729 693 * 730 694 * // This includes needed smp_rmb() 731 - * } while (raw_read_seqcount_latch_retry(&latch->seq, seq)); 695 + * } while (read_seqcount_latch_retry(&latch->seq, seq)); 732 696 * 733 697 * return entry; 734 698 * } ··· 752 716 * When data is a dynamic data structure; one should use regular RCU 753 717 * patterns to manage the lifetimes of the objects within. 754 718 */ 755 - static inline void raw_write_seqcount_latch(seqcount_latch_t *s) 719 + static __always_inline void write_seqcount_latch_begin(seqcount_latch_t *s) 756 720 { 757 - smp_wmb(); /* prior stores before incrementing "sequence" */ 758 - s->seqcount.sequence++; 759 - smp_wmb(); /* increment "sequence" before following stores */ 721 + kcsan_nestable_atomic_begin(); 722 + raw_write_seqcount_latch(s); 723 + } 724 + 725 + /** 726 + * write_seqcount_latch() - redirect latch readers to even copy 727 + * @s: Pointer to seqcount_latch_t 728 + */ 729 + static __always_inline void write_seqcount_latch(seqcount_latch_t *s) 730 + { 731 + raw_write_seqcount_latch(s); 732 + } 733 + 734 + /** 735 + * write_seqcount_latch_end() - end a seqcount_latch_t write section 736 + * @s: Pointer to seqcount_latch_t 737 + * 738 + * Marks the end of a seqcount_latch_t writer section, after all copies of the 739 + * latch-protected data have been updated. 740 + */ 741 + static __always_inline void write_seqcount_latch_end(seqcount_latch_t *s) 742 + { 743 + kcsan_nestable_atomic_end(); 760 744 } 761 745 762 746 #define __SEQLOCK_UNLOCKED(lockname) \ ··· 810 754 */ 811 755 static inline unsigned read_seqbegin(const seqlock_t *sl) 812 756 { 813 - unsigned ret = read_seqcount_begin(&sl->seqcount); 814 - 815 - kcsan_atomic_next(0); /* non-raw usage, assume closing read_seqretry() */ 816 - kcsan_flat_atomic_begin(); 817 - return ret; 757 + return read_seqcount_begin(&sl->seqcount); 818 758 } 819 759 820 760 /** ··· 826 774 */ 827 775 static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) 828 776 { 829 - /* 830 - * Assume not nested: read_seqretry() may be called multiple times when 831 - * completing read critical section. 832 - */ 833 - kcsan_flat_atomic_end(); 834 - 835 777 return read_seqcount_retry(&sl->seqcount, start); 836 778 } 837 779
+12 -16
include/linux/spinlock_rt.h
··· 16 16 } 17 17 #endif 18 18 19 - #define spin_lock_init(slock) \ 19 + #define __spin_lock_init(slock, name, key, percpu) \ 20 20 do { \ 21 - static struct lock_class_key __key; \ 22 - \ 23 21 rt_mutex_base_init(&(slock)->lock); \ 24 - __rt_spin_lock_init(slock, #slock, &__key, false); \ 22 + __rt_spin_lock_init(slock, name, key, percpu); \ 25 23 } while (0) 26 24 27 - #define local_spin_lock_init(slock) \ 25 + #define _spin_lock_init(slock, percpu) \ 28 26 do { \ 29 27 static struct lock_class_key __key; \ 30 - \ 31 - rt_mutex_base_init(&(slock)->lock); \ 32 - __rt_spin_lock_init(slock, #slock, &__key, true); \ 28 + __spin_lock_init(slock, #slock, &__key, percpu); \ 33 29 } while (0) 34 30 35 - extern void rt_spin_lock(spinlock_t *lock); 36 - extern void rt_spin_lock_nested(spinlock_t *lock, int subclass); 37 - extern void rt_spin_lock_nest_lock(spinlock_t *lock, struct lockdep_map *nest_lock); 38 - extern void rt_spin_unlock(spinlock_t *lock); 31 + #define spin_lock_init(slock) _spin_lock_init(slock, false) 32 + #define local_spin_lock_init(slock) _spin_lock_init(slock, true) 33 + 34 + extern void rt_spin_lock(spinlock_t *lock) __acquires(lock); 35 + extern void rt_spin_lock_nested(spinlock_t *lock, int subclass) __acquires(lock); 36 + extern void rt_spin_lock_nest_lock(spinlock_t *lock, struct lockdep_map *nest_lock) __acquires(lock); 37 + extern void rt_spin_unlock(spinlock_t *lock) __releases(lock); 39 38 extern void rt_spin_lock_unlock(spinlock_t *lock); 40 39 extern int rt_spin_trylock_bh(spinlock_t *lock); 41 40 extern int rt_spin_trylock(spinlock_t *lock); ··· 131 132 #define spin_trylock_irq(lock) \ 132 133 __cond_lock(lock, rt_spin_trylock(lock)) 133 134 134 - #define __spin_trylock_irqsave(lock, flags) \ 135 + #define spin_trylock_irqsave(lock, flags) \ 135 136 ({ \ 136 137 int __locked; \ 137 138 \ ··· 140 141 __locked = spin_trylock(lock); \ 141 142 __locked; \ 142 143 }) 143 - 144 - #define spin_trylock_irqsave(lock, flags) \ 145 - __cond_lock(lock, __spin_trylock_irqsave(lock, flags)) 146 144 147 145 #define spin_is_contended(lock) (((void)(lock), 0)) 148 146
+14
include/linux/ww_mutex.h
··· 65 65 #endif 66 66 #ifdef CONFIG_DEBUG_LOCK_ALLOC 67 67 struct lockdep_map dep_map; 68 + /** 69 + * @first_lock_dep_map: fake lockdep_map for first locked ww_mutex. 70 + * 71 + * lockdep requires the lockdep_map for the first locked ww_mutex 72 + * in a ww transaction to remain in memory until all ww_mutexes of 73 + * the transaction have been unlocked. Ensure this by keeping a 74 + * fake locked ww_mutex lockdep map between ww_acquire_init() and 75 + * ww_acquire_fini(). 76 + */ 77 + struct lockdep_map first_lock_dep_map; 68 78 #endif 69 79 #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH 70 80 unsigned int deadlock_inject_interval; ··· 156 146 debug_check_no_locks_freed((void *)ctx, sizeof(*ctx)); 157 147 lockdep_init_map(&ctx->dep_map, ww_class->acquire_name, 158 148 &ww_class->acquire_key, 0); 149 + lockdep_init_map(&ctx->first_lock_dep_map, ww_class->mutex_name, 150 + &ww_class->mutex_key, 0); 159 151 mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_); 152 + mutex_acquire_nest(&ctx->first_lock_dep_map, 0, 0, &ctx->dep_map, _RET_IP_); 160 153 #endif 161 154 #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH 162 155 ctx->deadlock_inject_interval = 1; ··· 198 185 static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) 199 186 { 200 187 #ifdef CONFIG_DEBUG_LOCK_ALLOC 188 + mutex_release(&ctx->first_lock_dep_map, _THIS_IP_); 201 189 mutex_release(&ctx->dep_map, _THIS_IP_); 202 190 #endif 203 191 #ifdef DEBUG_WW_MUTEXES
+1
kernel/cpu.c
··· 1338 1338 1339 1339 cpuhp_bp_sync_dead(cpu); 1340 1340 1341 + lockdep_cleanup_dead_cpu(cpu, idle_thread_get(cpu)); 1341 1342 tick_cleanup_dead_cpu(cpu); 1342 1343 1343 1344 /*
+3 -3
kernel/futex/core.c
··· 181 181 return old; 182 182 183 183 for (;;) { 184 - u64 new = atomic64_add_return(1, &i_seq); 184 + u64 new = atomic64_inc_return(&i_seq); 185 185 if (WARN_ON_ONCE(!new)) 186 186 continue; 187 187 188 - old = atomic64_cmpxchg_relaxed(&inode->i_sequence, 0, new); 189 - if (old) 188 + old = 0; 189 + if (!atomic64_try_cmpxchg_relaxed(&inode->i_sequence, &old, new)) 190 190 return old; 191 191 return new; 192 192 }
+35 -11
kernel/locking/lockdep.c
··· 4586 4586 debug_atomic_inc(redundant_softirqs_off); 4587 4587 } 4588 4588 4589 + /** 4590 + * lockdep_cleanup_dead_cpu - Ensure CPU lockdep state is cleanly stopped 4591 + * 4592 + * @cpu: index of offlined CPU 4593 + * @idle: task pointer for offlined CPU's idle thread 4594 + * 4595 + * Invoked after the CPU is dead. Ensures that the tracing infrastructure 4596 + * is left in a suitable state for the CPU to be subsequently brought 4597 + * online again. 4598 + */ 4599 + void lockdep_cleanup_dead_cpu(unsigned int cpu, struct task_struct *idle) 4600 + { 4601 + if (unlikely(!debug_locks)) 4602 + return; 4603 + 4604 + if (unlikely(per_cpu(hardirqs_enabled, cpu))) { 4605 + pr_warn("CPU %u left hardirqs enabled!", cpu); 4606 + if (idle) 4607 + print_irqtrace_events(idle); 4608 + /* Clean it up for when the CPU comes online again. */ 4609 + per_cpu(hardirqs_enabled, cpu) = 0; 4610 + } 4611 + } 4612 + 4589 4613 static int 4590 4614 mark_usage(struct task_struct *curr, struct held_lock *hlock, int check) 4591 4615 { ··· 6600 6576 6601 6577 void __init lockdep_init(void) 6602 6578 { 6603 - printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n"); 6579 + pr_info("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n"); 6604 6580 6605 - printk("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES); 6606 - printk("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH); 6607 - printk("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS); 6608 - printk("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE); 6609 - printk("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES); 6610 - printk("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS); 6611 - printk("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE); 6581 + pr_info("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES); 6582 + pr_info("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH); 6583 + pr_info("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS); 6584 + pr_info("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE); 6585 + pr_info("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES); 6586 + pr_info("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS); 6587 + pr_info("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE); 6612 6588 6613 - printk(" memory used by lock dependency info: %zu kB\n", 6589 + pr_info(" memory used by lock dependency info: %zu kB\n", 6614 6590 (sizeof(lock_classes) + 6615 6591 sizeof(lock_classes_in_use) + 6616 6592 sizeof(classhash_table) + ··· 6628 6604 ); 6629 6605 6630 6606 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING) 6631 - printk(" memory used for stack traces: %zu kB\n", 6607 + pr_info(" memory used for stack traces: %zu kB\n", 6632 6608 (sizeof(stack_trace) + sizeof(stack_trace_hash)) / 1024 6633 6609 ); 6634 6610 #endif 6635 6611 6636 - printk(" per task-struct memory footprint: %zu bytes\n", 6612 + pr_info(" per task-struct memory footprint: %zu bytes\n", 6637 6613 sizeof(((struct task_struct *)NULL)->held_locks)); 6638 6614 } 6639 6615
+1 -2
kernel/locking/osq_lock.c
··· 215 215 /* 216 216 * Fast path for the uncontended case. 217 217 */ 218 - if (likely(atomic_cmpxchg_release(&lock->tail, curr, 219 - OSQ_UNLOCKED_VAL) == curr)) 218 + if (atomic_try_cmpxchg_release(&lock->tail, &curr, OSQ_UNLOCKED_VAL)) 220 219 return; 221 220 222 221 /*
+18 -18
kernel/locking/qspinlock_paravirt.h
··· 38 38 #define PV_PREV_CHECK_MASK 0xff 39 39 40 40 /* 41 - * Queue node uses: vcpu_running & vcpu_halted. 42 - * Queue head uses: vcpu_running & vcpu_hashed. 41 + * Queue node uses: VCPU_RUNNING & VCPU_HALTED. 42 + * Queue head uses: VCPU_RUNNING & VCPU_HASHED. 43 43 */ 44 44 enum vcpu_state { 45 - vcpu_running = 0, 46 - vcpu_halted, /* Used only in pv_wait_node */ 47 - vcpu_hashed, /* = pv_hash'ed + vcpu_halted */ 45 + VCPU_RUNNING = 0, 46 + VCPU_HALTED, /* Used only in pv_wait_node */ 47 + VCPU_HASHED, /* = pv_hash'ed + VCPU_HALTED */ 48 48 }; 49 49 50 50 struct pv_node { ··· 266 266 if ((loop & PV_PREV_CHECK_MASK) != 0) 267 267 return false; 268 268 269 - return READ_ONCE(prev->state) != vcpu_running; 269 + return READ_ONCE(prev->state) != VCPU_RUNNING; 270 270 } 271 271 272 272 /* ··· 279 279 BUILD_BUG_ON(sizeof(struct pv_node) > sizeof(struct qnode)); 280 280 281 281 pn->cpu = smp_processor_id(); 282 - pn->state = vcpu_running; 282 + pn->state = VCPU_RUNNING; 283 283 } 284 284 285 285 /* ··· 308 308 /* 309 309 * Order pn->state vs pn->locked thusly: 310 310 * 311 - * [S] pn->state = vcpu_halted [S] next->locked = 1 311 + * [S] pn->state = VCPU_HALTED [S] next->locked = 1 312 312 * MB MB 313 - * [L] pn->locked [RmW] pn->state = vcpu_hashed 313 + * [L] pn->locked [RmW] pn->state = VCPU_HASHED 314 314 * 315 315 * Matches the cmpxchg() from pv_kick_node(). 316 316 */ 317 - smp_store_mb(pn->state, vcpu_halted); 317 + smp_store_mb(pn->state, VCPU_HALTED); 318 318 319 319 if (!READ_ONCE(node->locked)) { 320 320 lockevent_inc(pv_wait_node); 321 321 lockevent_cond_inc(pv_wait_early, wait_early); 322 - pv_wait(&pn->state, vcpu_halted); 322 + pv_wait(&pn->state, VCPU_HALTED); 323 323 } 324 324 325 325 /* 326 - * If pv_kick_node() changed us to vcpu_hashed, retain that 326 + * If pv_kick_node() changed us to VCPU_HASHED, retain that 327 327 * value so that pv_wait_head_or_lock() knows to not also try 328 328 * to hash this lock. 329 329 */ 330 - cmpxchg(&pn->state, vcpu_halted, vcpu_running); 330 + cmpxchg(&pn->state, VCPU_HALTED, VCPU_RUNNING); 331 331 332 332 /* 333 333 * If the locked flag is still not set after wakeup, it is a ··· 357 357 static void pv_kick_node(struct qspinlock *lock, struct mcs_spinlock *node) 358 358 { 359 359 struct pv_node *pn = (struct pv_node *)node; 360 - u8 old = vcpu_halted; 360 + u8 old = VCPU_HALTED; 361 361 /* 362 362 * If the vCPU is indeed halted, advance its state to match that of 363 363 * pv_wait_node(). If OTOH this fails, the vCPU was running and will ··· 374 374 * subsequent writes. 375 375 */ 376 376 smp_mb__before_atomic(); 377 - if (!try_cmpxchg_relaxed(&pn->state, &old, vcpu_hashed)) 377 + if (!try_cmpxchg_relaxed(&pn->state, &old, VCPU_HASHED)) 378 378 return; 379 379 380 380 /* ··· 407 407 * If pv_kick_node() already advanced our state, we don't need to 408 408 * insert ourselves into the hash table anymore. 409 409 */ 410 - if (READ_ONCE(pn->state) == vcpu_hashed) 410 + if (READ_ONCE(pn->state) == VCPU_HASHED) 411 411 lp = (struct qspinlock **)1; 412 412 413 413 /* ··· 420 420 * Set correct vCPU state to be used by queue node wait-early 421 421 * mechanism. 422 422 */ 423 - WRITE_ONCE(pn->state, vcpu_running); 423 + WRITE_ONCE(pn->state, VCPU_RUNNING); 424 424 425 425 /* 426 426 * Set the pending bit in the active lock spinning loop to ··· 460 460 goto gotlock; 461 461 } 462 462 } 463 - WRITE_ONCE(pn->state, vcpu_hashed); 463 + WRITE_ONCE(pn->state, VCPU_HASHED); 464 464 lockevent_inc(pv_wait_head); 465 465 lockevent_cond_inc(pv_wait_again, waitcnt); 466 466 pv_wait(&lock->locked, _Q_SLOW_VAL);
+2
kernel/locking/rtmutex.c
··· 1601 1601 unsigned int state, 1602 1602 struct hrtimer_sleeper *timeout, 1603 1603 struct rt_mutex_waiter *waiter) 1604 + __releases(&lock->wait_lock) __acquires(&lock->wait_lock) 1604 1605 { 1605 1606 struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex); 1606 1607 struct task_struct *owner; ··· 1806 1805 * @lock: The underlying RT mutex 1807 1806 */ 1808 1807 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock) 1808 + __releases(&lock->wait_lock) __acquires(&lock->wait_lock) 1809 1809 { 1810 1810 struct rt_mutex_waiter waiter; 1811 1811 struct task_struct *owner;
+4 -4
kernel/locking/rtmutex_api.c
··· 175 175 } 176 176 177 177 /* 178 - * We've already deboosted, mark_wakeup_next_waiter() will 179 - * retain preempt_disabled when we drop the wait_lock, to 180 - * avoid inversion prior to the wakeup. preempt_disable() 181 - * therein pairs with rt_mutex_postunlock(). 178 + * mark_wakeup_next_waiter() deboosts and retains preemption 179 + * disabled when dropping the wait_lock, to avoid inversion prior 180 + * to the wakeup. preempt_disable() therein pairs with the 181 + * preempt_enable() in rt_mutex_postunlock(). 182 182 */ 183 183 mark_wakeup_next_waiter(wqh, lock); 184 184
+4 -4
kernel/locking/spinlock.c
··· 65 65 * towards that other CPU that it should break the lock ASAP. 66 66 */ 67 67 #define BUILD_LOCK_OPS(op, locktype) \ 68 - void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ 68 + static void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ 69 69 { \ 70 70 for (;;) { \ 71 71 preempt_disable(); \ ··· 77 77 } \ 78 78 } \ 79 79 \ 80 - unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ 80 + static unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ 81 81 { \ 82 82 unsigned long flags; \ 83 83 \ ··· 95 95 return flags; \ 96 96 } \ 97 97 \ 98 - void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ 98 + static void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ 99 99 { \ 100 100 _raw_##op##_lock_irqsave(lock); \ 101 101 } \ 102 102 \ 103 - void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ 103 + static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ 104 104 { \ 105 105 unsigned long flags; \ 106 106 \
+7 -7
kernel/locking/spinlock_rt.c
··· 51 51 migrate_disable(); 52 52 } 53 53 54 - void __sched rt_spin_lock(spinlock_t *lock) 54 + void __sched rt_spin_lock(spinlock_t *lock) __acquires(RCU) 55 55 { 56 56 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 57 57 __rt_spin_lock(lock); ··· 75 75 EXPORT_SYMBOL(rt_spin_lock_nest_lock); 76 76 #endif 77 77 78 - void __sched rt_spin_unlock(spinlock_t *lock) 78 + void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU) 79 79 { 80 80 spin_release(&lock->dep_map, _RET_IP_); 81 81 migrate_enable(); ··· 225 225 } 226 226 EXPORT_SYMBOL(rt_write_trylock); 227 227 228 - void __sched rt_read_lock(rwlock_t *rwlock) 228 + void __sched rt_read_lock(rwlock_t *rwlock) __acquires(RCU) 229 229 { 230 230 rtlock_might_resched(); 231 231 rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_); ··· 235 235 } 236 236 EXPORT_SYMBOL(rt_read_lock); 237 237 238 - void __sched rt_write_lock(rwlock_t *rwlock) 238 + void __sched rt_write_lock(rwlock_t *rwlock) __acquires(RCU) 239 239 { 240 240 rtlock_might_resched(); 241 241 rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); ··· 246 246 EXPORT_SYMBOL(rt_write_lock); 247 247 248 248 #ifdef CONFIG_DEBUG_LOCK_ALLOC 249 - void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) 249 + void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(RCU) 250 250 { 251 251 rtlock_might_resched(); 252 252 rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_); ··· 257 257 EXPORT_SYMBOL(rt_write_lock_nested); 258 258 #endif 259 259 260 - void __sched rt_read_unlock(rwlock_t *rwlock) 260 + void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU) 261 261 { 262 262 rwlock_release(&rwlock->dep_map, _RET_IP_); 263 263 migrate_enable(); ··· 266 266 } 267 267 EXPORT_SYMBOL(rt_read_unlock); 268 268 269 - void __sched rt_write_unlock(rwlock_t *rwlock) 269 + void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU) 270 270 { 271 271 rwlock_release(&rwlock->dep_map, _RET_IP_); 272 272 rcu_read_unlock();
+5 -3
kernel/locking/test-ww_mutex.c
··· 62 62 int ret; 63 63 64 64 ww_mutex_init(&mtx.mutex, &ww_class); 65 - ww_acquire_init(&ctx, &ww_class); 65 + if (flags & TEST_MTX_CTX) 66 + ww_acquire_init(&ctx, &ww_class); 66 67 67 68 INIT_WORK_ONSTACK(&mtx.work, test_mutex_work); 68 69 init_completion(&mtx.ready); ··· 91 90 ret = wait_for_completion_timeout(&mtx.done, TIMEOUT); 92 91 } 93 92 ww_mutex_unlock(&mtx.mutex); 94 - ww_acquire_fini(&ctx); 93 + if (flags & TEST_MTX_CTX) 94 + ww_acquire_fini(&ctx); 95 95 96 96 if (ret) { 97 97 pr_err("%s(flags=%x): mutual exclusion failure\n", ··· 681 679 if (ret) 682 680 return ret; 683 681 684 - ret = stress(2047, hweight32(STRESS_ALL)*ncpus, STRESS_ALL); 682 + ret = stress(2046, hweight32(STRESS_ALL)*ncpus, STRESS_ALL); 685 683 if (ret) 686 684 return ret; 687 685
+5 -4
kernel/printk/printk.c
··· 560 560 /* Must be called under syslog_lock. */ 561 561 static void latched_seq_write(struct latched_seq *ls, u64 val) 562 562 { 563 - raw_write_seqcount_latch(&ls->latch); 563 + write_seqcount_latch_begin(&ls->latch); 564 564 ls->val[0] = val; 565 - raw_write_seqcount_latch(&ls->latch); 565 + write_seqcount_latch(&ls->latch); 566 566 ls->val[1] = val; 567 + write_seqcount_latch_end(&ls->latch); 567 568 } 568 569 569 570 /* Can be called from any context. */ ··· 575 574 u64 val; 576 575 577 576 do { 578 - seq = raw_read_seqcount_latch(&ls->latch); 577 + seq = read_seqcount_latch(&ls->latch); 579 578 idx = seq & 0x1; 580 579 val = ls->val[idx]; 581 - } while (raw_read_seqcount_latch_retry(&ls->latch, seq)); 580 + } while (read_seqcount_latch_retry(&ls->latch, seq)); 582 581 583 582 return val; 584 583 }
+24 -10
kernel/time/sched_clock.c
··· 71 71 72 72 notrace struct clock_read_data *sched_clock_read_begin(unsigned int *seq) 73 73 { 74 - *seq = raw_read_seqcount_latch(&cd.seq); 74 + *seq = read_seqcount_latch(&cd.seq); 75 75 return cd.read_data + (*seq & 1); 76 76 } 77 77 78 78 notrace int sched_clock_read_retry(unsigned int seq) 79 79 { 80 - return raw_read_seqcount_latch_retry(&cd.seq, seq); 80 + return read_seqcount_latch_retry(&cd.seq, seq); 81 81 } 82 82 83 - unsigned long long noinstr sched_clock_noinstr(void) 83 + static __always_inline unsigned long long __sched_clock(void) 84 84 { 85 85 struct clock_read_data *rd; 86 86 unsigned int seq; ··· 98 98 return res; 99 99 } 100 100 101 + unsigned long long noinstr sched_clock_noinstr(void) 102 + { 103 + return __sched_clock(); 104 + } 105 + 101 106 unsigned long long notrace sched_clock(void) 102 107 { 103 108 unsigned long long ns; 104 109 preempt_disable_notrace(); 105 - ns = sched_clock_noinstr(); 110 + /* 111 + * All of __sched_clock() is a seqcount_latch reader critical section, 112 + * but relies on the raw helpers which are uninstrumented. For KCSAN, 113 + * mark all accesses in __sched_clock() as atomic. 114 + */ 115 + kcsan_nestable_atomic_begin(); 116 + ns = __sched_clock(); 117 + kcsan_nestable_atomic_end(); 106 118 preempt_enable_notrace(); 107 119 return ns; 108 120 } ··· 131 119 */ 132 120 static void update_clock_read_data(struct clock_read_data *rd) 133 121 { 134 - /* update the backup (odd) copy with the new data */ 135 - cd.read_data[1] = *rd; 136 - 137 122 /* steer readers towards the odd copy */ 138 - raw_write_seqcount_latch(&cd.seq); 123 + write_seqcount_latch_begin(&cd.seq); 139 124 140 125 /* now its safe for us to update the normal (even) copy */ 141 126 cd.read_data[0] = *rd; 142 127 143 128 /* switch readers back to the even copy */ 144 - raw_write_seqcount_latch(&cd.seq); 129 + write_seqcount_latch(&cd.seq); 130 + 131 + /* update the backup (odd) copy with the new data */ 132 + cd.read_data[1] = *rd; 133 + 134 + write_seqcount_latch_end(&cd.seq); 145 135 } 146 136 147 137 /* ··· 281 267 */ 282 268 static u64 notrace suspended_sched_clock_read(void) 283 269 { 284 - unsigned int seq = raw_read_seqcount_latch(&cd.seq); 270 + unsigned int seq = read_seqcount_latch(&cd.seq); 285 271 286 272 return cd.read_data[seq & 1].epoch_cyc; 287 273 }
+7 -5
kernel/time/timekeeping.c
··· 428 428 * We want to use this from any context including NMI and tracing / 429 429 * instrumenting the timekeeping code itself. 430 430 * 431 - * Employ the latch technique; see @raw_write_seqcount_latch. 431 + * Employ the latch technique; see @write_seqcount_latch. 432 432 * 433 433 * So if a NMI hits the update of base[0] then it will use base[1] 434 434 * which is still consistent. In the worst case this can result is a ··· 441 441 struct tk_read_base *base = tkf->base; 442 442 443 443 /* Force readers off to base[1] */ 444 - raw_write_seqcount_latch(&tkf->seq); 444 + write_seqcount_latch_begin(&tkf->seq); 445 445 446 446 /* Update base[0] */ 447 447 memcpy(base, tkr, sizeof(*base)); 448 448 449 449 /* Force readers back to base[0] */ 450 - raw_write_seqcount_latch(&tkf->seq); 450 + write_seqcount_latch(&tkf->seq); 451 451 452 452 /* Update base[1] */ 453 453 memcpy(base + 1, base, sizeof(*base)); 454 + 455 + write_seqcount_latch_end(&tkf->seq); 454 456 } 455 457 456 458 static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf) ··· 462 460 u64 now; 463 461 464 462 do { 465 - seq = raw_read_seqcount_latch(&tkf->seq); 463 + seq = read_seqcount_latch(&tkf->seq); 466 464 tkr = tkf->base + (seq & 0x01); 467 465 now = ktime_to_ns(tkr->base); 468 466 now += __timekeeping_get_ns(tkr); 469 - } while (raw_read_seqcount_latch_retry(&tkf->seq, seq)); 467 + } while (read_seqcount_latch_retry(&tkf->seq, seq)); 470 468 471 469 return now; 472 470 }
+2 -10
lib/Kconfig.debug
··· 1409 1409 For more details, see Documentation/locking/lockdep-design.rst. 1410 1410 1411 1411 config PROVE_RAW_LOCK_NESTING 1412 - bool "Enable raw_spinlock - spinlock nesting checks" 1412 + bool 1413 1413 depends on PROVE_LOCKING 1414 - default n 1414 + default y 1415 1415 help 1416 1416 Enable the raw_spinlock vs. spinlock nesting checks which ensure 1417 1417 that the lock nesting rules for PREEMPT_RT enabled kernels are 1418 1418 not violated. 1419 - 1420 - NOTE: There are known nesting problems. So if you enable this 1421 - option expect lockdep splats until these problems have been fully 1422 - addressed which is work in progress. This config switch allows to 1423 - identify and analyze these problems. It will be removed and the 1424 - check permanently enabled once the main issues have been fixed. 1425 - 1426 - If unsure, select N. 1427 1419 1428 1420 config LOCK_STAT 1429 1421 bool "Lock usage statistics"
+39
lib/locking-selftest.c
··· 2710 2710 2711 2711 } 2712 2712 2713 + #ifdef CONFIG_DEBUG_LOCK_ALLOC 2714 + static inline const char *rw_semaphore_lockdep_name(struct rw_semaphore *rwsem) 2715 + { 2716 + return rwsem->dep_map.name; 2717 + } 2718 + #else 2719 + static inline const char *rw_semaphore_lockdep_name(struct rw_semaphore *rwsem) 2720 + { 2721 + return NULL; 2722 + } 2723 + #endif 2724 + 2725 + static void test_lockdep_set_subclass_name(void) 2726 + { 2727 + const char *name_before = rw_semaphore_lockdep_name(&rwsem_X1); 2728 + const char *name_after; 2729 + 2730 + lockdep_set_subclass(&rwsem_X1, 1); 2731 + name_after = rw_semaphore_lockdep_name(&rwsem_X1); 2732 + DEBUG_LOCKS_WARN_ON(name_before != name_after); 2733 + } 2734 + 2735 + /* 2736 + * lockdep_set_subclass() should reuse the existing lock class name instead 2737 + * of creating a new one. 2738 + */ 2739 + static void lockdep_set_subclass_name_test(void) 2740 + { 2741 + printk(" --------------------------------------------------------------------------\n"); 2742 + printk(" | lockdep_set_subclass() name test|\n"); 2743 + printk(" -----------------------------------\n"); 2744 + 2745 + print_testname("compare name before and after"); 2746 + dotest(test_lockdep_set_subclass_name, SUCCESS, LOCKTYPE_RWSEM); 2747 + pr_cont("\n"); 2748 + } 2749 + 2713 2750 static void local_lock_tests(void) 2714 2751 { 2715 2752 printk(" --------------------------------------------------------------------------\n"); ··· 2956 2919 print_testname("hardirq_unsafe_softirq_safe"); 2957 2920 dotest(hardirq_deadlock_softirq_not_deadlock, FAILURE, LOCKTYPE_SPECIAL); 2958 2921 pr_cont("\n"); 2922 + 2923 + lockdep_set_subclass_name_test(); 2959 2924 2960 2925 if (unexpected_testcase_failures) { 2961 2926 printk("-----------------------------------------------------------------\n");
+6 -2
rust/helpers/spinlock.c
··· 7 7 struct lock_class_key *key) 8 8 { 9 9 #ifdef CONFIG_DEBUG_SPINLOCK 10 + # if defined(CONFIG_PREEMPT_RT) 11 + __spin_lock_init(lock, name, key, false); 12 + # else /*!CONFIG_PREEMPT_RT */ 10 13 __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); 11 - #else 14 + # endif /* CONFIG_PREEMPT_RT */ 15 + #else /* !CONFIG_DEBUG_SPINLOCK */ 12 16 spin_lock_init(lock); 13 - #endif 17 + #endif /* CONFIG_DEBUG_SPINLOCK */ 14 18 } 15 19 16 20 void rust_helper_spin_lock(spinlock_t *lock)