Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
locking: Add __lockfunc to slow path functions
So that we can skip the functions in the perf lock contention and other
places like /proc/PID/wchan.
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Link: https://lore.kernel.org/r/20220810220346.1919485-1-namhyung@kernel.org
authored by
Namhyung Kim
and committed by
Peter Zijlstra
501f7f69
9aeaf5bc
+12
-11
+7
-6
arch/x86/include/asm/qspinlock_paravirt.h
+7
-6
arch/x86/include/asm/qspinlock_paravirt.h
···
12
12
*/
13
13
#ifdef CONFIG_64BIT
14
14
15
-
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
15
+
__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
16
16
#define __pv_queued_spin_unlock __pv_queued_spin_unlock
17
17
#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
18
18
#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
···
20
20
/*
21
21
* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
22
22
* which combines the registers saving trunk and the body of the following
23
-
* C code:
23
+
* C code. Note that it puts the code in the .spinlock.text section which
24
+
* is equivalent to adding __lockfunc in the C code:
24
25
*
25
-
* void __pv_queued_spin_unlock(struct qspinlock *lock)
26
+
* void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
26
27
* {
27
28
* u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
28
29
*
···
37
36
* rsi = lockval (second argument)
38
37
* rdx = internal variable (set to 0)
39
38
*/
40
-
asm (".pushsection .text;"
39
+
asm (".pushsection .spinlock.text;"
41
40
".globl " PV_UNLOCK ";"
42
41
".type " PV_UNLOCK ", @function;"
43
42
".align 4,0x90;"
···
66
65
67
66
#else /* CONFIG_64BIT */
68
67
69
-
extern void __pv_queued_spin_unlock(struct qspinlock *lock);
70
-
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
68
+
extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
69
+
__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
71
70
72
71
#endif /* CONFIG_64BIT */
73
72
#endif
+2
-2
kernel/locking/qrwlock.c
+2
-2
kernel/locking/qrwlock.c
···
18
18
* queued_read_lock_slowpath - acquire read lock of a queued rwlock
19
19
* @lock: Pointer to queued rwlock structure
20
20
*/
21
-
void queued_read_lock_slowpath(struct qrwlock *lock)
21
+
void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
22
22
{
23
23
/*
24
24
* Readers come here when they cannot get the lock without waiting
···
63
63
* queued_write_lock_slowpath - acquire write lock of a queued rwlock
64
64
* @lock : Pointer to queued rwlock structure
65
65
*/
66
-
void queued_write_lock_slowpath(struct qrwlock *lock)
66
+
void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
67
67
{
68
68
int cnts;
69
69
+1
-1
kernel/locking/qspinlock.c
+1
-1
kernel/locking/qspinlock.c
···
313
313
* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
314
314
* queue : ^--' :
315
315
*/
316
-
void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
316
+
void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
317
317
{
318
318
struct mcs_spinlock *prev, *next, *node;
319
319
u32 old, tail;
+2
-2
kernel/locking/qspinlock_paravirt.h
+2
-2
kernel/locking/qspinlock_paravirt.h
···
489
489
* PV versions of the unlock fastpath and slowpath functions to be used
490
490
* instead of queued_spin_unlock().
491
491
*/
492
-
__visible void
492
+
__visible __lockfunc void
493
493
__pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
494
494
{
495
495
struct pv_node *node;
···
544
544
#include <asm/qspinlock_paravirt.h>
545
545
546
546
#ifndef __pv_queued_spin_unlock
547
-
__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
547
+
__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
548
548
{
549
549
u8 locked;
550
550