Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-core-2022-08-01' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar:
"This was a fairly quiet cycle for the locking subsystem:
- lockdep: Fix a handful of the more complex lockdep_init_map_*()
primitives that can lose the lock_type & cause false reports. No
such mishap was observed in the wild.
- jump_label improvements: simplify the cross-arch support of initial
NOP patching by making it arch-specific code (used on MIPS only),
and remove the s390 initial NOP patching that was superfluous"
* tag 'locking-core-2022-08-01' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
locking/lockdep: Fix lockdep_init_map_*() confusion
jump_label: make initial NOP patching the special case
jump_label: mips: move module NOP patching into arch code
jump_label: s390: avoid pointless initial NOP patching
+59
-163
-3
Documentation/staging/static-keys.rst
-3
Documentation/staging/static-keys.rst
···
201
201
* ``void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)``,
202
202
see: arch/x86/kernel/jump_label.c
203
203
204
-
* ``__init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, enum jump_label_type type)``,
205
-
see: arch/x86/kernel/jump_label.c
206
-
207
204
* ``struct jump_entry``,
208
205
see: arch/x86/include/asm/jump_label.h
209
206
-13
arch/arc/kernel/jump_label.c
-13
arch/arc/kernel/jump_label.c
···
96
96
flush_icache_range(entry->code, entry->code + JUMP_LABEL_NOP_SIZE);
97
97
}
98
98
99
-
void arch_jump_label_transform_static(struct jump_entry *entry,
100
-
enum jump_label_type type)
101
-
{
102
-
/*
103
-
* We use only one NOP type (1x, 4 byte) in arch_static_branch, so
104
-
* there's no need to patch an identical NOP over the top of it here.
105
-
* The generic code calls 'arch_jump_label_transform' if the NOP needs
106
-
* to be replaced by a branch, so 'arch_jump_label_transform_static' is
107
-
* never called with type other than JUMP_LABEL_NOP.
108
-
*/
109
-
BUG_ON(type != JUMP_LABEL_NOP);
110
-
}
111
-
112
99
#ifdef CONFIG_ARC_DBG_JUMP_LABEL
113
100
#define SELFTEST_MSG "ARC: instruction generation self-test: "
114
101
-6
arch/arm/kernel/jump_label.c
-6
arch/arm/kernel/jump_label.c
-11
arch/arm64/kernel/jump_label.c
-11
arch/arm64/kernel/jump_label.c
···
26
26
27
27
aarch64_insn_patch_text_nosync(addr, insn);
28
28
}
29
-
30
-
void arch_jump_label_transform_static(struct jump_entry *entry,
31
-
enum jump_label_type type)
32
-
{
33
-
/*
34
-
* We use the architected A64 NOP in arch_static_branch, so there's no
35
-
* need to patch an identical A64 NOP over the top of it here. The core
36
-
* will call arch_jump_label_transform from a module notifier if the
37
-
* NOP needs to be replaced by a branch.
38
-
*/
39
-
}
+2
arch/mips/include/asm/jump_label.h
+2
arch/mips/include/asm/jump_label.h
+19
arch/mips/kernel/jump_label.c
+19
arch/mips/kernel/jump_label.c
···
88
88
89
89
mutex_unlock(&text_mutex);
90
90
}
91
+
92
+
#ifdef CONFIG_MODULES
93
+
void jump_label_apply_nops(struct module *mod)
94
+
{
95
+
struct jump_entry *iter_start = mod->jump_entries;
96
+
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
97
+
struct jump_entry *iter;
98
+
99
+
/* if the module doesn't have jump label entries, just return */
100
+
if (iter_start == iter_stop)
101
+
return;
102
+
103
+
for (iter = iter_start; iter < iter_stop; iter++) {
104
+
/* Only write NOPs for arch_branch_static(). */
105
+
if (jump_label_init_type(iter) == JUMP_LABEL_NOP)
106
+
arch_jump_label_transform(iter, JUMP_LABEL_NOP);
107
+
}
108
+
}
109
+
#endif
+3
-2
arch/mips/kernel/module.c
+3
-2
arch/mips/kernel/module.c
···
21
21
#include <linux/spinlock.h>
22
22
#include <linux/jump_label.h>
23
23
24
+
extern void jump_label_apply_nops(struct module *mod);
24
25
25
26
struct mips_hi16 {
26
27
struct mips_hi16 *next;
···
429
428
const Elf_Shdr *s;
430
429
char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
431
430
432
-
/* Make jump label nops. */
433
-
jump_label_apply_nops(me);
431
+
if (IS_ENABLED(CONFIG_JUMP_LABEL))
432
+
jump_label_apply_nops(me);
434
433
435
434
INIT_LIST_HEAD(&me->arch.dbe_list);
436
435
for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
-11
arch/parisc/kernel/jump_label.c
-11
arch/parisc/kernel/jump_label.c
···
42
42
43
43
patch_text(addr, insn);
44
44
}
45
-
46
-
void arch_jump_label_transform_static(struct jump_entry *entry,
47
-
enum jump_label_type type)
48
-
{
49
-
/*
50
-
* We use the architected NOP in arch_static_branch, so there's no
51
-
* need to patch an identical NOP over the top of it here. The core
52
-
* will call arch_jump_label_transform from a module notifier if the
53
-
* NOP needs to be replaced by a branch.
54
-
*/
55
-
}
-12
arch/riscv/kernel/jump_label.c
-12
arch/riscv/kernel/jump_label.c
···
39
39
patch_text_nosync(addr, &insn, sizeof(insn));
40
40
mutex_unlock(&text_mutex);
41
41
}
42
-
43
-
void arch_jump_label_transform_static(struct jump_entry *entry,
44
-
enum jump_label_type type)
45
-
{
46
-
/*
47
-
* We use the same instructions in the arch_static_branch and
48
-
* arch_static_branch_jump inline functions, so there's no
49
-
* need to patch them up here.
50
-
* The core will call arch_jump_label_transform when those
51
-
* instructions need to be replaced.
52
-
*/
53
-
}
+2
-3
arch/s390/include/asm/jump_label.h
+2
-3
arch/s390/include/asm/jump_label.h
···
10
10
#include <linux/stringify.h>
11
11
12
12
#define JUMP_LABEL_NOP_SIZE 6
13
-
#define JUMP_LABEL_NOP_OFFSET 2
14
13
15
14
#ifdef CONFIG_CC_IS_CLANG
16
15
#define JUMP_LABEL_STATIC_KEY_CONSTRAINT "i"
···
20
21
#endif
21
22
22
23
/*
23
-
* We use a brcl 0,2 instruction for jump labels at compile time so it
24
+
* We use a brcl 0,<offset> instruction for jump labels so it
24
25
* can be easily distinguished from a hotpatch generated instruction.
25
26
*/
26
27
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
27
28
{
28
-
asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n"
29
+
asm_volatile_goto("0: brcl 0,%l[label]\n"
29
30
".pushsection __jump_table,\"aw\"\n"
30
31
".balign 8\n"
31
32
".long 0b-.,%l[label]-.\n"
+5
-23
arch/s390/kernel/jump_label.c
+5
-23
arch/s390/kernel/jump_label.c
···
44
44
panic("Corrupted kernel text");
45
45
}
46
46
47
-
static struct insn orignop = {
48
-
.opcode = 0xc004,
49
-
.offset = JUMP_LABEL_NOP_OFFSET >> 1,
50
-
};
51
-
52
47
static void jump_label_transform(struct jump_entry *entry,
53
-
enum jump_label_type type,
54
-
int init)
48
+
enum jump_label_type type)
55
49
{
56
50
void *code = (void *)jump_entry_code(entry);
57
51
struct insn old, new;
···
57
63
jump_label_make_branch(entry, &old);
58
64
jump_label_make_nop(entry, &new);
59
65
}
60
-
if (init) {
61
-
if (memcmp(code, &orignop, sizeof(orignop)))
62
-
jump_label_bug(entry, &orignop, &new);
63
-
} else {
64
-
if (memcmp(code, &old, sizeof(old)))
65
-
jump_label_bug(entry, &old, &new);
66
-
}
66
+
if (memcmp(code, &old, sizeof(old)))
67
+
jump_label_bug(entry, &old, &new);
67
68
s390_kernel_write(code, &new, sizeof(new));
68
69
}
69
70
70
71
void arch_jump_label_transform(struct jump_entry *entry,
71
72
enum jump_label_type type)
72
73
{
73
-
jump_label_transform(entry, type, 0);
74
+
jump_label_transform(entry, type);
74
75
text_poke_sync();
75
76
}
76
77
77
78
bool arch_jump_label_transform_queue(struct jump_entry *entry,
78
79
enum jump_label_type type)
79
80
{
80
-
jump_label_transform(entry, type, 0);
81
+
jump_label_transform(entry, type);
81
82
return true;
82
83
}
83
84
84
85
void arch_jump_label_transform_apply(void)
85
86
{
86
-
text_poke_sync();
87
-
}
88
-
89
-
void __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,
90
-
enum jump_label_type type)
91
-
{
92
-
jump_label_transform(entry, type, 1);
93
87
text_poke_sync();
94
88
}
-1
arch/s390/kernel/module.c
-1
arch/s390/kernel/module.c
-3
arch/sparc/kernel/module.c
-3
arch/sparc/kernel/module.c
-13
arch/x86/kernel/jump_label.c
-13
arch/x86/kernel/jump_label.c
···
146
146
text_poke_finish();
147
147
mutex_unlock(&text_mutex);
148
148
}
149
-
150
-
static enum {
151
-
JL_STATE_START,
152
-
JL_STATE_NO_UPDATE,
153
-
JL_STATE_UPDATE,
154
-
} jlstate __initdata_or_module = JL_STATE_START;
155
-
156
-
__init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
157
-
enum jump_label_type type)
158
-
{
159
-
if (jlstate == JL_STATE_UPDATE)
160
-
jump_label_transform(entry, type, 1);
161
-
}
-3
arch/x86/kernel/module.c
-3
arch/x86/kernel/module.c
+1
-8
include/linux/jump_label.h
+1
-8
include/linux/jump_label.h
···
220
220
extern void jump_label_unlock(void);
221
221
extern void arch_jump_label_transform(struct jump_entry *entry,
222
222
enum jump_label_type type);
223
-
extern void arch_jump_label_transform_static(struct jump_entry *entry,
224
-
enum jump_label_type type);
225
223
extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
226
224
enum jump_label_type type);
227
225
extern void arch_jump_label_transform_apply(void);
···
228
230
extern void static_key_slow_dec(struct static_key *key);
229
231
extern void static_key_slow_inc_cpuslocked(struct static_key *key);
230
232
extern void static_key_slow_dec_cpuslocked(struct static_key *key);
231
-
extern void jump_label_apply_nops(struct module *mod);
232
233
extern int static_key_count(struct static_key *key);
233
234
extern void static_key_enable(struct static_key *key);
234
235
extern void static_key_disable(struct static_key *key);
235
236
extern void static_key_enable_cpuslocked(struct static_key *key);
236
237
extern void static_key_disable_cpuslocked(struct static_key *key);
238
+
extern enum jump_label_type jump_label_init_type(struct jump_entry *entry);
237
239
238
240
/*
239
241
* We should be using ATOMIC_INIT() for initializing .enabled, but
···
300
302
301
303
static inline void jump_label_lock(void) {}
302
304
static inline void jump_label_unlock(void) {}
303
-
304
-
static inline int jump_label_apply_nops(struct module *mod)
305
-
{
306
-
return 0;
307
-
}
308
305
309
306
static inline void static_key_enable(struct static_key *key)
310
307
{
+17
-13
include/linux/lockdep.h
+17
-13
include/linux/lockdep.h
···
188
188
lockdep_init_map_waits(struct lockdep_map *lock, const char *name,
189
189
struct lock_class_key *key, int subclass, u8 inner, u8 outer)
190
190
{
191
-
lockdep_init_map_type(lock, name, key, subclass, inner, LD_WAIT_INV, LD_LOCK_NORMAL);
191
+
lockdep_init_map_type(lock, name, key, subclass, inner, outer, LD_LOCK_NORMAL);
192
192
}
193
193
194
194
static inline void
···
211
211
* or they are too narrow (they suffer from a false class-split):
212
212
*/
213
213
#define lockdep_set_class(lock, key) \
214
-
lockdep_init_map_waits(&(lock)->dep_map, #key, key, 0, \
215
-
(lock)->dep_map.wait_type_inner, \
216
-
(lock)->dep_map.wait_type_outer)
214
+
lockdep_init_map_type(&(lock)->dep_map, #key, key, 0, \
215
+
(lock)->dep_map.wait_type_inner, \
216
+
(lock)->dep_map.wait_type_outer, \
217
+
(lock)->dep_map.lock_type)
217
218
218
219
#define lockdep_set_class_and_name(lock, key, name) \
219
-
lockdep_init_map_waits(&(lock)->dep_map, name, key, 0, \
220
-
(lock)->dep_map.wait_type_inner, \
221
-
(lock)->dep_map.wait_type_outer)
220
+
lockdep_init_map_type(&(lock)->dep_map, name, key, 0, \
221
+
(lock)->dep_map.wait_type_inner, \
222
+
(lock)->dep_map.wait_type_outer, \
223
+
(lock)->dep_map.lock_type)
222
224
223
225
#define lockdep_set_class_and_subclass(lock, key, sub) \
224
-
lockdep_init_map_waits(&(lock)->dep_map, #key, key, sub,\
225
-
(lock)->dep_map.wait_type_inner, \
226
-
(lock)->dep_map.wait_type_outer)
226
+
lockdep_init_map_type(&(lock)->dep_map, #key, key, sub, \
227
+
(lock)->dep_map.wait_type_inner, \
228
+
(lock)->dep_map.wait_type_outer, \
229
+
(lock)->dep_map.lock_type)
227
230
228
231
#define lockdep_set_subclass(lock, sub) \
229
-
lockdep_init_map_waits(&(lock)->dep_map, #lock, (lock)->dep_map.key, sub,\
230
-
(lock)->dep_map.wait_type_inner, \
231
-
(lock)->dep_map.wait_type_outer)
232
+
lockdep_init_map_type(&(lock)->dep_map, #lock, (lock)->dep_map.key, sub,\
233
+
(lock)->dep_map.wait_type_inner, \
234
+
(lock)->dep_map.wait_type_outer, \
235
+
(lock)->dep_map.lock_type)
232
236
233
237
#define lockdep_set_novalidate_class(lock) \
234
238
lockdep_set_class_and_name(lock, &__lockdep_no_validate__, #lock)
+6
-35
kernel/jump_label.c
+6
-35
kernel/jump_label.c
···
332
332
return 0;
333
333
}
334
334
335
-
/*
336
-
* Update code which is definitely not currently executing.
337
-
* Architectures which need heavyweight synchronization to modify
338
-
* running code can override this to make the non-live update case
339
-
* cheaper.
340
-
*/
341
-
void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,
342
-
enum jump_label_type type)
335
+
#ifndef arch_jump_label_transform_static
336
+
static void arch_jump_label_transform_static(struct jump_entry *entry,
337
+
enum jump_label_type type)
343
338
{
344
-
arch_jump_label_transform(entry, type);
339
+
/* nothing to do on most architectures */
345
340
}
341
+
#endif
346
342
347
343
static inline struct jump_entry *static_key_entries(struct static_key *key)
348
344
{
···
504
508
505
509
#ifdef CONFIG_MODULES
506
510
507
-
static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
511
+
enum jump_label_type jump_label_init_type(struct jump_entry *entry)
508
512
{
509
513
struct static_key *key = jump_entry_key(entry);
510
514
bool type = static_key_type(key);
···
589
593
stop = m->jump_entries + m->num_jump_entries;
590
594
__jump_label_update(key, mod->entries, stop,
591
595
m && m->state == MODULE_STATE_COMING);
592
-
}
593
-
}
594
-
595
-
/***
596
-
* apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
597
-
* @mod: module to patch
598
-
*
599
-
* Allow for run-time selection of the optimal nops. Before the module
600
-
* loads patch these with arch_get_jump_label_nop(), which is specified by
601
-
* the arch specific jump label code.
602
-
*/
603
-
void jump_label_apply_nops(struct module *mod)
604
-
{
605
-
struct jump_entry *iter_start = mod->jump_entries;
606
-
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
607
-
struct jump_entry *iter;
608
-
609
-
/* if the module doesn't have jump label entries, just return */
610
-
if (iter_start == iter_stop)
611
-
return;
612
-
613
-
for (iter = iter_start; iter < iter_stop; iter++) {
614
-
/* Only write NOPs for arch_branch_static(). */
615
-
if (jump_label_init_type(iter) == JUMP_LABEL_NOP)
616
-
arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
617
596
}
618
597
}
619
598
+4
-3
kernel/locking/lockdep.c
+4
-3
kernel/locking/lockdep.c
···
5238
5238
return 0;
5239
5239
}
5240
5240
5241
-
lockdep_init_map_waits(lock, name, key, 0,
5242
-
lock->wait_type_inner,
5243
-
lock->wait_type_outer);
5241
+
lockdep_init_map_type(lock, name, key, 0,
5242
+
lock->wait_type_inner,
5243
+
lock->wait_type_outer,
5244
+
lock->lock_type);
5244
5245
class = register_lock_class(lock, subclass, 0);
5245
5246
hlock->class_idx = class - lock_classes;
5246
5247