Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched/preempt: Refactor sched_dynamic_update()
Currently sched_dynamic_update needs to open-code the enabled/disabled
function names for each preemption model it supports, when in practice
this is a boolean enabled/disabled state for each function.
Make this clearer and avoid repetition by defining the enabled/disabled
states at the function definition, and using helper macros to perform the
static_call_update(). Where x86 currently overrides the enabled
function, it is made to provide both the enabled and disabled states for
consistency, with defaults provided by the core code otherwise.
In subsequent patches this will allow us to support PREEMPT_DYNAMIC
without static calls.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Acked-by: Frederic Weisbecker <frederic@kernel.org>
Link: https://lore.kernel.org/r/20220214165216.2231574-3-mark.rutland@arm.com
authored by
Mark Rutland
and committed by
Peter Zijlstra
8a69fe0b
4c748558
+45
-26
+6
-4
arch/x86/include/asm/preempt.h
+6
-4
arch/x86/include/asm/preempt.h
···
108
108
extern asmlinkage void preempt_schedule(void);
109
109
extern asmlinkage void preempt_schedule_thunk(void);
110
110
111
-
#define __preempt_schedule_func preempt_schedule_thunk
111
+
#define preempt_schedule_dynamic_enabled preempt_schedule_thunk
112
+
#define preempt_schedule_dynamic_disabled NULL
112
113
113
114
extern asmlinkage void preempt_schedule_notrace(void);
114
115
extern asmlinkage void preempt_schedule_notrace_thunk(void);
115
116
116
-
#define __preempt_schedule_notrace_func preempt_schedule_notrace_thunk
117
+
#define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace_thunk
118
+
#define preempt_schedule_notrace_dynamic_disabled NULL
117
119
118
120
#ifdef CONFIG_PREEMPT_DYNAMIC
119
121
120
-
DECLARE_STATIC_CALL(preempt_schedule, __preempt_schedule_func);
122
+
DECLARE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
121
123
122
124
#define __preempt_schedule() \
123
125
do { \
···
127
125
asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \
128
126
} while (0)
129
127
130
-
DECLARE_STATIC_CALL(preempt_schedule_notrace, __preempt_schedule_notrace_func);
128
+
DECLARE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
131
129
132
130
#define __preempt_schedule_notrace() \
133
131
do { \
+2
include/linux/entry-common.h
+2
include/linux/entry-common.h
···
456
456
*/
457
457
void irqentry_exit_cond_resched(void);
458
458
#ifdef CONFIG_PREEMPT_DYNAMIC
459
+
#define irqentry_exit_cond_resched_dynamic_enabled irqentry_exit_cond_resched
460
+
#define irqentry_exit_cond_resched_dynamic_disabled NULL
459
461
DECLARE_STATIC_CALL(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
460
462
#endif
461
463
+37
-22
kernel/sched/core.c
+37
-22
kernel/sched/core.c
···
6491
6491
EXPORT_SYMBOL(preempt_schedule);
6492
6492
6493
6493
#ifdef CONFIG_PREEMPT_DYNAMIC
6494
-
DEFINE_STATIC_CALL(preempt_schedule, __preempt_schedule_func);
6494
+
#ifndef preempt_schedule_dynamic_enabled
6495
+
#define preempt_schedule_dynamic_enabled preempt_schedule
6496
+
#define preempt_schedule_dynamic_disabled NULL
6497
+
#endif
6498
+
DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
6495
6499
EXPORT_STATIC_CALL_TRAMP(preempt_schedule);
6496
6500
#endif
6497
6501
···
6553
6549
EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
6554
6550
6555
6551
#ifdef CONFIG_PREEMPT_DYNAMIC
6556
-
DEFINE_STATIC_CALL(preempt_schedule_notrace, __preempt_schedule_notrace_func);
6552
+
#ifndef preempt_schedule_notrace_dynamic_enabled
6553
+
#define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace
6554
+
#define preempt_schedule_notrace_dynamic_disabled NULL
6555
+
#endif
6556
+
DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
6557
6557
EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace);
6558
6558
#endif
6559
6559
···
8068
8060
#endif
8069
8061
8070
8062
#ifdef CONFIG_PREEMPT_DYNAMIC
8063
+
#define cond_resched_dynamic_enabled __cond_resched
8064
+
#define cond_resched_dynamic_disabled ((void *)&__static_call_return0)
8071
8065
DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched);
8072
8066
EXPORT_STATIC_CALL_TRAMP(cond_resched);
8073
8067
8068
+
#define might_resched_dynamic_enabled __cond_resched
8069
+
#define might_resched_dynamic_disabled ((void *)&__static_call_return0)
8074
8070
DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched);
8075
8071
EXPORT_STATIC_CALL_TRAMP(might_resched);
8076
8072
#endif
···
8204
8192
return -EINVAL;
8205
8193
}
8206
8194
8195
+
#define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled)
8196
+
#define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled)
8197
+
8207
8198
void sched_dynamic_update(int mode)
8208
8199
{
8209
8200
/*
8210
8201
* Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
8211
8202
* the ZERO state, which is invalid.
8212
8203
*/
8213
-
static_call_update(cond_resched, __cond_resched);
8214
-
static_call_update(might_resched, __cond_resched);
8215
-
static_call_update(preempt_schedule, __preempt_schedule_func);
8216
-
static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func);
8217
-
static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
8204
+
preempt_dynamic_enable(cond_resched);
8205
+
preempt_dynamic_enable(might_resched);
8206
+
preempt_dynamic_enable(preempt_schedule);
8207
+
preempt_dynamic_enable(preempt_schedule_notrace);
8208
+
preempt_dynamic_enable(irqentry_exit_cond_resched);
8218
8209
8219
8210
switch (mode) {
8220
8211
case preempt_dynamic_none:
8221
-
static_call_update(cond_resched, __cond_resched);
8222
-
static_call_update(might_resched, (void *)&__static_call_return0);
8223
-
static_call_update(preempt_schedule, NULL);
8224
-
static_call_update(preempt_schedule_notrace, NULL);
8225
-
static_call_update(irqentry_exit_cond_resched, NULL);
8212
+
preempt_dynamic_enable(cond_resched);
8213
+
preempt_dynamic_disable(might_resched);
8214
+
preempt_dynamic_disable(preempt_schedule);
8215
+
preempt_dynamic_disable(preempt_schedule_notrace);
8216
+
preempt_dynamic_disable(irqentry_exit_cond_resched);
8226
8217
pr_info("Dynamic Preempt: none\n");
8227
8218
break;
8228
8219
8229
8220
case preempt_dynamic_voluntary:
8230
-
static_call_update(cond_resched, __cond_resched);
8231
-
static_call_update(might_resched, __cond_resched);
8232
-
static_call_update(preempt_schedule, NULL);
8233
-
static_call_update(preempt_schedule_notrace, NULL);
8234
-
static_call_update(irqentry_exit_cond_resched, NULL);
8221
+
preempt_dynamic_enable(cond_resched);
8222
+
preempt_dynamic_enable(might_resched);
8223
+
preempt_dynamic_disable(preempt_schedule);
8224
+
preempt_dynamic_disable(preempt_schedule_notrace);
8225
+
preempt_dynamic_disable(irqentry_exit_cond_resched);
8235
8226
pr_info("Dynamic Preempt: voluntary\n");
8236
8227
break;
8237
8228
8238
8229
case preempt_dynamic_full:
8239
-
static_call_update(cond_resched, (void *)&__static_call_return0);
8240
-
static_call_update(might_resched, (void *)&__static_call_return0);
8241
-
static_call_update(preempt_schedule, __preempt_schedule_func);
8242
-
static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func);
8243
-
static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
8230
+
preempt_dynamic_disable(cond_resched);
8231
+
preempt_dynamic_disable(might_resched);
8232
+
preempt_dynamic_enable(preempt_schedule);
8233
+
preempt_dynamic_enable(preempt_schedule_notrace);
8234
+
preempt_dynamic_enable(irqentry_exit_cond_resched);
8244
8235
pr_info("Dynamic Preempt: full\n");
8245
8236
break;
8246
8237
}