Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched/preempt: Move PREEMPT_DYNAMIC logic later
The PREEMPT_DYNAMIC logic in kernel/sched/core.c patches static calls
for a bunch of preemption functions. While most are defined prior to
this, the definition of cond_resched() is later in the file, and so we
only have its declarations from include/linux/sched.h.
In subsequent patches we'd like to define some macros alongside the
definition of each of the preemption functions, which we can use within
sched_dynamic_update(). For this to be possible, the PREEMPT_DYNAMIC
logic needs to be placed after the various preemption functions.
As a preparatory step, this patch moves the PREEMPT_DYNAMIC logic after
the various preemption functions, with no other changes -- this is
purely a move.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Acked-by: Frederic Weisbecker <frederic@kernel.org>
Link: https://lore.kernel.org/r/20220214165216.2231574-2-mark.rutland@arm.com
authored by
Mark Rutland
and committed by
Peter Zijlstra
4c748558
ed3b362d
+136
-136
+136
-136
kernel/sched/core.c
+136
-136
kernel/sched/core.c
···
6555
6555
6556
6556
#endif /* CONFIG_PREEMPTION */
6557
6557
6558
-
#ifdef CONFIG_PREEMPT_DYNAMIC
6559
-
6560
-
#include <linux/entry-common.h>
6561
-
6562
-
/*
6563
-
* SC:cond_resched
6564
-
* SC:might_resched
6565
-
* SC:preempt_schedule
6566
-
* SC:preempt_schedule_notrace
6567
-
* SC:irqentry_exit_cond_resched
6568
-
*
6569
-
*
6570
-
* NONE:
6571
-
* cond_resched <- __cond_resched
6572
-
* might_resched <- RET0
6573
-
* preempt_schedule <- NOP
6574
-
* preempt_schedule_notrace <- NOP
6575
-
* irqentry_exit_cond_resched <- NOP
6576
-
*
6577
-
* VOLUNTARY:
6578
-
* cond_resched <- __cond_resched
6579
-
* might_resched <- __cond_resched
6580
-
* preempt_schedule <- NOP
6581
-
* preempt_schedule_notrace <- NOP
6582
-
* irqentry_exit_cond_resched <- NOP
6583
-
*
6584
-
* FULL:
6585
-
* cond_resched <- RET0
6586
-
* might_resched <- RET0
6587
-
* preempt_schedule <- preempt_schedule
6588
-
* preempt_schedule_notrace <- preempt_schedule_notrace
6589
-
* irqentry_exit_cond_resched <- irqentry_exit_cond_resched
6590
-
*/
6591
-
6592
-
enum {
6593
-
preempt_dynamic_undefined = -1,
6594
-
preempt_dynamic_none,
6595
-
preempt_dynamic_voluntary,
6596
-
preempt_dynamic_full,
6597
-
};
6598
-
6599
-
int preempt_dynamic_mode = preempt_dynamic_undefined;
6600
-
6601
-
int sched_dynamic_mode(const char *str)
6602
-
{
6603
-
if (!strcmp(str, "none"))
6604
-
return preempt_dynamic_none;
6605
-
6606
-
if (!strcmp(str, "voluntary"))
6607
-
return preempt_dynamic_voluntary;
6608
-
6609
-
if (!strcmp(str, "full"))
6610
-
return preempt_dynamic_full;
6611
-
6612
-
return -EINVAL;
6613
-
}
6614
-
6615
-
void sched_dynamic_update(int mode)
6616
-
{
6617
-
/*
6618
-
* Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
6619
-
* the ZERO state, which is invalid.
6620
-
*/
6621
-
static_call_update(cond_resched, __cond_resched);
6622
-
static_call_update(might_resched, __cond_resched);
6623
-
static_call_update(preempt_schedule, __preempt_schedule_func);
6624
-
static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func);
6625
-
static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
6626
-
6627
-
switch (mode) {
6628
-
case preempt_dynamic_none:
6629
-
static_call_update(cond_resched, __cond_resched);
6630
-
static_call_update(might_resched, (void *)&__static_call_return0);
6631
-
static_call_update(preempt_schedule, NULL);
6632
-
static_call_update(preempt_schedule_notrace, NULL);
6633
-
static_call_update(irqentry_exit_cond_resched, NULL);
6634
-
pr_info("Dynamic Preempt: none\n");
6635
-
break;
6636
-
6637
-
case preempt_dynamic_voluntary:
6638
-
static_call_update(cond_resched, __cond_resched);
6639
-
static_call_update(might_resched, __cond_resched);
6640
-
static_call_update(preempt_schedule, NULL);
6641
-
static_call_update(preempt_schedule_notrace, NULL);
6642
-
static_call_update(irqentry_exit_cond_resched, NULL);
6643
-
pr_info("Dynamic Preempt: voluntary\n");
6644
-
break;
6645
-
6646
-
case preempt_dynamic_full:
6647
-
static_call_update(cond_resched, (void *)&__static_call_return0);
6648
-
static_call_update(might_resched, (void *)&__static_call_return0);
6649
-
static_call_update(preempt_schedule, __preempt_schedule_func);
6650
-
static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func);
6651
-
static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
6652
-
pr_info("Dynamic Preempt: full\n");
6653
-
break;
6654
-
}
6655
-
6656
-
preempt_dynamic_mode = mode;
6657
-
}
6658
-
6659
-
static int __init setup_preempt_mode(char *str)
6660
-
{
6661
-
int mode = sched_dynamic_mode(str);
6662
-
if (mode < 0) {
6663
-
pr_warn("Dynamic Preempt: unsupported mode: %s\n", str);
6664
-
return 0;
6665
-
}
6666
-
6667
-
sched_dynamic_update(mode);
6668
-
return 1;
6669
-
}
6670
-
__setup("preempt=", setup_preempt_mode);
6671
-
6672
-
static void __init preempt_dynamic_init(void)
6673
-
{
6674
-
if (preempt_dynamic_mode == preempt_dynamic_undefined) {
6675
-
if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
6676
-
sched_dynamic_update(preempt_dynamic_none);
6677
-
} else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
6678
-
sched_dynamic_update(preempt_dynamic_voluntary);
6679
-
} else {
6680
-
/* Default static call setting, nothing to do */
6681
-
WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
6682
-
preempt_dynamic_mode = preempt_dynamic_full;
6683
-
pr_info("Dynamic Preempt: full\n");
6684
-
}
6685
-
}
6686
-
}
6687
-
6688
-
#else /* !CONFIG_PREEMPT_DYNAMIC */
6689
-
6690
-
static inline void preempt_dynamic_init(void) { }
6691
-
6692
-
#endif /* #ifdef CONFIG_PREEMPT_DYNAMIC */
6693
-
6694
6558
/*
6695
6559
* This is the entry point to schedule() from kernel preemption
6696
6560
* off of irq context.
···
8134
8270
return ret;
8135
8271
}
8136
8272
EXPORT_SYMBOL(__cond_resched_rwlock_write);
8273
+
8274
+
#ifdef CONFIG_PREEMPT_DYNAMIC
8275
+
8276
+
#include <linux/entry-common.h>
8277
+
8278
+
/*
8279
+
* SC:cond_resched
8280
+
* SC:might_resched
8281
+
* SC:preempt_schedule
8282
+
* SC:preempt_schedule_notrace
8283
+
* SC:irqentry_exit_cond_resched
8284
+
*
8285
+
*
8286
+
* NONE:
8287
+
* cond_resched <- __cond_resched
8288
+
* might_resched <- RET0
8289
+
* preempt_schedule <- NOP
8290
+
* preempt_schedule_notrace <- NOP
8291
+
* irqentry_exit_cond_resched <- NOP
8292
+
*
8293
+
* VOLUNTARY:
8294
+
* cond_resched <- __cond_resched
8295
+
* might_resched <- __cond_resched
8296
+
* preempt_schedule <- NOP
8297
+
* preempt_schedule_notrace <- NOP
8298
+
* irqentry_exit_cond_resched <- NOP
8299
+
*
8300
+
* FULL:
8301
+
* cond_resched <- RET0
8302
+
* might_resched <- RET0
8303
+
* preempt_schedule <- preempt_schedule
8304
+
* preempt_schedule_notrace <- preempt_schedule_notrace
8305
+
* irqentry_exit_cond_resched <- irqentry_exit_cond_resched
8306
+
*/
8307
+
8308
+
enum {
8309
+
preempt_dynamic_undefined = -1,
8310
+
preempt_dynamic_none,
8311
+
preempt_dynamic_voluntary,
8312
+
preempt_dynamic_full,
8313
+
};
8314
+
8315
+
int preempt_dynamic_mode = preempt_dynamic_undefined;
8316
+
8317
+
int sched_dynamic_mode(const char *str)
8318
+
{
8319
+
if (!strcmp(str, "none"))
8320
+
return preempt_dynamic_none;
8321
+
8322
+
if (!strcmp(str, "voluntary"))
8323
+
return preempt_dynamic_voluntary;
8324
+
8325
+
if (!strcmp(str, "full"))
8326
+
return preempt_dynamic_full;
8327
+
8328
+
return -EINVAL;
8329
+
}
8330
+
8331
+
void sched_dynamic_update(int mode)
8332
+
{
8333
+
/*
8334
+
* Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
8335
+
* the ZERO state, which is invalid.
8336
+
*/
8337
+
static_call_update(cond_resched, __cond_resched);
8338
+
static_call_update(might_resched, __cond_resched);
8339
+
static_call_update(preempt_schedule, __preempt_schedule_func);
8340
+
static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func);
8341
+
static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
8342
+
8343
+
switch (mode) {
8344
+
case preempt_dynamic_none:
8345
+
static_call_update(cond_resched, __cond_resched);
8346
+
static_call_update(might_resched, (void *)&__static_call_return0);
8347
+
static_call_update(preempt_schedule, NULL);
8348
+
static_call_update(preempt_schedule_notrace, NULL);
8349
+
static_call_update(irqentry_exit_cond_resched, NULL);
8350
+
pr_info("Dynamic Preempt: none\n");
8351
+
break;
8352
+
8353
+
case preempt_dynamic_voluntary:
8354
+
static_call_update(cond_resched, __cond_resched);
8355
+
static_call_update(might_resched, __cond_resched);
8356
+
static_call_update(preempt_schedule, NULL);
8357
+
static_call_update(preempt_schedule_notrace, NULL);
8358
+
static_call_update(irqentry_exit_cond_resched, NULL);
8359
+
pr_info("Dynamic Preempt: voluntary\n");
8360
+
break;
8361
+
8362
+
case preempt_dynamic_full:
8363
+
static_call_update(cond_resched, (void *)&__static_call_return0);
8364
+
static_call_update(might_resched, (void *)&__static_call_return0);
8365
+
static_call_update(preempt_schedule, __preempt_schedule_func);
8366
+
static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func);
8367
+
static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
8368
+
pr_info("Dynamic Preempt: full\n");
8369
+
break;
8370
+
}
8371
+
8372
+
preempt_dynamic_mode = mode;
8373
+
}
8374
+
8375
+
static int __init setup_preempt_mode(char *str)
8376
+
{
8377
+
int mode = sched_dynamic_mode(str);
8378
+
if (mode < 0) {
8379
+
pr_warn("Dynamic Preempt: unsupported mode: %s\n", str);
8380
+
return 0;
8381
+
}
8382
+
8383
+
sched_dynamic_update(mode);
8384
+
return 1;
8385
+
}
8386
+
__setup("preempt=", setup_preempt_mode);
8387
+
8388
+
static void __init preempt_dynamic_init(void)
8389
+
{
8390
+
if (preempt_dynamic_mode == preempt_dynamic_undefined) {
8391
+
if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
8392
+
sched_dynamic_update(preempt_dynamic_none);
8393
+
} else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
8394
+
sched_dynamic_update(preempt_dynamic_voluntary);
8395
+
} else {
8396
+
/* Default static call setting, nothing to do */
8397
+
WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
8398
+
preempt_dynamic_mode = preempt_dynamic_full;
8399
+
pr_info("Dynamic Preempt: full\n");
8400
+
}
8401
+
}
8402
+
}
8403
+
8404
+
#else /* !CONFIG_PREEMPT_DYNAMIC */
8405
+
8406
+
static inline void preempt_dynamic_init(void) { }
8407
+
8408
+
#endif /* #ifdef CONFIG_PREEMPT_DYNAMIC */
8137
8409
8138
8410
/**
8139
8411
* yield - yield the current processor to other threads.