tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

sched/core: Introduce SM_IDLE and an idle re-entry fast-path in __schedule()

Since commit b2a02fc43a1f ("smp: Optimize
send_call_function_single_ipi()") an idle CPU in TIF_POLLING_NRFLAG mode
can be pulled out of idle by setting TIF_NEED_RESCHED flag to service an
IPI without actually sending an interrupt. Even in cases where the IPI
handler does not queue a task on the idle CPU, do_idle() will call
__schedule() since need_resched() returns true in these cases.

Introduce and use SM_IDLE to identify call to __schedule() from
schedule_idle() and shorten the idle re-entry time by skipping
pick_next_task() when nr_running is 0 and the previous task is the idle
task.

With the SM_IDLE fast-path, the time taken to complete a fixed set of
IPIs using ipistorm improves noticeably. Following are the numbers
from a dual socket Intel Ice Lake Xeon server (2 x 32C/64T) and
3rd Generation AMD EPYC system (2 x 64C/128T) (boost on, C2 disabled)
running ipistorm between CPU8 and CPU16:

cmdline: insmod ipistorm.ko numipi=100000 single=1 offset=8 cpulist=8 wait=1

==================================================================
Test : ipistorm (modified)
Units : Normalized runtime
Interpretation: Lower is better
Statistic : AMean
======================= Intel Ice Lake Xeon ======================
kernel: time [pct imp]
tip:sched/core 1.00 [baseline]
tip:sched/core + SM_IDLE 0.80 [20.51%]
==================== 3rd Generation AMD EPYC =====================
kernel: time [pct imp]
tip:sched/core 1.00 [baseline]
tip:sched/core + SM_IDLE 0.90 [10.17%]
==================================================================

[ kprateek: Commit message, SM_RTLOCK_WAIT fix ]

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Not-yet-signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lore.kernel.org/r/20240809092240.6921-1-kprateek.nayak@amd.com

Peter Zijlstra 3dcac251 b2d70222

+26 -19
+26 -19
kernel/sched/core.c
··· 6410 6410 * Constants for the sched_mode argument of __schedule(). 6411 6411 * 6412 6412 * The mode argument allows RT enabled kernels to differentiate a 6413 - * preemption from blocking on an 'sleeping' spin/rwlock. Note that 6414 - * SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to 6415 - * optimize the AND operation out and just check for zero. 6413 + * preemption from blocking on an 'sleeping' spin/rwlock. 6416 6414 */ 6417 - #define SM_NONE 0x0 6418 - #define SM_PREEMPT 0x1 6419 - #define SM_RTLOCK_WAIT 0x2 6420 - 6421 - #ifndef CONFIG_PREEMPT_RT 6422 - # define SM_MASK_PREEMPT (~0U) 6423 - #else 6424 - # define SM_MASK_PREEMPT SM_PREEMPT 6425 - #endif 6415 + #define SM_IDLE (-1) 6416 + #define SM_NONE 0 6417 + #define SM_PREEMPT 1 6418 + #define SM_RTLOCK_WAIT 2 6426 6419 6427 6420 /* 6428 6421 * __schedule() is the main scheduler function. ··· 6456 6463 * 6457 6464 * WARNING: must be called with preemption disabled! 6458 6465 */ 6459 - static void __sched notrace __schedule(unsigned int sched_mode) 6466 + static void __sched notrace __schedule(int sched_mode) 6460 6467 { 6461 6468 struct task_struct *prev, *next; 6469 + /* 6470 + * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted 6471 + * as a preemption by schedule_debug() and RCU. 6472 + */ 6473 + bool preempt = sched_mode > SM_NONE; 6462 6474 unsigned long *switch_count; 6463 6475 unsigned long prev_state; 6464 6476 struct rq_flags rf; ··· 6474 6476 rq = cpu_rq(cpu); 6475 6477 prev = rq->curr; 6476 6478 6477 - schedule_debug(prev, !!sched_mode); 6479 + schedule_debug(prev, preempt); 6478 6480 6479 6481 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) 6480 6482 hrtick_clear(rq); 6481 6483 6482 6484 local_irq_disable(); 6483 - rcu_note_context_switch(!!sched_mode); 6485 + rcu_note_context_switch(preempt); 6484 6486 6485 6487 /* 6486 6488 * Make sure that signal_pending_state()->signal_pending() below ··· 6509 6511 6510 6512 switch_count = &prev->nivcsw; 6511 6513 6514 + /* Task state changes only considers SM_PREEMPT as preemption */ 6515 + preempt = sched_mode == SM_PREEMPT; 6516 + 6512 6517 /* 6513 6518 * We must load prev->state once (task_struct::state is volatile), such 6514 6519 * that we form a control dependency vs deactivate_task() below. 6515 6520 */ 6516 6521 prev_state = READ_ONCE(prev->__state); 6517 - if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) { 6522 + if (sched_mode == SM_IDLE) { 6523 + if (!rq->nr_running) { 6524 + next = prev; 6525 + goto picked; 6526 + } 6527 + } else if (!preempt && prev_state) { 6518 6528 if (signal_pending_state(prev_state, prev)) { 6519 6529 WRITE_ONCE(prev->__state, TASK_RUNNING); 6520 6530 } else { ··· 6553 6547 } 6554 6548 6555 6549 next = pick_next_task(rq, prev, &rf); 6550 + picked: 6556 6551 clear_tsk_need_resched(prev); 6557 6552 clear_preempt_need_resched(); 6558 6553 #ifdef CONFIG_SCHED_DEBUG ··· 6595 6588 psi_account_irqtime(rq, prev, next); 6596 6589 psi_sched_switch(prev, next, !task_on_rq_queued(prev)); 6597 6590 6598 - trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); 6591 + trace_sched_switch(preempt, prev, next, prev_state); 6599 6592 6600 6593 /* Also unlocks the rq: */ 6601 6594 rq = context_switch(rq, prev, next, &rf); ··· 6671 6664 } 6672 6665 } 6673 6666 6674 - static __always_inline void __schedule_loop(unsigned int sched_mode) 6667 + static __always_inline void __schedule_loop(int sched_mode) 6675 6668 { 6676 6669 do { 6677 6670 preempt_disable(); ··· 6716 6709 */ 6717 6710 WARN_ON_ONCE(current->__state); 6718 6711 do { 6719 - __schedule(SM_NONE); 6712 + __schedule(SM_IDLE); 6720 6713 } while (need_resched()); 6721 6714 } 6722 6715