sched: Rework pick_next_task() · tjh.dev/kernel@fd03c5b

+15 -6

kernel/sched/core.c

··· 5893 5893 5894 5894 /* Assume the next prioritized class is idle_sched_class */ 5895 5895 if (!p) { 5896 + p = pick_task_idle(rq); 5896 5897 put_prev_task(rq, prev); 5897 - p = pick_next_task_idle(rq); 5898 + set_next_task_first(rq, p); 5898 5899 } 5899 5900 5900 5901 /* ··· 5917 5916 5918 5917 restart: 5919 5918 prev_balance(rq, prev, rf); 5920 - put_prev_task(rq, prev); 5921 5919 5922 5920 for_each_class(class) { 5923 - p = class->pick_next_task(rq); 5924 - if (p) 5925 - return p; 5921 + if (class->pick_next_task) { 5922 + p = class->pick_next_task(rq, prev); 5923 + if (p) 5924 + return p; 5925 + } else { 5926 + p = class->pick_task(rq); 5927 + if (p) { 5928 + put_prev_task(rq, prev); 5929 + set_next_task_first(rq, p); 5930 + return p; 5931 + } 5932 + } 5926 5933 } 5927 5934 5928 5935 BUG(); /* The idle class should always have a runnable task. */ ··· 6026 6017 } 6027 6018 6028 6019 prev_balance(rq, prev, rf); 6029 - put_prev_task(rq, prev); 6030 6020 6031 6021 smt_mask = cpu_smt_mask(cpu); 6032 6022 need_sync = !!rq->core->core_cookie; ··· 6192 6184 } 6193 6185 6194 6186 out_set_next: 6187 + put_prev_task(rq, prev); 6195 6188 set_next_task_first(rq, next); 6196 6189 out: 6197 6190 if (rq->core->core_forceidle_count && next == rq->idle)

+1 -20

kernel/sched/deadline.c

··· 2431 2431 return p; 2432 2432 } 2433 2433 2434 - #ifdef CONFIG_SMP 2435 2434 static struct task_struct *pick_task_dl(struct rq *rq) 2436 2435 { 2437 2436 return __pick_task_dl(rq); 2438 - } 2439 - #endif 2440 - 2441 - static struct task_struct *pick_next_task_dl(struct rq *rq) 2442 - { 2443 - struct task_struct *p; 2444 - 2445 - p = __pick_task_dl(rq); 2446 - if (!p) 2447 - return p; 2448 - 2449 - if (p->dl_server) 2450 - p->sched_class->set_next_task(rq, p, true); 2451 - else 2452 - set_next_task_dl(rq, p, true); 2453 - 2454 - return p; 2455 2437 } 2456 2438 2457 2439 static void put_prev_task_dl(struct rq *rq, struct task_struct *p) ··· 3128 3146 3129 3147 .wakeup_preempt = wakeup_preempt_dl, 3130 3148 3131 - .pick_next_task = pick_next_task_dl, 3149 + .pick_task = pick_task_dl, 3132 3150 .put_prev_task = put_prev_task_dl, 3133 3151 .set_next_task = set_next_task_dl, 3134 3152 3135 3153 #ifdef CONFIG_SMP 3136 3154 .balance = balance_dl, 3137 - .pick_task = pick_task_dl, 3138 3155 .select_task_rq = select_task_rq_dl, 3139 3156 .migrate_task_rq = migrate_task_rq_dl, 3140 3157 .set_cpus_allowed = set_cpus_allowed_dl,

+5 -6

kernel/sched/fair.c

··· 8777 8777 se = &p->se; 8778 8778 8779 8779 #ifdef CONFIG_FAIR_GROUP_SCHED 8780 - if (!prev || prev->sched_class != &fair_sched_class) 8780 + if (prev->sched_class != &fair_sched_class) 8781 8781 goto simple; 8782 8782 8783 8783 /* ··· 8819 8819 8820 8820 simple: 8821 8821 #endif 8822 - if (prev) 8823 - put_prev_task(rq, prev); 8822 + put_prev_task(rq, prev); 8824 8823 set_next_task_fair(rq, p, true); 8825 8824 return p; 8826 8825 ··· 8849 8850 return NULL; 8850 8851 } 8851 8852 8852 - static struct task_struct *__pick_next_task_fair(struct rq *rq) 8853 + static struct task_struct *__pick_next_task_fair(struct rq *rq, struct task_struct *prev) 8853 8854 { 8854 - return pick_next_task_fair(rq, NULL, NULL); 8855 + return pick_next_task_fair(rq, prev, NULL); 8855 8856 } 8856 8857 8857 8858 static bool fair_server_has_tasks(struct sched_dl_entity *dl_se) ··· 13489 13490 13490 13491 .wakeup_preempt = check_preempt_wakeup_fair, 13491 13492 13493 + .pick_task = pick_task_fair, 13492 13494 .pick_next_task = __pick_next_task_fair, 13493 13495 .put_prev_task = put_prev_task_fair, 13494 13496 .set_next_task = set_next_task_fair, 13495 13497 13496 13498 #ifdef CONFIG_SMP 13497 13499 .balance = balance_fair, 13498 - .pick_task = pick_task_fair, 13499 13500 .select_task_rq = select_task_rq_fair, 13500 13501 .migrate_task_rq = migrate_task_rq_fair, 13501 13502

+2 -14

kernel/sched/idle.c

··· 462 462 next->se.exec_start = rq_clock_task(rq); 463 463 } 464 464 465 - #ifdef CONFIG_SMP 466 - static struct task_struct *pick_task_idle(struct rq *rq) 465 + struct task_struct *pick_task_idle(struct rq *rq) 467 466 { 468 467 return rq->idle; 469 - } 470 - #endif 471 - 472 - struct task_struct *pick_next_task_idle(struct rq *rq) 473 - { 474 - struct task_struct *next = rq->idle; 475 - 476 - set_next_task_idle(rq, next, true); 477 - 478 - return next; 479 468 } 480 469 481 470 /* ··· 520 531 521 532 .wakeup_preempt = wakeup_preempt_idle, 522 533 523 - .pick_next_task = pick_next_task_idle, 534 + .pick_task = pick_task_idle, 524 535 .put_prev_task = put_prev_task_idle, 525 536 .set_next_task = set_next_task_idle, 526 537 527 538 #ifdef CONFIG_SMP 528 539 .balance = balance_idle, 529 - .pick_task = pick_task_idle, 530 540 .select_task_rq = select_task_rq_idle, 531 541 .set_cpus_allowed = set_cpus_allowed_common, 532 542 #endif

+1 -12

kernel/sched/rt.c

··· 1748 1748 return p; 1749 1749 } 1750 1750 1751 - static struct task_struct *pick_next_task_rt(struct rq *rq) 1752 - { 1753 - struct task_struct *p = pick_task_rt(rq); 1754 - 1755 - if (p) 1756 - set_next_task_rt(rq, p, true); 1757 - 1758 - return p; 1759 - } 1760 - 1761 1751 static void put_prev_task_rt(struct rq *rq, struct task_struct *p) 1762 1752 { 1763 1753 struct sched_rt_entity *rt_se = &p->rt; ··· 2635 2645 2636 2646 .wakeup_preempt = wakeup_preempt_rt, 2637 2647 2638 - .pick_next_task = pick_next_task_rt, 2648 + .pick_task = pick_task_rt, 2639 2649 .put_prev_task = put_prev_task_rt, 2640 2650 .set_next_task = set_next_task_rt, 2641 2651 2642 2652 #ifdef CONFIG_SMP 2643 2653 .balance = balance_rt, 2644 - .pick_task = pick_task_rt, 2645 2654 .select_task_rq = select_task_rq_rt, 2646 2655 .set_cpus_allowed = set_cpus_allowed_common, 2647 2656 .rq_online = rq_online_rt,

+12 -4

kernel/sched/sched.h

··· 2300 2300 2301 2301 void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags); 2302 2302 2303 - struct task_struct *(*pick_next_task)(struct rq *rq); 2303 + struct task_struct *(*pick_task)(struct rq *rq); 2304 + /* 2305 + * Optional! When implemented pick_next_task() should be equivalent to: 2306 + * 2307 + * next = pick_task(); 2308 + * if (next) { 2309 + * put_prev_task(prev); 2310 + * set_next_task_first(next); 2311 + * } 2312 + */ 2313 + struct task_struct *(*pick_next_task)(struct rq *rq, struct task_struct *prev); 2304 2314 2305 2315 void (*put_prev_task)(struct rq *rq, struct task_struct *p); 2306 2316 void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); ··· 2318 2308 #ifdef CONFIG_SMP 2319 2309 int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); 2320 2310 int (*select_task_rq)(struct task_struct *p, int task_cpu, int flags); 2321 - 2322 - struct task_struct * (*pick_task)(struct rq *rq); 2323 2311 2324 2312 void (*migrate_task_rq)(struct task_struct *p, int new_cpu); 2325 2313 ··· 2429 2421 } 2430 2422 2431 2423 extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); 2432 - extern struct task_struct *pick_next_task_idle(struct rq *rq); 2424 + extern struct task_struct *pick_task_idle(struct rq *rq); 2433 2425 2434 2426 #define SCA_CHECK 0x01 2435 2427 #define SCA_MIGRATE_DISABLE 0x02

+1 -12

kernel/sched/stop_task.c

··· 41 41 return rq->stop; 42 42 } 43 43 44 - static struct task_struct *pick_next_task_stop(struct rq *rq) 45 - { 46 - struct task_struct *p = pick_task_stop(rq); 47 - 48 - if (p) 49 - set_next_task_stop(rq, p, true); 50 - 51 - return p; 52 - } 53 - 54 44 static void 55 45 enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) 56 46 { ··· 102 112 103 113 .wakeup_preempt = wakeup_preempt_stop, 104 114 105 - .pick_next_task = pick_next_task_stop, 115 + .pick_task = pick_task_stop, 106 116 .put_prev_task = put_prev_task_stop, 107 117 .set_next_task = set_next_task_stop, 108 118 109 119 #ifdef CONFIG_SMP 110 120 .balance = balance_stop, 111 - .pick_task = pick_task_stop, 112 121 .select_task_rq = select_task_rq_stop, 113 122 .set_cpus_allowed = set_cpus_allowed_common, 114 123 #endif

Configure Feed

Configure Feed