Merge branch 'tip/sched/core' into for-6.12

-1

include/linux/sched.h

··· 694 694 */ 695 695 struct rq *rq; 696 696 dl_server_has_tasks_f server_has_tasks; 697 - dl_server_pick_f server_pick_next; 698 697 dl_server_pick_f server_pick_task; 699 698 700 699 #ifdef CONFIG_RT_MUTEXES

+42 -47

kernel/sched/core.c

··· 3690 3690 rq->idle_stamp = 0; 3691 3691 } 3692 3692 #endif 3693 - 3694 - p->dl_server = NULL; 3695 3693 } 3696 3694 3697 3695 /* ··· 5893 5895 schedstat_inc(this_rq()->sched_count); 5894 5896 } 5895 5897 5896 - static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, 5897 - struct rq_flags *rf) 5898 + static void prev_balance(struct rq *rq, struct task_struct *prev, 5899 + struct rq_flags *rf) 5898 5900 { 5899 5901 const struct sched_class *start_class = prev->sched_class; 5900 5902 const struct sched_class *class; ··· 5921 5923 if (class->balance && class->balance(rq, prev, rf)) 5922 5924 break; 5923 5925 } 5924 - 5925 - put_prev_task(rq, prev); 5926 - 5927 - /* 5928 - * We've updated @prev and no longer need the server link, clear it. 5929 - * Must be done before ->pick_next_task() because that can (re)set 5930 - * ->dl_server. 5931 - */ 5932 - if (prev->dl_server) 5933 - prev->dl_server = NULL; 5934 5926 } 5935 5927 5936 5928 /* ··· 5931 5943 { 5932 5944 const struct sched_class *class; 5933 5945 struct task_struct *p; 5946 + 5947 + rq->dl_server = NULL; 5934 5948 5935 5949 if (scx_enabled()) 5936 5950 goto restart; ··· 5952 5962 5953 5963 /* Assume the next prioritized class is idle_sched_class */ 5954 5964 if (!p) { 5955 - put_prev_task(rq, prev); 5956 - p = pick_next_task_idle(rq); 5965 + p = pick_task_idle(rq); 5966 + put_prev_set_next_task(rq, prev, p); 5957 5967 } 5958 - 5959 - /* 5960 - * This is a normal CFS pick, but the previous could be a DL pick. 5961 - * Clear it as previous is no longer picked. 5962 - */ 5963 - if (prev->dl_server) 5964 - prev->dl_server = NULL; 5965 - 5966 - /* 5967 - * This is the fast path; it cannot be a DL server pick; 5968 - * therefore even if @p == @prev, ->dl_server must be NULL. 5969 - */ 5970 - if (p->dl_server) 5971 - p->dl_server = NULL; 5972 5968 5973 5969 return p; 5974 5970 } 5975 5971 5976 5972 restart: 5977 - put_prev_task_balance(rq, prev, rf); 5973 + prev_balance(rq, prev, rf); 5978 5974 5979 5975 for_each_active_class(class) { 5980 - p = class->pick_next_task(rq); 5981 - if (p) { 5982 - const struct sched_class *prev_class = prev->sched_class; 5976 + if (class->pick_next_task) { 5977 + p = class->pick_next_task(rq, prev); 5978 + if (p) { 5979 + const struct sched_class *prev_class = prev->sched_class; 5983 5980 5984 - if (class != prev_class && prev_class->switch_class) 5985 - prev_class->switch_class(rq, p); 5986 - return p; 5981 + if (class != prev_class && prev_class->switch_class) 5982 + prev_class->switch_class(rq, p); 5983 + return p; 5984 + } 5985 + } else { 5986 + p = class->pick_task(rq); 5987 + if (p) { 5988 + const struct sched_class *prev_class = prev->sched_class; 5989 + 5990 + put_prev_set_next_task(rq, prev, p); 5991 + 5992 + if (class != prev_class && prev_class->switch_class) 5993 + prev_class->switch_class(rq, p); 5994 + return p; 5995 + } 5987 5996 } 5988 5997 } 5989 5998 ··· 6012 6023 { 6013 6024 const struct sched_class *class; 6014 6025 struct task_struct *p; 6026 + 6027 + rq->dl_server = NULL; 6015 6028 6016 6029 for_each_active_class(class) { 6017 6030 p = class->pick_task(rq); ··· 6053 6062 * another cpu during offline. 6054 6063 */ 6055 6064 rq->core_pick = NULL; 6065 + rq->core_dl_server = NULL; 6056 6066 return __pick_next_task(rq, prev, rf); 6057 6067 } 6058 6068 ··· 6072 6080 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); 6073 6081 6074 6082 next = rq->core_pick; 6075 - if (next != prev) { 6076 - put_prev_task(rq, prev); 6077 - set_next_task(rq, next); 6078 - } 6079 - 6083 + rq->dl_server = rq->core_dl_server; 6080 6084 rq->core_pick = NULL; 6081 - goto out; 6085 + rq->core_dl_server = NULL; 6086 + goto out_set_next; 6082 6087 } 6083 6088 6084 - put_prev_task_balance(rq, prev, rf); 6089 + prev_balance(rq, prev, rf); 6085 6090 6086 6091 smt_mask = cpu_smt_mask(cpu); 6087 6092 need_sync = !!rq->core->core_cookie; ··· 6119 6130 next = pick_task(rq); 6120 6131 if (!next->core_cookie) { 6121 6132 rq->core_pick = NULL; 6133 + rq->core_dl_server = NULL; 6122 6134 /* 6123 6135 * For robustness, update the min_vruntime_fi for 6124 6136 * unconstrained picks as well. ··· 6147 6157 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) 6148 6158 update_rq_clock(rq_i); 6149 6159 6150 - p = rq_i->core_pick = pick_task(rq_i); 6160 + rq_i->core_pick = p = pick_task(rq_i); 6161 + rq_i->core_dl_server = rq_i->dl_server; 6162 + 6151 6163 if (!max || prio_less(max, p, fi_before)) 6152 6164 max = p; 6153 6165 } ··· 6173 6181 } 6174 6182 6175 6183 rq_i->core_pick = p; 6184 + rq_i->core_dl_server = NULL; 6176 6185 6177 6186 if (p == rq_i->idle) { 6178 6187 if (rq_i->nr_running) { ··· 6234 6241 6235 6242 if (i == cpu) { 6236 6243 rq_i->core_pick = NULL; 6244 + rq_i->core_dl_server = NULL; 6237 6245 continue; 6238 6246 } 6239 6247 ··· 6243 6249 6244 6250 if (rq_i->curr == rq_i->core_pick) { 6245 6251 rq_i->core_pick = NULL; 6252 + rq_i->core_dl_server = NULL; 6246 6253 continue; 6247 6254 } 6248 6255 ··· 6251 6256 } 6252 6257 6253 6258 out_set_next: 6254 - set_next_task(rq, next); 6255 - out: 6259 + put_prev_set_next_task(rq, prev, next); 6256 6260 if (rq->core->core_forceidle_count && next == rq->idle) 6257 6261 queue_core_balance(rq); 6258 6262 ··· 8481 8487 #ifdef CONFIG_SCHED_CORE 8482 8488 rq->core = rq; 8483 8489 rq->core_pick = NULL; 8490 + rq->core_dl_server = NULL; 8484 8491 rq->core_enabled = 0; 8485 8492 rq->core_tree = RB_ROOT; 8486 8493 rq->core_forceidle_count = 0;

+25 -54

kernel/sched/deadline.c

··· 1665 1665 1666 1666 void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq, 1667 1667 dl_server_has_tasks_f has_tasks, 1668 - dl_server_pick_f pick_next, 1669 1668 dl_server_pick_f pick_task) 1670 1669 { 1671 1670 dl_se->rq = rq; 1672 1671 dl_se->server_has_tasks = has_tasks; 1673 - dl_se->server_pick_next = pick_next; 1674 1672 dl_se->server_pick_task = pick_task; 1675 1673 } 1676 1674 ··· 1894 1896 return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline); 1895 1897 } 1896 1898 1897 - static inline struct sched_statistics * 1899 + static __always_inline struct sched_statistics * 1898 1900 __schedstats_from_dl_se(struct sched_dl_entity *dl_se) 1899 1901 { 1902 + if (!schedstat_enabled()) 1903 + return NULL; 1904 + 1905 + if (dl_server(dl_se)) 1906 + return NULL; 1907 + 1900 1908 return &dl_task_of(dl_se)->stats; 1901 1909 } 1902 1910 1903 1911 static inline void 1904 1912 update_stats_wait_start_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se) 1905 1913 { 1906 - struct sched_statistics *stats; 1907 - 1908 - if (!schedstat_enabled()) 1909 - return; 1910 - 1911 - stats = __schedstats_from_dl_se(dl_se); 1912 - __update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); 1914 + struct sched_statistics *stats = __schedstats_from_dl_se(dl_se); 1915 + if (stats) 1916 + __update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); 1913 1917 } 1914 1918 1915 1919 static inline void 1916 1920 update_stats_wait_end_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se) 1917 1921 { 1918 - struct sched_statistics *stats; 1919 - 1920 - if (!schedstat_enabled()) 1921 - return; 1922 - 1923 - stats = __schedstats_from_dl_se(dl_se); 1924 - __update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); 1922 + struct sched_statistics *stats = __schedstats_from_dl_se(dl_se); 1923 + if (stats) 1924 + __update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); 1925 1925 } 1926 1926 1927 1927 static inline void 1928 1928 update_stats_enqueue_sleeper_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se) 1929 1929 { 1930 - struct sched_statistics *stats; 1931 - 1932 - if (!schedstat_enabled()) 1933 - return; 1934 - 1935 - stats = __schedstats_from_dl_se(dl_se); 1936 - __update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); 1930 + struct sched_statistics *stats = __schedstats_from_dl_se(dl_se); 1931 + if (stats) 1932 + __update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); 1937 1933 } 1938 1934 1939 1935 static inline void ··· 2384 2392 update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0); 2385 2393 2386 2394 deadline_queue_push_tasks(rq); 2395 + 2396 + if (hrtick_enabled(rq)) 2397 + start_hrtick_dl(rq, &p->dl); 2387 2398 } 2388 2399 2389 2400 static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq) ··· 2402 2407 /* 2403 2408 * __pick_next_task_dl - Helper to pick the next -deadline task to run. 2404 2409 * @rq: The runqueue to pick the next task from. 2405 - * @peek: If true, just peek at the next task. Only relevant for dlserver. 2406 2410 */ 2407 - static struct task_struct *__pick_next_task_dl(struct rq *rq, bool peek) 2411 + static struct task_struct *__pick_task_dl(struct rq *rq) 2408 2412 { 2409 2413 struct sched_dl_entity *dl_se; 2410 2414 struct dl_rq *dl_rq = &rq->dl; ··· 2417 2423 WARN_ON_ONCE(!dl_se); 2418 2424 2419 2425 if (dl_server(dl_se)) { 2420 - if (IS_ENABLED(CONFIG_SMP) && peek) 2421 - p = dl_se->server_pick_task(dl_se); 2422 - else 2423 - p = dl_se->server_pick_next(dl_se); 2426 + p = dl_se->server_pick_task(dl_se); 2424 2427 if (!p) { 2425 2428 dl_se->dl_yielded = 1; 2426 2429 update_curr_dl_se(rq, dl_se, 0); 2427 2430 goto again; 2428 2431 } 2429 - p->dl_server = dl_se; 2432 + rq->dl_server = dl_se; 2430 2433 } else { 2431 2434 p = dl_task_of(dl_se); 2432 2435 } ··· 2431 2440 return p; 2432 2441 } 2433 2442 2434 - #ifdef CONFIG_SMP 2435 2443 static struct task_struct *pick_task_dl(struct rq *rq) 2436 2444 { 2437 - return __pick_next_task_dl(rq, true); 2438 - } 2439 - #endif 2440 - 2441 - static struct task_struct *pick_next_task_dl(struct rq *rq) 2442 - { 2443 - struct task_struct *p; 2444 - 2445 - p = __pick_next_task_dl(rq, false); 2446 - if (!p) 2447 - return p; 2448 - 2449 - if (!p->dl_server) 2450 - set_next_task_dl(rq, p, true); 2451 - 2452 - if (hrtick_enabled(rq)) 2453 - start_hrtick_dl(rq, &p->dl); 2454 - 2455 - return p; 2445 + return __pick_task_dl(rq); 2456 2446 } 2457 2447 2458 - static void put_prev_task_dl(struct rq *rq, struct task_struct *p) 2448 + static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_struct *next) 2459 2449 { 2460 2450 struct sched_dl_entity *dl_se = &p->dl; 2461 2451 struct dl_rq *dl_rq = &rq->dl; ··· 3128 3156 3129 3157 .wakeup_preempt = wakeup_preempt_dl, 3130 3158 3131 - .pick_next_task = pick_next_task_dl, 3159 + .pick_task = pick_task_dl, 3132 3160 .put_prev_task = put_prev_task_dl, 3133 3161 .set_next_task = set_next_task_dl, 3134 3162 3135 3163 #ifdef CONFIG_SMP 3136 3164 .balance = balance_dl, 3137 - .pick_task = pick_task_dl, 3138 3165 .select_task_rq = select_task_rq_dl, 3139 3166 .migrate_task_rq = migrate_task_rq_dl, 3140 3167 .set_cpus_allowed = set_cpus_allowed_dl,

+10 -2

kernel/sched/ext.c

··· 2719 2719 } 2720 2720 } 2721 2721 2722 - static void put_prev_task_scx(struct rq *rq, struct task_struct *p) 2722 + static void put_prev_task_scx(struct rq *rq, struct task_struct *p, 2723 + struct task_struct *next) 2723 2724 { 2724 2725 update_curr_scx(rq); 2725 2726 ··· 2775 2774 struct task_struct, scx.dsq_list.node); 2776 2775 } 2777 2776 2778 - static struct task_struct *pick_next_task_scx(struct rq *rq) 2777 + static struct task_struct *pick_next_task_scx(struct rq *rq, 2778 + struct task_struct *prev) 2779 2779 { 2780 2780 struct task_struct *p; 2781 + 2782 + if (prev->sched_class == &ext_sched_class) 2783 + put_prev_task_scx(rq, prev, NULL); 2781 2784 2782 2785 p = first_local_task(rq); 2783 2786 if (!p) 2784 2787 return NULL; 2788 + 2789 + if (prev->sched_class != &ext_sched_class) 2790 + prev->sched_class->put_prev_task(rq, prev, p); 2785 2791 2786 2792 set_next_task_scx(rq, p, true); 2787 2793

+56 -66

kernel/sched/fair.c

··· 5457 5457 5458 5458 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5459 5459 5460 + static inline void finish_delayed_dequeue_entity(struct sched_entity *se) 5461 + { 5462 + se->sched_delayed = 0; 5463 + if (sched_feat(DELAY_ZERO) && se->vlag > 0) 5464 + se->vlag = 0; 5465 + } 5466 + 5460 5467 static bool 5461 5468 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5462 5469 { ··· 5539 5532 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) 5540 5533 update_min_vruntime(cfs_rq); 5541 5534 5542 - if (flags & DEQUEUE_DELAYED) { 5543 - se->sched_delayed = 0; 5544 - if (sched_feat(DELAY_ZERO) && se->vlag > 0) 5545 - se->vlag = 0; 5546 - } 5535 + if (flags & DEQUEUE_DELAYED) 5536 + finish_delayed_dequeue_entity(se); 5547 5537 5548 5538 if (cfs_rq->nr_running == 0) 5549 5539 update_idle_cfs_rq_clock_pelt(cfs_rq); ··· 8750 8746 cfs_rq = group_cfs_rq(se); 8751 8747 } while (cfs_rq); 8752 8748 8753 - /* 8754 - * This can be called from directly from CFS's ->pick_task() or indirectly 8755 - * from DL's ->pick_task when fair server is enabled. In the indirect case, 8756 - * DL will set ->dl_server just after this function is called, so its Ok to 8757 - * clear. In the direct case, we are picking directly so we must clear it. 8758 - */ 8759 - task_of(se)->dl_server = NULL; 8760 - 8761 8749 return task_of(se); 8762 8750 } 8751 + 8752 + static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 8753 + static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 8763 8754 8764 8755 struct task_struct * 8765 8756 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) ··· 8770 8771 se = &p->se; 8771 8772 8772 8773 #ifdef CONFIG_FAIR_GROUP_SCHED 8773 - if (!prev || prev->sched_class != &fair_sched_class) 8774 + if (prev->sched_class != &fair_sched_class) 8774 8775 goto simple; 8776 + 8777 + __put_prev_set_next_dl_server(rq, prev, p); 8775 8778 8776 8779 /* 8777 8780 * Because of the set_next_buddy() in dequeue_task_fair() it is rather ··· 8806 8805 8807 8806 put_prev_entity(cfs_rq, pse); 8808 8807 set_next_entity(cfs_rq, se); 8808 + 8809 + __set_next_task_fair(rq, p, true); 8809 8810 } 8810 8811 8811 - goto done; 8812 + return p; 8813 + 8812 8814 simple: 8813 8815 #endif 8814 - if (prev) 8815 - put_prev_task(rq, prev); 8816 - 8817 - for_each_sched_entity(se) 8818 - set_next_entity(cfs_rq_of(se), se); 8819 - 8820 - done: __maybe_unused; 8821 - #ifdef CONFIG_SMP 8822 - /* 8823 - * Move the next running task to the front of 8824 - * the list, so our cfs_tasks list becomes MRU 8825 - * one. 8826 - */ 8827 - list_move(&p->se.group_node, &rq->cfs_tasks); 8828 - #endif 8829 - 8830 - if (hrtick_enabled_fair(rq)) 8831 - hrtick_start_fair(rq, p); 8832 - 8833 - update_misfit_status(p, rq); 8834 - sched_fair_update_stop_tick(rq, p); 8835 - 8816 + put_prev_set_next_task(rq, prev, p); 8836 8817 return p; 8837 8818 8838 8819 idle: ··· 8843 8860 return NULL; 8844 8861 } 8845 8862 8846 - static struct task_struct *__pick_next_task_fair(struct rq *rq) 8863 + static struct task_struct *__pick_next_task_fair(struct rq *rq, struct task_struct *prev) 8847 8864 { 8848 - return pick_next_task_fair(rq, NULL, NULL); 8865 + return pick_next_task_fair(rq, prev, NULL); 8849 8866 } 8850 8867 8851 8868 static bool fair_server_has_tasks(struct sched_dl_entity *dl_se) ··· 8855 8872 8856 8873 static struct task_struct *fair_server_pick_task(struct sched_dl_entity *dl_se) 8857 8874 { 8858 - #ifdef CONFIG_SMP 8859 8875 return pick_task_fair(dl_se->rq); 8860 - #else 8861 - return NULL; 8862 - #endif 8863 - } 8864 - 8865 - static struct task_struct *fair_server_pick_next(struct sched_dl_entity *dl_se) 8866 - { 8867 - return pick_next_task_fair(dl_se->rq, NULL, NULL); 8868 8876 } 8869 8877 8870 8878 void fair_server_init(struct rq *rq) ··· 8864 8890 8865 8891 init_dl_entity(dl_se); 8866 8892 8867 - dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_next, 8868 - fair_server_pick_task); 8869 - 8893 + dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_task); 8870 8894 } 8871 8895 8872 8896 /* 8873 8897 * Account for a descheduled task: 8874 8898 */ 8875 - static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) 8899 + static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) 8876 8900 { 8877 8901 struct sched_entity *se = &prev->se; 8878 8902 struct cfs_rq *cfs_rq; ··· 13070 13098 * and we cannot use DEQUEUE_DELAYED. 13071 13099 */ 13072 13100 if (p->se.sched_delayed) { 13101 + /* First, dequeue it from its new class' structures */ 13073 13102 dequeue_task(rq, p, DEQUEUE_NOCLOCK | DEQUEUE_SLEEP); 13074 - p->se.sched_delayed = 0; 13103 + /* 13104 + * Now, clean up the fair_sched_class side of things 13105 + * related to sched_delayed being true and that wasn't done 13106 + * due to the generic dequeue not using DEQUEUE_DELAYED. 13107 + */ 13108 + finish_delayed_dequeue_entity(&p->se); 13075 13109 p->se.rel_deadline = 0; 13076 - if (sched_feat(DELAY_ZERO) && p->se.vlag > 0) 13077 - p->se.vlag = 0; 13110 + __block_task(rq, p); 13078 13111 } 13079 13112 } 13080 13113 ··· 13104 13127 } 13105 13128 } 13106 13129 13107 - /* Account for a task changing its policy or group. 13108 - * 13109 - * This routine is mostly called to set cfs_rq->curr field when a task 13110 - * migrates between groups/classes. 13111 - */ 13112 - static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13130 + static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13113 13131 { 13114 13132 struct sched_entity *se = &p->se; 13115 13133 ··· 13117 13145 list_move(&se->group_node, &rq->cfs_tasks); 13118 13146 } 13119 13147 #endif 13148 + if (!first) 13149 + return; 13150 + 13151 + SCHED_WARN_ON(se->sched_delayed); 13152 + 13153 + if (hrtick_enabled_fair(rq)) 13154 + hrtick_start_fair(rq, p); 13155 + 13156 + update_misfit_status(p, rq); 13157 + sched_fair_update_stop_tick(rq, p); 13158 + } 13159 + 13160 + /* 13161 + * Account for a task changing its policy or group. 13162 + * 13163 + * This routine is mostly called to set cfs_rq->curr field when a task 13164 + * migrates between groups/classes. 13165 + */ 13166 + static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13167 + { 13168 + struct sched_entity *se = &p->se; 13120 13169 13121 13170 for_each_sched_entity(se) { 13122 13171 struct cfs_rq *cfs_rq = cfs_rq_of(se); ··· 13147 13154 account_cfs_rq_runtime(cfs_rq, 0); 13148 13155 } 13149 13156 13150 - if (!first) 13151 - return; 13152 - 13153 - SCHED_WARN_ON(se->sched_delayed); 13157 + __set_next_task_fair(rq, p, first); 13154 13158 } 13155 13159 13156 13160 void init_cfs_rq(struct cfs_rq *cfs_rq) ··· 13473 13483 13474 13484 .wakeup_preempt = check_preempt_wakeup_fair, 13475 13485 13486 + .pick_task = pick_task_fair, 13476 13487 .pick_next_task = __pick_next_task_fair, 13477 13488 .put_prev_task = put_prev_task_fair, 13478 13489 .set_next_task = set_next_task_fair, 13479 13490 13480 13491 #ifdef CONFIG_SMP 13481 13492 .balance = balance_fair, 13482 - .pick_task = pick_task_fair, 13483 13493 .select_task_rq = select_task_rq_fair, 13484 13494 .migrate_task_rq = migrate_task_rq_fair, 13485 13495

+3 -15

kernel/sched/idle.c

··· 450 450 resched_curr(rq); 451 451 } 452 452 453 - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) 453 + static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct task_struct *next) 454 454 { 455 455 dl_server_update_idle_time(rq, prev); 456 456 scx_update_idle(rq, false); ··· 464 464 next->se.exec_start = rq_clock_task(rq); 465 465 } 466 466 467 - #ifdef CONFIG_SMP 468 - static struct task_struct *pick_task_idle(struct rq *rq) 467 + struct task_struct *pick_task_idle(struct rq *rq) 469 468 { 470 469 return rq->idle; 471 - } 472 - #endif 473 - 474 - struct task_struct *pick_next_task_idle(struct rq *rq) 475 - { 476 - struct task_struct *next = rq->idle; 477 - 478 - set_next_task_idle(rq, next, true); 479 - 480 - return next; 481 470 } 482 471 483 472 /* ··· 522 533 523 534 .wakeup_preempt = wakeup_preempt_idle, 524 535 525 - .pick_next_task = pick_next_task_idle, 536 + .pick_task = pick_task_idle, 526 537 .put_prev_task = put_prev_task_idle, 527 538 .set_next_task = set_next_task_idle, 528 539 529 540 #ifdef CONFIG_SMP 530 541 .balance = balance_idle, 531 - .pick_task = pick_task_idle, 532 542 .select_task_rq = select_task_rq_idle, 533 543 .set_cpus_allowed = set_cpus_allowed_common, 534 544 #endif

+2 -13

kernel/sched/rt.c

··· 1748 1748 return p; 1749 1749 } 1750 1750 1751 - static struct task_struct *pick_next_task_rt(struct rq *rq) 1752 - { 1753 - struct task_struct *p = pick_task_rt(rq); 1754 - 1755 - if (p) 1756 - set_next_task_rt(rq, p, true); 1757 - 1758 - return p; 1759 - } 1760 - 1761 - static void put_prev_task_rt(struct rq *rq, struct task_struct *p) 1751 + static void put_prev_task_rt(struct rq *rq, struct task_struct *p, struct task_struct *next) 1762 1752 { 1763 1753 struct sched_rt_entity *rt_se = &p->rt; 1764 1754 struct rt_rq *rt_rq = &rq->rt; ··· 2635 2645 2636 2646 .wakeup_preempt = wakeup_preempt_rt, 2637 2647 2638 - .pick_next_task = pick_next_task_rt, 2648 + .pick_task = pick_task_rt, 2639 2649 .put_prev_task = put_prev_task_rt, 2640 2650 .set_next_task = set_next_task_rt, 2641 2651 2642 2652 #ifdef CONFIG_SMP 2643 2653 .balance = balance_rt, 2644 - .pick_task = pick_task_rt, 2645 2654 .select_task_rq = select_task_rq_rt, 2646 2655 .set_cpus_allowed = set_cpus_allowed_common, 2647 2656 .rq_online = rq_online_rt,

+40 -7

kernel/sched/sched.h

··· 389 389 extern void dl_server_stop(struct sched_dl_entity *dl_se); 390 390 extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq, 391 391 dl_server_has_tasks_f has_tasks, 392 - dl_server_pick_f pick_next, 393 392 dl_server_pick_f pick_task); 394 393 395 394 extern void dl_server_update_idle_time(struct rq *rq, ··· 1132 1133 unsigned int nr_uninterruptible; 1133 1134 1134 1135 struct task_struct __rcu *curr; 1136 + struct sched_dl_entity *dl_server; 1135 1137 struct task_struct *idle; 1136 1138 struct task_struct *stop; 1137 1139 unsigned long next_balance; ··· 1260 1260 /* per rq */ 1261 1261 struct rq *core; 1262 1262 struct task_struct *core_pick; 1263 + struct sched_dl_entity *core_dl_server; 1263 1264 unsigned int core_enabled; 1264 1265 unsigned int core_sched_seq; 1265 1266 struct rb_root core_tree; ··· 2369 2368 void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags); 2370 2369 2371 2370 int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); 2372 - struct task_struct *(*pick_next_task)(struct rq *rq); 2371 + struct task_struct *(*pick_task)(struct rq *rq); 2372 + /* 2373 + * Optional! When implemented pick_next_task() should be equivalent to: 2374 + * 2375 + * next = pick_task(); 2376 + * if (next) { 2377 + * put_prev_task(prev); 2378 + * set_next_task_first(next); 2379 + * } 2380 + */ 2381 + struct task_struct *(*pick_next_task)(struct rq *rq, struct task_struct *prev); 2373 2382 2374 - void (*put_prev_task)(struct rq *rq, struct task_struct *p); 2383 + void (*put_prev_task)(struct rq *rq, struct task_struct *p, struct task_struct *next); 2375 2384 void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); 2376 2385 2377 2386 void (*switch_class)(struct rq *rq, struct task_struct *next); 2378 2387 2379 2388 #ifdef CONFIG_SMP 2380 2389 int (*select_task_rq)(struct task_struct *p, int task_cpu, int flags); 2381 - 2382 - struct task_struct * (*pick_task)(struct rq *rq); 2383 2390 2384 2391 void (*migrate_task_rq)(struct task_struct *p, int new_cpu); 2385 2392 ··· 2435 2426 static inline void put_prev_task(struct rq *rq, struct task_struct *prev) 2436 2427 { 2437 2428 WARN_ON_ONCE(rq->curr != prev); 2438 - prev->sched_class->put_prev_task(rq, prev); 2429 + prev->sched_class->put_prev_task(rq, prev, NULL); 2439 2430 } 2440 2431 2441 2432 static inline void set_next_task(struct rq *rq, struct task_struct *next) ··· 2443 2434 next->sched_class->set_next_task(rq, next, false); 2444 2435 } 2445 2436 2437 + static inline void 2438 + __put_prev_set_next_dl_server(struct rq *rq, 2439 + struct task_struct *prev, 2440 + struct task_struct *next) 2441 + { 2442 + prev->dl_server = NULL; 2443 + next->dl_server = rq->dl_server; 2444 + rq->dl_server = NULL; 2445 + } 2446 + 2447 + static inline void put_prev_set_next_task(struct rq *rq, 2448 + struct task_struct *prev, 2449 + struct task_struct *next) 2450 + { 2451 + WARN_ON_ONCE(rq->curr != prev); 2452 + 2453 + __put_prev_set_next_dl_server(rq, prev, next); 2454 + 2455 + if (next == prev) 2456 + return; 2457 + 2458 + prev->sched_class->put_prev_task(rq, prev, next); 2459 + next->sched_class->set_next_task(rq, next, true); 2460 + } 2446 2461 2447 2462 /* 2448 2463 * Helper to define a sched_class instance; each one is placed in a separate ··· 2557 2524 } 2558 2525 2559 2526 extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); 2560 - extern struct task_struct *pick_next_task_idle(struct rq *rq); 2527 + extern struct task_struct *pick_task_idle(struct rq *rq); 2561 2528 2562 2529 #define SCA_CHECK 0x01 2563 2530 #define SCA_MIGRATE_DISABLE 0x02

+2 -13

kernel/sched/stop_task.c

··· 41 41 return rq->stop; 42 42 } 43 43 44 - static struct task_struct *pick_next_task_stop(struct rq *rq) 45 - { 46 - struct task_struct *p = pick_task_stop(rq); 47 - 48 - if (p) 49 - set_next_task_stop(rq, p, true); 50 - 51 - return p; 52 - } 53 - 54 44 static void 55 45 enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) 56 46 { ··· 59 69 BUG(); /* the stop task should never yield, its pointless. */ 60 70 } 61 71 62 - static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) 72 + static void put_prev_task_stop(struct rq *rq, struct task_struct *prev, struct task_struct *next) 63 73 { 64 74 update_curr_common(rq); 65 75 } ··· 102 112 103 113 .wakeup_preempt = wakeup_preempt_stop, 104 114 105 - .pick_next_task = pick_next_task_stop, 115 + .pick_task = pick_task_stop, 106 116 .put_prev_task = put_prev_task_stop, 107 117 .set_next_task = set_next_task_stop, 108 118 109 119 #ifdef CONFIG_SMP 110 120 .balance = balance_stop, 111 - .pick_task = pick_task_stop, 112 121 .select_task_rq = select_task_rq_stop, 113 122 .set_cpus_allowed = set_cpus_allowed_common, 114 123 #endif

Configure Feed

Configure Feed