tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

sched: Rework dl_server

When a task is selected through a dl_server, it will have p->dl_server
set, such that it can account runtime to the dl_server, see
update_curr_task().

Currently p->dl_server is set in pick*task() whenever it goes through
the dl_server, clearing it is a bit of a mess though. The trivial
solution is clearing it on the final put (now that we have this
location).

However, this gives a problem when:

p = pick_task(rq);
if (p)
put_prev_set_next_task(rq, prev, next);

picks the same task but through a different path, notably when it goes
from picking through the dl_server to a direct pick or vice-versa. In
that case we cannot readily determine wether we should clear or
preserve p->dl_server.

An additional complication is pick_*task() setting p->dl_server for a
remote pick, it might still need to update runtime before it schedules
the core_pick.

Close all these holes and remove all the random clearing of
p->dl_server by:

- having pick_*task() manage rq->dl_server

- having the final put_prev_task() clear p->dl_server

- having the first set_next_task() set p->dl_server = rq->dl_server

- complicate the core_sched code to save/restore rq->dl_server where
appropriate.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20240813224016.259853414@infradead.org

Peter Zijlstra bd9bbc96 436f3eed

+32 -34
+15 -25
kernel/sched/core.c
··· 3668 3668 rq->idle_stamp = 0; 3669 3669 } 3670 3670 #endif 3671 - 3672 - p->dl_server = NULL; 3673 3671 } 3674 3672 3675 3673 /* ··· 5857 5859 break; 5858 5860 } 5859 5861 #endif 5860 - 5861 - /* 5862 - * We've updated @prev and no longer need the server link, clear it. 5863 - * Must be done before ->pick_next_task() because that can (re)set 5864 - * ->dl_server. 5865 - */ 5866 - if (prev->dl_server) 5867 - prev->dl_server = NULL; 5868 5862 } 5869 5863 5870 5864 /* ··· 5867 5877 { 5868 5878 const struct sched_class *class; 5869 5879 struct task_struct *p; 5880 + 5881 + rq->dl_server = NULL; 5870 5882 5871 5883 /* 5872 5884 * Optimization: we know that if all tasks are in the fair class we can ··· 5888 5896 p = pick_task_idle(rq); 5889 5897 put_prev_set_next_task(rq, prev, p); 5890 5898 } 5891 - 5892 - /* 5893 - * This is a normal CFS pick, but the previous could be a DL pick. 5894 - * Clear it as previous is no longer picked. 5895 - */ 5896 - if (prev->dl_server) 5897 - prev->dl_server = NULL; 5898 - 5899 - /* 5900 - * This is the fast path; it cannot be a DL server pick; 5901 - * therefore even if @p == @prev, ->dl_server must be NULL. 5902 - */ 5903 - if (p->dl_server) 5904 - p->dl_server = NULL; 5905 5899 5906 5900 return p; 5907 5901 } ··· 5936 5958 const struct sched_class *class; 5937 5959 struct task_struct *p; 5938 5960 5961 + rq->dl_server = NULL; 5962 + 5939 5963 for_each_class(class) { 5940 5964 p = class->pick_task(rq); 5941 5965 if (p) ··· 5976 5996 * another cpu during offline. 5977 5997 */ 5978 5998 rq->core_pick = NULL; 5999 + rq->core_dl_server = NULL; 5979 6000 return __pick_next_task(rq, prev, rf); 5980 6001 } 5981 6002 ··· 5995 6014 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); 5996 6015 5997 6016 next = rq->core_pick; 6017 + rq->dl_server = rq->core_dl_server; 5998 6018 rq->core_pick = NULL; 6019 + rq->core_dl_server = NULL; 5999 6020 goto out_set_next; 6000 6021 } 6001 6022 ··· 6042 6059 next = pick_task(rq); 6043 6060 if (!next->core_cookie) { 6044 6061 rq->core_pick = NULL; 6062 + rq->core_dl_server = NULL; 6045 6063 /* 6046 6064 * For robustness, update the min_vruntime_fi for 6047 6065 * unconstrained picks as well. ··· 6070 6086 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) 6071 6087 update_rq_clock(rq_i); 6072 6088 6073 - p = rq_i->core_pick = pick_task(rq_i); 6089 + rq_i->core_pick = p = pick_task(rq_i); 6090 + rq_i->core_dl_server = rq_i->dl_server; 6091 + 6074 6092 if (!max || prio_less(max, p, fi_before)) 6075 6093 max = p; 6076 6094 } ··· 6096 6110 } 6097 6111 6098 6112 rq_i->core_pick = p; 6113 + rq_i->core_dl_server = NULL; 6099 6114 6100 6115 if (p == rq_i->idle) { 6101 6116 if (rq_i->nr_running) { ··· 6157 6170 6158 6171 if (i == cpu) { 6159 6172 rq_i->core_pick = NULL; 6173 + rq_i->core_dl_server = NULL; 6160 6174 continue; 6161 6175 } 6162 6176 ··· 6166 6178 6167 6179 if (rq_i->curr == rq_i->core_pick) { 6168 6180 rq_i->core_pick = NULL; 6181 + rq_i->core_dl_server = NULL; 6169 6182 continue; 6170 6183 } 6171 6184 ··· 8390 8401 #ifdef CONFIG_SCHED_CORE 8391 8402 rq->core = rq; 8392 8403 rq->core_pick = NULL; 8404 + rq->core_dl_server = NULL; 8393 8405 rq->core_enabled = 0; 8394 8406 rq->core_tree = RB_ROOT; 8395 8407 rq->core_forceidle_count = 0;
+1 -1
kernel/sched/deadline.c
··· 2423 2423 update_curr_dl_se(rq, dl_se, 0); 2424 2424 goto again; 2425 2425 } 2426 - p->dl_server = dl_se; 2426 + rq->dl_server = dl_se; 2427 2427 } else { 2428 2428 p = dl_task_of(dl_se); 2429 2429 }
+2 -8
kernel/sched/fair.c
··· 8749 8749 cfs_rq = group_cfs_rq(se); 8750 8750 } while (cfs_rq); 8751 8751 8752 - /* 8753 - * This can be called from directly from CFS's ->pick_task() or indirectly 8754 - * from DL's ->pick_task when fair server is enabled. In the indirect case, 8755 - * DL will set ->dl_server just after this function is called, so its Ok to 8756 - * clear. In the direct case, we are picking directly so we must clear it. 8757 - */ 8758 - task_of(se)->dl_server = NULL; 8759 - 8760 8752 return task_of(se); 8761 8753 } 8762 8754 ··· 8771 8779 #ifdef CONFIG_FAIR_GROUP_SCHED 8772 8780 if (prev->sched_class != &fair_sched_class) 8773 8781 goto simple; 8782 + 8783 + __put_prev_set_next_dl_server(rq, prev, p); 8774 8784 8775 8785 /* 8776 8786 * Because of the set_next_buddy() in dequeue_task_fair() it is rather
+14
kernel/sched/sched.h
··· 1066 1066 unsigned int nr_uninterruptible; 1067 1067 1068 1068 struct task_struct __rcu *curr; 1069 + struct sched_dl_entity *dl_server; 1069 1070 struct task_struct *idle; 1070 1071 struct task_struct *stop; 1071 1072 unsigned long next_balance; ··· 1194 1193 /* per rq */ 1195 1194 struct rq *core; 1196 1195 struct task_struct *core_pick; 1196 + struct sched_dl_entity *core_dl_server; 1197 1197 unsigned int core_enabled; 1198 1198 unsigned int core_sched_seq; 1199 1199 struct rb_root core_tree; ··· 2372 2370 next->sched_class->set_next_task(rq, next, false); 2373 2371 } 2374 2372 2373 + static inline void 2374 + __put_prev_set_next_dl_server(struct rq *rq, 2375 + struct task_struct *prev, 2376 + struct task_struct *next) 2377 + { 2378 + prev->dl_server = NULL; 2379 + next->dl_server = rq->dl_server; 2380 + rq->dl_server = NULL; 2381 + } 2382 + 2375 2383 static inline void put_prev_set_next_task(struct rq *rq, 2376 2384 struct task_struct *prev, 2377 2385 struct task_struct *next) 2378 2386 { 2379 2387 WARN_ON_ONCE(rq->curr != prev); 2388 + 2389 + __put_prev_set_next_dl_server(rq, prev, next); 2380 2390 2381 2391 if (next == prev) 2382 2392 return;