sched/fair: Implement throttle task work and related helpers

Implement throttle_cfs_rq_work() task work which gets executed on task's
ret2user path where the task is dequeued and marked as throttled.

Signed-off-by: Valentin Schneider <vschneid@redhat.com>
Signed-off-by: Aaron Lu <ziqianlu@bytedance.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Chengming Zhou <chengming.zhou@linux.dev>
Tested-by: Valentin Schneider <vschneid@redhat.com>
Tested-by: Matteo Martelli <matteo.martelli@codethink.co.uk>
Tested-by: K Prateek Nayak <kprateek.nayak@amd.com>
Link: https://lore.kernel.org/r/20250829081120.806-3-ziqianlu@bytedance.com

authored by

Valentin Schneider and committed by

Peter Zijlstra 9 months ago 7fc2d143 2cd57124

+65

1 changed file

expand all

kernel

sched

fair.c

+65

kernel/sched/fair.c

··· 5748 5748 throttled_hierarchy(dest_cfs_rq); 5749 5749 } 5750 5750 5751 + static inline bool task_is_throttled(struct task_struct *p) 5752 + { 5753 + return cfs_bandwidth_used() && p->throttled; 5754 + } 5755 + 5756 + static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5751 5757 static void throttle_cfs_rq_work(struct callback_head *work) 5752 5758 { 5759 + struct task_struct *p = container_of(work, struct task_struct, sched_throttle_work); 5760 + struct sched_entity *se; 5761 + struct cfs_rq *cfs_rq; 5762 + struct rq *rq; 5763 + 5764 + WARN_ON_ONCE(p != current); 5765 + p->sched_throttle_work.next = &p->sched_throttle_work; 5766 + 5767 + /* 5768 + * If task is exiting, then there won't be a return to userspace, so we 5769 + * don't have to bother with any of this. 5770 + */ 5771 + if ((p->flags & PF_EXITING)) 5772 + return; 5773 + 5774 + scoped_guard(task_rq_lock, p) { 5775 + se = &p->se; 5776 + cfs_rq = cfs_rq_of(se); 5777 + 5778 + /* Raced, forget */ 5779 + if (p->sched_class != &fair_sched_class) 5780 + return; 5781 + 5782 + /* 5783 + * If not in limbo, then either replenish has happened or this 5784 + * task got migrated out of the throttled cfs_rq, move along. 5785 + */ 5786 + if (!cfs_rq->throttle_count) 5787 + return; 5788 + rq = scope.rq; 5789 + update_rq_clock(rq); 5790 + WARN_ON_ONCE(p->throttled || !list_empty(&p->throttle_node)); 5791 + dequeue_task_fair(rq, p, DEQUEUE_SLEEP | DEQUEUE_SPECIAL); 5792 + list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5793 + p->throttled = true; 5794 + resched_curr(rq); 5795 + } 5753 5796 } 5754 5797 5755 5798 void init_cfs_throttle_work(struct task_struct *p) ··· 5830 5787 } 5831 5788 5832 5789 return 0; 5790 + } 5791 + 5792 + static inline bool task_has_throttle_work(struct task_struct *p) 5793 + { 5794 + return p->sched_throttle_work.next != &p->sched_throttle_work; 5795 + } 5796 + 5797 + static inline void task_throttle_setup_work(struct task_struct *p) 5798 + { 5799 + if (task_has_throttle_work(p)) 5800 + return; 5801 + 5802 + /* 5803 + * Kthreads and exiting tasks don't return to userspace, so adding the 5804 + * work is pointless 5805 + */ 5806 + if ((p->flags & (PF_EXITING | PF_KTHREAD))) 5807 + return; 5808 + 5809 + task_work_add(p, &p->sched_throttle_work, TWA_RESUME); 5833 5810 } 5834 5811 5835 5812 static int tg_throttle_down(struct task_group *tg, void *data) ··· 6715 6652 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 6716 6653 static inline void sync_throttle(struct task_group *tg, int cpu) {} 6717 6654 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6655 + static void task_throttle_setup_work(struct task_struct *p) {} 6656 + static bool task_is_throttled(struct task_struct *p) { return false; } 6718 6657 6719 6658 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6720 6659 {

Configure Feed

Configure Feed