Merge branch 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

+16 -7

kernel/sched/core.c

··· 1073 1073 task_rq_unlock(rq, p, &rf); 1074 1074 } 1075 1075 1076 + #ifdef CONFIG_UCLAMP_TASK_GROUP 1076 1077 static inline void 1077 1078 uclamp_update_active_tasks(struct cgroup_subsys_state *css, 1078 1079 unsigned int clamps) ··· 1092 1091 css_task_iter_end(&it); 1093 1092 } 1094 1093 1095 - #ifdef CONFIG_UCLAMP_TASK_GROUP 1096 1094 static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1097 1095 static void uclamp_update_root_tg(void) 1098 1096 { ··· 3929 3929 } 3930 3930 3931 3931 restart: 3932 + #ifdef CONFIG_SMP 3932 3933 /* 3933 - * Ensure that we put DL/RT tasks before the pick loop, such that they 3934 - * can PULL higher prio tasks when we lower the RQ 'priority'. 3934 + * We must do the balancing pass before put_next_task(), such 3935 + * that when we release the rq->lock the task is in the same 3936 + * state as before we took rq->lock. 3937 + * 3938 + * We can terminate the balance pass as soon as we know there is 3939 + * a runnable task of @class priority or higher. 3935 3940 */ 3936 - prev->sched_class->put_prev_task(rq, prev, rf); 3937 - if (!rq->nr_running) 3938 - newidle_balance(rq, rf); 3941 + for_class_range(class, prev->sched_class, &idle_sched_class) { 3942 + if (class->balance(rq, prev, rf)) 3943 + break; 3944 + } 3945 + #endif 3946 + 3947 + put_prev_task(rq, prev); 3939 3948 3940 3949 for_each_class(class) { 3941 3950 p = class->pick_next_task(rq, NULL, NULL); ··· 6210 6201 for_each_class(class) { 6211 6202 next = class->pick_next_task(rq, NULL, NULL); 6212 6203 if (next) { 6213 - next->sched_class->put_prev_task(rq, next, NULL); 6204 + next->sched_class->put_prev_task(rq, next); 6214 6205 return next; 6215 6206 } 6216 6207 }

+20 -20

kernel/sched/deadline.c

··· 1691 1691 resched_curr(rq); 1692 1692 } 1693 1693 1694 + static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf) 1695 + { 1696 + if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) { 1697 + /* 1698 + * This is OK, because current is on_cpu, which avoids it being 1699 + * picked for load-balance and preemption/IRQs are still 1700 + * disabled avoiding further scheduler activity on it and we've 1701 + * not yet started the picking loop. 1702 + */ 1703 + rq_unpin_lock(rq, rf); 1704 + pull_dl_task(rq); 1705 + rq_repin_lock(rq, rf); 1706 + } 1707 + 1708 + return sched_stop_runnable(rq) || sched_dl_runnable(rq); 1709 + } 1694 1710 #endif /* CONFIG_SMP */ 1695 1711 1696 1712 /* ··· 1774 1758 pick_next_task_dl(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 1775 1759 { 1776 1760 struct sched_dl_entity *dl_se; 1761 + struct dl_rq *dl_rq = &rq->dl; 1777 1762 struct task_struct *p; 1778 - struct dl_rq *dl_rq; 1779 1763 1780 1764 WARN_ON_ONCE(prev || rf); 1781 1765 1782 - dl_rq = &rq->dl; 1783 - 1784 - if (unlikely(!dl_rq->dl_nr_running)) 1766 + if (!sched_dl_runnable(rq)) 1785 1767 return NULL; 1786 1768 1787 1769 dl_se = pick_next_dl_entity(rq, dl_rq); 1788 1770 BUG_ON(!dl_se); 1789 - 1790 1771 p = dl_task_of(dl_se); 1791 - 1792 1772 set_next_task_dl(rq, p); 1793 - 1794 1773 return p; 1795 1774 } 1796 1775 1797 - static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf) 1776 + static void put_prev_task_dl(struct rq *rq, struct task_struct *p) 1798 1777 { 1799 1778 update_curr_dl(rq); 1800 1779 1801 1780 update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 1); 1802 1781 if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1) 1803 1782 enqueue_pushable_dl_task(rq, p); 1804 - 1805 - if (rf && !on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) { 1806 - /* 1807 - * This is OK, because current is on_cpu, which avoids it being 1808 - * picked for load-balance and preemption/IRQs are still 1809 - * disabled avoiding further scheduler activity on it and we've 1810 - * not yet started the picking loop. 1811 - */ 1812 - rq_unpin_lock(rq, rf); 1813 - pull_dl_task(rq); 1814 - rq_repin_lock(rq, rf); 1815 - } 1816 1783 } 1817 1784 1818 1785 /* ··· 2441 2442 .set_next_task = set_next_task_dl, 2442 2443 2443 2444 #ifdef CONFIG_SMP 2445 + .balance = balance_dl, 2444 2446 .select_task_rq = select_task_rq_dl, 2445 2447 .migrate_task_rq = migrate_task_rq_dl, 2446 2448 .set_cpus_allowed = set_cpus_allowed_dl,

+12 -3

kernel/sched/fair.c

··· 6570 6570 { 6571 6571 remove_entity_load_avg(&p->se); 6572 6572 } 6573 + 6574 + static int 6575 + balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6576 + { 6577 + if (rq->nr_running) 6578 + return 1; 6579 + 6580 + return newidle_balance(rq, rf) != 0; 6581 + } 6573 6582 #endif /* CONFIG_SMP */ 6574 6583 6575 6584 static unsigned long wakeup_gran(struct sched_entity *se) ··· 6755 6746 int new_tasks; 6756 6747 6757 6748 again: 6758 - if (!cfs_rq->nr_running) 6749 + if (!sched_fair_runnable(rq)) 6759 6750 goto idle; 6760 6751 6761 6752 #ifdef CONFIG_FAIR_GROUP_SCHED ··· 6893 6884 /* 6894 6885 * Account for a descheduled task: 6895 6886 */ 6896 - static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6887 + static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) 6897 6888 { 6898 6889 struct sched_entity *se = &prev->se; 6899 6890 struct cfs_rq *cfs_rq; ··· 10423 10414 .check_preempt_curr = check_preempt_wakeup, 10424 10415 10425 10416 .pick_next_task = pick_next_task_fair, 10426 - 10427 10417 .put_prev_task = put_prev_task_fair, 10428 10418 .set_next_task = set_next_task_fair, 10429 10419 10430 10420 #ifdef CONFIG_SMP 10421 + .balance = balance_fair, 10431 10422 .select_task_rq = select_task_rq_fair, 10432 10423 .migrate_task_rq = migrate_task_rq_fair, 10433 10424

+8 -1

kernel/sched/idle.c

··· 365 365 { 366 366 return task_cpu(p); /* IDLE tasks as never migrated */ 367 367 } 368 + 369 + static int 370 + balance_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 371 + { 372 + return WARN_ON_ONCE(1); 373 + } 368 374 #endif 369 375 370 376 /* ··· 381 375 resched_curr(rq); 382 376 } 383 377 384 - static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 378 + static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) 385 379 { 386 380 } 387 381 ··· 466 460 .set_next_task = set_next_task_idle, 467 461 468 462 #ifdef CONFIG_SMP 463 + .balance = balance_idle, 469 464 .select_task_rq = select_task_rq_idle, 470 465 .set_cpus_allowed = set_cpus_allowed_common, 471 466 #endif

+19 -18

kernel/sched/rt.c

··· 1469 1469 resched_curr(rq); 1470 1470 } 1471 1471 1472 + static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf) 1473 + { 1474 + if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) { 1475 + /* 1476 + * This is OK, because current is on_cpu, which avoids it being 1477 + * picked for load-balance and preemption/IRQs are still 1478 + * disabled avoiding further scheduler activity on it and we've 1479 + * not yet started the picking loop. 1480 + */ 1481 + rq_unpin_lock(rq, rf); 1482 + pull_rt_task(rq); 1483 + rq_repin_lock(rq, rf); 1484 + } 1485 + 1486 + return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq); 1487 + } 1472 1488 #endif /* CONFIG_SMP */ 1473 1489 1474 1490 /* ··· 1568 1552 pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 1569 1553 { 1570 1554 struct task_struct *p; 1571 - struct rt_rq *rt_rq = &rq->rt; 1572 1555 1573 1556 WARN_ON_ONCE(prev || rf); 1574 1557 1575 - if (!rt_rq->rt_queued) 1558 + if (!sched_rt_runnable(rq)) 1576 1559 return NULL; 1577 1560 1578 1561 p = _pick_next_task_rt(rq); 1579 - 1580 1562 set_next_task_rt(rq, p); 1581 - 1582 1563 return p; 1583 1564 } 1584 1565 1585 - static void put_prev_task_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf) 1566 + static void put_prev_task_rt(struct rq *rq, struct task_struct *p) 1586 1567 { 1587 1568 update_curr_rt(rq); 1588 1569 ··· 1591 1578 */ 1592 1579 if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1) 1593 1580 enqueue_pushable_task(rq, p); 1594 - 1595 - if (rf && !on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) { 1596 - /* 1597 - * This is OK, because current is on_cpu, which avoids it being 1598 - * picked for load-balance and preemption/IRQs are still 1599 - * disabled avoiding further scheduler activity on it and we've 1600 - * not yet started the picking loop. 1601 - */ 1602 - rq_unpin_lock(rq, rf); 1603 - pull_rt_task(rq); 1604 - rq_repin_lock(rq, rf); 1605 - } 1606 1581 } 1607 1582 1608 1583 #ifdef CONFIG_SMP ··· 2367 2366 .set_next_task = set_next_task_rt, 2368 2367 2369 2368 #ifdef CONFIG_SMP 2369 + .balance = balance_rt, 2370 2370 .select_task_rq = select_task_rq_rt, 2371 - 2372 2371 .set_cpus_allowed = set_cpus_allowed_common, 2373 2372 .rq_online = rq_online_rt, 2374 2373 .rq_offline = rq_offline_rt,

+27 -3

kernel/sched/sched.h

··· 1727 1727 struct task_struct * (*pick_next_task)(struct rq *rq, 1728 1728 struct task_struct *prev, 1729 1729 struct rq_flags *rf); 1730 - void (*put_prev_task)(struct rq *rq, struct task_struct *p, struct rq_flags *rf); 1730 + void (*put_prev_task)(struct rq *rq, struct task_struct *p); 1731 1731 void (*set_next_task)(struct rq *rq, struct task_struct *p); 1732 1732 1733 1733 #ifdef CONFIG_SMP 1734 + int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); 1734 1735 int (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags); 1735 1736 void (*migrate_task_rq)(struct task_struct *p, int new_cpu); 1736 1737 ··· 1774 1773 static inline void put_prev_task(struct rq *rq, struct task_struct *prev) 1775 1774 { 1776 1775 WARN_ON_ONCE(rq->curr != prev); 1777 - prev->sched_class->put_prev_task(rq, prev, NULL); 1776 + prev->sched_class->put_prev_task(rq, prev); 1778 1777 } 1779 1778 1780 1779 static inline void set_next_task(struct rq *rq, struct task_struct *next) ··· 1788 1787 #else 1789 1788 #define sched_class_highest (&dl_sched_class) 1790 1789 #endif 1790 + 1791 + #define for_class_range(class, _from, _to) \ 1792 + for (class = (_from); class != (_to); class = class->next) 1793 + 1791 1794 #define for_each_class(class) \ 1792 - for (class = sched_class_highest; class; class = class->next) 1795 + for_class_range(class, sched_class_highest, NULL) 1793 1796 1794 1797 extern const struct sched_class stop_sched_class; 1795 1798 extern const struct sched_class dl_sched_class; ··· 1801 1796 extern const struct sched_class fair_sched_class; 1802 1797 extern const struct sched_class idle_sched_class; 1803 1798 1799 + static inline bool sched_stop_runnable(struct rq *rq) 1800 + { 1801 + return rq->stop && task_on_rq_queued(rq->stop); 1802 + } 1803 + 1804 + static inline bool sched_dl_runnable(struct rq *rq) 1805 + { 1806 + return rq->dl.dl_nr_running > 0; 1807 + } 1808 + 1809 + static inline bool sched_rt_runnable(struct rq *rq) 1810 + { 1811 + return rq->rt.rt_queued > 0; 1812 + } 1813 + 1814 + static inline bool sched_fair_runnable(struct rq *rq) 1815 + { 1816 + return rq->cfs.nr_running > 0; 1817 + } 1804 1818 1805 1819 #ifdef CONFIG_SMP 1806 1820

+11 -7

kernel/sched/stop_task.c

··· 15 15 { 16 16 return task_cpu(p); /* stop tasks as never migrate */ 17 17 } 18 + 19 + static int 20 + balance_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 21 + { 22 + return sched_stop_runnable(rq); 23 + } 18 24 #endif /* CONFIG_SMP */ 19 25 20 26 static void ··· 37 31 static struct task_struct * 38 32 pick_next_task_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 39 33 { 40 - struct task_struct *stop = rq->stop; 41 - 42 34 WARN_ON_ONCE(prev || rf); 43 35 44 - if (!stop || !task_on_rq_queued(stop)) 36 + if (!sched_stop_runnable(rq)) 45 37 return NULL; 46 38 47 - set_next_task_stop(rq, stop); 48 - 49 - return stop; 39 + set_next_task_stop(rq, rq->stop); 40 + return rq->stop; 50 41 } 51 42 52 43 static void ··· 63 60 BUG(); /* the stop task should never yield, its pointless. */ 64 61 } 65 62 66 - static void put_prev_task_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 63 + static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) 67 64 { 68 65 struct task_struct *curr = rq->curr; 69 66 u64 delta_exec; ··· 132 129 .set_next_task = set_next_task_stop, 133 130 134 131 #ifdef CONFIG_SMP 132 + .balance = balance_stop, 135 133 .select_task_rq = select_task_rq_stop, 136 134 .set_cpus_allowed = set_cpus_allowed_common, 137 135 #endif

Configure Feed

Configure Feed