Merge tag 'sched-urgent-2020-12-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

+20 -33

2 changed files

expand all

kernel

sched

core.c

sched.h

+15 -25

kernel/sched/core.c

··· 3985 3985 } 3986 3986 } 3987 3987 3988 + static void balance_push(struct rq *rq); 3989 + 3990 + struct callback_head balance_push_callback = { 3991 + .next = NULL, 3992 + .func = (void (*)(struct callback_head *))balance_push, 3993 + }; 3994 + 3988 3995 static inline struct callback_head *splice_balance_callbacks(struct rq *rq) 3989 3996 { 3990 3997 struct callback_head *head = rq->balance_callback; 3991 3998 3992 3999 lockdep_assert_held(&rq->lock); 3993 - if (head) { 4000 + if (head) 3994 4001 rq->balance_callback = NULL; 3995 - rq->balance_flags &= ~BALANCE_WORK; 3996 - } 3997 4002 3998 4003 return head; 3999 4004 } ··· 4019 4014 } 4020 4015 } 4021 4016 4022 - static void balance_push(struct rq *rq); 4023 - 4024 - static inline void balance_switch(struct rq *rq) 4025 - { 4026 - if (likely(!rq->balance_flags)) 4027 - return; 4028 - 4029 - if (rq->balance_flags & BALANCE_PUSH) { 4030 - balance_push(rq); 4031 - return; 4032 - } 4033 - 4034 - __balance_callbacks(rq); 4035 - } 4036 - 4037 4017 #else 4038 4018 4039 4019 static inline void __balance_callbacks(struct rq *rq) ··· 4031 4041 } 4032 4042 4033 4043 static inline void balance_callbacks(struct rq *rq, struct callback_head *head) 4034 - { 4035 - } 4036 - 4037 - static inline void balance_switch(struct rq *rq) 4038 4044 { 4039 4045 } 4040 4046 ··· 4061 4075 * prev into current: 4062 4076 */ 4063 4077 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); 4064 - balance_switch(rq); 4078 + __balance_callbacks(rq); 4065 4079 raw_spin_unlock_irq(&rq->lock); 4066 4080 } 4067 4081 ··· 7268 7282 7269 7283 lockdep_assert_held(&rq->lock); 7270 7284 SCHED_WARN_ON(rq->cpu != smp_processor_id()); 7285 + /* 7286 + * Ensure the thing is persistent until balance_push_set(.on = false); 7287 + */ 7288 + rq->balance_callback = &balance_push_callback; 7271 7289 7272 7290 /* 7273 7291 * Both the cpu-hotplug and stop task are in this case and are ··· 7321 7331 7322 7332 rq_lock_irqsave(rq, &rf); 7323 7333 if (on) 7324 - rq->balance_flags |= BALANCE_PUSH; 7334 + rq->balance_callback = &balance_push_callback; 7325 7335 else 7326 - rq->balance_flags &= ~BALANCE_PUSH; 7336 + rq->balance_callback = NULL; 7327 7337 rq_unlock_irqrestore(rq, &rf); 7328 7338 } 7329 7339

+5 -8

kernel/sched/sched.h

··· 975 975 unsigned long cpu_capacity_orig; 976 976 977 977 struct callback_head *balance_callback; 978 - unsigned char balance_flags; 979 978 980 979 unsigned char nohz_idle_balance; 981 980 unsigned char idle_balance; ··· 1225 1226 #endif 1226 1227 }; 1227 1228 1229 + extern struct callback_head balance_push_callback; 1230 + 1228 1231 /* 1229 1232 * Lockdep annotation that avoids accidental unlocks; it's like a 1230 1233 * sticky/continuous lockdep_assert_held(). ··· 1244 1243 #ifdef CONFIG_SCHED_DEBUG 1245 1244 rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP); 1246 1245 rf->clock_update_flags = 0; 1247 - #endif 1248 1246 #ifdef CONFIG_SMP 1249 - SCHED_WARN_ON(rq->balance_callback); 1247 + SCHED_WARN_ON(rq->balance_callback && rq->balance_callback != &balance_push_callback); 1248 + #endif 1250 1249 #endif 1251 1250 } 1252 1251 ··· 1409 1408 1410 1409 #ifdef CONFIG_SMP 1411 1410 1412 - #define BALANCE_WORK 0x01 1413 - #define BALANCE_PUSH 0x02 1414 - 1415 1411 static inline void 1416 1412 queue_balance_callback(struct rq *rq, 1417 1413 struct callback_head *head, ··· 1416 1418 { 1417 1419 lockdep_assert_held(&rq->lock); 1418 1420 1419 - if (unlikely(head->next || (rq->balance_flags & BALANCE_PUSH))) 1421 + if (unlikely(head->next || rq->balance_callback == &balance_push_callback)) 1420 1422 return; 1421 1423 1422 1424 head->func = (void (*)(struct callback_head *))func; 1423 1425 head->next = rq->balance_callback; 1424 1426 rq->balance_callback = head; 1425 - rq->balance_flags |= BALANCE_WORK; 1426 1427 } 1427 1428 1428 1429 #define rcu_dereference_check_sched_domain(p) \

Configure Feed

Configure Feed