Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
signal: align __lock_task_sighand() irq disabling and RCU
softirq,rcu: Inform RCU of irq_exit() activity
sched: Add irq_{enter,exit}() to scheduler_ipi()
rcu: protect __rcu_read_unlock() against scheduler-using irq handlers
rcu: Streamline code produced by __rcu_read_unlock()
rcu: Fix RCU_BOOST race handling current->rcu_read_unlock_special
rcu: decrease rcu_report_exp_rnp coupling with scheduler
+103
-28
+3
include/linux/sched.h
+3
include/linux/sched.h
···
1260
1260
#ifdef CONFIG_PREEMPT_RCU
1261
1261
int rcu_read_lock_nesting;
1262
1262
char rcu_read_unlock_special;
1263
+
#if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU)
1264
+
int rcu_boosted;
1265
+
#endif /* #if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU) */
1263
1266
struct list_head rcu_node_entry;
1264
1267
#endif /* #ifdef CONFIG_PREEMPT_RCU */
1265
1268
#ifdef CONFIG_TREE_PREEMPT_RCU
+39
-14
kernel/rcutree_plugin.h
+39
-14
kernel/rcutree_plugin.h
···
68
68
DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
69
69
static struct rcu_state *rcu_state = &rcu_preempt_state;
70
70
71
+
static void rcu_read_unlock_special(struct task_struct *t);
71
72
static int rcu_preempted_readers_exp(struct rcu_node *rnp);
72
73
73
74
/*
···
148
147
struct rcu_data *rdp;
149
148
struct rcu_node *rnp;
150
149
151
-
if (t->rcu_read_lock_nesting &&
150
+
if (t->rcu_read_lock_nesting > 0 &&
152
151
(t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
153
152
154
153
/* Possibly blocking in an RCU read-side critical section. */
···
191
190
rnp->gp_tasks = &t->rcu_node_entry;
192
191
}
193
192
raw_spin_unlock_irqrestore(&rnp->lock, flags);
193
+
} else if (t->rcu_read_lock_nesting < 0 &&
194
+
t->rcu_read_unlock_special) {
195
+
196
+
/*
197
+
* Complete exit from RCU read-side critical section on
198
+
* behalf of preempted instance of __rcu_read_unlock().
199
+
*/
200
+
rcu_read_unlock_special(t);
194
201
}
195
202
196
203
/*
···
293
284
* notify RCU core processing or task having blocked during the RCU
294
285
* read-side critical section.
295
286
*/
296
-
static void rcu_read_unlock_special(struct task_struct *t)
287
+
static noinline void rcu_read_unlock_special(struct task_struct *t)
297
288
{
298
289
int empty;
299
290
int empty_exp;
···
318
309
}
319
310
320
311
/* Hardware IRQ handlers cannot block. */
321
-
if (in_irq()) {
312
+
if (in_irq() || in_serving_softirq()) {
322
313
local_irq_restore(flags);
323
314
return;
324
315
}
···
351
342
#ifdef CONFIG_RCU_BOOST
352
343
if (&t->rcu_node_entry == rnp->boost_tasks)
353
344
rnp->boost_tasks = np;
345
+
/* Snapshot and clear ->rcu_boosted with rcu_node lock held. */
346
+
if (t->rcu_boosted) {
347
+
special |= RCU_READ_UNLOCK_BOOSTED;
348
+
t->rcu_boosted = 0;
349
+
}
354
350
#endif /* #ifdef CONFIG_RCU_BOOST */
355
351
t->rcu_blocked_node = NULL;
356
352
···
372
358
#ifdef CONFIG_RCU_BOOST
373
359
/* Unboost if we were boosted. */
374
360
if (special & RCU_READ_UNLOCK_BOOSTED) {
375
-
t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BOOSTED;
376
361
rt_mutex_unlock(t->rcu_boost_mutex);
377
362
t->rcu_boost_mutex = NULL;
378
363
}
···
400
387
struct task_struct *t = current;
401
388
402
389
barrier(); /* needed if we ever invoke rcu_read_unlock in rcutree.c */
403
-
--t->rcu_read_lock_nesting;
404
-
barrier(); /* decrement before load of ->rcu_read_unlock_special */
405
-
if (t->rcu_read_lock_nesting == 0 &&
406
-
unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
407
-
rcu_read_unlock_special(t);
390
+
if (t->rcu_read_lock_nesting != 1)
391
+
--t->rcu_read_lock_nesting;
392
+
else {
393
+
t->rcu_read_lock_nesting = INT_MIN;
394
+
barrier(); /* assign before ->rcu_read_unlock_special load */
395
+
if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
396
+
rcu_read_unlock_special(t);
397
+
barrier(); /* ->rcu_read_unlock_special load before assign */
398
+
t->rcu_read_lock_nesting = 0;
399
+
}
408
400
#ifdef CONFIG_PROVE_LOCKING
409
-
WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
401
+
{
402
+
int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
403
+
404
+
WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
405
+
}
410
406
#endif /* #ifdef CONFIG_PROVE_LOCKING */
411
407
}
412
408
EXPORT_SYMBOL_GPL(__rcu_read_unlock);
···
611
589
rcu_preempt_qs(cpu);
612
590
return;
613
591
}
614
-
if (per_cpu(rcu_preempt_data, cpu).qs_pending)
592
+
if (t->rcu_read_lock_nesting > 0 &&
593
+
per_cpu(rcu_preempt_data, cpu).qs_pending)
615
594
t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
616
595
}
617
596
···
718
695
719
696
raw_spin_lock_irqsave(&rnp->lock, flags);
720
697
for (;;) {
721
-
if (!sync_rcu_preempt_exp_done(rnp))
698
+
if (!sync_rcu_preempt_exp_done(rnp)) {
699
+
raw_spin_unlock_irqrestore(&rnp->lock, flags);
722
700
break;
701
+
}
723
702
if (rnp->parent == NULL) {
703
+
raw_spin_unlock_irqrestore(&rnp->lock, flags);
724
704
wake_up(&sync_rcu_preempt_exp_wq);
725
705
break;
726
706
}
···
733
707
raw_spin_lock(&rnp->lock); /* irqs already disabled */
734
708
rnp->expmask &= ~mask;
735
709
}
736
-
raw_spin_unlock_irqrestore(&rnp->lock, flags);
737
710
}
738
711
739
712
/*
···
1199
1174
t = container_of(tb, struct task_struct, rcu_node_entry);
1200
1175
rt_mutex_init_proxy_locked(&mtx, t);
1201
1176
t->rcu_boost_mutex = &mtx;
1202
-
t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
1177
+
t->rcu_boosted = 1;
1203
1178
raw_spin_unlock_irqrestore(&rnp->lock, flags);
1204
1179
rt_mutex_lock(&mtx); /* Side effect: boosts task t's priority. */
1205
1180
rt_mutex_unlock(&mtx); /* Keep lockdep happy. */
+38
-6
kernel/sched.c
+38
-6
kernel/sched.c
···
2544
2544
}
2545
2545
2546
2546
#ifdef CONFIG_SMP
2547
-
static void sched_ttwu_pending(void)
2547
+
static void sched_ttwu_do_pending(struct task_struct *list)
2548
2548
{
2549
2549
struct rq *rq = this_rq();
2550
-
struct task_struct *list = xchg(&rq->wake_list, NULL);
2551
-
2552
-
if (!list)
2553
-
return;
2554
2550
2555
2551
raw_spin_lock(&rq->lock);
2556
2552
···
2559
2563
raw_spin_unlock(&rq->lock);
2560
2564
}
2561
2565
2566
+
#ifdef CONFIG_HOTPLUG_CPU
2567
+
2568
+
static void sched_ttwu_pending(void)
2569
+
{
2570
+
struct rq *rq = this_rq();
2571
+
struct task_struct *list = xchg(&rq->wake_list, NULL);
2572
+
2573
+
if (!list)
2574
+
return;
2575
+
2576
+
sched_ttwu_do_pending(list);
2577
+
}
2578
+
2579
+
#endif /* CONFIG_HOTPLUG_CPU */
2580
+
2562
2581
void scheduler_ipi(void)
2563
2582
{
2564
-
sched_ttwu_pending();
2583
+
struct rq *rq = this_rq();
2584
+
struct task_struct *list = xchg(&rq->wake_list, NULL);
2585
+
2586
+
if (!list)
2587
+
return;
2588
+
2589
+
/*
2590
+
* Not all reschedule IPI handlers call irq_enter/irq_exit, since
2591
+
* traditionally all their work was done from the interrupt return
2592
+
* path. Now that we actually do some work, we need to make sure
2593
+
* we do call them.
2594
+
*
2595
+
* Some archs already do call them, luckily irq_enter/exit nest
2596
+
* properly.
2597
+
*
2598
+
* Arguably we should visit all archs and update all handlers,
2599
+
* however a fair share of IPIs are still resched only so this would
2600
+
* somewhat pessimize the simple resched case.
2601
+
*/
2602
+
irq_enter();
2603
+
sched_ttwu_do_pending(list);
2604
+
irq_exit();
2565
2605
}
2566
2606
2567
2607
static void ttwu_queue_remote(struct task_struct *p, int cpu)
+13
-6
kernel/signal.c
+13
-6
kernel/signal.c
···
1178
1178
{
1179
1179
struct sighand_struct *sighand;
1180
1180
1181
-
rcu_read_lock();
1182
1181
for (;;) {
1182
+
local_irq_save(*flags);
1183
+
rcu_read_lock();
1183
1184
sighand = rcu_dereference(tsk->sighand);
1184
-
if (unlikely(sighand == NULL))
1185
+
if (unlikely(sighand == NULL)) {
1186
+
rcu_read_unlock();
1187
+
local_irq_restore(*flags);
1185
1188
break;
1189
+
}
1186
1190
1187
-
spin_lock_irqsave(&sighand->siglock, *flags);
1188
-
if (likely(sighand == tsk->sighand))
1191
+
spin_lock(&sighand->siglock);
1192
+
if (likely(sighand == tsk->sighand)) {
1193
+
rcu_read_unlock();
1189
1194
break;
1190
-
spin_unlock_irqrestore(&sighand->siglock, *flags);
1195
+
}
1196
+
spin_unlock(&sighand->siglock);
1197
+
rcu_read_unlock();
1198
+
local_irq_restore(*flags);
1191
1199
}
1192
-
rcu_read_unlock();
1193
1200
1194
1201
return sighand;
1195
1202
}
+10
-2
kernel/softirq.c
+10
-2
kernel/softirq.c
···
315
315
{
316
316
if (!force_irqthreads)
317
317
__do_softirq();
318
-
else
318
+
else {
319
+
__local_bh_disable((unsigned long)__builtin_return_address(0),
320
+
SOFTIRQ_OFFSET);
319
321
wakeup_softirqd();
322
+
__local_bh_enable(SOFTIRQ_OFFSET);
323
+
}
320
324
}
321
325
#else
322
326
static inline void invoke_softirq(void)
323
327
{
324
328
if (!force_irqthreads)
325
329
do_softirq();
326
-
else
330
+
else {
331
+
__local_bh_disable((unsigned long)__builtin_return_address(0),
332
+
SOFTIRQ_OFFSET);
327
333
wakeup_softirqd();
334
+
__local_bh_enable(SOFTIRQ_OFFSET);
335
+
}
328
336
}
329
337
#endif
330
338