Merge branch 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

+5 -12

Documentation/RCU/trace.txt

··· 99 99 100 100 o "dt" is the current value of the dyntick counter that is incremented 101 101 when entering or leaving dynticks idle state, either by the 102 - scheduler or by irq. The number after the "/" is the interrupt 103 - nesting depth when in dyntick-idle state, or one greater than 104 - the interrupt-nesting depth otherwise. 105 - 106 - This field is displayed only for CONFIG_NO_HZ kernels. 107 - 108 - o "dn" is the current value of the dyntick counter that is incremented 109 - when entering or leaving dynticks idle state via NMI. If both 110 - the "dt" and "dn" values are even, then this CPU is in dynticks 111 - idle mode and may be ignored by RCU. If either of these two 112 - counters is odd, then RCU must be alert to the possibility of 113 - an RCU read-side critical section running on this CPU. 102 + scheduler or by irq. This number is even if the CPU is in 103 + dyntick idle mode and odd otherwise. The number after the first 104 + "/" is the interrupt nesting depth when in dyntick-idle state, 105 + or one greater than the interrupt-nesting depth otherwise. 106 + The number after the second "/" is the NMI nesting depth. 114 107 115 108 This field is displayed only for CONFIG_NO_HZ kernels. 116 109

+34 -2

Documentation/lockstat.txt

··· 12 12 - HOW 13 13 14 14 Lockdep already has hooks in the lock functions and maps lock instances to 15 - lock classes. We build on that. The graph below shows the relation between 16 - the lock functions and the various hooks therein. 15 + lock classes. We build on that (see Documentation/lockdep-design.txt). 16 + The graph below shows the relation between the lock functions and the various 17 + hooks therein. 17 18 18 19 __acquire 19 20 | ··· 128 127 points are the points we're contending with. 129 128 130 129 The integer part of the time values is in us. 130 + 131 + Dealing with nested locks, subclasses may appear: 132 + 133 + 32............................................................................................................................................................................................... 134 + 33 135 + 34 &rq->lock: 13128 13128 0.43 190.53 103881.26 97454 3453404 0.00 401.11 13224683.11 136 + 35 --------- 137 + 36 &rq->lock 645 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75 138 + 37 &rq->lock 297 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a 139 + 38 &rq->lock 360 [<ffffffff8103c4c5>] select_task_rq_fair+0x1f0/0x74a 140 + 39 &rq->lock 428 [<ffffffff81045f98>] scheduler_tick+0x46/0x1fb 141 + 40 --------- 142 + 41 &rq->lock 77 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75 143 + 42 &rq->lock 174 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a 144 + 43 &rq->lock 4715 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54 145 + 44 &rq->lock 893 [<ffffffff81340524>] schedule+0x157/0x7b8 146 + 45 147 + 46............................................................................................................................................................................................... 148 + 47 149 + 48 &rq->lock/1: 11526 11488 0.33 388.73 136294.31 21461 38404 0.00 37.93 109388.53 150 + 49 ----------- 151 + 50 &rq->lock/1 11526 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54 152 + 51 ----------- 153 + 52 &rq->lock/1 5645 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54 154 + 53 &rq->lock/1 1224 [<ffffffff81340524>] schedule+0x157/0x7b8 155 + 54 &rq->lock/1 4336 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54 156 + 55 &rq->lock/1 181 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a 157 + 158 + Line 48 shows statistics for the second subclass (/1) of &rq->lock class 159 + (subclass starts from 0), since in this case, as line 50 suggests, 160 + double_rq_lock actually acquires a nested lock of two spinlocks. 131 161 132 162 View the top contending locks: 133 163

+13

include/linux/atomic.h

··· 34 34 } 35 35 #endif 36 36 37 + #ifndef CONFIG_ARCH_HAS_ATOMIC_OR 38 + static inline void atomic_or(int i, atomic_t *v) 39 + { 40 + int old; 41 + int new; 42 + 43 + do { 44 + old = atomic_read(v); 45 + new = old | i; 46 + } while (atomic_cmpxchg(v, old, new) != old); 47 + } 48 + #endif /* #ifndef CONFIG_ARCH_HAS_ATOMIC_OR */ 49 + 37 50 #endif /* _LINUX_ATOMIC_H */

+67 -97

kernel/rcutree.c

··· 36 36 #include <linux/interrupt.h> 37 37 #include <linux/sched.h> 38 38 #include <linux/nmi.h> 39 - #include <asm/atomic.h> 39 + #include <linux/atomic.h> 40 40 #include <linux/bitops.h> 41 41 #include <linux/module.h> 42 42 #include <linux/completion.h> ··· 95 95 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); 96 96 DEFINE_PER_CPU(int, rcu_cpu_kthread_cpu); 97 97 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); 98 - static DEFINE_PER_CPU(wait_queue_head_t, rcu_cpu_wq); 99 98 DEFINE_PER_CPU(char, rcu_cpu_has_work); 100 99 static char rcu_kthreads_spawnable; 101 100 ··· 162 163 #ifdef CONFIG_NO_HZ 163 164 DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { 164 165 .dynticks_nesting = 1, 165 - .dynticks = 1, 166 + .dynticks = ATOMIC_INIT(1), 166 167 }; 167 168 #endif /* #ifdef CONFIG_NO_HZ */ 168 169 ··· 321 322 unsigned long flags; 322 323 struct rcu_dynticks *rdtp; 323 324 324 - smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ 325 325 local_irq_save(flags); 326 326 rdtp = &__get_cpu_var(rcu_dynticks); 327 - rdtp->dynticks++; 328 - rdtp->dynticks_nesting--; 329 - WARN_ON_ONCE(rdtp->dynticks & 0x1); 327 + if (--rdtp->dynticks_nesting) { 328 + local_irq_restore(flags); 329 + return; 330 + } 331 + /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ 332 + smp_mb__before_atomic_inc(); /* See above. */ 333 + atomic_inc(&rdtp->dynticks); 334 + smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */ 335 + WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); 330 336 local_irq_restore(flags); 337 + 338 + /* If the interrupt queued a callback, get out of dyntick mode. */ 339 + if (in_irq() && 340 + (__get_cpu_var(rcu_sched_data).nxtlist || 341 + __get_cpu_var(rcu_bh_data).nxtlist || 342 + rcu_preempt_needs_cpu(smp_processor_id()))) 343 + set_need_resched(); 331 344 } 332 345 333 346 /* ··· 355 344 356 345 local_irq_save(flags); 357 346 rdtp = &__get_cpu_var(rcu_dynticks); 358 - rdtp->dynticks++; 359 - rdtp->dynticks_nesting++; 360 - WARN_ON_ONCE(!(rdtp->dynticks & 0x1)); 347 + if (rdtp->dynticks_nesting++) { 348 + local_irq_restore(flags); 349 + return; 350 + } 351 + smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */ 352 + atomic_inc(&rdtp->dynticks); 353 + /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ 354 + smp_mb__after_atomic_inc(); /* See above. */ 355 + WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); 361 356 local_irq_restore(flags); 362 - smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */ 363 357 } 364 358 365 359 /** ··· 378 362 { 379 363 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); 380 364 381 - if (rdtp->dynticks & 0x1) 365 + if (rdtp->dynticks_nmi_nesting == 0 && 366 + (atomic_read(&rdtp->dynticks) & 0x1)) 382 367 return; 383 - rdtp->dynticks_nmi++; 384 - WARN_ON_ONCE(!(rdtp->dynticks_nmi & 0x1)); 385 - smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */ 368 + rdtp->dynticks_nmi_nesting++; 369 + smp_mb__before_atomic_inc(); /* Force delay from prior write. */ 370 + atomic_inc(&rdtp->dynticks); 371 + /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ 372 + smp_mb__after_atomic_inc(); /* See above. */ 373 + WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); 386 374 } 387 375 388 376 /** ··· 400 380 { 401 381 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); 402 382 403 - if (rdtp->dynticks & 0x1) 383 + if (rdtp->dynticks_nmi_nesting == 0 || 384 + --rdtp->dynticks_nmi_nesting != 0) 404 385 return; 405 - smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ 406 - rdtp->dynticks_nmi++; 407 - WARN_ON_ONCE(rdtp->dynticks_nmi & 0x1); 386 + /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ 387 + smp_mb__before_atomic_inc(); /* See above. */ 388 + atomic_inc(&rdtp->dynticks); 389 + smp_mb__after_atomic_inc(); /* Force delay to next write. */ 390 + WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); 408 391 } 409 392 410 393 /** ··· 418 395 */ 419 396 void rcu_irq_enter(void) 420 397 { 421 - struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); 422 - 423 - if (rdtp->dynticks_nesting++) 424 - return; 425 - rdtp->dynticks++; 426 - WARN_ON_ONCE(!(rdtp->dynticks & 0x1)); 427 - smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */ 398 + rcu_exit_nohz(); 428 399 } 429 400 430 401 /** ··· 430 413 */ 431 414 void rcu_irq_exit(void) 432 415 { 433 - struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); 434 - 435 - if (--rdtp->dynticks_nesting) 436 - return; 437 - smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ 438 - rdtp->dynticks++; 439 - WARN_ON_ONCE(rdtp->dynticks & 0x1); 440 - 441 - /* If the interrupt queued a callback, get out of dyntick mode. */ 442 - if (__this_cpu_read(rcu_sched_data.nxtlist) || 443 - __this_cpu_read(rcu_bh_data.nxtlist)) 444 - set_need_resched(); 416 + rcu_enter_nohz(); 445 417 } 446 418 447 419 #ifdef CONFIG_SMP ··· 442 436 */ 443 437 static int dyntick_save_progress_counter(struct rcu_data *rdp) 444 438 { 445 - int ret; 446 - int snap; 447 - int snap_nmi; 448 - 449 - snap = rdp->dynticks->dynticks; 450 - snap_nmi = rdp->dynticks->dynticks_nmi; 451 - smp_mb(); /* Order sampling of snap with end of grace period. */ 452 - rdp->dynticks_snap = snap; 453 - rdp->dynticks_nmi_snap = snap_nmi; 454 - ret = ((snap & 0x1) == 0) && ((snap_nmi & 0x1) == 0); 455 - if (ret) 456 - rdp->dynticks_fqs++; 457 - return ret; 439 + rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks); 440 + return 0; 458 441 } 459 442 460 443 /* ··· 454 459 */ 455 460 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) 456 461 { 457 - long curr; 458 - long curr_nmi; 459 - long snap; 460 - long snap_nmi; 462 + unsigned long curr; 463 + unsigned long snap; 461 464 462 - curr = rdp->dynticks->dynticks; 463 - snap = rdp->dynticks_snap; 464 - curr_nmi = rdp->dynticks->dynticks_nmi; 465 - snap_nmi = rdp->dynticks_nmi_snap; 466 - smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ 465 + curr = (unsigned long)atomic_add_return(0, &rdp->dynticks->dynticks); 466 + snap = (unsigned long)rdp->dynticks_snap; 467 467 468 468 /* 469 469 * If the CPU passed through or entered a dynticks idle phase with ··· 468 478 * read-side critical section that started before the beginning 469 479 * of the current RCU grace period. 470 480 */ 471 - if ((curr != snap || (curr & 0x1) == 0) && 472 - (curr_nmi != snap_nmi || (curr_nmi & 0x1) == 0)) { 481 + if ((curr & 0x1) == 0 || ULONG_CMP_GE(curr, snap + 2)) { 473 482 rdp->dynticks_fqs++; 474 483 return 1; 475 484 } ··· 897 908 unsigned long gp_duration; 898 909 899 910 WARN_ON_ONCE(!rcu_gp_in_progress(rsp)); 911 + 912 + /* 913 + * Ensure that all grace-period and pre-grace-period activity 914 + * is seen before the assignment to rsp->completed. 915 + */ 916 + smp_mb(); /* See above block comment. */ 900 917 gp_duration = jiffies - rsp->gp_start; 901 918 if (gp_duration > rsp->gp_max) 902 919 rsp->gp_max = gp_duration; ··· 1450 1455 */ 1451 1456 static void rcu_process_callbacks(void) 1452 1457 { 1453 - /* 1454 - * Memory references from any prior RCU read-side critical sections 1455 - * executed by the interrupted code must be seen before any RCU 1456 - * grace-period manipulations below. 1457 - */ 1458 - smp_mb(); /* See above block comment. */ 1459 - 1460 1458 __rcu_process_callbacks(&rcu_sched_state, 1461 1459 &__get_cpu_var(rcu_sched_data)); 1462 1460 __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data)); 1463 1461 rcu_preempt_process_callbacks(); 1464 - 1465 - /* 1466 - * Memory references from any later RCU read-side critical sections 1467 - * executed by the interrupted code must be seen after any RCU 1468 - * grace-period manipulations above. 1469 - */ 1470 - smp_mb(); /* See above block comment. */ 1471 1462 1472 1463 /* If we are last CPU on way to dyntick-idle mode, accelerate it. */ 1473 1464 rcu_needs_cpu_flush(); ··· 1475 1494 local_irq_restore(flags); 1476 1495 return; 1477 1496 } 1478 - wake_up(&__get_cpu_var(rcu_cpu_wq)); 1497 + wake_up_process(__this_cpu_read(rcu_cpu_kthread_task)); 1479 1498 local_irq_restore(flags); 1480 1499 } 1481 1500 ··· 1525 1544 */ 1526 1545 static void rcu_cpu_kthread_timer(unsigned long arg) 1527 1546 { 1528 - unsigned long flags; 1529 1547 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, arg); 1530 1548 struct rcu_node *rnp = rdp->mynode; 1531 1549 1532 - raw_spin_lock_irqsave(&rnp->lock, flags); 1533 - rnp->wakemask |= rdp->grpmask; 1534 - raw_spin_unlock_irqrestore(&rnp->lock, flags); 1550 + atomic_or(rdp->grpmask, &rnp->wakemask); 1535 1551 invoke_rcu_node_kthread(rnp); 1536 1552 } 1537 1553 ··· 1595 1617 unsigned long flags; 1596 1618 int spincnt = 0; 1597 1619 unsigned int *statusp = &per_cpu(rcu_cpu_kthread_status, cpu); 1598 - wait_queue_head_t *wqp = &per_cpu(rcu_cpu_wq, cpu); 1599 1620 char work; 1600 1621 char *workp = &per_cpu(rcu_cpu_has_work, cpu); 1601 1622 1602 1623 for (;;) { 1603 1624 *statusp = RCU_KTHREAD_WAITING; 1604 - wait_event_interruptible(*wqp, 1605 - *workp != 0 || kthread_should_stop()); 1625 + rcu_wait(*workp != 0 || kthread_should_stop()); 1606 1626 local_bh_disable(); 1607 1627 if (rcu_cpu_kthread_should_stop(cpu)) { 1608 1628 local_bh_enable(); ··· 1648 1672 if (IS_ERR(t)) 1649 1673 return PTR_ERR(t); 1650 1674 kthread_bind(t, cpu); 1675 + set_task_state(t, TASK_INTERRUPTIBLE); 1651 1676 per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu; 1652 1677 WARN_ON_ONCE(per_cpu(rcu_cpu_kthread_task, cpu) != NULL); 1653 1678 per_cpu(rcu_cpu_kthread_task, cpu) = t; 1654 - wake_up_process(t); 1655 1679 sp.sched_priority = RCU_KTHREAD_PRIO; 1656 1680 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 1657 1681 return 0; ··· 1674 1698 1675 1699 for (;;) { 1676 1700 rnp->node_kthread_status = RCU_KTHREAD_WAITING; 1677 - wait_event_interruptible(rnp->node_wq, rnp->wakemask != 0); 1701 + rcu_wait(atomic_read(&rnp->wakemask) != 0); 1678 1702 rnp->node_kthread_status = RCU_KTHREAD_RUNNING; 1679 1703 raw_spin_lock_irqsave(&rnp->lock, flags); 1680 - mask = rnp->wakemask; 1681 - rnp->wakemask = 0; 1704 + mask = atomic_xchg(&rnp->wakemask, 0); 1682 1705 rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */ 1683 1706 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) { 1684 1707 if ((mask & 0x1) == 0) ··· 1756 1781 if (IS_ERR(t)) 1757 1782 return PTR_ERR(t); 1758 1783 raw_spin_lock_irqsave(&rnp->lock, flags); 1784 + set_task_state(t, TASK_INTERRUPTIBLE); 1759 1785 rnp->node_kthread_task = t; 1760 1786 raw_spin_unlock_irqrestore(&rnp->lock, flags); 1761 - wake_up_process(t); 1762 1787 sp.sched_priority = 99; 1763 1788 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 1764 1789 } ··· 1775 1800 1776 1801 rcu_kthreads_spawnable = 1; 1777 1802 for_each_possible_cpu(cpu) { 1778 - init_waitqueue_head(&per_cpu(rcu_cpu_wq, cpu)); 1779 1803 per_cpu(rcu_cpu_has_work, cpu) = 0; 1780 1804 if (cpu_online(cpu)) 1781 1805 (void)rcu_spawn_one_cpu_kthread(cpu); 1782 1806 } 1783 1807 rnp = rcu_get_root(rcu_state); 1784 - init_waitqueue_head(&rnp->node_wq); 1785 - rcu_init_boost_waitqueue(rnp); 1786 1808 (void)rcu_spawn_one_node_kthread(rcu_state, rnp); 1787 - if (NUM_RCU_NODES > 1) 1788 - rcu_for_each_leaf_node(rcu_state, rnp) { 1789 - init_waitqueue_head(&rnp->node_wq); 1790 - rcu_init_boost_waitqueue(rnp); 1809 + if (NUM_RCU_NODES > 1) { 1810 + rcu_for_each_leaf_node(rcu_state, rnp) 1791 1811 (void)rcu_spawn_one_node_kthread(rcu_state, rnp); 1792 - } 1812 + } 1793 1813 return 0; 1794 1814 } 1795 1815 early_initcall(rcu_spawn_kthreads);

+16 -14

kernel/rcutree.h

··· 84 84 * Dynticks per-CPU state. 85 85 */ 86 86 struct rcu_dynticks { 87 - int dynticks_nesting; /* Track nesting level, sort of. */ 88 - int dynticks; /* Even value for dynticks-idle, else odd. */ 89 - int dynticks_nmi; /* Even value for either dynticks-idle or */ 90 - /* not in nmi handler, else odd. So this */ 91 - /* remains even for nmi from irq handler. */ 87 + int dynticks_nesting; /* Track irq/process nesting level. */ 88 + int dynticks_nmi_nesting; /* Track NMI nesting level. */ 89 + atomic_t dynticks; /* Even value for dynticks-idle, else odd. */ 92 90 }; 93 91 94 92 /* RCU's kthread states for tracing. */ ··· 119 121 /* elements that need to drain to allow the */ 120 122 /* current expedited grace period to */ 121 123 /* complete (only for TREE_PREEMPT_RCU). */ 122 - unsigned long wakemask; /* CPUs whose kthread needs to be awakened. */ 124 + atomic_t wakemask; /* CPUs whose kthread needs to be awakened. */ 125 + /* Since this has meaning only for leaf */ 126 + /* rcu_node structures, 32 bits suffices. */ 123 127 unsigned long qsmaskinit; 124 128 /* Per-GP initial value for qsmask & expmask. */ 125 129 unsigned long grpmask; /* Mask to apply to parent qsmask. */ ··· 159 159 struct task_struct *boost_kthread_task; 160 160 /* kthread that takes care of priority */ 161 161 /* boosting for this rcu_node structure. */ 162 - wait_queue_head_t boost_wq; 163 - /* Wait queue on which to park the boost */ 164 - /* kthread. */ 165 162 unsigned int boost_kthread_status; 166 163 /* State of boost_kthread_task for tracing. */ 167 164 unsigned long n_tasks_boosted; ··· 185 188 /* kthread that takes care of this rcu_node */ 186 189 /* structure, for example, awakening the */ 187 190 /* per-CPU kthreads as needed. */ 188 - wait_queue_head_t node_wq; 189 - /* Wait queue on which to park the per-node */ 190 - /* kthread. */ 191 191 unsigned int node_kthread_status; 192 192 /* State of node_kthread_task for tracing. */ 193 193 } ____cacheline_internodealigned_in_smp; ··· 278 284 /* 3) dynticks interface. */ 279 285 struct rcu_dynticks *dynticks; /* Shared per-CPU dynticks state. */ 280 286 int dynticks_snap; /* Per-GP tracking for dynticks. */ 281 - int dynticks_nmi_snap; /* Per-GP tracking for dynticks_nmi. */ 282 287 #endif /* #ifdef CONFIG_NO_HZ */ 283 288 284 289 /* 4) reasons this CPU needed to be kicked by force_quiescent_state */ ··· 330 337 /* scheduling clock irq */ 331 338 /* before ratting on them. */ 332 339 340 + #define rcu_wait(cond) \ 341 + do { \ 342 + for (;;) { \ 343 + set_current_state(TASK_INTERRUPTIBLE); \ 344 + if (cond) \ 345 + break; \ 346 + schedule(); \ 347 + } \ 348 + __set_current_state(TASK_RUNNING); \ 349 + } while (0) 333 350 334 351 /* 335 352 * RCU global state, including node hierarchy. This hierarchy is ··· 449 446 static void rcu_preempt_send_cbs_to_online(void); 450 447 static void __init __rcu_init_preempt(void); 451 448 static void rcu_needs_cpu_flush(void); 452 - static void __init rcu_init_boost_waitqueue(struct rcu_node *rnp); 453 449 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); 454 450 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, 455 451 cpumask_var_t cm);

+5 -19

kernel/rcutree_plugin.h

··· 1196 1196 1197 1197 for (;;) { 1198 1198 rnp->boost_kthread_status = RCU_KTHREAD_WAITING; 1199 - wait_event_interruptible(rnp->boost_wq, rnp->boost_tasks || 1200 - rnp->exp_tasks); 1199 + rcu_wait(rnp->boost_tasks || rnp->exp_tasks); 1201 1200 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING; 1202 1201 more2boost = rcu_boost(rnp); 1203 1202 if (more2boost) ··· 1274 1275 } 1275 1276 1276 1277 /* 1277 - * Initialize the RCU-boost waitqueue. 1278 - */ 1279 - static void __init rcu_init_boost_waitqueue(struct rcu_node *rnp) 1280 - { 1281 - init_waitqueue_head(&rnp->boost_wq); 1282 - } 1283 - 1284 - /* 1285 1278 * Create an RCU-boost kthread for the specified node if one does not 1286 1279 * already exist. We only create this kthread for preemptible RCU. 1287 1280 * Returns zero if all is well, a negated errno otherwise. ··· 1295 1304 if (IS_ERR(t)) 1296 1305 return PTR_ERR(t); 1297 1306 raw_spin_lock_irqsave(&rnp->lock, flags); 1307 + set_task_state(t, TASK_INTERRUPTIBLE); 1298 1308 rnp->boost_kthread_task = t; 1299 1309 raw_spin_unlock_irqrestore(&rnp->lock, flags); 1300 - wake_up_process(t); 1301 1310 sp.sched_priority = RCU_KTHREAD_PRIO; 1302 1311 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 1303 1312 return 0; ··· 1316 1325 } 1317 1326 1318 1327 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) 1319 - { 1320 - } 1321 - 1322 - static void __init rcu_init_boost_waitqueue(struct rcu_node *rnp) 1323 1328 { 1324 1329 } 1325 1330 ··· 1507 1520 { 1508 1521 int c = 0; 1509 1522 int snap; 1510 - int snap_nmi; 1511 1523 int thatcpu; 1512 1524 1513 1525 /* Check for being in the holdoff period. */ ··· 1517 1531 for_each_online_cpu(thatcpu) { 1518 1532 if (thatcpu == cpu) 1519 1533 continue; 1520 - snap = per_cpu(rcu_dynticks, thatcpu).dynticks; 1521 - snap_nmi = per_cpu(rcu_dynticks, thatcpu).dynticks_nmi; 1534 + snap = atomic_add_return(0, &per_cpu(rcu_dynticks, 1535 + thatcpu).dynticks); 1522 1536 smp_mb(); /* Order sampling of snap with end of grace period. */ 1523 - if (((snap & 0x1) != 0) || ((snap_nmi & 0x1) != 0)) { 1537 + if ((snap & 0x1) != 0) { 1524 1538 per_cpu(rcu_dyntick_drain, cpu) = 0; 1525 1539 per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1; 1526 1540 return rcu_needs_cpu_quick_check(cpu);

+6 -6

kernel/rcutree_trace.c

··· 69 69 rdp->passed_quiesc, rdp->passed_quiesc_completed, 70 70 rdp->qs_pending); 71 71 #ifdef CONFIG_NO_HZ 72 - seq_printf(m, " dt=%d/%d dn=%d df=%lu", 73 - rdp->dynticks->dynticks, 72 + seq_printf(m, " dt=%d/%d/%d df=%lu", 73 + atomic_read(&rdp->dynticks->dynticks), 74 74 rdp->dynticks->dynticks_nesting, 75 - rdp->dynticks->dynticks_nmi, 75 + rdp->dynticks->dynticks_nmi_nesting, 76 76 rdp->dynticks_fqs); 77 77 #endif /* #ifdef CONFIG_NO_HZ */ 78 78 seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi); ··· 141 141 rdp->qs_pending); 142 142 #ifdef CONFIG_NO_HZ 143 143 seq_printf(m, ",%d,%d,%d,%lu", 144 - rdp->dynticks->dynticks, 144 + atomic_read(&rdp->dynticks->dynticks), 145 145 rdp->dynticks->dynticks_nesting, 146 - rdp->dynticks->dynticks_nmi, 146 + rdp->dynticks->dynticks_nmi_nesting, 147 147 rdp->dynticks_fqs); 148 148 #endif /* #ifdef CONFIG_NO_HZ */ 149 149 seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi); ··· 167 167 { 168 168 seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pqc\",\"pq\","); 169 169 #ifdef CONFIG_NO_HZ 170 - seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\","); 170 + seq_puts(m, "\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\","); 171 171 #endif /* #ifdef CONFIG_NO_HZ */ 172 172 seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\",\"ci\",\"co\",\"ca\"\n"); 173 173 #ifdef CONFIG_TREE_PREEMPT_RCU

+1 -1

lib/locking-selftest.c

··· 144 144 145 145 #define HARDIRQ_ENTER() \ 146 146 local_irq_disable(); \ 147 - irq_enter(); \ 147 + __irq_enter(); \ 148 148 WARN_ON(!in_irq()); 149 149 150 150 #define HARDIRQ_EXIT() \

Configure Feed

Configure Feed