tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'rcu.2021.11.01a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu

Pull RCU updates from Paul McKenney:

- Miscellaneous fixes

- Torture-test updates for smp_call_function(), most notably improved
checking of module parameters.

- Tasks-trace RCU updates that fix a number of rare but important
race-condition bugs.

- Other torture-test updates, most notably better checking of module
parameters. In addition, rcutorture may once again be run on
CONFIG_PREEMPT_RT kernels.

- Torture-test scripting updates, most notably specifying the new
CONFIG_KCSAN_STRICT kconfig option rather than maintaining an
ever-changing list of individual KCSAN kconfig options.

* tag 'rcu.2021.11.01a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu: (46 commits)
rcu: Fix rcu_dynticks_curr_cpu_in_eqs() vs noinstr
rcu: Always inline rcu_dynticks_task*_{enter,exit}()
torture: Make kvm-remote.sh print size of downloaded tarball
torture: Allot 1G of memory for scftorture runs
tools/rcu: Add an extract-stall script
scftorture: Warn on individual scf_torture_init() error conditions
scftorture: Count reschedule IPIs
scftorture: Account for weight_resched when checking for all zeroes
scftorture: Shut down if nonsensical arguments given
scftorture: Allow zero weight to exclude an smp_call_function*() category
rcu: Avoid unneeded function call in rcu_read_unlock()
rcu-tasks: Update comments to cond_resched_tasks_rcu_qs()
rcu-tasks: Fix IPI failure handling in trc_wait_for_one_reader
rcu-tasks: Fix read-side primitives comment for call_rcu_tasks_trace
rcu-tasks: Clarify read side section info for rcu_tasks_rude GP primitives
rcu-tasks: Correct comparisons for CPU numbers in show_stalled_task_trace
rcu-tasks: Correct firstreport usage in check_all_holdout_tasks_trace
rcu-tasks: Fix s/rcu_add_holdout/trc_add_holdout/ typo in comment
rcu-tasks: Move RTGS_WAIT_CBS to beginning of rcu_tasks_kthread() loop
rcu-tasks: Fix s/instruction/instructions/ typo in comment
...

Linus Torvalds 6fedc280 79ef0c00

+273 -200
+32 -37
Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst
··· 202 202 1 static void rcu_prepare_for_idle(void) 203 203 2 { 204 204 3 bool needwake; 205 - 4 struct rcu_data *rdp; 206 - 5 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); 207 - 6 struct rcu_node *rnp; 208 - 7 struct rcu_state *rsp; 209 - 8 int tne; 210 - 9 211 - 10 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) || 212 - 11 rcu_is_nocb_cpu(smp_processor_id())) 213 - 12 return; 205 + 4 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 206 + 5 struct rcu_node *rnp; 207 + 6 int tne; 208 + 7 209 + 8 lockdep_assert_irqs_disabled(); 210 + 9 if (rcu_rdp_is_offloaded(rdp)) 211 + 10 return; 212 + 11 213 + 12 /* Handle nohz enablement switches conservatively. */ 214 214 13 tne = READ_ONCE(tick_nohz_active); 215 - 14 if (tne != rdtp->tick_nohz_enabled_snap) { 216 - 15 if (rcu_cpu_has_callbacks(NULL)) 217 - 16 invoke_rcu_core(); 218 - 17 rdtp->tick_nohz_enabled_snap = tne; 215 + 14 if (tne != rdp->tick_nohz_enabled_snap) { 216 + 15 if (!rcu_segcblist_empty(&rdp->cblist)) 217 + 16 invoke_rcu_core(); /* force nohz to see update. */ 218 + 17 rdp->tick_nohz_enabled_snap = tne; 219 219 18 return; 220 - 19 } 220 + 19 } 221 221 20 if (!tne) 222 222 21 return; 223 - 22 if (rdtp->all_lazy && 224 - 23 rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) { 225 - 24 rdtp->all_lazy = false; 226 - 25 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted; 227 - 26 invoke_rcu_core(); 228 - 27 return; 229 - 28 } 230 - 29 if (rdtp->last_accelerate == jiffies) 231 - 30 return; 232 - 31 rdtp->last_accelerate = jiffies; 233 - 32 for_each_rcu_flavor(rsp) { 234 - 33 rdp = this_cpu_ptr(rsp->rda); 235 - 34 if (rcu_segcblist_pend_cbs(&rdp->cblist)) 236 - 35 continue; 237 - 36 rnp = rdp->mynode; 238 - 37 raw_spin_lock_rcu_node(rnp); 239 - 38 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); 240 - 39 raw_spin_unlock_rcu_node(rnp); 241 - 40 if (needwake) 242 - 41 rcu_gp_kthread_wake(rsp); 243 - 42 } 244 - 43 } 223 + 22 224 + 23 /* 225 + 24 * If we have not yet accelerated this jiffy, accelerate all 226 + 25 * callbacks on this CPU. 227 + 26 */ 228 + 27 if (rdp->last_accelerate == jiffies) 229 + 28 return; 230 + 29 rdp->last_accelerate = jiffies; 231 + 30 if (rcu_segcblist_pend_cbs(&rdp->cblist)) { 232 + 31 rnp = rdp->mynode; 233 + 32 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ 234 + 33 needwake = rcu_accelerate_cbs(rnp, rdp); 235 + 34 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ 236 + 35 if (needwake) 237 + 36 rcu_gp_kthread_wake(); 238 + 37 } 239 + 38 } 245 240 246 241 But the only part of ``rcu_prepare_for_idle()`` that really matters for 247 - this discussion are lines 37–39. We will therefore abbreviate this 242 + this discussion are lines 32–34. We will therefore abbreviate this 248 243 function as follows: 249 244 250 245 .. kernel-figure:: rcu_node-lock.svg
+10
Documentation/RCU/stallwarn.rst
··· 96 96 the ``rcu_.*timer wakeup didn't happen for`` console-log message, 97 97 which will include additional debugging information. 98 98 99 + - A low-level kernel issue that either fails to invoke one of the 100 + variants of rcu_user_enter(), rcu_user_exit(), rcu_idle_enter(), 101 + rcu_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one 102 + hand, or that invokes one of them too many times on the other. 103 + Historically, the most frequent issue has been an omission 104 + of either irq_enter() or irq_exit(), which in turn invoke 105 + rcu_irq_enter() or rcu_irq_exit(), respectively. Building your 106 + kernel with CONFIG_RCU_EQS_DEBUG=y can help track down these types 107 + of issues, which sometimes arise in architecture-specific code. 108 + 99 109 - A bug in the RCU implementation. 100 110 101 111 - A hardware failure. This is quite unlikely, but has occurred
-1
arch/sh/configs/sdk7786_defconfig
··· 5 5 CONFIG_BSD_PROCESS_ACCT_V3=y 6 6 CONFIG_AUDIT=y 7 7 CONFIG_AUDITSYSCALL=y 8 - CONFIG_TREE_PREEMPT_RCU=y 9 8 CONFIG_RCU_TRACE=y 10 9 CONFIG_IKCONFIG=y 11 10 CONFIG_IKCONFIG_PROC=y
-1
arch/xtensa/configs/nommu_kc705_defconfig
··· 119 119 CONFIG_DEBUG_MUTEXES=y 120 120 CONFIG_DEBUG_ATOMIC_SLEEP=y 121 121 CONFIG_STACKTRACE=y 122 - # CONFIG_RCU_CPU_STALL_INFO is not set 123 122 CONFIG_RCU_TRACE=y 124 123 # CONFIG_FTRACE is not set 125 124 # CONFIG_LD_NO_RELAX is not set
+2 -1
include/linux/rcupdate.h
··· 71 71 static inline void __rcu_read_unlock(void) 72 72 { 73 73 preempt_enable(); 74 - rcu_read_unlock_strict(); 74 + if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) 75 + rcu_read_unlock_strict(); 75 76 } 76 77 77 78 static inline int rcu_preempt_depth(void)
+3 -2
include/linux/rcupdate_trace.h
··· 31 31 32 32 #ifdef CONFIG_TASKS_TRACE_RCU 33 33 34 - void rcu_read_unlock_trace_special(struct task_struct *t, int nesting); 34 + void rcu_read_unlock_trace_special(struct task_struct *t); 35 35 36 36 /** 37 37 * rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section ··· 80 80 WRITE_ONCE(t->trc_reader_nesting, nesting); 81 81 return; // We assume shallow reader nesting. 82 82 } 83 - rcu_read_unlock_trace_special(t, nesting); 83 + WARN_ON_ONCE(nesting != 0); 84 + rcu_read_unlock_trace_special(t); 84 85 } 85 86 86 87 void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func);
+8
include/linux/torture.h
··· 47 47 } while (0) 48 48 void verbose_torout_sleep(void); 49 49 50 + #define torture_init_error(firsterr) \ 51 + ({ \ 52 + int ___firsterr = (firsterr); \ 53 + \ 54 + WARN_ONCE(!IS_MODULE(CONFIG_RCU_TORTURE_TEST) && ___firsterr < 0, "Torture-test initialization failed with error code %d\n", ___firsterr); \ 55 + ___firsterr < 0; \ 56 + }) 57 + 50 58 /* Definitions for online/offline exerciser. */ 51 59 #ifdef CONFIG_HOTPLUG_CPU 52 60 int torture_num_online_cpus(void);
+7 -7
kernel/locking/locktorture.c
··· 1022 1022 if (onoff_interval > 0) { 1023 1023 firsterr = torture_onoff_init(onoff_holdoff * HZ, 1024 1024 onoff_interval * HZ, NULL); 1025 - if (firsterr) 1025 + if (torture_init_error(firsterr)) 1026 1026 goto unwind; 1027 1027 } 1028 1028 if (shuffle_interval > 0) { 1029 1029 firsterr = torture_shuffle_init(shuffle_interval); 1030 - if (firsterr) 1030 + if (torture_init_error(firsterr)) 1031 1031 goto unwind; 1032 1032 } 1033 1033 if (shutdown_secs > 0) { 1034 1034 firsterr = torture_shutdown_init(shutdown_secs, 1035 1035 lock_torture_cleanup); 1036 - if (firsterr) 1036 + if (torture_init_error(firsterr)) 1037 1037 goto unwind; 1038 1038 } 1039 1039 if (stutter > 0) { 1040 1040 firsterr = torture_stutter_init(stutter, stutter); 1041 - if (firsterr) 1041 + if (torture_init_error(firsterr)) 1042 1042 goto unwind; 1043 1043 } 1044 1044 ··· 1082 1082 /* Create writer. */ 1083 1083 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i], 1084 1084 writer_tasks[i]); 1085 - if (firsterr) 1085 + if (torture_init_error(firsterr)) 1086 1086 goto unwind; 1087 1087 1088 1088 create_reader: ··· 1091 1091 /* Create reader. */ 1092 1092 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j], 1093 1093 reader_tasks[j]); 1094 - if (firsterr) 1094 + if (torture_init_error(firsterr)) 1095 1095 goto unwind; 1096 1096 } 1097 1097 if (stat_interval > 0) { 1098 1098 firsterr = torture_create_kthread(lock_torture_stats, NULL, 1099 1099 stats_task); 1100 - if (firsterr) 1100 + if (torture_init_error(firsterr)) 1101 1101 goto unwind; 1102 1102 } 1103 1103 torture_init_end();
+5 -5
kernel/rcu/rcuscale.c
··· 758 758 init_waitqueue_head(&shutdown_wq); 759 759 firsterr = torture_create_kthread(kfree_scale_shutdown, NULL, 760 760 shutdown_task); 761 - if (firsterr) 761 + if (torture_init_error(firsterr)) 762 762 goto unwind; 763 763 schedule_timeout_uninterruptible(1); 764 764 } ··· 775 775 for (i = 0; i < kfree_nrealthreads; i++) { 776 776 firsterr = torture_create_kthread(kfree_scale_thread, (void *)i, 777 777 kfree_reader_tasks[i]); 778 - if (firsterr) 778 + if (torture_init_error(firsterr)) 779 779 goto unwind; 780 780 } 781 781 ··· 838 838 init_waitqueue_head(&shutdown_wq); 839 839 firsterr = torture_create_kthread(rcu_scale_shutdown, NULL, 840 840 shutdown_task); 841 - if (firsterr) 841 + if (torture_init_error(firsterr)) 842 842 goto unwind; 843 843 schedule_timeout_uninterruptible(1); 844 844 } ··· 852 852 for (i = 0; i < nrealreaders; i++) { 853 853 firsterr = torture_create_kthread(rcu_scale_reader, (void *)i, 854 854 reader_tasks[i]); 855 - if (firsterr) 855 + if (torture_init_error(firsterr)) 856 856 goto unwind; 857 857 } 858 858 while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders) ··· 879 879 } 880 880 firsterr = torture_create_kthread(rcu_scale_writer, (void *)i, 881 881 writer_tasks[i]); 882 - if (firsterr) 882 + if (torture_init_error(firsterr)) 883 883 goto unwind; 884 884 } 885 885 torture_init_end();
+55 -31
kernel/rcu/rcutorture.c
··· 1432 1432 /* First, put new protection in place to avoid critical-section gap. */ 1433 1433 if (statesnew & RCUTORTURE_RDR_BH) 1434 1434 local_bh_disable(); 1435 + if (statesnew & RCUTORTURE_RDR_RBH) 1436 + rcu_read_lock_bh(); 1435 1437 if (statesnew & RCUTORTURE_RDR_IRQ) 1436 1438 local_irq_disable(); 1437 1439 if (statesnew & RCUTORTURE_RDR_PREEMPT) 1438 1440 preempt_disable(); 1439 - if (statesnew & RCUTORTURE_RDR_RBH) 1440 - rcu_read_lock_bh(); 1441 1441 if (statesnew & RCUTORTURE_RDR_SCHED) 1442 1442 rcu_read_lock_sched(); 1443 1443 if (statesnew & RCUTORTURE_RDR_RCU) 1444 1444 idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT; 1445 1445 1446 - /* Next, remove old protection, irq first due to bh conflict. */ 1446 + /* 1447 + * Next, remove old protection, in decreasing order of strength 1448 + * to avoid unlock paths that aren't safe in the stronger 1449 + * context. Namely: BH can not be enabled with disabled interrupts. 1450 + * Additionally PREEMPT_RT requires that BH is enabled in preemptible 1451 + * context. 1452 + */ 1447 1453 if (statesold & RCUTORTURE_RDR_IRQ) 1448 1454 local_irq_enable(); 1449 - if (statesold & RCUTORTURE_RDR_BH) 1450 - local_bh_enable(); 1451 1455 if (statesold & RCUTORTURE_RDR_PREEMPT) 1452 1456 preempt_enable(); 1453 - if (statesold & RCUTORTURE_RDR_RBH) 1454 - rcu_read_unlock_bh(); 1455 1457 if (statesold & RCUTORTURE_RDR_SCHED) 1456 1458 rcu_read_unlock_sched(); 1459 + if (statesold & RCUTORTURE_RDR_BH) 1460 + local_bh_enable(); 1461 + if (statesold & RCUTORTURE_RDR_RBH) 1462 + rcu_read_unlock_bh(); 1457 1463 if (statesold & RCUTORTURE_RDR_RCU) { 1458 1464 bool lockit = !statesnew && !(torture_random(trsp) & 0xffff); 1459 1465 ··· 1502 1496 int mask = rcutorture_extend_mask_max(); 1503 1497 unsigned long randmask1 = torture_random(trsp) >> 8; 1504 1498 unsigned long randmask2 = randmask1 >> 3; 1499 + unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 1500 + unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; 1501 + unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 1505 1502 1506 1503 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT); 1507 1504 /* Mostly only one bit (need preemption!), sometimes lots of bits. */ ··· 1512 1503 mask = mask & randmask2; 1513 1504 else 1514 1505 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); 1515 - /* Can't enable bh w/irq disabled. */ 1516 - if ((mask & RCUTORTURE_RDR_IRQ) && 1517 - ((!(mask & RCUTORTURE_RDR_BH) && (oldmask & RCUTORTURE_RDR_BH)) || 1518 - (!(mask & RCUTORTURE_RDR_RBH) && (oldmask & RCUTORTURE_RDR_RBH)))) 1519 - mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 1506 + 1507 + /* 1508 + * Can't enable bh w/irq disabled. 1509 + */ 1510 + if (mask & RCUTORTURE_RDR_IRQ) 1511 + mask |= oldmask & bhs; 1512 + 1513 + /* 1514 + * Ideally these sequences would be detected in debug builds 1515 + * (regardless of RT), but until then don't stop testing 1516 + * them on non-RT. 1517 + */ 1518 + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 1519 + /* Can't modify BH in atomic context */ 1520 + if (oldmask & preempts_irq) 1521 + mask &= ~bhs; 1522 + if ((oldmask | mask) & preempts_irq) 1523 + mask |= oldmask & bhs; 1524 + } 1525 + 1520 1526 return mask ?: RCUTORTURE_RDR_RCU; 1521 1527 } 1522 1528 ··· 2473 2449 } 2474 2450 if (stall_cpu > 0) { 2475 2451 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing"); 2476 - if (IS_MODULE(CONFIG_RCU_TORTURE_TESTS)) 2452 + if (IS_MODULE(CONFIG_RCU_TORTURE_TEST)) 2477 2453 return -EINVAL; /* In module, can fail back to user. */ 2478 2454 WARN_ON(1); /* Make sure rcutorture notices conflict. */ 2479 2455 return 0; ··· 2765 2741 static int rcu_torture_read_exit_init(void) 2766 2742 { 2767 2743 if (read_exit_burst <= 0) 2768 - return -EINVAL; 2744 + return 0; 2769 2745 init_waitqueue_head(&read_exit_wq); 2770 2746 read_exit_child_stop = false; 2771 2747 read_exit_child_stopped = false; ··· 2843 2819 rcutorture_seq_diff(gp_seq, start_gp_seq)); 2844 2820 torture_stop_kthread(rcu_torture_stats, stats_task); 2845 2821 torture_stop_kthread(rcu_torture_fqs, fqs_task); 2846 - if (rcu_torture_can_boost()) 2822 + if (rcu_torture_can_boost() && rcutor_hp >= 0) 2847 2823 cpuhp_remove_state(rcutor_hp); 2848 2824 2849 2825 /* ··· 3061 3037 rcu_torture_write_types(); 3062 3038 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 3063 3039 writer_task); 3064 - if (firsterr) 3040 + if (torture_init_error(firsterr)) 3065 3041 goto unwind; 3066 3042 if (nfakewriters > 0) { 3067 3043 fakewriter_tasks = kcalloc(nfakewriters, ··· 3076 3052 for (i = 0; i < nfakewriters; i++) { 3077 3053 firsterr = torture_create_kthread(rcu_torture_fakewriter, 3078 3054 NULL, fakewriter_tasks[i]); 3079 - if (firsterr) 3055 + if (torture_init_error(firsterr)) 3080 3056 goto unwind; 3081 3057 } 3082 3058 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), ··· 3092 3068 rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; 3093 3069 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, 3094 3070 reader_tasks[i]); 3095 - if (firsterr) 3071 + if (torture_init_error(firsterr)) 3096 3072 goto unwind; 3097 3073 } 3098 3074 nrealnocbers = nocbs_nthreads; ··· 3112 3088 } 3113 3089 for (i = 0; i < nrealnocbers; i++) { 3114 3090 firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]); 3115 - if (firsterr) 3091 + if (torture_init_error(firsterr)) 3116 3092 goto unwind; 3117 3093 } 3118 3094 if (stat_interval > 0) { 3119 3095 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 3120 3096 stats_task); 3121 - if (firsterr) 3097 + if (torture_init_error(firsterr)) 3122 3098 goto unwind; 3123 3099 } 3124 3100 if (test_no_idle_hz && shuffle_interval > 0) { 3125 3101 firsterr = torture_shuffle_init(shuffle_interval * HZ); 3126 - if (firsterr) 3102 + if (torture_init_error(firsterr)) 3127 3103 goto unwind; 3128 3104 } 3129 3105 if (stutter < 0) ··· 3133 3109 3134 3110 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; 3135 3111 firsterr = torture_stutter_init(stutter * HZ, t); 3136 - if (firsterr) 3112 + if (torture_init_error(firsterr)) 3137 3113 goto unwind; 3138 3114 } 3139 3115 if (fqs_duration < 0) ··· 3142 3118 /* Create the fqs thread */ 3143 3119 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 3144 3120 fqs_task); 3145 - if (firsterr) 3121 + if (torture_init_error(firsterr)) 3146 3122 goto unwind; 3147 3123 } 3148 3124 if (test_boost_interval < 1) ··· 3156 3132 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 3157 3133 rcutorture_booster_init, 3158 3134 rcutorture_booster_cleanup); 3159 - if (firsterr < 0) 3160 - goto unwind; 3161 3135 rcutor_hp = firsterr; 3136 + if (torture_init_error(firsterr)) 3137 + goto unwind; 3162 3138 3163 3139 // Testing RCU priority boosting requires rcutorture do 3164 3140 // some serious abuse. Counter this by running ksoftirqd ··· 3177 3153 } 3178 3154 shutdown_jiffies = jiffies + shutdown_secs * HZ; 3179 3155 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 3180 - if (firsterr) 3156 + if (torture_init_error(firsterr)) 3181 3157 goto unwind; 3182 3158 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, 3183 3159 rcutorture_sync); 3184 - if (firsterr) 3160 + if (torture_init_error(firsterr)) 3185 3161 goto unwind; 3186 3162 firsterr = rcu_torture_stall_init(); 3187 - if (firsterr) 3163 + if (torture_init_error(firsterr)) 3188 3164 goto unwind; 3189 3165 firsterr = rcu_torture_fwd_prog_init(); 3190 - if (firsterr) 3166 + if (torture_init_error(firsterr)) 3191 3167 goto unwind; 3192 3168 firsterr = rcu_torture_barrier_init(); 3193 - if (firsterr) 3169 + if (torture_init_error(firsterr)) 3194 3170 goto unwind; 3195 3171 firsterr = rcu_torture_read_exit_init(); 3196 - if (firsterr) 3172 + if (torture_init_error(firsterr)) 3197 3173 goto unwind; 3198 3174 if (object_debug) 3199 3175 rcu_test_debug_objects();
+3 -3
kernel/rcu/refscale.c
··· 824 824 init_waitqueue_head(&shutdown_wq); 825 825 firsterr = torture_create_kthread(ref_scale_shutdown, NULL, 826 826 shutdown_task); 827 - if (firsterr) 827 + if (torture_init_error(firsterr)) 828 828 goto unwind; 829 829 schedule_timeout_uninterruptible(1); 830 830 } ··· 851 851 for (i = 0; i < nreaders; i++) { 852 852 firsterr = torture_create_kthread(ref_scale_reader, (void *)i, 853 853 reader_tasks[i].task); 854 - if (firsterr) 854 + if (torture_init_error(firsterr)) 855 855 goto unwind; 856 856 857 857 init_waitqueue_head(&(reader_tasks[i].wq)); ··· 860 860 // Main Task 861 861 init_waitqueue_head(&main_wq); 862 862 firsterr = torture_create_kthread(main_func, NULL, main_task); 863 - if (firsterr) 863 + if (torture_init_error(firsterr)) 864 864 goto unwind; 865 865 866 866 torture_init_end();
+53 -56
kernel/rcu/tasks.h
··· 197 197 * This loop is terminated by the system going down. ;-) 198 198 */ 199 199 for (;;) { 200 + set_tasks_gp_state(rtp, RTGS_WAIT_CBS); 200 201 201 202 /* Pick up any new callbacks. */ 202 203 raw_spin_lock_irqsave(&rtp->cbs_lock, flags); ··· 237 236 } 238 237 /* Paranoid sleep to keep this from entering a tight loop */ 239 238 schedule_timeout_idle(rtp->gp_sleep); 240 - 241 - set_tasks_gp_state(rtp, RTGS_WAIT_CBS); 242 239 } 243 240 } 244 241 ··· 368 369 //////////////////////////////////////////////////////////////////////// 369 370 // 370 371 // Simple variant of RCU whose quiescent states are voluntary context 371 - // switch, cond_resched_rcu_qs(), user-space execution, and idle. 372 + // switch, cond_resched_tasks_rcu_qs(), user-space execution, and idle. 372 373 // As such, grace periods can take one good long time. There are no 373 374 // read-side primitives similar to rcu_read_lock() and rcu_read_unlock() 374 375 // because this implementation is intended to get the system into a safe ··· 539 540 * period elapses, in other words after all currently executing RCU 540 541 * read-side critical sections have completed. call_rcu_tasks() assumes 541 542 * that the read-side critical sections end at a voluntary context 542 - * switch (not a preemption!), cond_resched_rcu_qs(), entry into idle, 543 + * switch (not a preemption!), cond_resched_tasks_rcu_qs(), entry into idle, 543 544 * or transition to usermode execution. As such, there are no read-side 544 545 * primitives analogous to rcu_read_lock() and rcu_read_unlock() because 545 546 * this primitive is intended to determine that all tasks have passed ··· 677 678 * period elapses, in other words after all currently executing RCU 678 679 * read-side critical sections have completed. call_rcu_tasks_rude() 679 680 * assumes that the read-side critical sections end at context switch, 680 - * cond_resched_rcu_qs(), or transition to usermode execution. As such, 681 - * there are no read-side primitives analogous to rcu_read_lock() and 682 - * rcu_read_unlock() because this primitive is intended to determine 683 - * that all tasks have passed through a safe state, not so much for 684 - * data-structure synchronization. 681 + * cond_resched_tasks_rcu_qs(), or transition to usermode execution (as 682 + * usermode execution is schedulable). As such, there are no read-side 683 + * primitives analogous to rcu_read_lock() and rcu_read_unlock() because 684 + * this primitive is intended to determine that all tasks have passed 685 + * through a safe state, not so much for data-structure synchronization. 685 686 * 686 687 * See the description of call_rcu() for more detailed information on 687 688 * memory ordering guarantees. ··· 699 700 * grace period has elapsed, in other words after all currently 700 701 * executing rcu-tasks read-side critical sections have elapsed. These 701 702 * read-side critical sections are delimited by calls to schedule(), 702 - * cond_resched_tasks_rcu_qs(), userspace execution, and (in theory, 703 - * anyway) cond_resched(). 703 + * cond_resched_tasks_rcu_qs(), userspace execution (which is a schedulable 704 + * context), and (in theory, anyway) cond_resched(). 704 705 * 705 706 * This is a very specialized primitive, intended only for a few uses in 706 707 * tracing and other situations requiring manipulation of function preambles ··· 757 758 // 2. Protects code in the idle loop, exception entry/exit, and 758 759 // CPU-hotplug code paths, similar to the capabilities of SRCU. 759 760 // 760 - // 3. Avoids expensive read-side instruction, having overhead similar 761 + // 3. Avoids expensive read-side instructions, having overhead similar 761 762 // to that of Preemptible RCU. 762 763 // 763 764 // There are of course downsides. The grace-period code can send IPIs to ··· 847 848 static DEFINE_IRQ_WORK(rcu_tasks_trace_iw, rcu_read_unlock_iw); 848 849 849 850 /* If we are the last reader, wake up the grace-period kthread. */ 850 - void rcu_read_unlock_trace_special(struct task_struct *t, int nesting) 851 + void rcu_read_unlock_trace_special(struct task_struct *t) 851 852 { 852 853 int nq = READ_ONCE(t->trc_reader_special.b.need_qs); 853 854 ··· 857 858 // Update .need_qs before ->trc_reader_nesting for irq/NMI handlers. 858 859 if (nq) 859 860 WRITE_ONCE(t->trc_reader_special.b.need_qs, false); 860 - WRITE_ONCE(t->trc_reader_nesting, nesting); 861 + WRITE_ONCE(t->trc_reader_nesting, 0); 861 862 if (nq && atomic_dec_and_test(&trc_n_readers_need_end)) 862 863 irq_work_queue(&rcu_tasks_trace_iw); 863 864 } ··· 889 890 890 891 // If the task is no longer running on this CPU, leave. 891 892 if (unlikely(texp != t)) { 892 - if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end))) 893 - wake_up(&trc_wait); 894 893 goto reset_ipi; // Already on holdout list, so will check later. 895 894 } 896 895 897 896 // If the task is not in a read-side critical section, and 898 897 // if this is the last reader, awaken the grace-period kthread. 899 898 if (likely(!READ_ONCE(t->trc_reader_nesting))) { 900 - if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end))) 901 - wake_up(&trc_wait); 902 - // Mark as checked after decrement to avoid false 903 - // positives on the above WARN_ON_ONCE(). 904 899 WRITE_ONCE(t->trc_reader_checked, true); 905 900 goto reset_ipi; 906 901 } 907 902 // If we are racing with an rcu_read_unlock_trace(), try again later. 908 - if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) { 909 - if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end))) 910 - wake_up(&trc_wait); 903 + if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) 911 904 goto reset_ipi; 912 - } 913 905 WRITE_ONCE(t->trc_reader_checked, true); 914 906 915 907 // Get here if the task is in a read-side critical section. Set 916 908 // its state so that it will awaken the grace-period kthread upon 917 909 // exit from that critical section. 910 + atomic_inc(&trc_n_readers_need_end); // One more to wait on. 918 911 WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)); 919 912 WRITE_ONCE(t->trc_reader_special.b.need_qs, true); 920 913 ··· 922 931 static int trc_inspect_reader(struct task_struct *t, void *arg) 923 932 { 924 933 int cpu = task_cpu(t); 925 - bool in_qs = false; 934 + int nesting; 926 935 bool ofl = cpu_is_offline(cpu); 927 936 928 937 if (task_curr(t)) { ··· 942 951 n_heavy_reader_updates++; 943 952 if (ofl) 944 953 n_heavy_reader_ofl_updates++; 945 - in_qs = true; 954 + nesting = 0; 946 955 } else { 947 956 // The task is not running, so C-language access is safe. 948 - in_qs = likely(!t->trc_reader_nesting); 957 + nesting = t->trc_reader_nesting; 949 958 } 950 959 951 - // Mark as checked so that the grace-period kthread will 952 - // remove it from the holdout list. 953 - t->trc_reader_checked = true; 954 - 955 - if (in_qs) 956 - return 0; // Already in quiescent state, done!!! 960 + // If not exiting a read-side critical section, mark as checked 961 + // so that the grace-period kthread will remove it from the 962 + // holdout list. 963 + t->trc_reader_checked = nesting >= 0; 964 + if (nesting <= 0) 965 + return nesting ? -EINVAL : 0; // If in QS, done, otherwise try again later. 957 966 958 967 // The task is in a read-side critical section, so set up its 959 968 // state so that it will awaken the grace-period kthread upon exit ··· 991 1000 992 1001 // If this task is not yet on the holdout list, then we are in 993 1002 // an RCU read-side critical section. Otherwise, the invocation of 994 - // rcu_add_holdout() that added it to the list did the necessary 1003 + // trc_add_holdout() that added it to the list did the necessary 995 1004 // get_task_struct(). Either way, the task cannot be freed out 996 1005 // from under this code. 997 1006 ··· 1006 1015 if (per_cpu(trc_ipi_to_cpu, cpu) || t->trc_ipi_to_cpu >= 0) 1007 1016 return; 1008 1017 1009 - atomic_inc(&trc_n_readers_need_end); 1010 1018 per_cpu(trc_ipi_to_cpu, cpu) = true; 1011 1019 t->trc_ipi_to_cpu = cpu; 1012 1020 rcu_tasks_trace.n_ipis++; 1013 - if (smp_call_function_single(cpu, 1014 - trc_read_check_handler, t, 0)) { 1021 + if (smp_call_function_single(cpu, trc_read_check_handler, t, 0)) { 1015 1022 // Just in case there is some other reason for 1016 1023 // failure than the target CPU being offline. 1024 + WARN_ONCE(1, "%s(): smp_call_function_single() failed for CPU: %d\n", 1025 + __func__, cpu); 1017 1026 rcu_tasks_trace.n_ipis_fails++; 1018 1027 per_cpu(trc_ipi_to_cpu, cpu) = false; 1019 - t->trc_ipi_to_cpu = cpu; 1020 - if (atomic_dec_and_test(&trc_n_readers_need_end)) { 1021 - WARN_ON_ONCE(1); 1022 - wake_up(&trc_wait); 1023 - } 1028 + t->trc_ipi_to_cpu = -1; 1024 1029 } 1025 1030 } 1026 1031 } ··· 1086 1099 cpu = task_cpu(t); 1087 1100 pr_alert("P%d: %c%c%c nesting: %d%c cpu: %d\n", 1088 1101 t->pid, 1089 - ".I"[READ_ONCE(t->trc_ipi_to_cpu) > 0], 1102 + ".I"[READ_ONCE(t->trc_ipi_to_cpu) >= 0], 1090 1103 ".i"[is_idle_task(t)], 1091 - ".N"[cpu > 0 && tick_nohz_full_cpu(cpu)], 1104 + ".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)], 1092 1105 READ_ONCE(t->trc_reader_nesting), 1093 1106 " N"[!!READ_ONCE(t->trc_reader_special.b.need_qs)], 1094 1107 cpu); ··· 1131 1144 cpus_read_unlock(); 1132 1145 1133 1146 if (needreport) { 1134 - if (firstreport) 1147 + if (*firstreport) 1135 1148 pr_err("INFO: rcu_tasks_trace detected stalls? (Late IPI?)\n"); 1136 1149 show_stalled_ipi_trace(); 1137 1150 } 1138 1151 } 1139 1152 1153 + static void rcu_tasks_trace_empty_fn(void *unused) 1154 + { 1155 + } 1156 + 1140 1157 /* Wait for grace period to complete and provide ordering. */ 1141 1158 static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) 1142 1159 { 1160 + int cpu; 1143 1161 bool firstreport; 1144 1162 struct task_struct *g, *t; 1145 1163 LIST_HEAD(holdouts); 1146 1164 long ret; 1165 + 1166 + // Wait for any lingering IPI handlers to complete. Note that 1167 + // if a CPU has gone offline or transitioned to userspace in the 1168 + // meantime, all IPI handlers should have been drained beforehand. 1169 + // Yes, this assumes that CPUs process IPIs in order. If that ever 1170 + // changes, there will need to be a recheck and/or timed wait. 1171 + for_each_online_cpu(cpu) 1172 + if (smp_load_acquire(per_cpu_ptr(&trc_ipi_to_cpu, cpu))) 1173 + smp_call_function_single(cpu, rcu_tasks_trace_empty_fn, NULL, 1); 1147 1174 1148 1175 // Remove the safety count. 1149 1176 smp_mb__before_atomic(); // Order vs. earlier atomics ··· 1201 1200 WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); 1202 1201 WRITE_ONCE(t->trc_reader_nesting, 0); 1203 1202 if (WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs))) 1204 - rcu_read_unlock_trace_special(t, 0); 1203 + rcu_read_unlock_trace_special(t); 1205 1204 } 1206 1205 1207 1206 /** ··· 1209 1208 * @rhp: structure to be used for queueing the RCU updates. 1210 1209 * @func: actual callback function to be invoked after the grace period 1211 1210 * 1212 - * The callback function will be invoked some time after a full grace 1213 - * period elapses, in other words after all currently executing RCU 1214 - * read-side critical sections have completed. call_rcu_tasks_trace() 1215 - * assumes that the read-side critical sections end at context switch, 1216 - * cond_resched_rcu_qs(), or transition to usermode execution. As such, 1217 - * there are no read-side primitives analogous to rcu_read_lock() and 1218 - * rcu_read_unlock() because this primitive is intended to determine 1219 - * that all tasks have passed through a safe state, not so much for 1220 - * data-structure synchronization. 1211 + * The callback function will be invoked some time after a trace rcu-tasks 1212 + * grace period elapses, in other words after all currently executing 1213 + * trace rcu-tasks read-side critical sections have completed. These 1214 + * read-side critical sections are delimited by calls to rcu_read_lock_trace() 1215 + * and rcu_read_unlock_trace(). 1221 1216 * 1222 1217 * See the description of call_rcu() for more detailed information on 1223 1218 * memory ordering guarantees. ··· 1229 1232 * 1230 1233 * Control will return to the caller some time after a trace rcu-tasks 1231 1234 * grace period has elapsed, in other words after all currently executing 1232 - * rcu-tasks read-side critical sections have elapsed. These read-side 1235 + * trace rcu-tasks read-side critical sections have elapsed. These read-side 1233 1236 * critical sections are delimited by calls to rcu_read_lock_trace() 1234 1237 * and rcu_read_unlock_trace(). 1235 1238 *
+16 -20
kernel/rcu/tree.c
··· 327 327 */ 328 328 static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) 329 329 { 330 - return !(atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1); 330 + return !(arch_atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1); 331 331 } 332 332 333 333 /* ··· 1219 1219 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) 1220 1220 { 1221 1221 unsigned long jtsq; 1222 - bool *rnhqp; 1223 - bool *ruqp; 1224 1222 struct rcu_node *rnp = rdp->mynode; 1225 1223 1226 1224 /* ··· 1283 1285 * is set way high. 1284 1286 */ 1285 1287 jtsq = READ_ONCE(jiffies_to_sched_qs); 1286 - ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu); 1287 - rnhqp = per_cpu_ptr(&rcu_data.rcu_need_heavy_qs, rdp->cpu); 1288 - if (!READ_ONCE(*rnhqp) && 1288 + if (!READ_ONCE(rdp->rcu_need_heavy_qs) && 1289 1289 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) || 1290 1290 time_after(jiffies, rcu_state.jiffies_resched) || 1291 1291 rcu_state.cbovld)) { 1292 - WRITE_ONCE(*rnhqp, true); 1292 + WRITE_ONCE(rdp->rcu_need_heavy_qs, true); 1293 1293 /* Store rcu_need_heavy_qs before rcu_urgent_qs. */ 1294 - smp_store_release(ruqp, true); 1294 + smp_store_release(&rdp->rcu_urgent_qs, true); 1295 1295 } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) { 1296 - WRITE_ONCE(*ruqp, true); 1296 + WRITE_ONCE(rdp->rcu_urgent_qs, true); 1297 1297 } 1298 1298 1299 1299 /* ··· 1305 1309 if (tick_nohz_full_cpu(rdp->cpu) && 1306 1310 (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) || 1307 1311 rcu_state.cbovld)) { 1308 - WRITE_ONCE(*ruqp, true); 1312 + WRITE_ONCE(rdp->rcu_urgent_qs, true); 1309 1313 resched_cpu(rdp->cpu); 1310 1314 WRITE_ONCE(rdp->last_fqs_resched, jiffies); 1311 1315 } ··· 1775 1779 */ 1776 1780 WRITE_ONCE(rcu_state.gp_state, RCU_GP_ONOFF); 1777 1781 rcu_for_each_leaf_node(rnp) { 1782 + // Wait for CPU-hotplug operations that might have 1783 + // started before this grace period did. 1778 1784 smp_mb(); // Pair with barriers used when updating ->ofl_seq to odd values. 1779 1785 firstseq = READ_ONCE(rnp->ofl_seq); 1780 1786 if (firstseq & 0x1) ··· 1905 1907 struct rcu_node *rnp = rcu_get_root(); 1906 1908 1907 1909 WRITE_ONCE(rcu_state.gp_activity, jiffies); 1908 - rcu_state.n_force_qs++; 1910 + WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1); 1909 1911 if (first_time) { 1910 1912 /* Collect dyntick-idle snapshots. */ 1911 1913 force_qs_rnp(dyntick_save_progress_counter); ··· 2356 2358 int rcutree_dying_cpu(unsigned int cpu) 2357 2359 { 2358 2360 bool blkd; 2359 - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 2361 + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); 2360 2362 struct rcu_node *rnp = rdp->mynode; 2361 2363 2362 2364 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) ··· 2548 2550 /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */ 2549 2551 if (count == 0 && rdp->qlen_last_fqs_check != 0) { 2550 2552 rdp->qlen_last_fqs_check = 0; 2551 - rdp->n_force_qs_snap = rcu_state.n_force_qs; 2553 + rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); 2552 2554 } else if (count < rdp->qlen_last_fqs_check - qhimark) 2553 2555 rdp->qlen_last_fqs_check = count; 2554 2556 ··· 2896 2898 } else { 2897 2899 /* Give the grace period a kick. */ 2898 2900 rdp->blimit = DEFAULT_MAX_RCU_BLIMIT; 2899 - if (rcu_state.n_force_qs == rdp->n_force_qs_snap && 2901 + if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap && 2900 2902 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) 2901 2903 rcu_force_quiescent_state(); 2902 - rdp->n_force_qs_snap = rcu_state.n_force_qs; 2904 + rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); 2903 2905 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); 2904 2906 } 2905 2907 } ··· 4126 4128 /* Set up local state, ensuring consistent view of global state. */ 4127 4129 raw_spin_lock_irqsave_rcu_node(rnp, flags); 4128 4130 rdp->qlen_last_fqs_check = 0; 4129 - rdp->n_force_qs_snap = rcu_state.n_force_qs; 4131 + rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); 4130 4132 rdp->blimit = blimit; 4131 4133 rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */ 4132 - rcu_dynticks_eqs_online(); 4133 4134 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ 4134 4135 4135 4136 /* ··· 4248 4251 mask = rdp->grpmask; 4249 4252 WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1); 4250 4253 WARN_ON_ONCE(!(rnp->ofl_seq & 0x1)); 4254 + rcu_dynticks_eqs_online(); 4251 4255 smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier(). 4252 4256 raw_spin_lock_irqsave_rcu_node(rnp, flags); 4253 4257 WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext | mask); ··· 4294 4296 do_nocb_deferred_wakeup(rdp); 4295 4297 4296 4298 /* QS for any half-done expedited grace period. */ 4297 - preempt_disable(); 4298 - rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); 4299 - preempt_enable(); 4299 + rcu_report_exp_rdp(rdp); 4300 4300 rcu_preempt_deferred_qs(current); 4301 4301 4302 4302 /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
+1 -2
kernel/rcu/tree_exp.h
··· 512 512 j = READ_ONCE(jiffies_till_first_fqs); 513 513 if (synchronize_rcu_expedited_wait_once(j + HZ)) 514 514 return; 515 - WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT)); 516 515 } 517 516 518 517 for (;;) { ··· 759 760 my_cpu = get_cpu(); 760 761 /* Quiescent state either not needed or already requested, leave. */ 761 762 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || 762 - __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) { 763 + rdp->cpu_no_qs.b.exp) { 763 764 put_cpu(); 764 765 return; 765 766 }
+1 -1
kernel/rcu/tree_nocb.h
··· 549 549 rcu_nocb_unlock_irqrestore(rdp, flags); 550 550 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); 551 551 } 552 - return; 553 552 } 554 553 555 554 /* ··· 766 767 static inline bool nocb_cb_can_run(struct rcu_data *rdp) 767 768 { 768 769 u8 flags = SEGCBLIST_OFFLOADED | SEGCBLIST_KTHREAD_CB; 770 + 769 771 return rcu_segcblist_test_flags(&rdp->cblist, flags); 770 772 } 771 773
+5 -6
kernel/rcu/tree_plugin.h
··· 814 814 { 815 815 struct rcu_data *rdp; 816 816 817 - if (!IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) || 818 - irqs_disabled() || preempt_count() || !rcu_state.gp_kthread) 817 + if (irqs_disabled() || preempt_count() || !rcu_state.gp_kthread) 819 818 return; 820 819 rdp = this_cpu_ptr(&rcu_data); 821 820 rcu_report_qs_rdp(rdp); ··· 1479 1480 } 1480 1481 1481 1482 /* Record the current task on dyntick-idle entry. */ 1482 - static void noinstr rcu_dynticks_task_enter(void) 1483 + static __always_inline void rcu_dynticks_task_enter(void) 1483 1484 { 1484 1485 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) 1485 1486 WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id()); ··· 1487 1488 } 1488 1489 1489 1490 /* Record no current task on dyntick-idle exit. */ 1490 - static void noinstr rcu_dynticks_task_exit(void) 1491 + static __always_inline void rcu_dynticks_task_exit(void) 1491 1492 { 1492 1493 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) 1493 1494 WRITE_ONCE(current->rcu_tasks_idle_cpu, -1); ··· 1495 1496 } 1496 1497 1497 1498 /* Turn on heavyweight RCU tasks trace readers on idle/user entry. */ 1498 - static void rcu_dynticks_task_trace_enter(void) 1499 + static __always_inline void rcu_dynticks_task_trace_enter(void) 1499 1500 { 1500 1501 #ifdef CONFIG_TASKS_TRACE_RCU 1501 1502 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) ··· 1504 1505 } 1505 1506 1506 1507 /* Turn off heavyweight RCU tasks trace readers on idle/user exit. */ 1507 - static void rcu_dynticks_task_trace_exit(void) 1508 + static __always_inline void rcu_dynticks_task_trace_exit(void) 1508 1509 { 1509 1510 #ifdef CONFIG_TASKS_TRACE_RCU 1510 1511 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB))
+4 -4
kernel/rcu/update.c
··· 54 54 #define MODULE_PARAM_PREFIX "rcupdate." 55 55 56 56 #ifndef CONFIG_TINY_RCU 57 - module_param(rcu_expedited, int, 0); 58 - module_param(rcu_normal, int, 0); 57 + module_param(rcu_expedited, int, 0444); 58 + module_param(rcu_normal, int, 0444); 59 59 static int rcu_normal_after_boot = IS_ENABLED(CONFIG_PREEMPT_RT); 60 - #ifndef CONFIG_PREEMPT_RT 61 - module_param(rcu_normal_after_boot, int, 0); 60 + #if !defined(CONFIG_PREEMPT_RT) || defined(CONFIG_NO_HZ_FULL) 61 + module_param(rcu_normal_after_boot, int, 0444); 62 62 #endif 63 63 #endif /* #ifndef CONFIG_TINY_RCU */ 64 64
+24 -19
kernel/scftorture.c
··· 341 341 cpu = torture_random(trsp) % nr_cpu_ids; 342 342 scfp->n_resched++; 343 343 resched_cpu(cpu); 344 + this_cpu_inc(scf_invoked_count); 344 345 } 345 346 break; 346 347 case SCF_PRIM_SINGLE: ··· 554 553 555 554 scftorture_print_module_parms("Start of test"); 556 555 557 - if (weight_resched == -1 && 558 - weight_single == -1 && weight_single_rpc == -1 && weight_single_wait == -1 && 559 - weight_many == -1 && weight_many_wait == -1 && 560 - weight_all == -1 && weight_all_wait == -1) { 561 - weight_resched1 = 2 * nr_cpu_ids; 562 - weight_single1 = 2 * nr_cpu_ids; 563 - weight_single_rpc1 = 2 * nr_cpu_ids; 564 - weight_single_wait1 = 2 * nr_cpu_ids; 565 - weight_many1 = 2; 566 - weight_many_wait1 = 2; 567 - weight_all1 = 1; 568 - weight_all_wait1 = 1; 556 + if (weight_resched <= 0 && 557 + weight_single <= 0 && weight_single_rpc <= 0 && weight_single_wait <= 0 && 558 + weight_many <= 0 && weight_many_wait <= 0 && 559 + weight_all <= 0 && weight_all_wait <= 0) { 560 + weight_resched1 = weight_resched == 0 ? 0 : 2 * nr_cpu_ids; 561 + weight_single1 = weight_single == 0 ? 0 : 2 * nr_cpu_ids; 562 + weight_single_rpc1 = weight_single_rpc == 0 ? 0 : 2 * nr_cpu_ids; 563 + weight_single_wait1 = weight_single_wait == 0 ? 0 : 2 * nr_cpu_ids; 564 + weight_many1 = weight_many == 0 ? 0 : 2; 565 + weight_many_wait1 = weight_many_wait == 0 ? 0 : 2; 566 + weight_all1 = weight_all == 0 ? 0 : 1; 567 + weight_all_wait1 = weight_all_wait == 0 ? 0 : 1; 569 568 } else { 570 569 if (weight_resched == -1) 571 570 weight_resched1 = 0; ··· 584 583 if (weight_all_wait == -1) 585 584 weight_all_wait1 = 0; 586 585 } 587 - if (weight_single1 == 0 && weight_single_rpc1 == 0 && weight_single_wait1 == 0 && 588 - weight_many1 == 0 && weight_many_wait1 == 0 && 586 + if (weight_resched1 == 0 && weight_single1 == 0 && weight_single_rpc1 == 0 && 587 + weight_single_wait1 == 0 && weight_many1 == 0 && weight_many_wait1 == 0 && 589 588 weight_all1 == 0 && weight_all_wait1 == 0) { 590 589 VERBOSE_SCFTORTOUT_ERRSTRING("all zero weights makes no sense"); 591 590 firsterr = -EINVAL; ··· 606 605 607 606 if (onoff_interval > 0) { 608 607 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL); 609 - if (firsterr) 608 + if (torture_init_error(firsterr)) 610 609 goto unwind; 611 610 } 612 611 if (shutdown_secs > 0) { 613 612 firsterr = torture_shutdown_init(shutdown_secs, scf_torture_cleanup); 614 - if (firsterr) 613 + if (torture_init_error(firsterr)) 615 614 goto unwind; 616 615 } 617 616 if (stutter > 0) { 618 617 firsterr = torture_stutter_init(stutter, stutter); 619 - if (firsterr) 618 + if (torture_init_error(firsterr)) 620 619 goto unwind; 621 620 } 622 621 ··· 637 636 scf_stats_p[i].cpu = i; 638 637 firsterr = torture_create_kthread(scftorture_invoker, (void *)&scf_stats_p[i], 639 638 scf_stats_p[i].task); 640 - if (firsterr) 639 + if (torture_init_error(firsterr)) 641 640 goto unwind; 642 641 } 643 642 if (stat_interval > 0) { 644 643 firsterr = torture_create_kthread(scf_torture_stats, NULL, scf_torture_stats_task); 645 - if (firsterr) 644 + if (torture_init_error(firsterr)) 646 645 goto unwind; 647 646 } 648 647 ··· 652 651 unwind: 653 652 torture_init_end(); 654 653 scf_torture_cleanup(); 654 + if (shutdown_secs) { 655 + WARN_ON(!IS_MODULE(CONFIG_SCF_TORTURE_TEST)); 656 + kernel_power_off(); 657 + } 655 658 return firsterr; 656 659 } 657 660
+34
tools/rcu/extract-stall.sh
··· 1 + #!/bin/sh 2 + # SPDX-License-Identifier: GPL-2.0+ 3 + # 4 + # Extract any RCU CPU stall warnings present in specified file. 5 + # Filter out clocksource lines. Note that preceding-lines excludes the 6 + # initial line of the stall warning but trailing-lines includes it. 7 + # 8 + # Usage: extract-stall.sh dmesg-file [ preceding-lines [ trailing-lines ] ] 9 + 10 + echo $1 11 + preceding_lines="${2-3}" 12 + trailing_lines="${3-10}" 13 + 14 + awk -v preceding_lines="$preceding_lines" -v trailing_lines="$trailing_lines" ' 15 + suffix <= 0 { 16 + for (i = preceding_lines; i > 0; i--) 17 + last[i] = last[i - 1]; 18 + last[0] = $0; 19 + } 20 + 21 + suffix > 0 { 22 + print $0; 23 + suffix--; 24 + if (suffix <= 0) 25 + print ""; 26 + } 27 + 28 + suffix <= 0 && /detected stall/ { 29 + for (i = preceding_lines; i >= 0; i--) 30 + if (last[i] != "") 31 + print last[i]; 32 + suffix = trailing_lines; 33 + }' < "$1" | tr -d '\015' | grep -v clocksource 34 +
+1
tools/testing/selftests/rcutorture/bin/kvm-remote.sh
··· 149 149 done 150 150 151 151 # Download and expand the tarball on all systems. 152 + echo Build-products tarball: `du -h $T/binres.tgz` | tee -a "$oldrun/remote-log" 152 153 for i in $systems 153 154 do 154 155 echo Downloading tarball to $i `date` | tee -a "$oldrun/remote-log"
+1 -1
tools/testing/selftests/rcutorture/bin/kvm.sh
··· 184 184 TORTURE_KCONFIG_KASAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KASAN=y"; export TORTURE_KCONFIG_KASAN_ARG 185 185 ;; 186 186 --kcsan) 187 - TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KCSAN=y CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY=n CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_INTERRUPT_WATCHER=y CONFIG_KCSAN_VERBOSE=y CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y"; export TORTURE_KCONFIG_KCSAN_ARG 187 + TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KCSAN=y CONFIG_KCSAN_STRICT=y CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_VERBOSE=y CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y"; export TORTURE_KCONFIG_KCSAN_ARG 188 188 ;; 189 189 --kmake-arg|--kmake-args) 190 190 checkarg --kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
+8 -3
tools/testing/selftests/rcutorture/bin/torture.sh
··· 351 351 if test "$do_scftorture" = "yes" 352 352 then 353 353 torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot" 354 - torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make 354 + torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make 355 355 fi 356 356 357 357 if test "$do_refscale" = yes ··· 434 434 batchno=1 435 435 if test -s $T/xz-todo 436 436 then 437 - echo Size before compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 437 + for i in `cat $T/xz-todo` 438 + do 439 + find $i -name 'vmlinux*' -print 440 + done | wc -l | awk '{ print $1 }' > $T/xz-todo-count 441 + n2compress="`cat $T/xz-todo-count`" 442 + echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 438 443 for i in `cat $T/xz-todo` 439 444 do 440 445 echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1 ··· 461 456 echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log 462 457 fi 463 458 wait 464 - echo Size after compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 459 + echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 465 460 echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log 466 461 else 467 462 echo No compression needed: `date` >> "$tdir/log-xz" 2>&1