Merge tag 'rcu-fixes-v6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks

+2

kernel/rcu/rcu.h

··· 500 500 static inline void rcu_unexpedite_gp(void) { } 501 501 static inline void rcu_async_hurry(void) { } 502 502 static inline void rcu_async_relax(void) { } 503 + static inline bool rcu_cpu_online(int cpu) { return true; } 503 504 #else /* #ifdef CONFIG_TINY_RCU */ 504 505 bool rcu_gp_is_normal(void); /* Internal RCU use. */ 505 506 bool rcu_gp_is_expedited(void); /* Internal RCU use. */ ··· 510 509 void rcu_async_hurry(void); 511 510 void rcu_async_relax(void); 512 511 void rcupdate_announce_bootup_oddness(void); 512 + bool rcu_cpu_online(int cpu); 513 513 #ifdef CONFIG_TASKS_RCU_GENERIC 514 514 void show_rcu_tasks_gp_kthreads(void); 515 515 #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */

+29 -3

kernel/rcu/tasks.h

··· 895 895 synchronize_rcu(); 896 896 } 897 897 898 + /* Check for quiescent states since the pregp's synchronize_rcu() */ 899 + static bool rcu_tasks_is_holdout(struct task_struct *t) 900 + { 901 + int cpu; 902 + 903 + /* Has the task been seen voluntarily sleeping? */ 904 + if (!READ_ONCE(t->on_rq)) 905 + return false; 906 + 907 + /* 908 + * Idle tasks (or idle injection) within the idle loop are RCU-tasks 909 + * quiescent states. But CPU boot code performed by the idle task 910 + * isn't a quiescent state. 911 + */ 912 + if (is_idle_task(t)) 913 + return false; 914 + 915 + cpu = task_cpu(t); 916 + 917 + /* Idle tasks on offline CPUs are RCU-tasks quiescent states. */ 918 + if (t == idle_task(cpu) && !rcu_cpu_online(cpu)) 919 + return false; 920 + 921 + return true; 922 + } 923 + 898 924 /* Per-task initial processing. */ 899 925 static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop) 900 926 { 901 - if (t != current && READ_ONCE(t->on_rq) && !is_idle_task(t)) { 927 + if (t != current && rcu_tasks_is_holdout(t)) { 902 928 get_task_struct(t); 903 929 t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw); 904 930 WRITE_ONCE(t->rcu_tasks_holdout, true); ··· 973 947 974 948 if (!READ_ONCE(t->rcu_tasks_holdout) || 975 949 t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) || 976 - !READ_ONCE(t->on_rq) || 950 + !rcu_tasks_is_holdout(t) || 977 951 (IS_ENABLED(CONFIG_NO_HZ_FULL) && 978 952 !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) { 979 953 WRITE_ONCE(t->rcu_tasks_holdout, false); ··· 1551 1525 } else { 1552 1526 // The task is not running, so C-language access is safe. 1553 1527 nesting = t->trc_reader_nesting; 1554 - WARN_ON_ONCE(ofl && task_curr(t) && !is_idle_task(t)); 1528 + WARN_ON_ONCE(ofl && task_curr(t) && (t != idle_task(task_cpu(t)))); 1555 1529 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && ofl) 1556 1530 n_heavy_reader_ofl_updates++; 1557 1531 }

+32 -11

kernel/rcu/tree.c

··· 755 755 } 756 756 757 757 /* 758 - * Return true if the specified CPU has passed through a quiescent 759 - * state by virtue of being in or having passed through an dynticks 760 - * idle state since the last call to dyntick_save_progress_counter() 761 - * for this same CPU, or by virtue of having been offline. 758 + * Returns positive if the specified CPU has passed through a quiescent state 759 + * by virtue of being in or having passed through an dynticks idle state since 760 + * the last call to dyntick_save_progress_counter() for this same CPU, or by 761 + * virtue of having been offline. 762 + * 763 + * Returns negative if the specified CPU needs a force resched. 764 + * 765 + * Returns zero otherwise. 762 766 */ 763 767 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) 764 768 { 765 769 unsigned long jtsq; 770 + int ret = 0; 766 771 struct rcu_node *rnp = rdp->mynode; 767 772 768 773 /* ··· 853 848 (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) || 854 849 rcu_state.cbovld)) { 855 850 WRITE_ONCE(rdp->rcu_urgent_qs, true); 856 - resched_cpu(rdp->cpu); 857 851 WRITE_ONCE(rdp->last_fqs_resched, jiffies); 852 + ret = -1; 858 853 } 859 854 860 855 /* ··· 867 862 if (time_after(jiffies, rcu_state.jiffies_resched)) { 868 863 if (time_after(jiffies, 869 864 READ_ONCE(rdp->last_fqs_resched) + jtsq)) { 870 - resched_cpu(rdp->cpu); 871 865 WRITE_ONCE(rdp->last_fqs_resched, jiffies); 866 + ret = -1; 872 867 } 873 868 if (IS_ENABLED(CONFIG_IRQ_WORK) && 874 869 !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq && ··· 897 892 } 898 893 } 899 894 900 - return 0; 895 + return ret; 901 896 } 902 897 903 898 /* Trace-event wrapper function for trace_rcu_future_grace_period. */ ··· 2276 2271 { 2277 2272 int cpu; 2278 2273 unsigned long flags; 2279 - unsigned long mask; 2280 - struct rcu_data *rdp; 2281 2274 struct rcu_node *rnp; 2282 2275 2283 2276 rcu_state.cbovld = rcu_state.cbovldnext; 2284 2277 rcu_state.cbovldnext = false; 2285 2278 rcu_for_each_leaf_node(rnp) { 2279 + unsigned long mask = 0; 2280 + unsigned long rsmask = 0; 2281 + 2286 2282 cond_resched_tasks_rcu_qs(); 2287 - mask = 0; 2288 2283 raw_spin_lock_irqsave_rcu_node(rnp, flags); 2289 2284 rcu_state.cbovldnext |= !!rnp->cbovldmask; 2290 2285 if (rnp->qsmask == 0) { ··· 2302 2297 continue; 2303 2298 } 2304 2299 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) { 2300 + struct rcu_data *rdp; 2301 + int ret; 2302 + 2305 2303 rdp = per_cpu_ptr(&rcu_data, cpu); 2306 - if (f(rdp)) { 2304 + ret = f(rdp); 2305 + if (ret > 0) { 2307 2306 mask |= rdp->grpmask; 2308 2307 rcu_disable_urgency_upon_qs(rdp); 2309 2308 } 2309 + if (ret < 0) 2310 + rsmask |= rdp->grpmask; 2310 2311 } 2311 2312 if (mask != 0) { 2312 2313 /* Idle/offline CPUs, report (releases rnp->lock). */ ··· 2321 2310 /* Nothing to do here, so just drop the lock. */ 2322 2311 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 2323 2312 } 2313 + 2314 + for_each_leaf_node_cpu_mask(rnp, cpu, rsmask) 2315 + resched_cpu(cpu); 2324 2316 } 2325 2317 } 2326 2318 ··· 4207 4193 static bool rcu_rdp_cpu_online(struct rcu_data *rdp) 4208 4194 { 4209 4195 return !!(rdp->grpmask & rcu_rnp_online_cpus(rdp->mynode)); 4196 + } 4197 + 4198 + bool rcu_cpu_online(int cpu) 4199 + { 4200 + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); 4201 + 4202 + return rcu_rdp_cpu_online(rdp); 4210 4203 } 4211 4204 4212 4205 #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)

Configure Feed

Configure Feed