Merge branch 'srcu-next.2025.08.21a' into HEAD

+34

include/linux/srcu.h

··· 275 275 { 276 276 struct srcu_ctr __percpu *retval; 277 277 278 + RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast()."); 278 279 srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST); 279 280 retval = __srcu_read_lock_fast(ssp); 280 281 rcu_try_lock_acquire(&ssp->dep_map); 282 + return retval; 283 + } 284 + 285 + /* 286 + * Used by tracing, cannot be traced and cannot call lockdep. 287 + * See srcu_read_lock_fast() for more information. 288 + */ 289 + static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp) 290 + __acquires(ssp) 291 + { 292 + struct srcu_ctr __percpu *retval; 293 + 294 + srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST); 295 + retval = __srcu_read_lock_fast(ssp); 281 296 return retval; 282 297 } 283 298 ··· 310 295 static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp) 311 296 { 312 297 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); 298 + RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast()."); 313 299 srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST); 314 300 return __srcu_read_lock_fast(ssp); 315 301 } ··· 405 389 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 406 390 srcu_lock_release(&ssp->dep_map); 407 391 __srcu_read_unlock_fast(ssp, scp); 392 + RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast()."); 393 + } 394 + 395 + /* 396 + * Used by tracing, cannot be traced and cannot call lockdep. 397 + * See srcu_read_unlock_fast() for more information. 398 + */ 399 + static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp, 400 + struct srcu_ctr __percpu *scp) __releases(ssp) 401 + { 402 + srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 403 + __srcu_read_unlock_fast(ssp, scp); 408 404 } 409 405 410 406 /** ··· 433 405 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); 434 406 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); 435 407 __srcu_read_unlock_fast(ssp, scp); 408 + RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast()."); 436 409 } 437 410 438 411 /** ··· 513 484 DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct, 514 485 _T->scp = srcu_read_lock_fast(_T->lock), 515 486 srcu_read_unlock_fast(_T->lock, _T->scp), 487 + struct srcu_ctr __percpu *scp) 488 + 489 + DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct, 490 + _T->scp = srcu_read_lock_fast_notrace(_T->lock), 491 + srcu_read_unlock_fast_notrace(_T->lock, _T->scp), 516 492 struct srcu_ctr __percpu *scp) 517 493 518 494 #endif

+30 -19

include/linux/srcutree.h

··· 232 232 * srcu_read_unlock_fast(). 233 233 * 234 234 * Note that both this_cpu_inc() and atomic_long_inc() are RCU read-side 235 - * critical sections either because they disables interrupts, because they 236 - * are a single instruction, or because they are a read-modify-write atomic 237 - * operation, depending on the whims of the architecture. 235 + * critical sections either because they disables interrupts, because 236 + * they are a single instruction, or because they are read-modify-write 237 + * atomic operations, depending on the whims of the architecture. 238 + * This matters because the SRCU-fast grace-period mechanism uses either 239 + * synchronize_rcu() or synchronize_rcu_expedited(), that is, RCU, 240 + * *not* SRCU, in order to eliminate the need for the read-side smp_mb() 241 + * invocations that are used by srcu_read_lock() and srcu_read_unlock(). 242 + * The __srcu_read_unlock_fast() function also relies on this same RCU 243 + * (again, *not* SRCU) trick to eliminate the need for smp_mb(). 244 + * 245 + * The key point behind this RCU trick is that if any part of a given 246 + * RCU reader precedes the beginning of a given RCU grace period, then 247 + * the entirety of that RCU reader and everything preceding it happens 248 + * before the end of that same RCU grace period. Similarly, if any part 249 + * of a given RCU reader follows the end of a given RCU grace period, 250 + * then the entirety of that RCU reader and everything following it 251 + * happens after the beginning of that same RCU grace period. Therefore, 252 + * the operations labeled Y in __srcu_read_lock_fast() and those labeled Z 253 + * in __srcu_read_unlock_fast() are ordered against the corresponding SRCU 254 + * read-side critical section from the viewpoint of the SRCU grace period. 255 + * This is all the ordering that is required, hence no calls to smp_mb(). 238 256 * 239 257 * This means that __srcu_read_lock_fast() is not all that fast 240 258 * on architectures that support NMIs but do not supply NMI-safe 241 259 * implementations of this_cpu_inc(). 242 260 */ 243 - static inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct *ssp) 261 + static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct srcu_struct *ssp) 244 262 { 245 263 struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp); 246 264 247 - RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast()."); 248 265 if (!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE)) 249 - this_cpu_inc(scp->srcu_locks.counter); /* Y */ 266 + this_cpu_inc(scp->srcu_locks.counter); // Y, and implicit RCU reader. 250 267 else 251 - atomic_long_inc(raw_cpu_ptr(&scp->srcu_locks)); /* Z */ 268 + atomic_long_inc(raw_cpu_ptr(&scp->srcu_locks)); // Y, and implicit RCU reader. 252 269 barrier(); /* Avoid leaking the critical section. */ 253 270 return scp; 254 271 } ··· 276 259 * different CPU than that which was incremented by the corresponding 277 260 * srcu_read_lock_fast(), but it must be within the same task. 278 261 * 279 - * Note that both this_cpu_inc() and atomic_long_inc() are RCU read-side 280 - * critical sections either because they disables interrupts, because they 281 - * are a single instruction, or because they are a read-modify-write atomic 282 - * operation, depending on the whims of the architecture. 283 - * 284 - * This means that __srcu_read_unlock_fast() is not all that fast 285 - * on architectures that support NMIs but do not supply NMI-safe 286 - * implementations of this_cpu_inc(). 262 + * Please see the __srcu_read_lock_fast() function's header comment for 263 + * information on implicit RCU readers and NMI safety. 287 264 */ 288 - static inline void __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 265 + static inline void notrace 266 + __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) 289 267 { 290 268 barrier(); /* Avoid leaking the critical section. */ 291 269 if (!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE)) 292 - this_cpu_inc(scp->srcu_unlocks.counter); /* Z */ 270 + this_cpu_inc(scp->srcu_unlocks.counter); // Z, and implicit RCU reader. 293 271 else 294 - atomic_long_inc(raw_cpu_ptr(&scp->srcu_unlocks)); /* Z */ 295 - RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast()."); 272 + atomic_long_inc(raw_cpu_ptr(&scp->srcu_unlocks)); // Z, and implicit RCU reader. 296 273 } 297 274 298 275 void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor);

+1 -3

kernel/rcu/srcutiny.c

··· 176 176 { 177 177 unsigned long cookie; 178 178 179 - preempt_disable(); // Needed for PREEMPT_LAZY 179 + lockdep_assert_preemption_disabled(); // Needed for PREEMPT_LAZY 180 180 cookie = get_state_synchronize_srcu(ssp); 181 181 if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) { 182 - preempt_enable(); 183 182 return; 184 183 } 185 184 WRITE_ONCE(ssp->srcu_idx_max, cookie); ··· 188 189 else if (list_empty(&ssp->srcu_work.entry)) 189 190 list_add(&ssp->srcu_work.entry, &srcu_boot_list); 190 191 } 191 - preempt_enable(); 192 192 } 193 193 194 194 /*

+10

kernel/rcu/srcutree.c

··· 1168 1168 * counter update. Note that both this memory barrier and the 1169 1169 * one in srcu_readers_active_idx_check() provide the guarantee 1170 1170 * for __srcu_read_lock(). 1171 + * 1172 + * Note that this is a performance optimization, in which we spend 1173 + * an otherwise unnecessary smp_mb() in order to reduce the number 1174 + * of full per-CPU-variable scans in srcu_readers_lock_idx() and 1175 + * srcu_readers_unlock_idx(). But this performance optimization 1176 + * is not so optimal for SRCU-fast, where we would be spending 1177 + * not smp_mb(), but rather synchronize_rcu(). At the same time, 1178 + * the overhead of the smp_mb() is in the noise, so there is no 1179 + * point in omitting it in the SRCU-fast case. So the same code 1180 + * is executed either way. 1171 1181 */ 1172 1182 smp_mb(); /* D */ /* Pairs with C. */ 1173 1183 }

Configure Feed

Configure Feed