Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
rcu: Clean up after the SRCU-fastification of RCU Tasks Trace
Now that RCU Tasks Trace has been re-implemented in terms of SRCU-fast,
the ->trc_ipi_to_cpu, ->trc_blkd_cpu, ->trc_blkd_node, ->trc_holdout_list,
and ->trc_reader_special task_struct fields are no longer used.
In addition, the rcu_tasks_trace_qs(), rcu_tasks_trace_qs_blkd(),
exit_tasks_rcu_finish_trace(), and rcu_spawn_tasks_trace_kthread(),
show_rcu_tasks_trace_gp_kthread(), rcu_tasks_trace_get_gp_data(),
rcu_tasks_trace_torture_stats_print(), and get_rcu_tasks_trace_gp_kthread()
functions and all the other functions that they invoke are no longer used.
Also, the TRC_NEED_QS and TRC_NEED_QS_CHECKED CPP macros are no longer used.
Neither are the rcu_tasks_trace_lazy_ms and rcu_task_ipi_delay rcupdate
module parameters and the TASKS_TRACE_RCU_READ_MB Kconfig option.
This commit therefore removes all of them.
[ paulmck: Apply Alexei Starovoitov feedback. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: bpf@vger.kernel.org
Reviewed-by: Joel Fernandes <joelagnelf@nvidia.com>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
authored by
Paul E. McKenney
and committed by
Boqun Feng
a73fc3dc
46e32359
+2
-174
-15
Documentation/admin-guide/kernel-parameters.txt
-15
Documentation/admin-guide/kernel-parameters.txt
···
6249
6249
dynamically) adjusted. This parameter is intended
6250
6250
for use in testing.
6251
6251
6252
-
rcupdate.rcu_task_ipi_delay= [KNL]
6253
-
Set time in jiffies during which RCU tasks will
6254
-
avoid sending IPIs, starting with the beginning
6255
-
of a given grace period. Setting a large
6256
-
number avoids disturbing real-time workloads,
6257
-
but lengthens grace periods.
6258
-
6259
6252
rcupdate.rcu_task_lazy_lim= [KNL]
6260
6253
Number of callbacks on a given CPU that will
6261
6254
cancel laziness on that CPU. Use -1 to disable
···
6291
6298
A negative value will take the default. A value
6292
6299
of zero will disable batching. Batching is
6293
6300
always disabled for synchronize_rcu_tasks().
6294
-
6295
-
rcupdate.rcu_tasks_trace_lazy_ms= [KNL]
6296
-
Set timeout in milliseconds RCU Tasks
6297
-
Trace asynchronous callback batching for
6298
-
call_rcu_tasks_trace(). A negative value
6299
-
will take the default. A value of zero will
6300
-
disable batching. Batching is always disabled
6301
-
for synchronize_rcu_tasks_trace().
6302
6301
6303
6302
rcupdate.rcu_self_test= [KNL]
6304
6303
Run the RCU early boot self tests
+1
-30
include/linux/rcupdate.h
+1
-30
include/linux/rcupdate.h
···
175
175
# define synchronize_rcu_tasks synchronize_rcu
176
176
# endif
177
177
178
-
# ifdef CONFIG_TASKS_TRACE_RCU
179
-
// Bits for ->trc_reader_special.b.need_qs field.
180
-
#define TRC_NEED_QS 0x1 // Task needs a quiescent state.
181
-
#define TRC_NEED_QS_CHECKED 0x2 // Task has been checked for needing quiescent state.
182
-
183
-
u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new);
184
-
void rcu_tasks_trace_qs_blkd(struct task_struct *t);
185
-
186
-
# define rcu_tasks_trace_qs(t) \
187
-
do { \
188
-
int ___rttq_nesting = READ_ONCE((t)->trc_reader_nesting); \
189
-
\
190
-
if (unlikely(READ_ONCE((t)->trc_reader_special.b.need_qs) == TRC_NEED_QS) && \
191
-
likely(!___rttq_nesting)) { \
192
-
rcu_trc_cmpxchg_need_qs((t), TRC_NEED_QS, TRC_NEED_QS_CHECKED); \
193
-
} else if (___rttq_nesting && ___rttq_nesting != INT_MIN && \
194
-
!READ_ONCE((t)->trc_reader_special.b.blocked)) { \
195
-
rcu_tasks_trace_qs_blkd(t); \
196
-
} \
197
-
} while (0)
198
-
void rcu_tasks_trace_torture_stats_print(char *tt, char *tf);
199
-
# else
200
-
# define rcu_tasks_trace_qs(t) do { } while (0)
201
-
# endif
202
-
203
-
#define rcu_tasks_qs(t, preempt) \
204
-
do { \
205
-
rcu_tasks_classic_qs((t), (preempt)); \
206
-
rcu_tasks_trace_qs(t); \
207
-
} while (0)
178
+
#define rcu_tasks_qs(t, preempt) rcu_tasks_classic_qs((t), (preempt))
208
179
209
180
# ifdef CONFIG_TASKS_RUDE_RCU
210
181
void synchronize_rcu_tasks_rude(void);
-2
include/linux/rcupdate_trace.h
-2
include/linux/rcupdate_trace.h
···
136
136
}
137
137
138
138
// Placeholders to enable stepwise transition.
139
-
void rcu_tasks_trace_get_gp_data(int *flags, unsigned long *gp_seq);
140
139
void __init rcu_tasks_trace_suppress_unused(void);
141
-
struct task_struct *get_rcu_tasks_trace_gp_kthread(void);
142
140
143
141
#else
144
142
/*
-5
include/linux/sched.h
-5
include/linux/sched.h
···
946
946
#ifdef CONFIG_TASKS_TRACE_RCU
947
947
int trc_reader_nesting;
948
948
struct srcu_ctr __percpu *trc_reader_scp;
949
-
int trc_ipi_to_cpu;
950
-
union rcu_special trc_reader_special;
951
-
struct list_head trc_holdout_list;
952
-
struct list_head trc_blkd_node;
953
-
int trc_blkd_cpu;
954
949
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
955
950
956
951
struct sched_info sched_info;
-3
init/init_task.c
-3
init/init_task.c
···
195
195
#endif
196
196
#ifdef CONFIG_TASKS_TRACE_RCU
197
197
.trc_reader_nesting = 0,
198
-
.trc_reader_special.s = 0,
199
-
.trc_holdout_list = LIST_HEAD_INIT(init_task.trc_holdout_list),
200
-
.trc_blkd_node = LIST_HEAD_INIT(init_task.trc_blkd_node),
201
198
#endif
202
199
#ifdef CONFIG_CPUSETS
203
200
.mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq,
-3
kernel/fork.c
-3
kernel/fork.c
···
1828
1828
#endif /* #ifdef CONFIG_TASKS_RCU */
1829
1829
#ifdef CONFIG_TASKS_TRACE_RCU
1830
1830
p->trc_reader_nesting = 0;
1831
-
p->trc_reader_special.s = 0;
1832
-
INIT_LIST_HEAD(&p->trc_holdout_list);
1833
-
INIT_LIST_HEAD(&p->trc_blkd_node);
1834
1831
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
1835
1832
}
1836
1833
-18
kernel/rcu/Kconfig
-18
kernel/rcu/Kconfig
···
313
313
Say Y here if you want to set RT priority for offloading kthreads.
314
314
Say N here if you are building a !PREEMPT_RT kernel and are unsure.
315
315
316
-
config TASKS_TRACE_RCU_READ_MB
317
-
bool "Tasks Trace RCU readers use memory barriers in user and idle"
318
-
depends on RCU_EXPERT && TASKS_TRACE_RCU
319
-
default PREEMPT_RT || NR_CPUS < 8
320
-
help
321
-
Use this option to further reduce the number of IPIs sent
322
-
to CPUs executing in userspace or idle during tasks trace
323
-
RCU grace periods. Given that a reasonable setting of
324
-
the rcupdate.rcu_task_ipi_delay kernel boot parameter
325
-
eliminates such IPIs for many workloads, proper setting
326
-
of this Kconfig option is important mostly for aggressive
327
-
real-time installations and for battery-powered devices,
328
-
hence the default chosen above.
329
-
330
-
Say Y here if you hate IPIs.
331
-
Say N here if you hate read-side memory barriers.
332
-
Take the default if you are unsure.
333
-
334
316
config RCU_LAZY
335
317
bool "RCU callback lazy invocation functionality"
336
318
depends on RCU_NOCB_CPU
-9
kernel/rcu/rcu.h
-9
kernel/rcu/rcu.h
···
544
544
void rcu_tasks_rude_get_gp_data(int *flags, unsigned long *gp_seq);
545
545
#endif // # ifdef CONFIG_TASKS_RUDE_RCU
546
546
547
-
#ifdef CONFIG_TASKS_TRACE_RCU
548
-
void rcu_tasks_trace_get_gp_data(int *flags, unsigned long *gp_seq);
549
-
#endif
550
-
551
547
#ifdef CONFIG_TASKS_RCU_GENERIC
552
548
void tasks_cblist_init_generic(void);
553
549
#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
···
668
672
void show_rcu_tasks_rude_gp_kthread(void);
669
673
#else
670
674
static inline void show_rcu_tasks_rude_gp_kthread(void) {}
671
-
#endif
672
-
#if !defined(CONFIG_TINY_RCU) && defined(CONFIG_TASKS_TRACE_RCU)
673
-
void show_rcu_tasks_trace_gp_kthread(void);
674
-
#else
675
-
static inline void show_rcu_tasks_trace_gp_kthread(void) {}
676
675
#endif
677
676
678
677
#ifdef CONFIG_TINY_RCU
-7
kernel/rcu/rcuscale.c
-7
kernel/rcu/rcuscale.c
···
400
400
rcu_read_unlock_trace();
401
401
}
402
402
403
-
static void rcu_tasks_trace_scale_stats(void)
404
-
{
405
-
rcu_tasks_trace_torture_stats_print(scale_type, SCALE_FLAG);
406
-
}
407
-
408
403
static struct rcu_scale_ops tasks_tracing_ops = {
409
404
.ptype = RCU_TASKS_FLAVOR,
410
405
.init = rcu_sync_scale_init,
···
411
416
.gp_barrier = rcu_barrier_tasks_trace,
412
417
.sync = synchronize_rcu_tasks_trace,
413
418
.exp_sync = synchronize_rcu_tasks_trace,
414
-
.rso_gp_kthread = get_rcu_tasks_trace_gp_kthread,
415
-
.stats = IS_ENABLED(CONFIG_TINY_RCU) ? NULL : rcu_tasks_trace_scale_stats,
416
419
.name = "tasks-tracing"
417
420
};
418
421
-2
kernel/rcu/rcutorture.c
-2
kernel/rcu/rcutorture.c
···
1180
1180
.exp_sync = synchronize_rcu_tasks_trace,
1181
1181
.call = call_rcu_tasks_trace,
1182
1182
.cb_barrier = rcu_barrier_tasks_trace,
1183
-
.gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread,
1184
-
.get_gp_data = rcu_tasks_trace_get_gp_data,
1185
1183
.cbflood_max = 50000,
1186
1184
.irq_capable = 1,
1187
1185
.slow_gps = 1,
+1
-78
kernel/rcu/tasks.h
+1
-78
kernel/rcu/tasks.h
···
161
161
static DEFINE_TIMER(tasks_rcu_exit_srcu_stall_timer, tasks_rcu_exit_srcu_stall);
162
162
#endif
163
163
164
-
/* Avoid IPIing CPUs early in the grace period. */
165
-
#define RCU_TASK_IPI_DELAY (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) ? HZ / 2 : 0)
166
-
static int rcu_task_ipi_delay __read_mostly = RCU_TASK_IPI_DELAY;
167
-
module_param(rcu_task_ipi_delay, int, 0644);
168
-
169
164
/* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */
170
165
#define RCU_TASK_BOOT_STALL_TIMEOUT (HZ * 30)
171
166
#define RCU_TASK_STALL_TIMEOUT (HZ * 60 * 10)
···
795
800
796
801
#endif // #ifndef CONFIG_TINY_RCU
797
802
798
-
static void exit_tasks_rcu_finish_trace(struct task_struct *t);
799
-
800
803
#if defined(CONFIG_TASKS_RCU)
801
804
802
805
////////////////////////////////////////////////////////////////////////
···
1314
1321
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
1315
1322
list_del_init(&t->rcu_tasks_exit_list);
1316
1323
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
1317
-
1318
-
exit_tasks_rcu_finish_trace(t);
1319
1324
}
1320
1325
1321
1326
#else /* #ifdef CONFIG_TASKS_RCU */
1322
1327
void exit_tasks_rcu_start(void) { }
1323
-
void exit_tasks_rcu_finish(void) { exit_tasks_rcu_finish_trace(current); }
1328
+
void exit_tasks_rcu_finish(void) { }
1324
1329
#endif /* #else #ifdef CONFIG_TASKS_RCU */
1325
1330
1326
1331
#ifdef CONFIG_TASKS_RUDE_RCU
···
1466
1475
#endif // #ifndef CONFIG_TINY_RCU
1467
1476
}
1468
1477
1469
-
/*
1470
-
* Do a cmpxchg() on ->trc_reader_special.b.need_qs, allowing for
1471
-
* the four-byte operand-size restriction of some platforms.
1472
-
*
1473
-
* Returns the old value, which is often ignored.
1474
-
*/
1475
-
u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new)
1476
-
{
1477
-
return cmpxchg(&t->trc_reader_special.b.need_qs, old, new);
1478
-
}
1479
-
EXPORT_SYMBOL_GPL(rcu_trc_cmpxchg_need_qs);
1480
-
1481
-
/* Add a newly blocked reader task to its CPU's list. */
1482
-
void rcu_tasks_trace_qs_blkd(struct task_struct *t)
1483
-
{
1484
-
}
1485
-
EXPORT_SYMBOL_GPL(rcu_tasks_trace_qs_blkd);
1486
-
1487
-
/* Communicate task state back to the RCU tasks trace stall warning request. */
1488
-
struct trc_stall_chk_rdr {
1489
-
int nesting;
1490
-
int ipi_to_cpu;
1491
-
u8 needqs;
1492
-
};
1493
-
1494
-
/* Report any needed quiescent state for this exiting task. */
1495
-
static void exit_tasks_rcu_finish_trace(struct task_struct *t)
1496
-
{
1497
-
}
1498
-
1499
-
int rcu_tasks_trace_lazy_ms = -1;
1500
-
module_param(rcu_tasks_trace_lazy_ms, int, 0444);
1501
-
1502
-
static int __init rcu_spawn_tasks_trace_kthread(void)
1503
-
{
1504
-
return 0;
1505
-
}
1506
-
1507
-
#if !defined(CONFIG_TINY_RCU)
1508
-
void show_rcu_tasks_trace_gp_kthread(void)
1509
-
{
1510
-
}
1511
-
EXPORT_SYMBOL_GPL(show_rcu_tasks_trace_gp_kthread);
1512
-
1513
-
void rcu_tasks_trace_torture_stats_print(char *tt, char *tf)
1514
-
{
1515
-
}
1516
-
EXPORT_SYMBOL_GPL(rcu_tasks_trace_torture_stats_print);
1517
-
#endif // !defined(CONFIG_TINY_RCU)
1518
-
1519
-
struct task_struct *get_rcu_tasks_trace_gp_kthread(void)
1520
-
{
1521
-
return NULL;
1522
-
}
1523
-
EXPORT_SYMBOL_GPL(get_rcu_tasks_trace_gp_kthread);
1524
-
1525
-
void rcu_tasks_trace_get_gp_data(int *flags, unsigned long *gp_seq)
1526
-
{
1527
-
}
1528
-
EXPORT_SYMBOL_GPL(rcu_tasks_trace_get_gp_data);
1529
-
1530
-
#else /* #ifdef CONFIG_TASKS_TRACE_RCU */
1531
-
static void exit_tasks_rcu_finish_trace(struct task_struct *t) { }
1532
1478
#endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */
1533
1479
1534
1480
#ifndef CONFIG_TINY_RCU
···
1473
1545
{
1474
1546
show_rcu_tasks_classic_gp_kthread();
1475
1547
show_rcu_tasks_rude_gp_kthread();
1476
-
show_rcu_tasks_trace_gp_kthread();
1477
1548
}
1478
1549
#endif /* #ifndef CONFIG_TINY_RCU */
1479
1550
···
1609
1682
1610
1683
#ifdef CONFIG_TASKS_RUDE_RCU
1611
1684
rcu_spawn_tasks_rude_kthread();
1612
-
#endif
1613
-
1614
-
#ifdef CONFIG_TASKS_TRACE_RCU
1615
-
rcu_spawn_tasks_trace_kthread();
1616
1685
#endif
1617
1686
1618
1687
// Run the self-tests.
-1
tools/testing/selftests/rcutorture/configs/rcu/TRACE01
-1
tools/testing/selftests/rcutorture/configs/rcu/TRACE01