Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
srcu: Use raw spinlocks so call_srcu() can be used under preempt_disable()
Tree SRCU has used non-raw spinlocks for many years, motivated by a desire
to avoid unnecessary real-time latency and the absence of any reason to
use raw spinlocks. However, the recent use of SRCU in tracing as the
underlying implementation of RCU Tasks Trace means that call_srcu()
is invoked from preemption-disabled regions of code, which in turn
requires that any locks acquired by call_srcu() or its callees must be
raw spinlocks.
This commit therefore converts SRCU's spinlocks to raw spinlocks.
[boqun: Add Fixes tag]
Reported-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Fixes: c27cea4416a3 ("rcu: Re-implement RCU Tasks Trace in terms of SRCU-fast")
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
authored by
Paul E. McKenney
and committed by
Boqun Feng
175b45ed
f338e773
+82
-109
+4
-4
include/linux/srcutree.h
+4
-4
include/linux/srcutree.h
···
34
34
/* Values: SRCU_READ_FLAVOR_.* */
35
35
36
36
/* Update-side state. */
37
-
spinlock_t __private lock ____cacheline_internodealigned_in_smp;
37
+
raw_spinlock_t __private lock ____cacheline_internodealigned_in_smp;
38
38
struct rcu_segcblist srcu_cblist; /* List of callbacks.*/
39
39
unsigned long srcu_gp_seq_needed; /* Furthest future GP needed. */
40
40
unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
···
55
55
* Node in SRCU combining tree, similar in function to rcu_data.
56
56
*/
57
57
struct srcu_node {
58
-
spinlock_t __private lock;
58
+
raw_spinlock_t __private lock;
59
59
unsigned long srcu_have_cbs[4]; /* GP seq for children having CBs, but only */
60
60
/* if greater than ->srcu_gp_seq. */
61
61
unsigned long srcu_data_have_cbs[4]; /* Which srcu_data structs have CBs for given GP? */
···
74
74
/* First node at each level. */
75
75
int srcu_size_state; /* Small-to-big transition state. */
76
76
struct mutex srcu_cb_mutex; /* Serialize CB preparation. */
77
-
spinlock_t __private lock; /* Protect counters and size state. */
77
+
raw_spinlock_t __private lock; /* Protect counters and size state. */
78
78
struct mutex srcu_gp_mutex; /* Serialize GP work. */
79
79
unsigned long srcu_gp_seq; /* Grace-period seq #. */
80
80
unsigned long srcu_gp_seq_needed; /* Latest gp_seq needed. */
···
156
156
157
157
#define __SRCU_USAGE_INIT(name) \
158
158
{ \
159
-
.lock = __SPIN_LOCK_UNLOCKED(name.lock), \
159
+
.lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \
160
160
.srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL, \
161
161
.srcu_gp_seq_needed = SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE, \
162
162
.srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL, \
+9
kernel/rcu/rcu.h
+9
kernel/rcu/rcu.h
···
502
502
___locked; \
503
503
})
504
504
505
+
#define raw_spin_trylock_irqsave_rcu_node(p, flags) \
506
+
({ \
507
+
bool ___locked = raw_spin_trylock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \
508
+
\
509
+
if (___locked) \
510
+
smp_mb__after_unlock_lock(); \
511
+
___locked; \
512
+
})
513
+
505
514
#define raw_lockdep_assert_held_rcu_node(p) \
506
515
lockdep_assert_held(&ACCESS_PRIVATE(p, lock))
507
516
+69
-105
kernel/rcu/srcutree.c
+69
-105
kernel/rcu/srcutree.c
···
77
77
static void process_srcu(struct work_struct *work);
78
78
static void srcu_delay_timer(struct timer_list *t);
79
79
80
-
/* Wrappers for lock acquisition and release, see raw_spin_lock_rcu_node(). */
81
-
#define spin_lock_rcu_node(p) \
82
-
do { \
83
-
spin_lock(&ACCESS_PRIVATE(p, lock)); \
84
-
smp_mb__after_unlock_lock(); \
85
-
} while (0)
86
-
87
-
#define spin_unlock_rcu_node(p) spin_unlock(&ACCESS_PRIVATE(p, lock))
88
-
89
-
#define spin_lock_irq_rcu_node(p) \
90
-
do { \
91
-
spin_lock_irq(&ACCESS_PRIVATE(p, lock)); \
92
-
smp_mb__after_unlock_lock(); \
93
-
} while (0)
94
-
95
-
#define spin_unlock_irq_rcu_node(p) \
96
-
spin_unlock_irq(&ACCESS_PRIVATE(p, lock))
97
-
98
-
#define spin_lock_irqsave_rcu_node(p, flags) \
99
-
do { \
100
-
spin_lock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \
101
-
smp_mb__after_unlock_lock(); \
102
-
} while (0)
103
-
104
-
#define spin_trylock_irqsave_rcu_node(p, flags) \
105
-
({ \
106
-
bool ___locked = spin_trylock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \
107
-
\
108
-
if (___locked) \
109
-
smp_mb__after_unlock_lock(); \
110
-
___locked; \
111
-
})
112
-
113
-
#define spin_unlock_irqrestore_rcu_node(p, flags) \
114
-
spin_unlock_irqrestore(&ACCESS_PRIVATE(p, lock), flags) \
115
-
116
80
/*
117
81
* Initialize SRCU per-CPU data. Note that statically allocated
118
82
* srcu_struct structures might already have srcu_read_lock() and
···
95
131
*/
96
132
for_each_possible_cpu(cpu) {
97
133
sdp = per_cpu_ptr(ssp->sda, cpu);
98
-
spin_lock_init(&ACCESS_PRIVATE(sdp, lock));
134
+
raw_spin_lock_init(&ACCESS_PRIVATE(sdp, lock));
99
135
rcu_segcblist_init(&sdp->srcu_cblist);
100
136
sdp->srcu_cblist_invoking = false;
101
137
sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;
···
150
186
151
187
/* Each pass through this loop initializes one srcu_node structure. */
152
188
srcu_for_each_node_breadth_first(ssp, snp) {
153
-
spin_lock_init(&ACCESS_PRIVATE(snp, lock));
189
+
raw_spin_lock_init(&ACCESS_PRIVATE(snp, lock));
154
190
BUILD_BUG_ON(ARRAY_SIZE(snp->srcu_have_cbs) !=
155
191
ARRAY_SIZE(snp->srcu_data_have_cbs));
156
192
for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) {
···
206
242
if (!ssp->srcu_sup)
207
243
return -ENOMEM;
208
244
if (!is_static)
209
-
spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock));
245
+
raw_spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock));
210
246
ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;
211
247
ssp->srcu_sup->node = NULL;
212
248
mutex_init(&ssp->srcu_sup->srcu_cb_mutex);
···
358
394
/* Double-checked locking on ->srcu_size-state. */
359
395
if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL)
360
396
return;
361
-
spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);
397
+
raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);
362
398
if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) {
363
-
spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
399
+
raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
364
400
return;
365
401
}
366
402
__srcu_transition_to_big(ssp);
367
-
spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
403
+
raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
368
404
}
369
405
370
406
/*
371
407
* Check to see if the just-encountered contention event justifies
372
408
* a transition to SRCU_SIZE_BIG.
373
409
*/
374
-
static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)
410
+
static void raw_spin_lock_irqsave_check_contention(struct srcu_struct *ssp)
375
411
{
376
412
unsigned long j;
377
413
···
393
429
* to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module
394
430
* parameter permits this.
395
431
*/
396
-
static void spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags)
432
+
static void raw_spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags)
397
433
{
398
434
struct srcu_struct *ssp = sdp->ssp;
399
435
400
-
if (spin_trylock_irqsave_rcu_node(sdp, *flags))
436
+
if (raw_spin_trylock_irqsave_rcu_node(sdp, *flags))
401
437
return;
402
-
spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);
403
-
spin_lock_irqsave_check_contention(ssp);
404
-
spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags);
405
-
spin_lock_irqsave_rcu_node(sdp, *flags);
438
+
raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);
439
+
raw_spin_lock_irqsave_check_contention(ssp);
440
+
raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags);
441
+
raw_spin_lock_irqsave_rcu_node(sdp, *flags);
406
442
}
407
443
408
444
/*
···
411
447
* to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module
412
448
* parameter permits this.
413
449
*/
414
-
static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)
450
+
static void raw_spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)
415
451
{
416
-
if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags))
452
+
if (raw_spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags))
417
453
return;
418
-
spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);
419
-
spin_lock_irqsave_check_contention(ssp);
454
+
raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);
455
+
raw_spin_lock_irqsave_check_contention(ssp);
420
456
}
421
457
422
458
/*
···
434
470
/* The smp_load_acquire() pairs with the smp_store_release(). */
435
471
if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/
436
472
return; /* Already initialized. */
437
-
spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);
473
+
raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);
438
474
if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) {
439
-
spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
475
+
raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
440
476
return;
441
477
}
442
478
init_srcu_struct_fields(ssp, true);
443
-
spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
479
+
raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
444
480
}
445
481
446
482
/*
···
706
742
unsigned long delay;
707
743
struct srcu_usage *sup = ssp->srcu_sup;
708
744
709
-
spin_lock_irq_rcu_node(ssp->srcu_sup);
745
+
raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
710
746
delay = srcu_get_delay(ssp);
711
-
spin_unlock_irq_rcu_node(ssp->srcu_sup);
747
+
raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
712
748
if (WARN_ON(!delay))
713
749
return; /* Just leak it! */
714
750
if (WARN_ON(srcu_readers_active(ssp)))
···
924
960
mutex_lock(&sup->srcu_cb_mutex);
925
961
926
962
/* End the current grace period. */
927
-
spin_lock_irq_rcu_node(sup);
963
+
raw_spin_lock_irq_rcu_node(sup);
928
964
idx = rcu_seq_state(sup->srcu_gp_seq);
929
965
WARN_ON_ONCE(idx != SRCU_STATE_SCAN2);
930
966
if (srcu_gp_is_expedited(ssp))
···
935
971
gpseq = rcu_seq_current(&sup->srcu_gp_seq);
936
972
if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq))
937
973
WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq);
938
-
spin_unlock_irq_rcu_node(sup);
974
+
raw_spin_unlock_irq_rcu_node(sup);
939
975
mutex_unlock(&sup->srcu_gp_mutex);
940
976
/* A new grace period can start at this point. But only one. */
941
977
···
947
983
} else {
948
984
idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);
949
985
srcu_for_each_node_breadth_first(ssp, snp) {
950
-
spin_lock_irq_rcu_node(snp);
986
+
raw_spin_lock_irq_rcu_node(snp);
951
987
cbs = false;
952
988
last_lvl = snp >= sup->level[rcu_num_lvls - 1];
953
989
if (last_lvl)
···
962
998
else
963
999
mask = snp->srcu_data_have_cbs[idx];
964
1000
snp->srcu_data_have_cbs[idx] = 0;
965
-
spin_unlock_irq_rcu_node(snp);
1001
+
raw_spin_unlock_irq_rcu_node(snp);
966
1002
if (cbs)
967
1003
srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);
968
1004
}
···
972
1008
if (!(gpseq & counter_wrap_check))
973
1009
for_each_possible_cpu(cpu) {
974
1010
sdp = per_cpu_ptr(ssp->sda, cpu);
975
-
spin_lock_irq_rcu_node(sdp);
1011
+
raw_spin_lock_irq_rcu_node(sdp);
976
1012
if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100))
977
1013
sdp->srcu_gp_seq_needed = gpseq;
978
1014
if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100))
979
1015
sdp->srcu_gp_seq_needed_exp = gpseq;
980
-
spin_unlock_irq_rcu_node(sdp);
1016
+
raw_spin_unlock_irq_rcu_node(sdp);
981
1017
}
982
1018
983
1019
/* Callback initiation done, allow grace periods after next. */
984
1020
mutex_unlock(&sup->srcu_cb_mutex);
985
1021
986
1022
/* Start a new grace period if needed. */
987
-
spin_lock_irq_rcu_node(sup);
1023
+
raw_spin_lock_irq_rcu_node(sup);
988
1024
gpseq = rcu_seq_current(&sup->srcu_gp_seq);
989
1025
if (!rcu_seq_state(gpseq) &&
990
1026
ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) {
991
1027
srcu_gp_start(ssp);
992
-
spin_unlock_irq_rcu_node(sup);
1028
+
raw_spin_unlock_irq_rcu_node(sup);
993
1029
srcu_reschedule(ssp, 0);
994
1030
} else {
995
-
spin_unlock_irq_rcu_node(sup);
1031
+
raw_spin_unlock_irq_rcu_node(sup);
996
1032
}
997
1033
998
1034
/* Transition to big if needed. */
···
1023
1059
if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) ||
1024
1060
(!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s)))
1025
1061
return;
1026
-
spin_lock_irqsave_rcu_node(snp, flags);
1062
+
raw_spin_lock_irqsave_rcu_node(snp, flags);
1027
1063
sgsne = snp->srcu_gp_seq_needed_exp;
1028
1064
if (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s)) {
1029
-
spin_unlock_irqrestore_rcu_node(snp, flags);
1065
+
raw_spin_unlock_irqrestore_rcu_node(snp, flags);
1030
1066
return;
1031
1067
}
1032
1068
WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
1033
-
spin_unlock_irqrestore_rcu_node(snp, flags);
1069
+
raw_spin_unlock_irqrestore_rcu_node(snp, flags);
1034
1070
}
1035
-
spin_lock_irqsave_ssp_contention(ssp, &flags);
1071
+
raw_spin_lock_irqsave_ssp_contention(ssp, &flags);
1036
1072
if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s))
1037
1073
WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s);
1038
-
spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
1074
+
raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
1039
1075
}
1040
1076
1041
1077
/*
···
1073
1109
for (snp = snp_leaf; snp != NULL; snp = snp->srcu_parent) {
1074
1110
if (WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && snp != snp_leaf)
1075
1111
return; /* GP already done and CBs recorded. */
1076
-
spin_lock_irqsave_rcu_node(snp, flags);
1112
+
raw_spin_lock_irqsave_rcu_node(snp, flags);
1077
1113
snp_seq = snp->srcu_have_cbs[idx];
1078
1114
if (!srcu_invl_snp_seq(snp_seq) && ULONG_CMP_GE(snp_seq, s)) {
1079
1115
if (snp == snp_leaf && snp_seq == s)
1080
1116
snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
1081
-
spin_unlock_irqrestore_rcu_node(snp, flags);
1117
+
raw_spin_unlock_irqrestore_rcu_node(snp, flags);
1082
1118
if (snp == snp_leaf && snp_seq != s) {
1083
1119
srcu_schedule_cbs_sdp(sdp, do_norm ? SRCU_INTERVAL : 0);
1084
1120
return;
···
1093
1129
sgsne = snp->srcu_gp_seq_needed_exp;
1094
1130
if (!do_norm && (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, s)))
1095
1131
WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
1096
-
spin_unlock_irqrestore_rcu_node(snp, flags);
1132
+
raw_spin_unlock_irqrestore_rcu_node(snp, flags);
1097
1133
}
1098
1134
1099
1135
/* Top of tree, must ensure the grace period will be started. */
1100
-
spin_lock_irqsave_ssp_contention(ssp, &flags);
1136
+
raw_spin_lock_irqsave_ssp_contention(ssp, &flags);
1101
1137
if (ULONG_CMP_LT(sup->srcu_gp_seq_needed, s)) {
1102
1138
/*
1103
1139
* Record need for grace period s. Pair with load
···
1124
1160
else if (list_empty(&sup->work.work.entry))
1125
1161
list_add(&sup->work.work.entry, &srcu_boot_list);
1126
1162
}
1127
-
spin_unlock_irqrestore_rcu_node(sup, flags);
1163
+
raw_spin_unlock_irqrestore_rcu_node(sup, flags);
1128
1164
}
1129
1165
1130
1166
/*
···
1136
1172
{
1137
1173
unsigned long curdelay;
1138
1174
1139
-
spin_lock_irq_rcu_node(ssp->srcu_sup);
1175
+
raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
1140
1176
curdelay = !srcu_get_delay(ssp);
1141
-
spin_unlock_irq_rcu_node(ssp->srcu_sup);
1177
+
raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
1142
1178
1143
1179
for (;;) {
1144
1180
if (srcu_readers_active_idx_check(ssp, idx))
···
1249
1285
return false;
1250
1286
/* If the local srcu_data structure has callbacks, not idle. */
1251
1287
sdp = raw_cpu_ptr(ssp->sda);
1252
-
spin_lock_irqsave_rcu_node(sdp, flags);
1288
+
raw_spin_lock_irqsave_rcu_node(sdp, flags);
1253
1289
if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {
1254
-
spin_unlock_irqrestore_rcu_node(sdp, flags);
1290
+
raw_spin_unlock_irqrestore_rcu_node(sdp, flags);
1255
1291
return false; /* Callbacks already present, so not idle. */
1256
1292
}
1257
-
spin_unlock_irqrestore_rcu_node(sdp, flags);
1293
+
raw_spin_unlock_irqrestore_rcu_node(sdp, flags);
1258
1294
1259
1295
/*
1260
1296
* No local callbacks, so probabilistically probe global state.
···
1314
1350
sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());
1315
1351
else
1316
1352
sdp = raw_cpu_ptr(ssp->sda);
1317
-
spin_lock_irqsave_sdp_contention(sdp, &flags);
1353
+
raw_spin_lock_irqsave_sdp_contention(sdp, &flags);
1318
1354
if (rhp)
1319
1355
rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);
1320
1356
/*
···
1374
1410
sdp->srcu_gp_seq_needed_exp = s;
1375
1411
needexp = true;
1376
1412
}
1377
-
spin_unlock_irqrestore_rcu_node(sdp, flags);
1413
+
raw_spin_unlock_irqrestore_rcu_node(sdp, flags);
1378
1414
1379
1415
/* Ensure that snp node tree is fully initialized before traversing it */
1380
1416
if (ss_state < SRCU_SIZE_WAIT_BARRIER)
···
1486
1522
1487
1523
/*
1488
1524
* Make sure that later code is ordered after the SRCU grace
1489
-
* period. This pairs with the spin_lock_irq_rcu_node()
1525
+
* period. This pairs with the raw_spin_lock_irq_rcu_node()
1490
1526
* in srcu_invoke_callbacks(). Unlike Tree RCU, this is needed
1491
1527
* because the current CPU might have been totally uninvolved with
1492
1528
* (and thus unordered against) that grace period.
···
1665
1701
*/
1666
1702
static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp)
1667
1703
{
1668
-
spin_lock_irq_rcu_node(sdp);
1704
+
raw_spin_lock_irq_rcu_node(sdp);
1669
1705
atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt);
1670
1706
sdp->srcu_barrier_head.func = srcu_barrier_cb;
1671
1707
debug_rcu_head_queue(&sdp->srcu_barrier_head);
···
1674
1710
debug_rcu_head_unqueue(&sdp->srcu_barrier_head);
1675
1711
atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt);
1676
1712
}
1677
-
spin_unlock_irq_rcu_node(sdp);
1713
+
raw_spin_unlock_irq_rcu_node(sdp);
1678
1714
}
1679
1715
1680
1716
/**
···
1725
1761
bool needcb = false;
1726
1762
struct srcu_data *sdp = container_of(rhp, struct srcu_data, srcu_ec_head);
1727
1763
1728
-
spin_lock_irqsave_sdp_contention(sdp, &flags);
1764
+
raw_spin_lock_irqsave_sdp_contention(sdp, &flags);
1729
1765
if (sdp->srcu_ec_state == SRCU_EC_IDLE) {
1730
1766
WARN_ON_ONCE(1);
1731
1767
} else if (sdp->srcu_ec_state == SRCU_EC_PENDING) {
···
1735
1771
sdp->srcu_ec_state = SRCU_EC_PENDING;
1736
1772
needcb = true;
1737
1773
}
1738
-
spin_unlock_irqrestore_rcu_node(sdp, flags);
1774
+
raw_spin_unlock_irqrestore_rcu_node(sdp, flags);
1739
1775
// If needed, requeue ourselves as an expedited SRCU callback.
1740
1776
if (needcb)
1741
1777
__call_srcu(sdp->ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false);
···
1759
1795
1760
1796
migrate_disable();
1761
1797
sdp = this_cpu_ptr(ssp->sda);
1762
-
spin_lock_irqsave_sdp_contention(sdp, &flags);
1798
+
raw_spin_lock_irqsave_sdp_contention(sdp, &flags);
1763
1799
if (sdp->srcu_ec_state == SRCU_EC_IDLE) {
1764
1800
sdp->srcu_ec_state = SRCU_EC_PENDING;
1765
1801
needcb = true;
···
1768
1804
} else {
1769
1805
WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST);
1770
1806
}
1771
-
spin_unlock_irqrestore_rcu_node(sdp, flags);
1807
+
raw_spin_unlock_irqrestore_rcu_node(sdp, flags);
1772
1808
// If needed, queue an expedited SRCU callback.
1773
1809
if (needcb)
1774
1810
__call_srcu(ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false);
···
1812
1848
*/
1813
1849
idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */
1814
1850
if (idx == SRCU_STATE_IDLE) {
1815
-
spin_lock_irq_rcu_node(ssp->srcu_sup);
1851
+
raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
1816
1852
if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {
1817
1853
WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));
1818
-
spin_unlock_irq_rcu_node(ssp->srcu_sup);
1854
+
raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
1819
1855
mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
1820
1856
return;
1821
1857
}
1822
1858
idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq));
1823
1859
if (idx == SRCU_STATE_IDLE)
1824
1860
srcu_gp_start(ssp);
1825
-
spin_unlock_irq_rcu_node(ssp->srcu_sup);
1861
+
raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
1826
1862
if (idx != SRCU_STATE_IDLE) {
1827
1863
mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
1828
1864
return; /* Someone else started the grace period. */
···
1836
1872
return; /* readers present, retry later. */
1837
1873
}
1838
1874
srcu_flip(ssp);
1839
-
spin_lock_irq_rcu_node(ssp->srcu_sup);
1875
+
raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
1840
1876
rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);
1841
1877
ssp->srcu_sup->srcu_n_exp_nodelay = 0;
1842
-
spin_unlock_irq_rcu_node(ssp->srcu_sup);
1878
+
raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
1843
1879
}
1844
1880
1845
1881
if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) {
···
1877
1913
1878
1914
ssp = sdp->ssp;
1879
1915
rcu_cblist_init(&ready_cbs);
1880
-
spin_lock_irq_rcu_node(sdp);
1916
+
raw_spin_lock_irq_rcu_node(sdp);
1881
1917
WARN_ON_ONCE(!rcu_segcblist_segempty(&sdp->srcu_cblist, RCU_NEXT_TAIL));
1882
1918
rcu_segcblist_advance(&sdp->srcu_cblist,
1883
1919
rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));
···
1888
1924
*/
1889
1925
if (sdp->srcu_cblist_invoking ||
1890
1926
!rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) {
1891
-
spin_unlock_irq_rcu_node(sdp);
1927
+
raw_spin_unlock_irq_rcu_node(sdp);
1892
1928
return; /* Someone else on the job or nothing to do. */
1893
1929
}
1894
1930
···
1896
1932
sdp->srcu_cblist_invoking = true;
1897
1933
rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs);
1898
1934
len = ready_cbs.len;
1899
-
spin_unlock_irq_rcu_node(sdp);
1935
+
raw_spin_unlock_irq_rcu_node(sdp);
1900
1936
rhp = rcu_cblist_dequeue(&ready_cbs);
1901
1937
for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
1902
1938
debug_rcu_head_unqueue(rhp);
···
1911
1947
* Update counts, accelerate new callbacks, and if needed,
1912
1948
* schedule another round of callback invocation.
1913
1949
*/
1914
-
spin_lock_irq_rcu_node(sdp);
1950
+
raw_spin_lock_irq_rcu_node(sdp);
1915
1951
rcu_segcblist_add_len(&sdp->srcu_cblist, -len);
1916
1952
sdp->srcu_cblist_invoking = false;
1917
1953
more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist);
1918
-
spin_unlock_irq_rcu_node(sdp);
1954
+
raw_spin_unlock_irq_rcu_node(sdp);
1919
1955
/* An SRCU barrier or callbacks from previous nesting work pending */
1920
1956
if (more)
1921
1957
srcu_schedule_cbs_sdp(sdp, 0);
···
1929
1965
{
1930
1966
bool pushgp = true;
1931
1967
1932
-
spin_lock_irq_rcu_node(ssp->srcu_sup);
1968
+
raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
1933
1969
if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {
1934
1970
if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) {
1935
1971
/* All requests fulfilled, time to go idle. */
···
1939
1975
/* Outstanding request and no GP. Start one. */
1940
1976
srcu_gp_start(ssp);
1941
1977
}
1942
-
spin_unlock_irq_rcu_node(ssp->srcu_sup);
1978
+
raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
1943
1979
1944
1980
if (pushgp)
1945
1981
queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);
···
1959
1995
ssp = sup->srcu_ssp;
1960
1996
1961
1997
srcu_advance_state(ssp);
1962
-
spin_lock_irq_rcu_node(ssp->srcu_sup);
1998
+
raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
1963
1999
curdelay = srcu_get_delay(ssp);
1964
-
spin_unlock_irq_rcu_node(ssp->srcu_sup);
2000
+
raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
1965
2001
if (curdelay) {
1966
2002
WRITE_ONCE(sup->reschedule_count, 0);
1967
2003
} else {