Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
srcu: Add SRCU_READ_FLAVOR_FAST_UPDOWN CPP macro
This commit adds the SRCU_READ_FLAVOR_FAST_UPDOWN=0x8 macro
and adjusts rcutorture to make use of it. In this commit, both
SRCU_READ_FLAVOR_FAST=0x4 and the new SRCU_READ_FLAVOR_FAST_UPDOWN
test SRCU-fast. When the SRCU-fast-updown is added, the new
SRCU_READ_FLAVOR_FAST_UPDOWN macro will test it when passed to the
rcutorture.reader_flavor module parameter.
The old SRCU_READ_FLAVOR_FAST macro's value changed from 0x8 to 0x4.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
authored by
Paul E. McKenney
and committed by
Frederic Weisbecker
37827223
3ed04e3f
+27
-13
+9
-7
include/linux/srcu.h
+9
-7
include/linux/srcu.h
···
56
56
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
57
57
58
58
/* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */
59
-
#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().
60
-
#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().
61
-
// 0x4 // SRCU-lite is no longer with us.
62
-
#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().
63
-
#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
64
-
SRCU_READ_FLAVOR_FAST) // All of the above.
65
-
#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_FAST
59
+
#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().
60
+
#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().
61
+
// 0x4 // SRCU-lite is no longer with us.
62
+
#define SRCU_READ_FLAVOR_FAST 0x4 // srcu_read_lock_fast().
63
+
#define SRCU_READ_FLAVOR_FAST_UPDOWN 0x8 // srcu_read_lock_fast().
64
+
#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
65
+
SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
66
+
// All of the above.
67
+
#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
66
68
// Flavors requiring synchronize_rcu()
67
69
// instead of smp_mb().
68
70
void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
+18
-6
kernel/rcu/rcutorture.c
+18
-6
kernel/rcu/rcutorture.c
···
702
702
rcu_sync_torture_init();
703
703
if (reader_flavor & SRCU_READ_FLAVOR_FAST)
704
704
srcu_ctlp = &srcu_ctlf;
705
+
if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)
706
+
srcu_ctlp = &srcu_ctlf;
705
707
}
706
708
707
709
static void srcu_get_gp_data(int *flags, unsigned long *gp_seq)
···
730
728
ret += idx << 1;
731
729
}
732
730
if (reader_flavor & SRCU_READ_FLAVOR_FAST) {
731
+
scp = srcu_read_lock_fast(srcu_ctlp);
732
+
idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);
733
+
WARN_ON_ONCE(idx & ~0x1);
734
+
ret += idx << 2;
735
+
}
736
+
if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {
733
737
scp = srcu_read_lock_fast(srcu_ctlp);
734
738
idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);
735
739
WARN_ON_ONCE(idx & ~0x1);
···
766
758
static void srcu_torture_read_unlock(int idx)
767
759
{
768
760
WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));
769
-
if (reader_flavor & SRCU_READ_FLAVOR_FAST)
761
+
if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)
770
762
srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3));
763
+
if (reader_flavor & SRCU_READ_FLAVOR_FAST)
764
+
srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 2));
771
765
if (reader_flavor & SRCU_READ_FLAVOR_NMI)
772
766
srcu_read_unlock_nmisafe(srcu_ctlp, (idx & 0x2) >> 1);
773
767
if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL))
···
803
793
WARN_ON_ONCE(idx & ~0x1);
804
794
return idx;
805
795
}
806
-
if (reader_flavor & SRCU_READ_FLAVOR_FAST) {
796
+
if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {
807
797
scp = srcu_down_read_fast(srcu_ctlp);
808
798
idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);
809
799
WARN_ON_ONCE(idx & ~0x1);
···
816
806
static void srcu_torture_up_read(int idx)
817
807
{
818
808
WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));
819
-
if (reader_flavor & SRCU_READ_FLAVOR_FAST)
809
+
if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)
820
810
srcu_up_read_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3));
821
811
else if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) ||
822
812
!(reader_flavor & SRCU_READ_FLAVOR_ALL))
···
911
901
.no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU),
912
902
.debug_objects = 1,
913
903
.have_up_down = IS_ENABLED(CONFIG_TINY_SRCU)
914
-
? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST,
904
+
? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,
915
905
.name = "srcu"
916
906
};
917
907
918
908
static void srcud_torture_init(void)
919
909
{
920
910
rcu_sync_torture_init();
921
-
if (reader_flavor & SRCU_READ_FLAVOR_FAST)
911
+
if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)
912
+
WARN_ON(init_srcu_struct_fast(&srcu_ctld));
913
+
else if (reader_flavor & SRCU_READ_FLAVOR_FAST)
922
914
WARN_ON(init_srcu_struct_fast(&srcu_ctld));
923
915
else
924
916
WARN_ON(init_srcu_struct(&srcu_ctld));
···
965
953
.no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU),
966
954
.debug_objects = 1,
967
955
.have_up_down = IS_ENABLED(CONFIG_TINY_SRCU)
968
-
? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST,
956
+
? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,
969
957
.name = "srcud"
970
958
};
971
959