Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm/vma: introduce vma_test[_any[_mask]](), and make inlining consistent
Introduce helper functions and macros to make it convenient to test flags
and flag masks for VMAs, specifically:
* vma_test() - determine if a single VMA flag is set in a VMA.
* vma_test_any_mask() - determine if any flags in a vma_flags_t value are
set in a VMA.
* vma_test_any() - Helper macro to test if any of specific flags are set.
Also, there are a mix of 'inline's and '__always_inline's in VMA helper
function declarations, update to consistently use __always_inline.
Finally, update the VMA tests to reflect the changes.
Link: https://lkml.kernel.org/r/be1d71f08307d747a82232cbd8664a88c0f41419.1774034900.git.ljs@kernel.org
Signed-off-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Acked-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: "Borislav Petkov (AMD)" <bp@alien8.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christian Brauner <brauner@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: Kees Cook <kees@kernel.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Ondrej Mosnacek <omosnace@redhat.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Moore <paul@paul-moore.com>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: Richard Weinberger <richard@nod.at>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stephen Smalley <stephen.smalley.work@gmail.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vineet Gupta <vgupta@kernel.org>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Will Deacon <will@kernel.org>
Cc: xu xin <xu.xin16@zte.com.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Lorenzo Stoakes (Oracle)
and committed by
Andrew Morton
fb67bba5
a8add93f
+88
-34
+40
-9
include/linux/mm.h
+40
-9
include/linux/mm.h
···
994
994
__vm_flags_mod(vma, set, clear);
995
995
}
996
996
997
-
static inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, vma_flag_t bit)
997
+
static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma,
998
+
vma_flag_t bit)
998
999
{
999
1000
const vm_flags_t mask = BIT((__force int)bit);
1000
1001
···
1010
1009
* Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific
1011
1010
* valid flags are allowed to do this.
1012
1011
*/
1013
-
static inline void vma_set_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit)
1012
+
static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma,
1013
+
vma_flag_t bit)
1014
1014
{
1015
1015
unsigned long *bitmap = vma->flags.__vma_flags;
1016
1016
···
1027
1025
* This is necessarily racey, so callers must ensure that serialisation is
1028
1026
* achieved through some other means, or that races are permissible.
1029
1027
*/
1030
-
static inline bool vma_test_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit)
1028
+
static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma,
1029
+
vma_flag_t bit)
1031
1030
{
1032
1031
if (__vma_atomic_valid_flag(vma, bit))
1033
1032
return test_bit((__force int)bit, &vma->vm_flags);
···
1234
1231
vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__))
1235
1232
1236
1233
/*
1234
+
* Test whether a specific flag in the VMA is set, e.g.:
1235
+
*
1236
+
* if (vma_test(vma, VMA_READ_BIT)) { ... }
1237
+
*/
1238
+
static __always_inline bool vma_test(const struct vm_area_struct *vma,
1239
+
vma_flag_t bit)
1240
+
{
1241
+
return vma_flags_test(&vma->flags, bit);
1242
+
}
1243
+
1244
+
/* Helper to test any VMA flags in a VMA . */
1245
+
static __always_inline bool vma_test_any_mask(const struct vm_area_struct *vma,
1246
+
vma_flags_t flags)
1247
+
{
1248
+
return vma_flags_test_any_mask(&vma->flags, flags);
1249
+
}
1250
+
1251
+
/*
1252
+
* Helper macro for testing whether any VMA flags are set in a VMA,
1253
+
* e.g.:
1254
+
*
1255
+
* if (vma_test_any(vma, VMA_IO_BIT, VMA_PFNMAP_BIT,
1256
+
* VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... }
1257
+
*/
1258
+
#define vma_test_any(vma, ...) \
1259
+
vma_test_any_mask(vma, mk_vma_flags(__VA_ARGS__))
1260
+
1261
+
/*
1237
1262
* Helper to test that ALL specified flags are set in a VMA.
1238
1263
*
1239
1264
* Note: appropriate locks must be held, this function does not acquire them for
1240
1265
* you.
1241
1266
*/
1242
-
static inline bool vma_test_all_mask(const struct vm_area_struct *vma,
1267
+
static __always_inline bool vma_test_all_mask(const struct vm_area_struct *vma,
1243
1268
vma_flags_t flags)
1244
1269
{
1245
1270
return vma_flags_test_all_mask(&vma->flags, flags);
···
1287
1256
* Note: appropriate locks must be held, this function does not acquire them for
1288
1257
* you.
1289
1258
*/
1290
-
static inline void vma_set_flags_mask(struct vm_area_struct *vma,
1259
+
static __always_inline void vma_set_flags_mask(struct vm_area_struct *vma,
1291
1260
vma_flags_t flags)
1292
1261
{
1293
1262
vma_flags_set_mask(&vma->flags, flags);
···
1317
1286
}
1318
1287
1319
1288
/* Helper to test any VMA flags in a VMA descriptor. */
1320
-
static inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc,
1289
+
static __always_inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc,
1321
1290
vma_flags_t flags)
1322
1291
{
1323
1292
return vma_flags_test_any_mask(&desc->vma_flags, flags);
···
1334
1303
vma_desc_test_any_mask(desc, mk_vma_flags(__VA_ARGS__))
1335
1304
1336
1305
/* Helper to test all VMA flags in a VMA descriptor. */
1337
-
static inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc,
1306
+
static __always_inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc,
1338
1307
vma_flags_t flags)
1339
1308
{
1340
1309
return vma_flags_test_all_mask(&desc->vma_flags, flags);
···
1350
1319
vma_desc_test_all_mask(desc, mk_vma_flags(__VA_ARGS__))
1351
1320
1352
1321
/* Helper to set all VMA flags in a VMA descriptor. */
1353
-
static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
1322
+
static __always_inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
1354
1323
vma_flags_t flags)
1355
1324
{
1356
1325
vma_flags_set_mask(&desc->vma_flags, flags);
···
1367
1336
vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1368
1337
1369
1338
/* Helper to clear all VMA flags in a VMA descriptor. */
1370
-
static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
1339
+
static __always_inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
1371
1340
vma_flags_t flags)
1372
1341
{
1373
1342
vma_flags_clear_mask(&desc->vma_flags, flags);
+8
-4
include/linux/mm_types.h
+8
-4
include/linux/mm_types.h
···
1087
1087
* IMPORTANT: This does not overwrite bytes past the first system word. The
1088
1088
* caller must account for this.
1089
1089
*/
1090
-
static inline void vma_flags_overwrite_word(vma_flags_t *flags, unsigned long value)
1090
+
static __always_inline void vma_flags_overwrite_word(vma_flags_t *flags,
1091
+
unsigned long value)
1091
1092
{
1092
1093
unsigned long *bitmap = flags->__vma_flags;
1093
1094
···
1115
1114
* IMPORTANT: This does not overwrite bytes past the first system word. The
1116
1115
* caller must account for this.
1117
1116
*/
1118
-
static inline void vma_flags_overwrite_word_once(vma_flags_t *flags, unsigned long value)
1117
+
static __always_inline void vma_flags_overwrite_word_once(vma_flags_t *flags,
1118
+
unsigned long value)
1119
1119
{
1120
1120
unsigned long *bitmap = flags->__vma_flags;
1121
1121
···
1124
1122
}
1125
1123
1126
1124
/* Update the first system word of VMA flags setting bits, non-atomically. */
1127
-
static inline void vma_flags_set_word(vma_flags_t *flags, unsigned long value)
1125
+
static __always_inline void vma_flags_set_word(vma_flags_t *flags,
1126
+
unsigned long value)
1128
1127
{
1129
1128
unsigned long *bitmap = flags->__vma_flags;
1130
1129
···
1133
1130
}
1134
1131
1135
1132
/* Update the first system word of VMA flags clearing bits, non-atomically. */
1136
-
static inline void vma_flags_clear_word(vma_flags_t *flags, unsigned long value)
1133
+
static __always_inline void vma_flags_clear_word(vma_flags_t *flags,
1134
+
unsigned long value)
1137
1135
{
1138
1136
unsigned long *bitmap = flags->__vma_flags;
1139
1137
+40
-21
tools/testing/vma/include/dup.h
+40
-21
tools/testing/vma/include/dup.h
···
764
764
* IMPORTANT: This does not overwrite bytes past the first system word. The
765
765
* caller must account for this.
766
766
*/
767
-
static inline void vma_flags_overwrite_word(vma_flags_t *flags, unsigned long value)
767
+
static __always_inline void vma_flags_overwrite_word(vma_flags_t *flags,
768
+
unsigned long value)
768
769
{
769
770
unsigned long *bitmap = flags->__vma_flags;
770
771
···
778
777
* IMPORTANT: This does not overwrite bytes past the first system word. The
779
778
* caller must account for this.
780
779
*/
781
-
static inline void vma_flags_overwrite_word_once(vma_flags_t *flags, unsigned long value)
780
+
static __always_inline void vma_flags_overwrite_word_once(vma_flags_t *flags,
781
+
unsigned long value)
782
782
{
783
783
unsigned long *bitmap = flags->__vma_flags;
784
784
···
787
785
}
788
786
789
787
/* Update the first system word of VMA flags setting bits, non-atomically. */
790
-
static inline void vma_flags_set_word(vma_flags_t *flags, unsigned long value)
788
+
static __always_inline void vma_flags_set_word(vma_flags_t *flags,
789
+
unsigned long value)
791
790
{
792
791
unsigned long *bitmap = flags->__vma_flags;
793
792
···
796
793
}
797
794
798
795
/* Update the first system word of VMA flags clearing bits, non-atomically. */
799
-
static inline void vma_flags_clear_word(vma_flags_t *flags, unsigned long value)
796
+
static __always_inline void vma_flags_clear_word(vma_flags_t *flags,
797
+
unsigned long value)
800
798
{
801
799
unsigned long *bitmap = flags->__vma_flags;
802
800
···
1007
1003
#define vma_flags_same(flags, ...) \
1008
1004
vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__))
1009
1005
1010
-
static inline bool vma_test_all_mask(const struct vm_area_struct *vma,
1011
-
vma_flags_t flags)
1006
+
static __always_inline bool vma_test(const struct vm_area_struct *vma,
1007
+
vma_flag_t bit)
1008
+
{
1009
+
return vma_flags_test(&vma->flags, bit);
1010
+
}
1011
+
1012
+
static __always_inline bool vma_test_any_mask(const struct vm_area_struct *vma,
1013
+
vma_flags_t flags)
1014
+
{
1015
+
return vma_flags_test_any_mask(&vma->flags, flags);
1016
+
}
1017
+
1018
+
#define vma_test_any(vma, ...) \
1019
+
vma_test_any_mask(vma, mk_vma_flags(__VA_ARGS__))
1020
+
1021
+
static __always_inline bool vma_test_all_mask(const struct vm_area_struct *vma,
1022
+
vma_flags_t flags)
1012
1023
{
1013
1024
return vma_flags_test_all_mask(&vma->flags, flags);
1014
1025
}
···
1031
1012
#define vma_test_all(vma, ...) \
1032
1013
vma_test_all_mask(vma, mk_vma_flags(__VA_ARGS__))
1033
1014
1034
-
static inline bool is_shared_maywrite_vm_flags(vm_flags_t vm_flags)
1035
-
{
1036
-
return (vm_flags & (VM_SHARED | VM_MAYWRITE)) ==
1037
-
(VM_SHARED | VM_MAYWRITE);
1038
-
}
1039
-
1040
-
static inline void vma_set_flags_mask(struct vm_area_struct *vma,
1041
-
vma_flags_t flags)
1015
+
static __always_inline void vma_set_flags_mask(struct vm_area_struct *vma,
1016
+
vma_flags_t flags)
1042
1017
{
1043
1018
vma_flags_set_mask(&vma->flags, flags);
1044
1019
}
···
1046
1033
return vma_flags_test(&desc->vma_flags, bit);
1047
1034
}
1048
1035
1049
-
static inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc,
1050
-
vma_flags_t flags)
1036
+
static __always_inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc,
1037
+
vma_flags_t flags)
1051
1038
{
1052
1039
return vma_flags_test_any_mask(&desc->vma_flags, flags);
1053
1040
}
···
1055
1042
#define vma_desc_test_any(desc, ...) \
1056
1043
vma_desc_test_any_mask(desc, mk_vma_flags(__VA_ARGS__))
1057
1044
1058
-
static inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc,
1045
+
static __always_inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc,
1059
1046
vma_flags_t flags)
1060
1047
{
1061
1048
return vma_flags_test_all_mask(&desc->vma_flags, flags);
···
1064
1051
#define vma_desc_test_all(desc, ...) \
1065
1052
vma_desc_test_all_mask(desc, mk_vma_flags(__VA_ARGS__))
1066
1053
1067
-
static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
1068
-
vma_flags_t flags)
1054
+
static __always_inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
1055
+
vma_flags_t flags)
1069
1056
{
1070
1057
vma_flags_set_mask(&desc->vma_flags, flags);
1071
1058
}
···
1073
1060
#define vma_desc_set_flags(desc, ...) \
1074
1061
vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1075
1062
1076
-
static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
1077
-
vma_flags_t flags)
1063
+
static __always_inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
1064
+
vma_flags_t flags)
1078
1065
{
1079
1066
vma_flags_clear_mask(&desc->vma_flags, flags);
1080
1067
}
1081
1068
1082
1069
#define vma_desc_clear_flags(desc, ...) \
1083
1070
vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1071
+
1072
+
static inline bool is_shared_maywrite_vm_flags(vm_flags_t vm_flags)
1073
+
{
1074
+
return (vm_flags & (VM_SHARED | VM_MAYWRITE)) ==
1075
+
(VM_SHARED | VM_MAYWRITE);
1076
+
}
1084
1077
1085
1078
static inline bool is_shared_maywrite(const vma_flags_t *flags)
1086
1079
{