Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm: add basic VMA flag operation helper functions
Now we have the mk_vma_flags() macro helper which permits easy
specification of any number of VMA flags, add helper functions which
operate with vma_flags_t parameters.
This patch provides vma_flags_test[_mask](), vma_flags_set[_mask]() and
vma_flags_clear[_mask]() respectively testing, setting and clearing flags
with the _mask variants accepting vma_flag_t parameters, and the non-mask
variants implemented as macros which accept a list of flags.
This allows us to trivially test/set/clear aggregate VMA flag values as
necessary, for instance:
if (vma_flags_test(&flags, VMA_READ_BIT, VMA_WRITE_BIT))
goto readwrite;
vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT);
vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT);
We also add a function for testing that ALL flags are set for convenience,
e.g.:
if (vma_flags_test_all(&flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) {
/* Both READ and MAYREAD flags set */
...
}
The compiler generates optimal assembly for each such that they behave as
if the caller were setting the bitmap flags manually.
This is important for e.g. drivers which manipulate flag values rather
than a VMA's specific flag values.
We also add helpers for testing, setting and clearing flags for VMA's and
VMA descriptors to reduce boilerplate.
Also add the EMPTY_VMA_FLAGS define to aid initialisation of empty flags.
Finally, update the userland VMA tests to add the helpers there so they
can be utilised as part of userland testing.
Link: https://lkml.kernel.org/r/885d4897d67a6a57c0b07fa182a7055ad752df11.1769097829.git.lorenzo.stoakes@oracle.com
Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: Pedro Falcato <pfalcato@suse.de>
Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Zi Yan <ziy@nvidia.com>
Cc: Damien Le Moal <dlemoal@kernel.org>
Cc: "Darrick J. Wong" <djwong@kernel.org>
Cc: Jarkko Sakkinen <jarkko@kernel.org>
Cc: Yury Norov <ynorov@nvidia.com>
Cc: Chris Mason <clm@fb.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Lorenzo Stoakes
and committed by
Andrew Morton
bae0ba7c
21c8a5ba
+295
-21
+165
include/linux/mm.h
+165
include/linux/mm.h
···
1059
1059
#define mk_vma_flags(...) __mk_vma_flags(COUNT_ARGS(__VA_ARGS__), \
1060
1060
(const vma_flag_t []){__VA_ARGS__})
1061
1061
1062
+
/* Test each of to_test flags in flags, non-atomically. */
1063
+
static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags,
1064
+
vma_flags_t to_test)
1065
+
{
1066
+
const unsigned long *bitmap = flags->__vma_flags;
1067
+
const unsigned long *bitmap_to_test = to_test.__vma_flags;
1068
+
1069
+
return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
1070
+
}
1071
+
1072
+
/*
1073
+
* Test whether any specified VMA flag is set, e.g.:
1074
+
*
1075
+
* if (vma_flags_test(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
1076
+
*/
1077
+
#define vma_flags_test(flags, ...) \
1078
+
vma_flags_test_mask(flags, mk_vma_flags(__VA_ARGS__))
1079
+
1080
+
/* Test that ALL of the to_test flags are set, non-atomically. */
1081
+
static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags,
1082
+
vma_flags_t to_test)
1083
+
{
1084
+
const unsigned long *bitmap = flags->__vma_flags;
1085
+
const unsigned long *bitmap_to_test = to_test.__vma_flags;
1086
+
1087
+
return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
1088
+
}
1089
+
1090
+
/*
1091
+
* Test whether ALL specified VMA flags are set, e.g.:
1092
+
*
1093
+
* if (vma_flags_test_all(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
1094
+
*/
1095
+
#define vma_flags_test_all(flags, ...) \
1096
+
vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__))
1097
+
1098
+
/* Set each of the to_set flags in flags, non-atomically. */
1099
+
static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set)
1100
+
{
1101
+
unsigned long *bitmap = flags->__vma_flags;
1102
+
const unsigned long *bitmap_to_set = to_set.__vma_flags;
1103
+
1104
+
bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS);
1105
+
}
1106
+
1107
+
/*
1108
+
* Set all specified VMA flags, e.g.:
1109
+
*
1110
+
* vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT);
1111
+
*/
1112
+
#define vma_flags_set(flags, ...) \
1113
+
vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__))
1114
+
1115
+
/* Clear all of the to-clear flags in flags, non-atomically. */
1116
+
static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear)
1117
+
{
1118
+
unsigned long *bitmap = flags->__vma_flags;
1119
+
const unsigned long *bitmap_to_clear = to_clear.__vma_flags;
1120
+
1121
+
bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS);
1122
+
}
1123
+
1124
+
/*
1125
+
* Clear all specified individual flags, e.g.:
1126
+
*
1127
+
* vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT);
1128
+
*/
1129
+
#define vma_flags_clear(flags, ...) \
1130
+
vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__))
1131
+
1132
+
/*
1133
+
* Helper to test that ALL specified flags are set in a VMA.
1134
+
*
1135
+
* Note: appropriate locks must be held, this function does not acquire them for
1136
+
* you.
1137
+
*/
1138
+
static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma,
1139
+
vma_flags_t flags)
1140
+
{
1141
+
return vma_flags_test_all_mask(&vma->flags, flags);
1142
+
}
1143
+
1144
+
/*
1145
+
* Helper macro for checking that ALL specified flags are set in a VMA, e.g.:
1146
+
*
1147
+
* if (vma_test_all_flags(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... }
1148
+
*/
1149
+
#define vma_test_all_flags(vma, ...) \
1150
+
vma_test_all_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
1151
+
1152
+
/*
1153
+
* Helper to set all VMA flags in a VMA.
1154
+
*
1155
+
* Note: appropriate locks must be held, this function does not acquire them for
1156
+
* you.
1157
+
*/
1158
+
static inline void vma_set_flags_mask(struct vm_area_struct *vma,
1159
+
vma_flags_t flags)
1160
+
{
1161
+
vma_flags_set_mask(&vma->flags, flags);
1162
+
}
1163
+
1164
+
/*
1165
+
* Helper macro for specifying VMA flags in a VMA, e.g.:
1166
+
*
1167
+
* vma_set_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1168
+
* VMA_DONTDUMP_BIT);
1169
+
*
1170
+
* Note: appropriate locks must be held, this function does not acquire them for
1171
+
* you.
1172
+
*/
1173
+
#define vma_set_flags(vma, ...) \
1174
+
vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
1175
+
1176
+
/* Helper to test all VMA flags in a VMA descriptor. */
1177
+
static inline bool vma_desc_test_flags_mask(const struct vm_area_desc *desc,
1178
+
vma_flags_t flags)
1179
+
{
1180
+
return vma_flags_test_mask(&desc->vma_flags, flags);
1181
+
}
1182
+
1183
+
/*
1184
+
* Helper macro for testing VMA flags for an input pointer to a struct
1185
+
* vm_area_desc object describing a proposed VMA, e.g.:
1186
+
*
1187
+
* if (vma_desc_test_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT,
1188
+
* VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... }
1189
+
*/
1190
+
#define vma_desc_test_flags(desc, ...) \
1191
+
vma_desc_test_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1192
+
1193
+
/* Helper to set all VMA flags in a VMA descriptor. */
1194
+
static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
1195
+
vma_flags_t flags)
1196
+
{
1197
+
vma_flags_set_mask(&desc->vma_flags, flags);
1198
+
}
1199
+
1200
+
/*
1201
+
* Helper macro for specifying VMA flags for an input pointer to a struct
1202
+
* vm_area_desc object describing a proposed VMA, e.g.:
1203
+
*
1204
+
* vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1205
+
* VMA_DONTDUMP_BIT);
1206
+
*/
1207
+
#define vma_desc_set_flags(desc, ...) \
1208
+
vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1209
+
1210
+
/* Helper to clear all VMA flags in a VMA descriptor. */
1211
+
static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
1212
+
vma_flags_t flags)
1213
+
{
1214
+
vma_flags_clear_mask(&desc->vma_flags, flags);
1215
+
}
1216
+
1217
+
/*
1218
+
* Helper macro for clearing VMA flags for an input pointer to a struct
1219
+
* vm_area_desc object describing a proposed VMA, e.g.:
1220
+
*
1221
+
* vma_desc_clear_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1222
+
* VMA_DONTDUMP_BIT);
1223
+
*/
1224
+
#define vma_desc_clear_flags(desc, ...) \
1225
+
vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1226
+
1062
1227
static inline void vma_set_anonymous(struct vm_area_struct *vma)
1063
1228
{
1064
1229
vma->vm_ops = NULL;
+3
-1
include/linux/mm_types.h
+3
-1
include/linux/mm_types.h
···
844
844
845
845
/*
846
846
* If specified, this hook is invoked when an error occurred when
847
-
* attempting the selection action.
847
+
* attempting the selected action.
848
848
*
849
849
* The hook can return an error code in order to filter the error, but
850
850
* it is not valid to clear the error here.
···
867
867
typedef struct {
868
868
DECLARE_BITMAP(__vma_flags, NUM_VMA_FLAG_BITS);
869
869
} vma_flags_t;
870
+
871
+
#define EMPTY_VMA_FLAGS ((vma_flags_t){ })
870
872
871
873
/*
872
874
* Describes a VMA that is about to be mmap()'ed. Drivers may choose to
+127
-20
tools/testing/vma/vma_internal.h
+127
-20
tools/testing/vma/vma_internal.h
···
21
21
22
22
#include <stdlib.h>
23
23
24
+
#ifdef __CONCAT
25
+
#undef __CONCAT
26
+
#endif
27
+
28
+
#include <linux/args.h>
24
29
#include <linux/atomic.h>
30
+
#include <linux/bitmap.h>
25
31
#include <linux/list.h>
26
32
#include <linux/maple_tree.h>
27
33
#include <linux/mm.h>
···
43
37
#define mmap_min_addr 0UL
44
38
#define dac_mmap_min_addr 0UL
45
39
#endif
40
+
41
+
#define ACCESS_PRIVATE(p, member) ((p)->member)
46
42
47
43
#define VM_WARN_ON(_expr) (WARN_ON(_expr))
48
44
#define VM_WARN_ON_ONCE(_expr) (WARN_ON_ONCE(_expr))
···
541
533
DECLARE_BITMAP(__vma_flags, NUM_VMA_FLAG_BITS);
542
534
} __private vma_flags_t;
543
535
536
+
#define EMPTY_VMA_FLAGS ((vma_flags_t){ })
537
+
544
538
struct mm_struct {
545
539
struct maple_tree mm_mt;
546
540
int map_count; /* number of VMAs */
···
891
881
{
892
882
return __pgprot(vm_flags);
893
883
}
884
+
885
+
static inline void vma_flags_clear_all(vma_flags_t *flags)
886
+
{
887
+
bitmap_zero(flags->__vma_flags, NUM_VMA_FLAG_BITS);
888
+
}
889
+
890
+
static inline void vma_flag_set(vma_flags_t *flags, vma_flag_t bit)
891
+
{
892
+
unsigned long *bitmap = flags->__vma_flags;
893
+
894
+
__set_bit((__force int)bit, bitmap);
895
+
}
896
+
897
+
static inline vma_flags_t __mk_vma_flags(size_t count, const vma_flag_t *bits)
898
+
{
899
+
vma_flags_t flags;
900
+
int i;
901
+
902
+
vma_flags_clear_all(&flags);
903
+
for (i = 0; i < count; i++)
904
+
vma_flag_set(&flags, bits[i]);
905
+
return flags;
906
+
}
907
+
908
+
#define mk_vma_flags(...) __mk_vma_flags(COUNT_ARGS(__VA_ARGS__), \
909
+
(const vma_flag_t []){__VA_ARGS__})
910
+
911
+
static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags,
912
+
vma_flags_t to_test)
913
+
{
914
+
const unsigned long *bitmap = flags->__vma_flags;
915
+
const unsigned long *bitmap_to_test = to_test.__vma_flags;
916
+
917
+
return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
918
+
}
919
+
920
+
#define vma_flags_test(flags, ...) \
921
+
vma_flags_test_mask(flags, mk_vma_flags(__VA_ARGS__))
922
+
923
+
static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags,
924
+
vma_flags_t to_test)
925
+
{
926
+
const unsigned long *bitmap = flags->__vma_flags;
927
+
const unsigned long *bitmap_to_test = to_test.__vma_flags;
928
+
929
+
return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
930
+
}
931
+
932
+
#define vma_flags_test_all(flags, ...) \
933
+
vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__))
934
+
935
+
static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set)
936
+
{
937
+
unsigned long *bitmap = flags->__vma_flags;
938
+
const unsigned long *bitmap_to_set = to_set.__vma_flags;
939
+
940
+
bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS);
941
+
}
942
+
943
+
#define vma_flags_set(flags, ...) \
944
+
vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__))
945
+
946
+
static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear)
947
+
{
948
+
unsigned long *bitmap = flags->__vma_flags;
949
+
const unsigned long *bitmap_to_clear = to_clear.__vma_flags;
950
+
951
+
bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS);
952
+
}
953
+
954
+
#define vma_flags_clear(flags, ...) \
955
+
vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__))
956
+
957
+
static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma,
958
+
vma_flags_t flags)
959
+
{
960
+
return vma_flags_test_all_mask(&vma->flags, flags);
961
+
}
962
+
963
+
#define vma_test_all_flags(vma, ...) \
964
+
vma_test_all_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
965
+
966
+
static inline void vma_set_flags_mask(struct vm_area_struct *vma,
967
+
vma_flags_t flags)
968
+
{
969
+
vma_flags_set_mask(&vma->flags, flags);
970
+
}
971
+
972
+
#define vma_set_flags(vma, ...) \
973
+
vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
974
+
975
+
static inline bool vma_desc_test_flags_mask(const struct vm_area_desc *desc,
976
+
vma_flags_t flags)
977
+
{
978
+
return vma_flags_test_mask(&desc->vma_flags, flags);
979
+
}
980
+
981
+
#define vma_desc_test_flags(desc, ...) \
982
+
vma_desc_test_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
983
+
984
+
static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
985
+
vma_flags_t flags)
986
+
{
987
+
vma_flags_set_mask(&desc->vma_flags, flags);
988
+
}
989
+
990
+
#define vma_desc_set_flags(desc, ...) \
991
+
vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
992
+
993
+
static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
994
+
vma_flags_t flags)
995
+
{
996
+
vma_flags_clear_mask(&desc->vma_flags, flags);
997
+
}
998
+
999
+
#define vma_desc_clear_flags(desc, ...) \
1000
+
vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
894
1001
895
1002
static inline bool is_shared_maywrite(vm_flags_t vm_flags)
896
1003
{
···
1667
1540
{
1668
1541
}
1669
1542
1670
-
#define ACCESS_PRIVATE(p, member) ((p)->member)
1671
-
1672
-
#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)
1673
-
1674
-
static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
1675
-
{
1676
-
unsigned int len = bitmap_size(nbits);
1677
-
1678
-
if (small_const_nbits(nbits))
1679
-
*dst = 0;
1680
-
else
1681
-
memset(dst, 0, len);
1682
-
}
1683
-
1684
1543
static inline bool mm_flags_test(int flag, const struct mm_struct *mm)
1685
1544
{
1686
1545
return test_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));
1687
-
}
1688
-
1689
-
/* Clears all bits in the VMA flags bitmap, non-atomically. */
1690
-
static inline void vma_flags_clear_all(vma_flags_t *flags)
1691
-
{
1692
-
bitmap_zero(ACCESS_PRIVATE(flags, __vma_flags), NUM_VMA_FLAG_BITS);
1693
1546
}
1694
1547
1695
1548
/*