mm/vma: use new VMA flags for sticky flags logic

Use the new vma_flags_t flags implementation to perform the logic around
sticky flags and what flags are ignored on VMA merge.

We make use of the new vma_flags_empty(), vma_flags_diff_pair(), and
vma_flags_and_mask() functionality.

Also update the VMA tests accordingly.

Link: https://lkml.kernel.org/r/369574f06360ffa44707047e3b58eb4897345fba.1774034900.git.ljs@kernel.org
Signed-off-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Acked-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: "Borislav Petkov (AMD)" <bp@alien8.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christian Brauner <brauner@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: Kees Cook <kees@kernel.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Ondrej Mosnacek <omosnace@redhat.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Moore <paul@paul-moore.com>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: Richard Weinberger <richard@nod.at>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stephen Smalley <stephen.smalley.work@gmail.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vineet Gupta <vgupta@kernel.org>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Will Deacon <will@kernel.org>
Cc: xu xin <xu.xin16@zte.com.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Lorenzo Stoakes (Oracle) and committed by

Andrew Morton 2 months ago 7ec1885a bd44d91d

+62 -32

4 changed files

expand all

include

linux

mm.h

vma.c

tools

testing

vma

include

custom.h

dup.h

+19 -13

include/linux/mm.h

··· 540 540 541 541 /* VMA basic access permission flags */ 542 542 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 543 + #define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT) 543 544 544 545 /* 545 546 * Special vmas that are non-mergable, non-mlock()able. ··· 586 585 * possesses it but the other does not, the merged VMA should nonetheless have 587 586 * applied to it: 588 587 * 589 - * VM_SOFTDIRTY - if a VMA is marked soft-dirty, that is has not had its 590 - * references cleared via /proc/$pid/clear_refs, any merged VMA 591 - * should be considered soft-dirty also as it operates at a VMA 592 - * granularity. 588 + * VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its 589 + * references cleared via /proc/$pid/clear_refs, any 590 + * merged VMA should be considered soft-dirty also as it 591 + * operates at a VMA granularity. 593 592 * 594 - * VM_MAYBE_GUARD - If a VMA may have guard regions in place it implies that 595 - * mapped page tables may contain metadata not described by the 596 - * VMA and thus any merged VMA may also contain this metadata, 597 - * and thus we must make this flag sticky. 593 + * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies 594 + * that mapped page tables may contain metadata not 595 + * described by the VMA and thus any merged VMA may also 596 + * contain this metadata, and thus we must make this flag 597 + * sticky. 598 598 */ 599 - #define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD) 599 + #ifdef CONFIG_MEM_SOFT_DIRTY 600 + #define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT) 601 + #else 602 + #define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT) 603 + #endif 600 604 601 605 /* 602 606 * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one 603 607 * of these flags and the other not does not preclude a merge. 604 608 * 605 - * VM_STICKY - When merging VMAs, VMA flags must match, unless they are 606 - * 'sticky'. If any sticky flags exist in either VMA, we simply 607 - * set all of them on the merged VMA. 609 + * VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they 610 + * are 'sticky'. If any sticky flags exist in either VMA, 611 + * we simply set all of them on the merged VMA. 608 612 */ 609 - #define VM_IGNORE_MERGE VM_STICKY 613 + #define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS 610 614 611 615 /* 612 616 * Flags which should result in page tables being copied on fork. These are

+36 -12

mm/vma.c

··· 86 86 static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next) 87 87 { 88 88 struct vm_area_struct *vma = merge_next ? vmg->next : vmg->prev; 89 + vma_flags_t diff; 89 90 90 91 if (!mpol_equal(vmg->policy, vma_policy(vma))) 91 92 return false; 92 - if ((vma->vm_flags ^ vmg->vm_flags) & ~VM_IGNORE_MERGE) 93 + 94 + diff = vma_flags_diff_pair(&vma->flags, &vmg->vma_flags); 95 + vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS); 96 + 97 + if (!vma_flags_empty(&diff)) 93 98 return false; 94 99 if (vma->vm_file != vmg->file) 95 100 return false; ··· 810 805 static __must_check struct vm_area_struct *vma_merge_existing_range( 811 806 struct vma_merge_struct *vmg) 812 807 { 813 - vm_flags_t sticky_flags = vmg->vm_flags & VM_STICKY; 808 + vma_flags_t sticky_flags = vma_flags_and_mask(&vmg->vma_flags, 809 + VMA_STICKY_FLAGS); 814 810 struct vm_area_struct *middle = vmg->middle; 815 811 struct vm_area_struct *prev = vmg->prev; 816 812 struct vm_area_struct *next; ··· 904 898 vma_start_write(middle); 905 899 906 900 if (merge_right) { 901 + vma_flags_t next_sticky; 902 + 907 903 vma_start_write(next); 908 904 vmg->target = next; 909 - sticky_flags |= (next->vm_flags & VM_STICKY); 905 + next_sticky = vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS); 906 + vma_flags_set_mask(&sticky_flags, next_sticky); 910 907 } 911 908 912 909 if (merge_left) { 910 + vma_flags_t prev_sticky; 911 + 913 912 vma_start_write(prev); 914 913 vmg->target = prev; 915 - sticky_flags |= (prev->vm_flags & VM_STICKY); 914 + 915 + prev_sticky = vma_flags_and_mask(&prev->flags, VMA_STICKY_FLAGS); 916 + vma_flags_set_mask(&sticky_flags, prev_sticky); 916 917 } 917 918 918 919 if (merge_both) { ··· 989 976 if (err || commit_merge(vmg)) 990 977 goto abort; 991 978 992 - vm_flags_set(vmg->target, sticky_flags); 979 + vma_set_flags_mask(vmg->target, sticky_flags); 993 980 khugepaged_enter_vma(vmg->target, vmg->vm_flags); 994 981 vmg->state = VMA_MERGE_SUCCESS; 995 982 return vmg->target; ··· 1167 1154 struct vm_area_struct *target = vmg->target; 1168 1155 struct vm_area_struct *next = vmg->next; 1169 1156 bool remove_next = false; 1170 - vm_flags_t sticky_flags; 1157 + vma_flags_t sticky_flags = 1158 + vma_flags_and_mask(&vmg->vma_flags, VMA_STICKY_FLAGS); 1159 + vma_flags_t target_sticky; 1171 1160 int ret = 0; 1172 1161 1173 1162 mmap_assert_write_locked(vmg->mm); 1174 1163 vma_start_write(target); 1164 + 1165 + target_sticky = vma_flags_and_mask(&target->flags, VMA_STICKY_FLAGS); 1175 1166 1176 1167 if (next && target != next && vmg->end == next->vm_end) 1177 1168 remove_next = true; ··· 1191 1174 VM_WARN_ON_VMG(target->vm_start < vmg->start || 1192 1175 target->vm_end > vmg->end, vmg); 1193 1176 1194 - sticky_flags = vmg->vm_flags & VM_STICKY; 1195 - sticky_flags |= target->vm_flags & VM_STICKY; 1196 - if (remove_next) 1197 - sticky_flags |= next->vm_flags & VM_STICKY; 1177 + vma_flags_set_mask(&sticky_flags, target_sticky); 1198 1178 1199 1179 /* 1200 1180 * If we are removing the next VMA or copying from a VMA ··· 1208 1194 return ret; 1209 1195 1210 1196 if (remove_next) { 1197 + vma_flags_t next_sticky; 1198 + 1211 1199 vma_start_write(next); 1212 1200 vmg->__remove_next = true; 1201 + 1202 + next_sticky = vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS); 1203 + vma_flags_set_mask(&sticky_flags, next_sticky); 1213 1204 } 1214 1205 if (commit_merge(vmg)) 1215 1206 goto nomem; 1216 1207 1217 - vm_flags_set(target, sticky_flags); 1208 + vma_set_flags_mask(target, sticky_flags); 1218 1209 return 0; 1219 1210 1220 1211 nomem: ··· 1969 1950 */ 1970 1951 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) 1971 1952 { 1953 + vma_flags_t diff = vma_flags_diff_pair(&a->flags, &b->flags); 1954 + 1955 + vma_flags_clear_mask(&diff, VMA_ACCESS_FLAGS); 1956 + vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS); 1957 + 1972 1958 return a->vm_end == b->vm_start && 1973 1959 mpol_equal(vma_policy(a), vma_policy(b)) && 1974 1960 a->vm_file == b->vm_file && 1975 - !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_IGNORE_MERGE)) && 1961 + vma_flags_empty(&diff) && 1976 1962 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); 1977 1963 } 1978 1964

-5

tools/testing/vma/include/custom.h

··· 134 134 vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__)) 135 135 #define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \ 136 136 VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT) 137 - #ifdef CONFIG_MEM_SOFT_DIRTY 138 - #define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT) 139 - #else 140 - #define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT) 141 - #endif

+7 -2

tools/testing/vma/include/dup.h

··· 338 338 339 339 /* VMA basic access permission flags */ 340 340 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 341 + #define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT) 341 342 342 343 /* 343 344 * Special vmas that are non-mergable, non-mlock()able. ··· 364 363 365 364 #define CAP_IPC_LOCK 14 366 365 367 - #define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD) 366 + #ifdef CONFIG_MEM_SOFT_DIRTY 367 + #define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT) 368 + #else 369 + #define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT) 370 + #endif 368 371 369 - #define VM_IGNORE_MERGE VM_STICKY 372 + #define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS 370 373 371 374 #define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD) 372 375

Configure Feed

Configure Feed