Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm/vma: convert as much as we can in mm/vma.c to vma_flags_t
Now we have established a good foundation for vm_flags_t to vma_flags_t
changes, update mm/vma.c to utilise vma_flags_t wherever possible.
We are able to convert VM_STARTGAP_FLAGS entirely as this is only used in
mm/vma.c, and to account for the fact we can't use VM_NONE to make life
easier, place the definition of this within existing #ifdef's to be
cleaner.
Generally the remaining changes are mechanical.
Also update the VMA tests to reflect the changes.
Link: https://lkml.kernel.org/r/5fdeaf8af9a12c2a5d68497495f52fa627d05a5b.1774034900.git.ljs@kernel.org
Signed-off-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Acked-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: "Borislav Petkov (AMD)" <bp@alien8.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christian Brauner <brauner@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: Kees Cook <kees@kernel.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Ondrej Mosnacek <omosnace@redhat.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Moore <paul@paul-moore.com>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: Richard Weinberger <richard@nod.at>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stephen Smalley <stephen.smalley.work@gmail.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vineet Gupta <vgupta@kernel.org>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Will Deacon <will@kernel.org>
Cc: xu xin <xu.xin16@zte.com.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Lorenzo Stoakes (Oracle)
and committed by
Andrew Morton
769669bd
a6f14fb5
+59
-42
+4
-2
include/linux/mm.h
+4
-2
include/linux/mm.h
···
463
463
#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \
464
464
defined(CONFIG_RISCV_USER_CFI)
465
465
#define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK)
466
+
#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT)
466
467
#else
467
468
#define VM_SHADOW_STACK VM_NONE
469
+
#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT)
468
470
#endif
469
471
#if defined(CONFIG_PPC64)
470
472
#define VM_SAO INIT_VM_FLAG(SAO)
···
541
539
/* Temporary until VMA flags conversion complete. */
542
540
#define VM_STACK_FLAGS vma_flags_to_legacy(VMA_STACK_FLAGS)
543
541
544
-
#define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK)
545
-
546
542
#ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS
547
543
#define VM_SEALED_SYSMAP VM_SEALED
548
544
#else
···
583
583
584
584
/* This mask represents all the VMA flag bits used by mlock */
585
585
#define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT)
586
+
587
+
#define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT)
586
588
587
589
/* These flags can be updated atomically via VMA/mmap read lock. */
588
590
#define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD
+50
-39
mm/vma.c
+50
-39
mm/vma.c
···
185
185
}
186
186
187
187
/*
188
-
* Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
188
+
* Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff)
189
189
* in front of (at a lower virtual address and file offset than) the vma.
190
190
*
191
191
* We cannot merge two vmas if they have differently assigned (non-NULL)
···
211
211
}
212
212
213
213
/*
214
-
* Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
214
+
* Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff)
215
215
* beyond (at a higher virtual address and file offset than) the vma.
216
216
*
217
217
* We cannot merge two vmas if they have differently assigned (non-NULL)
···
850
850
* furthermost left or right side of the VMA, then we have no chance of
851
851
* merging and should abort.
852
852
*/
853
-
if (vmg->vm_flags & VM_SPECIAL || (!left_side && !right_side))
853
+
if (vma_flags_test_any_mask(&vmg->vma_flags, VMA_SPECIAL_FLAGS) ||
854
+
(!left_side && !right_side))
854
855
return NULL;
855
856
856
857
if (left_side)
···
1073
1072
vmg->state = VMA_MERGE_NOMERGE;
1074
1073
1075
1074
/* Special VMAs are unmergeable, also if no prev/next. */
1076
-
if ((vmg->vm_flags & VM_SPECIAL) || (!prev && !next))
1075
+
if (vma_flags_test_any_mask(&vmg->vma_flags, VMA_SPECIAL_FLAGS) ||
1076
+
(!prev && !next))
1077
1077
return NULL;
1078
1078
1079
1079
can_merge_left = can_vma_merge_left(vmg);
···
1461
1459
nrpages = vma_pages(next);
1462
1460
1463
1461
vms->nr_pages += nrpages;
1464
-
if (next->vm_flags & VM_LOCKED)
1462
+
if (vma_test(next, VMA_LOCKED_BIT))
1465
1463
vms->locked_vm += nrpages;
1466
1464
1467
-
if (next->vm_flags & VM_ACCOUNT)
1465
+
if (vma_test(next, VMA_ACCOUNT_BIT))
1468
1466
vms->nr_accounted += nrpages;
1469
1467
1470
1468
if (is_exec_mapping(next->vm_flags))
1471
1469
vms->exec_vm += nrpages;
1472
1470
else if (is_stack_mapping(next->vm_flags))
1473
1471
vms->stack_vm += nrpages;
1474
-
else if (is_data_mapping(next->vm_flags))
1472
+
else if (is_data_mapping_vma_flags(&next->flags))
1475
1473
vms->data_vm += nrpages;
1476
1474
1477
1475
if (vms->uf) {
···
2067
2065
2068
2066
static bool vma_is_shared_writable(struct vm_area_struct *vma)
2069
2067
{
2070
-
return (vma->vm_flags & (VM_WRITE | VM_SHARED)) ==
2071
-
(VM_WRITE | VM_SHARED);
2068
+
return vma_test_all(vma, VMA_WRITE_BIT, VMA_SHARED_BIT);
2072
2069
}
2073
2070
2074
2071
static bool vma_fs_can_writeback(struct vm_area_struct *vma)
2075
2072
{
2076
2073
/* No managed pages to writeback. */
2077
-
if (vma->vm_flags & VM_PFNMAP)
2074
+
if (vma_test(vma, VMA_PFNMAP_BIT))
2078
2075
return false;
2079
2076
2080
2077
return vma->vm_file && vma->vm_file->f_mapping &&
···
2339
2338
* We account for memory if it's a private writeable mapping,
2340
2339
* not hugepages and VM_NORESERVE wasn't set.
2341
2340
*/
2342
-
static bool accountable_mapping(struct file *file, vm_flags_t vm_flags)
2341
+
static bool accountable_mapping(struct mmap_state *map)
2343
2342
{
2343
+
const struct file *file = map->file;
2344
+
vma_flags_t mask;
2345
+
2344
2346
/*
2345
2347
* hugetlb has its own accounting separate from the core VM
2346
2348
* VM_HUGETLB may not be set yet so we cannot check for that flag.
···
2351
2347
if (file && is_file_hugepages(file))
2352
2348
return false;
2353
2349
2354
-
return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
2350
+
mask = vma_flags_and(&map->vma_flags, VMA_NORESERVE_BIT, VMA_SHARED_BIT,
2351
+
VMA_WRITE_BIT);
2352
+
return vma_flags_same(&mask, VMA_WRITE_BIT);
2355
2353
}
2356
2354
2357
2355
/*
···
2456
2450
return -ENOMEM;
2457
2451
2458
2452
/* Private writable mapping: check memory availability. */
2459
-
if (accountable_mapping(map->file, map->vm_flags)) {
2453
+
if (accountable_mapping(map)) {
2460
2454
map->charged = map->pglen;
2461
2455
map->charged -= vms->nr_accounted;
2462
2456
if (map->charged) {
···
2466
2460
}
2467
2461
2468
2462
vms->nr_accounted = 0;
2469
-
map->vm_flags |= VM_ACCOUNT;
2463
+
vma_flags_set(&map->vma_flags, VMA_ACCOUNT_BIT);
2470
2464
}
2471
2465
2472
2466
/*
···
2514
2508
* Drivers should not permit writability when previously it was
2515
2509
* disallowed.
2516
2510
*/
2517
-
VM_WARN_ON_ONCE(map->vm_flags != vma->vm_flags &&
2518
-
!(map->vm_flags & VM_MAYWRITE) &&
2519
-
(vma->vm_flags & VM_MAYWRITE));
2511
+
VM_WARN_ON_ONCE(!vma_flags_same_pair(&map->vma_flags, &vma->flags) &&
2512
+
!vma_flags_test(&map->vma_flags, VMA_MAYWRITE_BIT) &&
2513
+
vma_test(vma, VMA_MAYWRITE_BIT));
2520
2514
2521
2515
map->file = vma->vm_file;
2522
-
map->vm_flags = vma->vm_flags;
2516
+
map->vma_flags = vma->flags;
2523
2517
2524
2518
return 0;
2525
2519
}
···
2550
2544
2551
2545
vma_iter_config(vmi, map->addr, map->end);
2552
2546
vma_set_range(vma, map->addr, map->end, map->pgoff);
2553
-
vm_flags_init(vma, map->vm_flags);
2547
+
vma->flags = map->vma_flags;
2554
2548
vma->vm_page_prot = map->page_prot;
2555
2549
2556
2550
if (vma_iter_prealloc(vmi, vma)) {
···
2560
2554
2561
2555
if (map->file)
2562
2556
error = __mmap_new_file_vma(map, vma);
2563
-
else if (map->vm_flags & VM_SHARED)
2557
+
else if (vma_flags_test(&map->vma_flags, VMA_SHARED_BIT))
2564
2558
error = shmem_zero_setup(vma);
2565
2559
else
2566
2560
vma_set_anonymous(vma);
···
2570
2564
2571
2565
if (!map->check_ksm_early) {
2572
2566
update_ksm_flags(map);
2573
-
vm_flags_init(vma, map->vm_flags);
2567
+
vma->flags = map->vma_flags;
2574
2568
}
2575
2569
2576
2570
#ifdef CONFIG_SPARC64
···
2610
2604
static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma)
2611
2605
{
2612
2606
struct mm_struct *mm = map->mm;
2613
-
vm_flags_t vm_flags = vma->vm_flags;
2614
2607
2615
2608
perf_event_mmap(vma);
2616
2609
···
2617
2612
vms_complete_munmap_vmas(&map->vms, &map->mas_detach);
2618
2613
2619
2614
vm_stat_account(mm, vma->vm_flags, map->pglen);
2620
-
if (vm_flags & VM_LOCKED) {
2615
+
if (vma_test(vma, VMA_LOCKED_BIT)) {
2621
2616
if (!vma_supports_mlock(vma))
2622
-
vm_flags_clear(vma, VM_LOCKED_MASK);
2617
+
vma_clear_flags_mask(vma, VMA_LOCKED_MASK);
2623
2618
else
2624
2619
mm->locked_vm += map->pglen;
2625
2620
}
···
2635
2630
* a completely new data area).
2636
2631
*/
2637
2632
if (pgtable_supports_soft_dirty())
2638
-
vm_flags_set(vma, VM_SOFTDIRTY);
2633
+
vma_set_flags(vma, VMA_SOFTDIRTY_BIT);
2639
2634
2640
2635
vma_set_page_prot(vma);
2641
2636
}
···
2998
2993
gap = vma_iter_addr(&vmi) + info->start_gap;
2999
2994
gap += (info->align_offset - gap) & info->align_mask;
3000
2995
tmp = vma_next(&vmi);
3001
-
if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
2996
+
/* Avoid prev check if possible */
2997
+
if (tmp && vma_test_any_mask(tmp, VMA_STARTGAP_FLAGS)) {
3002
2998
if (vm_start_gap(tmp) < gap + length - 1) {
3003
2999
low_limit = tmp->vm_end;
3004
3000
vma_iter_reset(&vmi);
···
3051
3045
gap -= (gap - info->align_offset) & info->align_mask;
3052
3046
gap_end = vma_iter_end(&vmi);
3053
3047
tmp = vma_next(&vmi);
3054
-
if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
3048
+
/* Avoid prev check if possible */
3049
+
if (tmp && vma_test_any_mask(tmp, VMA_STARTGAP_FLAGS)) {
3055
3050
if (vm_start_gap(tmp) < gap_end) {
3056
3051
high_limit = vm_start_gap(tmp);
3057
3052
vma_iter_reset(&vmi);
···
3090
3083
return -ENOMEM;
3091
3084
3092
3085
/* mlock limit tests */
3093
-
if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, grow << PAGE_SHIFT))
3086
+
if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT),
3087
+
grow << PAGE_SHIFT))
3094
3088
return -ENOMEM;
3095
3089
3096
3090
/* Check to ensure the stack will not grow into a hugetlb-only region */
3097
-
new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
3098
-
vma->vm_end - size;
3091
+
new_start = vma->vm_end - size;
3092
+
#ifdef CONFIG_STACK_GROWSUP
3093
+
if (vma_test(vma, VMA_GROWSUP_BIT))
3094
+
new_start = vma->vm_start;
3095
+
#endif
3099
3096
if (is_hugepage_only_range(vma->vm_mm, new_start, size))
3100
3097
return -EFAULT;
3101
3098
···
3113
3102
return 0;
3114
3103
}
3115
3104
3116
-
#if defined(CONFIG_STACK_GROWSUP)
3105
+
#ifdef CONFIG_STACK_GROWSUP
3117
3106
/*
3118
3107
* PA-RISC uses this for its stack.
3119
3108
* vma is the last one with address > vma->vm_end. Have to extend vma.
···
3126
3115
int error = 0;
3127
3116
VMA_ITERATOR(vmi, mm, vma->vm_start);
3128
3117
3129
-
if (!(vma->vm_flags & VM_GROWSUP))
3118
+
if (!vma_test(vma, VMA_GROWSUP_BIT))
3130
3119
return -EFAULT;
3131
3120
3132
3121
mmap_assert_write_locked(mm);
···
3146
3135
3147
3136
next = find_vma_intersection(mm, vma->vm_end, gap_addr);
3148
3137
if (next && vma_is_accessible(next)) {
3149
-
if (!(next->vm_flags & VM_GROWSUP))
3138
+
if (!vma_test(next, VMA_GROWSUP_BIT))
3150
3139
return -ENOMEM;
3151
3140
/* Check that both stack segments have the same anon_vma? */
3152
3141
}
···
3180
3169
if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
3181
3170
error = acct_stack_growth(vma, size, grow);
3182
3171
if (!error) {
3183
-
if (vma->vm_flags & VM_LOCKED)
3172
+
if (vma_test(vma, VMA_LOCKED_BIT))
3184
3173
mm->locked_vm += grow;
3185
3174
vm_stat_account(mm, vma->vm_flags, grow);
3186
3175
anon_vma_interval_tree_pre_update_vma(vma);
···
3211
3200
int error = 0;
3212
3201
VMA_ITERATOR(vmi, mm, vma->vm_start);
3213
3202
3214
-
if (!(vma->vm_flags & VM_GROWSDOWN))
3203
+
if (!vma_test(vma, VMA_GROWSDOWN_BIT))
3215
3204
return -EFAULT;
3216
3205
3217
3206
mmap_assert_write_locked(mm);
···
3224
3213
prev = vma_prev(&vmi);
3225
3214
/* Check that both stack segments have the same anon_vma? */
3226
3215
if (prev) {
3227
-
if (!(prev->vm_flags & VM_GROWSDOWN) &&
3216
+
if (!vma_test(prev, VMA_GROWSDOWN_BIT) &&
3228
3217
vma_is_accessible(prev) &&
3229
3218
(address - prev->vm_end < stack_guard_gap))
3230
3219
return -ENOMEM;
···
3259
3248
if (grow <= vma->vm_pgoff) {
3260
3249
error = acct_stack_growth(vma, size, grow);
3261
3250
if (!error) {
3262
-
if (vma->vm_flags & VM_LOCKED)
3251
+
if (vma_test(vma, VMA_LOCKED_BIT))
3263
3252
mm->locked_vm += grow;
3264
3253
vm_stat_account(mm, vma->vm_flags, grow);
3265
3254
anon_vma_interval_tree_pre_update_vma(vma);
···
3308
3297
if (find_vma_intersection(mm, vma->vm_start, vma->vm_end))
3309
3298
return -ENOMEM;
3310
3299
3311
-
if ((vma->vm_flags & VM_ACCOUNT) &&
3300
+
if (vma_test(vma, VMA_ACCOUNT_BIT) &&
3312
3301
security_vm_enough_memory_mm(mm, charged))
3313
3302
return -ENOMEM;
3314
3303
···
3330
3319
}
3331
3320
3332
3321
if (vma_link(mm, vma)) {
3333
-
if (vma->vm_flags & VM_ACCOUNT)
3322
+
if (vma_test(vma, VMA_ACCOUNT_BIT))
3334
3323
vm_unacct_memory(charged);
3335
3324
return -ENOMEM;
3336
3325
}
+4
tools/testing/vma/include/dup.h
+4
tools/testing/vma/include/dup.h
···
267
267
#endif /* CONFIG_ARCH_HAS_PKEYS */
268
268
#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS)
269
269
#define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK)
270
+
#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT)
270
271
#else
271
272
#define VM_SHADOW_STACK VM_NONE
273
+
#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT)
272
274
#endif
273
275
#if defined(CONFIG_PPC64)
274
276
#define VM_SAO INIT_VM_FLAG(SAO)
···
367
365
368
366
/* This mask represents all the VMA flag bits used by mlock */
369
367
#define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT)
368
+
369
+
#define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT)
370
370
371
371
#define RLIMIT_STACK 3 /* max stack size */
372
372
#define RLIMIT_MEMLOCK 8 /* max locked-in-memory address space */