mm: introduce vma_iter_store_attached() to use with attached vmas

vma_iter_store() functions can be used both when adding a new vma and when
updating an existing one. However for existing ones we do not need to
mark them attached as they are already marked that way. With
vma->detached being a separate flag, double-marking a vmas as attached or
detached is not an issue because the flag will simply be overwritten with
the same value. However once we fold this flag into the refcount later in
this series, re-attaching or re-detaching a vma becomes an issue since
these operations will be incrementing/decrementing a refcount.

Introduce vma_iter_store_new() and vma_iter_store_overwrite() to replace
vma_iter_store() and avoid re-attaching a vma during vma update. Add
assertions in vma_mark_attached()/vma_mark_detached() to catch invalid
usage. Update vma tests to check for vma detached state correctness.

Link: https://lkml.kernel.org/r/20250213224655.1680278-5-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Tested-by: Shivank Garg <shivankg@amd.com>
Link: https://lkml.kernel.org/r/5e19ec93-8307-47c2-bb13-3ddf7150624e@amd.com
Reviewed-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christian Brauner <brauner@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Klara Modin <klarasmodin@gmail.com>
Cc: Lokesh Gidra <lokeshgidra@google.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mateusz Guzik <mjguzik@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Minchan Kim <minchan@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: "Paul E . McKenney" <paulmck@kernel.org>
Cc: Peter Xu <peterx@redhat.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Sourav Panda <souravpanda@google.com>
Cc: Wei Yang <richard.weiyang@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Suren Baghdasaryan and committed by

Andrew Morton 1 year ago 55e50223 8ef95d8f

+74 -19

6 changed files

expand all

include

linux

mm.h

nommu.c

vma.c

vma.h

tools

testing

vma

vma.c

vma_internal.h

+14

include/linux/mm.h

··· 821 821 vma_assert_write_locked(vma); 822 822 } 823 823 824 + static inline void vma_assert_attached(struct vm_area_struct *vma) 825 + { 826 + WARN_ON_ONCE(vma->detached); 827 + } 828 + 829 + static inline void vma_assert_detached(struct vm_area_struct *vma) 830 + { 831 + WARN_ON_ONCE(!vma->detached); 832 + } 833 + 824 834 static inline void vma_mark_attached(struct vm_area_struct *vma) 825 835 { 836 + vma_assert_detached(vma); 826 837 vma->detached = false; 827 838 } 828 839 ··· 841 830 { 842 831 /* When detaching vma should be write-locked */ 843 832 vma_assert_write_locked(vma); 833 + vma_assert_attached(vma); 844 834 vma->detached = true; 845 835 } 846 836 ··· 878 866 static inline void vma_start_write(struct vm_area_struct *vma) {} 879 867 static inline void vma_assert_write_locked(struct vm_area_struct *vma) 880 868 { mmap_assert_write_locked(vma->vm_mm); } 869 + static inline void vma_assert_attached(struct vm_area_struct *vma) {} 870 + static inline void vma_assert_detached(struct vm_area_struct *vma) {} 881 871 static inline void vma_mark_attached(struct vm_area_struct *vma) {} 882 872 static inline void vma_mark_detached(struct vm_area_struct *vma) {} 883 873

+2 -2

mm/nommu.c

··· 1191 1191 setup_vma_to_mm(vma, current->mm); 1192 1192 current->mm->map_count++; 1193 1193 /* add the VMA to the tree */ 1194 - vma_iter_store(&vmi, vma); 1194 + vma_iter_store_new(&vmi, vma); 1195 1195 1196 1196 /* we flush the region from the icache only when the first executable 1197 1197 * mapping of it is made */ ··· 1356 1356 1357 1357 setup_vma_to_mm(vma, mm); 1358 1358 setup_vma_to_mm(new, mm); 1359 - vma_iter_store(vmi, new); 1359 + vma_iter_store_new(vmi, new); 1360 1360 mm->map_count++; 1361 1361 return 0; 1362 1362

+6 -6

mm/vma.c

··· 320 320 * us to insert it before dropping the locks 321 321 * (it may either follow vma or precede it). 322 322 */ 323 - vma_iter_store(vmi, vp->insert); 323 + vma_iter_store_new(vmi, vp->insert); 324 324 mm->map_count++; 325 325 } 326 326 ··· 700 700 vmg->__adjust_middle_start ? vmg->middle : NULL); 701 701 vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); 702 702 vmg_adjust_set_range(vmg); 703 - vma_iter_store(vmg->vmi, vmg->target); 703 + vma_iter_store_overwrite(vmg->vmi, vmg->target); 704 704 705 705 vma_complete(&vp, vmg->vmi, vma->vm_mm); 706 706 ··· 1711 1711 return -ENOMEM; 1712 1712 1713 1713 vma_start_write(vma); 1714 - vma_iter_store(&vmi, vma); 1714 + vma_iter_store_new(&vmi, vma); 1715 1715 vma_link_file(vma); 1716 1716 mm->map_count++; 1717 1717 validate_mm(mm); ··· 2390 2390 2391 2391 /* Lock the VMA since it is modified after insertion into VMA tree */ 2392 2392 vma_start_write(vma); 2393 - vma_iter_store(vmi, vma); 2393 + vma_iter_store_new(vmi, vma); 2394 2394 map->mm->map_count++; 2395 2395 vma_link_file(vma); 2396 2396 ··· 2867 2867 anon_vma_interval_tree_pre_update_vma(vma); 2868 2868 vma->vm_end = address; 2869 2869 /* Overwrite old entry in mtree. */ 2870 - vma_iter_store(&vmi, vma); 2870 + vma_iter_store_overwrite(&vmi, vma); 2871 2871 anon_vma_interval_tree_post_update_vma(vma); 2872 2872 2873 2873 perf_event_mmap(vma); ··· 2947 2947 vma->vm_start = address; 2948 2948 vma->vm_pgoff -= grow; 2949 2949 /* Overwrite old entry in mtree. */ 2950 - vma_iter_store(&vmi, vma); 2950 + vma_iter_store_overwrite(&vmi, vma); 2951 2951 anon_vma_interval_tree_post_update_vma(vma); 2952 2952 2953 2953 perf_event_mmap(vma);

+9 -2

mm/vma.h

··· 413 413 } 414 414 415 415 /* Store a VMA with preallocated memory */ 416 - static inline void vma_iter_store(struct vma_iterator *vmi, 417 - struct vm_area_struct *vma) 416 + static inline void vma_iter_store_overwrite(struct vma_iterator *vmi, 417 + struct vm_area_struct *vma) 418 418 { 419 + vma_assert_attached(vma); 419 420 420 421 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 421 422 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && ··· 439 438 440 439 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 441 440 mas_store_prealloc(&vmi->mas, vma); 441 + } 442 + 443 + static inline void vma_iter_store_new(struct vma_iterator *vmi, 444 + struct vm_area_struct *vma) 445 + { 442 446 vma_mark_attached(vma); 447 + vma_iter_store_overwrite(vmi, vma); 443 448 } 444 449 445 450 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)

+33 -9

tools/testing/vma/vma.c

··· 74 74 ret->vm_end = end; 75 75 ret->vm_pgoff = pgoff; 76 76 ret->__vm_flags = flags; 77 + vma_assert_detached(ret); 77 78 78 79 return ret; 80 + } 81 + 82 + /* Helper function to allocate a VMA and link it to the tree. */ 83 + static int attach_vma(struct mm_struct *mm, struct vm_area_struct *vma) 84 + { 85 + int res; 86 + 87 + res = vma_link(mm, vma); 88 + if (!res) 89 + vma_assert_attached(vma); 90 + return res; 79 91 } 80 92 81 93 /* Helper function to allocate a VMA and link it to the tree. */ ··· 102 90 if (vma == NULL) 103 91 return NULL; 104 92 105 - if (vma_link(mm, vma)) { 93 + if (attach_vma(mm, vma)) { 106 94 vm_area_free(vma); 107 95 return NULL; 108 96 } ··· 120 108 /* Helper function which provides a wrapper around a merge new VMA operation. */ 121 109 static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg) 122 110 { 111 + struct vm_area_struct *vma; 123 112 /* 124 113 * For convenience, get prev and next VMAs. Which the new VMA operation 125 114 * requires. ··· 129 116 vmg->prev = vma_prev(vmg->vmi); 130 117 vma_iter_next_range(vmg->vmi); 131 118 132 - return vma_merge_new_range(vmg); 119 + vma = vma_merge_new_range(vmg); 120 + if (vma) 121 + vma_assert_attached(vma); 122 + 123 + return vma; 133 124 } 134 125 135 126 /* ··· 142 125 */ 143 126 static struct vm_area_struct *merge_existing(struct vma_merge_struct *vmg) 144 127 { 145 - return vma_merge_existing_range(vmg); 128 + struct vm_area_struct *vma; 129 + 130 + vma = vma_merge_existing_range(vmg); 131 + if (vma) 132 + vma_assert_attached(vma); 133 + return vma; 146 134 } 147 135 148 136 /* ··· 282 260 .pgoff = 1, 283 261 }; 284 262 285 - ASSERT_FALSE(vma_link(&mm, vma_left)); 286 - ASSERT_FALSE(vma_link(&mm, vma_right)); 263 + ASSERT_FALSE(attach_vma(&mm, vma_left)); 264 + ASSERT_FALSE(attach_vma(&mm, vma_right)); 287 265 288 266 vma = merge_new(&vmg); 289 267 ASSERT_NE(vma, NULL); ··· 307 285 struct vm_area_struct *init_vma = alloc_vma(&mm, 0, 0x3000, 0, flags); 308 286 VMA_ITERATOR(vmi, &mm, 0x1000); 309 287 310 - ASSERT_FALSE(vma_link(&mm, init_vma)); 288 + ASSERT_FALSE(attach_vma(&mm, init_vma)); 311 289 312 290 /* 313 291 * The flags will not be changed, the vma_modify_flags() function ··· 373 351 .pgoff = 0, 374 352 }; 375 353 376 - ASSERT_FALSE(vma_link(&mm, vma)); 354 + ASSERT_FALSE(attach_vma(&mm, vma)); 377 355 378 356 ASSERT_FALSE(expand_existing(&vmg)); 379 357 ··· 394 372 struct vm_area_struct *vma = alloc_vma(&mm, 0, 0x3000, 0, flags); 395 373 VMA_ITERATOR(vmi, &mm, 0); 396 374 397 - ASSERT_FALSE(vma_link(&mm, vma)); 375 + ASSERT_FALSE(attach_vma(&mm, vma)); 398 376 399 377 ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); 400 378 ··· 1544 1522 1545 1523 vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, flags); 1546 1524 vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); 1547 - 1548 1525 ASSERT_NE(vma_new, vma); 1549 1526 ASSERT_EQ(vma_new->vm_start, 0); 1550 1527 ASSERT_EQ(vma_new->vm_end, 0x2000); 1551 1528 ASSERT_EQ(vma_new->vm_pgoff, 0); 1529 + vma_assert_attached(vma_new); 1552 1530 1553 1531 cleanup_mm(&mm, &vmi); 1554 1532 ··· 1557 1535 vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, flags); 1558 1536 vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, flags); 1559 1537 vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); 1538 + vma_assert_attached(vma_new); 1560 1539 1561 1540 ASSERT_EQ(vma_new, vma_next); 1562 1541 ··· 1599 1576 ASSERT_EQ(vma->vm_pgoff, 3); 1600 1577 ASSERT_TRUE(vma_write_started(vma)); 1601 1578 ASSERT_EQ(vma_iter_addr(&vmi), 0x3000); 1579 + vma_assert_attached(vma); 1602 1580 1603 1581 cleanup_mm(&mm, &vmi); 1604 1582 return true;

+10

tools/testing/vma/vma_internal.h

··· 470 470 vma->vm_lock_seq = UINT_MAX; 471 471 } 472 472 473 + static inline void vma_assert_attached(struct vm_area_struct *vma) 474 + { 475 + WARN_ON_ONCE(vma->detached); 476 + } 477 + 478 + static inline void vma_assert_detached(struct vm_area_struct *vma) 479 + { 480 + WARN_ON_ONCE(!vma->detached); 481 + } 482 + 473 483 static inline void vma_assert_write_locked(struct vm_area_struct *); 474 484 static inline void vma_mark_attached(struct vm_area_struct *vma) 475 485 {

Configure Feed

Configure Feed