Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm: use unmap_desc struct for freeing page tables
Pass through the unmap_desc to free_pgtables() because it almost has
everything necessary and is already on the stack.
Updates testing code as necessary.
No functional changes intended.
[Liam.Howlett@oracle.com: fix up unmap desc use on exit_mmap()]
Link: https://lkml.kernel.org/r/20260210214214.364856-1-Liam.Howlett@oracle.com
Link: https://lkml.kernel.org/r/20260121164946.2093480-12-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Chris Li <chrisl@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Jann Horn <jannh@google.com>
Cc: Kairui Song <kasong@tencent.com>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: SeongJae Park <sj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Liam R. Howlett
and committed by
Andrew Morton
a8700d42
2314fe9b
+45
-35
+1
-4
mm/internal.h
+1
-4
mm/internal.h
···
512
512
void deactivate_file_folio(struct folio *folio);
513
513
void folio_activate(struct folio *folio);
514
514
515
-
void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
516
-
struct vm_area_struct *vma, unsigned long pg_start,
517
-
unsigned long pg_end, unsigned long vma_end,
518
-
bool mm_wr_locked);
515
+
void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *desc);
519
516
520
517
void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);
521
518
+13
-20
mm/memory.c
+13
-20
mm/memory.c
···
373
373
/**
374
374
* free_pgtables() - Free a range of page tables
375
375
* @tlb: The mmu gather
376
-
* @mas: The maple state
377
-
* @vma: The first vma
378
-
* @pg_start: The lowest page table address (floor)
379
-
* @pg_end: The highest page table address (ceiling)
380
-
* @vma_end: The highest vma tree search address
381
-
* @mm_wr_locked: boolean indicating if the mm is write locked
376
+
* @unmap: The unmap_desc
382
377
*
383
378
* Note: pg_start and pg_end are provided to indicate the absolute range of the
384
379
* page tables that should be removed. This can differ from the vma mappings on
···
383
388
* The vma_end differs from the pg_end when a dup_mmap() failed and the tree has
384
389
* unrelated data to the mm_struct being torn down.
385
390
*/
386
-
void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
387
-
struct vm_area_struct *vma, unsigned long pg_start,
388
-
unsigned long pg_end, unsigned long vma_end,
389
-
bool mm_wr_locked)
391
+
void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *unmap)
390
392
{
391
393
struct unlink_vma_file_batch vb;
394
+
struct ma_state *mas = unmap->mas;
395
+
struct vm_area_struct *vma = unmap->first;
392
396
393
397
/*
394
398
* Note: USER_PGTABLES_CEILING may be passed as the value of pg_end and
395
399
* may be 0. Underflow is expected in this case. Otherwise the
396
-
* pagetable end is exclusive.
397
-
* vma_end is exclusive.
398
-
* The last vma address should never be larger than the pagetable end.
400
+
* pagetable end is exclusive. vma_end is exclusive. The last vma
401
+
* address should never be larger than the pagetable end.
399
402
*/
400
-
WARN_ON_ONCE(vma_end - 1 > pg_end - 1);
403
+
WARN_ON_ONCE(unmap->vma_end - 1 > unmap->pg_end - 1);
401
404
402
405
tlb_free_vmas(tlb);
403
406
···
403
410
unsigned long addr = vma->vm_start;
404
411
struct vm_area_struct *next;
405
412
406
-
next = mas_find(mas, vma_end - 1);
413
+
next = mas_find(mas, unmap->tree_end - 1);
407
414
408
415
/*
409
416
* Hide vma from rmap and truncate_pagecache before freeing
410
417
* pgtables
411
418
*/
412
-
if (mm_wr_locked)
419
+
if (unmap->mm_wr_locked)
413
420
vma_start_write(vma);
414
421
unlink_anon_vmas(vma);
415
422
···
421
428
*/
422
429
while (next && next->vm_start <= vma->vm_end + PMD_SIZE) {
423
430
vma = next;
424
-
next = mas_find(mas, vma_end - 1);
425
-
if (mm_wr_locked)
431
+
next = mas_find(mas, unmap->tree_end - 1);
432
+
if (unmap->mm_wr_locked)
426
433
vma_start_write(vma);
427
434
unlink_anon_vmas(vma);
428
435
unlink_file_vma_batch_add(&vb, vma);
429
436
}
430
437
unlink_file_vma_batch_final(&vb);
431
438
432
-
free_pgd_range(tlb, addr, vma->vm_end,
433
-
pg_start, next ? next->vm_start : pg_end);
439
+
free_pgd_range(tlb, addr, vma->vm_end, unmap->pg_start,
440
+
next ? next->vm_start : unmap->pg_end);
434
441
vma = next;
435
442
} while (vma);
436
443
}
+3
-3
mm/mmap.c
+3
-3
mm/mmap.c
···
1307
1307
*/
1308
1308
mm_flags_set(MMF_OOM_SKIP, mm);
1309
1309
mmap_write_lock(mm);
1310
+
unmap.mm_wr_locked = true;
1310
1311
mt_clear_in_rcu(&mm->mm_mt);
1311
-
vma_iter_set(&vmi, vma->vm_end);
1312
-
free_pgtables(&tlb, &vmi.mas, vma, FIRST_USER_ADDRESS,
1313
-
USER_PGTABLES_CEILING, USER_PGTABLES_CEILING, true);
1312
+
unmap_pgtable_init(&unmap, &vmi);
1313
+
free_pgtables(&tlb, &unmap);
1314
1314
tlb_finish_mmu(&tlb);
1315
1315
1316
1316
/*
+2
-4
mm/vma.c
+2
-4
mm/vma.c
···
475
475
void unmap_region(struct unmap_desc *unmap)
476
476
{
477
477
struct mm_struct *mm = unmap->first->vm_mm;
478
-
struct ma_state *mas = unmap->mas;
479
478
struct mmu_gather tlb;
480
479
481
480
tlb_gather_mmu(&tlb, mm);
482
481
update_hiwater_rss(mm);
483
482
unmap_vmas(&tlb, unmap);
484
-
mas_set(mas, unmap->tree_reset);
485
-
free_pgtables(&tlb, mas, unmap->first, unmap->pg_start, unmap->pg_end,
486
-
unmap->tree_end, unmap->mm_wr_locked);
483
+
mas_set(unmap->mas, unmap->tree_reset);
484
+
free_pgtables(&tlb, unmap);
487
485
tlb_finish_mmu(&tlb);
488
486
}
489
487
+23
mm/vma.h
+23
mm/vma.h
···
167
167
bool mm_wr_locked; /* If the mmap write lock is held */
168
168
};
169
169
170
+
/*
171
+
* unmap_all_init() - Initialize unmap_desc to remove all vmas, point the
172
+
* pg_start and pg_end to a safe location.
173
+
*/
170
174
static inline void unmap_all_init(struct unmap_desc *unmap,
171
175
struct vma_iterator *vmi, struct vm_area_struct *vma)
172
176
{
···
183
179
unmap->tree_end = ULONG_MAX;
184
180
unmap->tree_reset = vma->vm_end;
185
181
unmap->mm_wr_locked = false;
182
+
}
183
+
184
+
/*
185
+
* unmap_pgtable_init() - Initialize unmap_desc to remove all page tables within
186
+
* the user range.
187
+
*
188
+
* ARM can have mappings outside of vmas.
189
+
* See: e2cdef8c847b4 ("[PATCH] freepgt: free_pgtables from FIRST_USER_ADDRESS")
190
+
*
191
+
* ARM LPAE uses page table mappings beyond the USER_PGTABLES_CEILING
192
+
* See: CONFIG_ARM_LPAE in arch/arm/include/asm/pgtable.h
193
+
*/
194
+
static inline void unmap_pgtable_init(struct unmap_desc *unmap,
195
+
struct vma_iterator *vmi)
196
+
{
197
+
vma_iter_set(vmi, unmap->tree_reset);
198
+
unmap->vma_start = FIRST_USER_ADDRESS;
199
+
unmap->vma_end = USER_PGTABLES_CEILING;
200
+
unmap->tree_end = USER_PGTABLES_CEILING;
186
201
}
187
202
188
203
#define UNMAP_STATE(name, _vmi, _vma, _vma_start, _vma_end, _prev, _next) \
+3
-4
tools/testing/vma/vma_internal.h
+3
-4
tools/testing/vma/vma_internal.h
···
1137
1137
{
1138
1138
}
1139
1139
1140
-
static inline void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
1141
-
struct vm_area_struct *vma, unsigned long floor,
1142
-
unsigned long ceiling, unsigned long tree_max,
1143
-
bool mm_wr_locked)
1140
+
static inline void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *desc)
1144
1141
{
1142
+
(void)tlb;
1143
+
(void)desc;
1145
1144
}
1146
1145
1147
1146
static inline void mapping_unmap_writable(struct address_space *mapping)