Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm/rmap: separate out fork-only logic on anon_vma_clone()
Specify which operation is being performed to anon_vma_clone(), which
allows us to do checks specific to each operation type, as well as to
separate out and make clear that the anon_vma reuse logic is absolutely
specific to fork only.
This opens the door to further refactorings and refinements later as we
have more information to work with.
Link: https://lkml.kernel.org/r/cf7da7a2d973cdc72a1b80dd9a73260519e8fa9f.1768746221.git.lorenzo.stoakes@oracle.com
Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: Chris Li <chriscli@google.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Harry Yoo <harry.yoo@oracle.com>
Cc: Jann Horn <jannh@google.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: Rik van Riel <riel@surriel.com>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Lorenzo Stoakes
and committed by
Andrew Morton
d17f0241
bfc2b13b
+74
-28
+10
-1
mm/internal.h
+10
-1
mm/internal.h
···
244
244
245
245
struct anon_vma *folio_get_anon_vma(const struct folio *folio);
246
246
247
-
int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src);
247
+
/* Operations which modify VMAs. */
248
+
enum vma_operation {
249
+
VMA_OP_SPLIT,
250
+
VMA_OP_MERGE_UNFAULTED,
251
+
VMA_OP_REMAP,
252
+
VMA_OP_FORK,
253
+
};
254
+
255
+
int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,
256
+
enum vma_operation operation);
248
257
int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma);
249
258
int __anon_vma_prepare(struct vm_area_struct *vma);
250
259
void unlink_anon_vmas(struct vm_area_struct *vma);
+51
-23
mm/rmap.c
+51
-23
mm/rmap.c
···
232
232
}
233
233
234
234
static void check_anon_vma_clone(struct vm_area_struct *dst,
235
-
struct vm_area_struct *src)
235
+
struct vm_area_struct *src,
236
+
enum vma_operation operation)
236
237
{
237
238
/* The write lock must be held. */
238
239
mmap_assert_write_locked(src->vm_mm);
239
-
/* If not a fork (implied by dst->anon_vma) then must be on same mm. */
240
-
VM_WARN_ON_ONCE(dst->anon_vma && dst->vm_mm != src->vm_mm);
240
+
/* If not a fork then must be on same mm. */
241
+
VM_WARN_ON_ONCE(operation != VMA_OP_FORK && dst->vm_mm != src->vm_mm);
241
242
242
243
/* If we have anything to do src->anon_vma must be provided. */
243
244
VM_WARN_ON_ONCE(!src->anon_vma && !list_empty(&src->anon_vma_chain));
···
250
249
* must be the same across dst and src.
251
250
*/
252
251
VM_WARN_ON_ONCE(dst->anon_vma && dst->anon_vma != src->anon_vma);
252
+
/*
253
+
* Essentially equivalent to above - if not a no-op, we should expect
254
+
* dst->anon_vma to be set for everything except a fork.
255
+
*/
256
+
VM_WARN_ON_ONCE(operation != VMA_OP_FORK && src->anon_vma &&
257
+
!dst->anon_vma);
258
+
/* For the anon_vma to be compatible, it can only be singular. */
259
+
VM_WARN_ON_ONCE(operation == VMA_OP_MERGE_UNFAULTED &&
260
+
!list_is_singular(&src->anon_vma_chain));
261
+
#ifdef CONFIG_PER_VMA_LOCK
262
+
/* Only merging an unfaulted VMA leaves the destination attached. */
263
+
VM_WARN_ON_ONCE(operation != VMA_OP_MERGE_UNFAULTED &&
264
+
vma_is_attached(dst));
265
+
#endif
266
+
}
267
+
268
+
static void maybe_reuse_anon_vma(struct vm_area_struct *dst,
269
+
struct anon_vma *anon_vma)
270
+
{
271
+
/* If already populated, nothing to do.*/
272
+
if (dst->anon_vma)
273
+
return;
274
+
275
+
/*
276
+
* We reuse an anon_vma if any linking VMAs were unmapped and it has
277
+
* only a single child at most.
278
+
*/
279
+
if (anon_vma->num_active_vmas > 0)
280
+
return;
281
+
if (anon_vma->num_children > 1)
282
+
return;
283
+
284
+
dst->anon_vma = anon_vma;
285
+
anon_vma->num_active_vmas++;
253
286
}
254
287
255
288
static void cleanup_partial_anon_vmas(struct vm_area_struct *vma);
···
293
258
* all of the anon_vma objects contained within @src anon_vma_chain's.
294
259
* @dst: The destination VMA with an empty anon_vma_chain.
295
260
* @src: The source VMA we wish to duplicate.
261
+
* @operation: The type of operation which resulted in the clone.
296
262
*
297
263
* This is the heart of the VMA side of the anon_vma implementation - we invoke
298
264
* this function whenever we need to set up a new VMA's anon_vma state.
···
316
280
*
317
281
* Returns: 0 on success, -ENOMEM on failure.
318
282
*/
319
-
int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
283
+
int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,
284
+
enum vma_operation operation)
320
285
{
321
286
struct anon_vma_chain *avc, *pavc;
287
+
struct anon_vma *active_anon_vma = src->anon_vma;
322
288
323
-
check_anon_vma_clone(dst, src);
289
+
check_anon_vma_clone(dst, src, operation);
324
290
325
-
if (!src->anon_vma)
291
+
if (!active_anon_vma)
326
292
return 0;
327
-
328
-
check_anon_vma_clone(dst, src);
329
293
330
294
/*
331
295
* Allocate AVCs. We don't need an anon_vma lock for this as we
···
354
318
struct anon_vma *anon_vma = avc->anon_vma;
355
319
356
320
anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
357
-
358
-
/*
359
-
* Reuse existing anon_vma if it has no vma and only one
360
-
* anon_vma child.
361
-
*
362
-
* Root anon_vma is never reused:
363
-
* it has self-parent reference and at least one child.
364
-
*/
365
-
if (!dst->anon_vma && src->anon_vma &&
366
-
anon_vma->num_children < 2 &&
367
-
anon_vma->num_active_vmas == 0)
368
-
dst->anon_vma = anon_vma;
321
+
if (operation == VMA_OP_FORK)
322
+
maybe_reuse_anon_vma(dst, anon_vma);
369
323
}
370
-
if (dst->anon_vma)
324
+
325
+
if (operation != VMA_OP_FORK)
371
326
dst->anon_vma->num_active_vmas++;
372
-
anon_vma_unlock_write(src->anon_vma);
327
+
328
+
anon_vma_unlock_write(active_anon_vma);
373
329
return 0;
374
330
375
331
enomem_failure:
···
400
372
* First, attach the new VMA to the parent VMA's anon_vmas,
401
373
* so rmap can find non-COWed pages in child processes.
402
374
*/
403
-
rc = anon_vma_clone(vma, pvma);
375
+
rc = anon_vma_clone(vma, pvma, VMA_OP_FORK);
404
376
/* An error arose or an existing anon_vma was reused, all done then. */
405
377
if (rc || vma->anon_vma) {
406
378
put_anon_vma(anon_vma);
+3
-3
mm/vma.c
+3
-3
mm/vma.c
···
530
530
if (err)
531
531
goto out_free_vmi;
532
532
533
-
err = anon_vma_clone(new, vma);
533
+
err = anon_vma_clone(new, vma, VMA_OP_SPLIT);
534
534
if (err)
535
535
goto out_free_mpol;
536
536
···
628
628
629
629
vma_assert_write_locked(dst);
630
630
dst->anon_vma = src->anon_vma;
631
-
ret = anon_vma_clone(dst, src);
631
+
ret = anon_vma_clone(dst, src, VMA_OP_MERGE_UNFAULTED);
632
632
if (ret)
633
633
return ret;
634
634
···
1901
1901
vma_set_range(new_vma, addr, addr + len, pgoff);
1902
1902
if (vma_dup_policy(vma, new_vma))
1903
1903
goto out_free_vma;
1904
-
if (anon_vma_clone(new_vma, vma))
1904
+
if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP))
1905
1905
goto out_free_mempol;
1906
1906
if (new_vma->vm_file)
1907
1907
get_file(new_vma->vm_file);
+10
-1
tools/testing/vma/vma_internal.h
+10
-1
tools/testing/vma/vma_internal.h
···
600
600
bool hide_from_rmap_until_complete :1;
601
601
};
602
602
603
+
/* Operations which modify VMAs. */
604
+
enum vma_operation {
605
+
VMA_OP_SPLIT,
606
+
VMA_OP_MERGE_UNFAULTED,
607
+
VMA_OP_REMAP,
608
+
VMA_OP_FORK,
609
+
};
610
+
603
611
/*
604
612
* Describes a VMA that is about to be mmap()'ed. Drivers may choose to
605
613
* manipulate mutable fields which will cause those fields to be updated in the
···
1165
1157
return 0;
1166
1158
}
1167
1159
1168
-
static inline int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
1160
+
static inline int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,
1161
+
enum vma_operation operation)
1169
1162
{
1170
1163
/* For testing purposes. We indicate that an anon_vma has been cloned. */
1171
1164
if (src->anon_vma != NULL) {