mm: defer second attempt at merge on mmap()

Rather than trying to merge again when ostensibly allocating a new VMA,
instead defer until the VMA is added and attempt to merge the existing
range.

This way we have no complicated unwinding logic midway through the process
of mapping the VMA.

In addition this removes limitations on the VMA not being able to be the
first in the virtual memory address space which was previously implicitly
required.

In theory, for this very same reason, we should unconditionally attempt
merge here, however this is likely to have a performance impact so it is
better to avoid this given the unlikely outcome of a merge.

[lorenzo.stoakes@oracle.com: remove unnecessary indirection]
Link: https://lkml.kernel.org/r/5106696d-e625-4d8a-8545-9d1430301730@lucifer.local
Link: https://lkml.kernel.org/r/d4f84502605d7651ac114587f507395c0fc76004.1729858176.git.lorenzo.stoakes@oracle.com
Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Liam R. Howlett <Liam.Howlett@Oracle.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Lorenzo Stoakes and committed by

Andrew Morton 2 years ago 5ac87a88 5a689bac

+14 -42

1 changed file

expand all

vma.c

+14 -42

mm/vma.c

··· 19 19 struct file *file; 20 20 21 21 unsigned long charged; 22 + bool retry_merge; 22 23 23 24 struct vm_area_struct *prev; 24 25 struct vm_area_struct *next; ··· 2279 2278 return 0; 2280 2279 } 2281 2280 2281 + 2282 2282 static int __mmap_new_file_vma(struct mmap_state *map, 2283 - struct vm_area_struct **vmap, bool *mergedp) 2283 + struct vm_area_struct *vma) 2284 2284 { 2285 2285 struct vma_iterator *vmi = map->vmi; 2286 - struct vm_area_struct *vma = *vmap; 2287 2286 int error; 2288 2287 2289 2288 vma->vm_file = get_file(map->file); ··· 2309 2308 !(map->flags & VM_MAYWRITE) && 2310 2309 (vma->vm_flags & VM_MAYWRITE)); 2311 2310 2312 - /* mmap_file() might have changed VMA flags. */ 2311 + /* If the flags change (and are mergeable), let's retry later. */ 2312 + map->retry_merge = vma->vm_flags != map->flags && !(vma->vm_flags & VM_SPECIAL); 2313 2313 map->flags = vma->vm_flags; 2314 2314 2315 - vma_iter_config(vmi, map->addr, map->end); 2316 - /* 2317 - * If flags changed after mmap_file(), we should try merge 2318 - * vma again as we may succeed this time. 2319 - */ 2320 - if (unlikely(map->flags != vma->vm_flags && map->prev)) { 2321 - struct vm_area_struct *merge; 2322 - VMG_MMAP_STATE(vmg, map, /* vma = */ NULL); 2323 - 2324 - merge = vma_merge_new_range(&vmg); 2325 - if (merge) { 2326 - /* 2327 - * ->mmap() can change vma->vm_file and fput 2328 - * the original file. So fput the vma->vm_file 2329 - * here or we would add an extra fput for file 2330 - * and cause general protection fault 2331 - * ultimately. 2332 - */ 2333 - fput(vma->vm_file); 2334 - vm_area_free(vma); 2335 - vma = merge; 2336 - *mergedp = true; 2337 - } else { 2338 - vma_iter_config(vmi, map->addr, map->end); 2339 - } 2340 - } 2341 - 2342 - *vmap = vma; 2343 2315 return 0; 2344 2316 } 2345 2317 2346 2318 /* 2347 2319 * __mmap_new_vma() - Allocate a new VMA for the region, as merging was not 2348 2320 * possible. 2349 - * 2350 - * An exception to this is if the mapping is file-backed, and the underlying 2351 - * driver changes the VMA flags, permitting a subsequent merge of the VMA, in 2352 - * which case the returned VMA is one that was merged on a second attempt. 2353 2321 * 2354 2322 * @map: Mapping state. 2355 2323 * @vmap: Output pointer for the new VMA. ··· 2329 2359 { 2330 2360 struct vma_iterator *vmi = map->vmi; 2331 2361 int error = 0; 2332 - bool merged = false; 2333 2362 struct vm_area_struct *vma; 2334 2363 2335 2364 /* ··· 2351 2382 } 2352 2383 2353 2384 if (map->file) 2354 - error = __mmap_new_file_vma(map, &vma, &merged); 2385 + error = __mmap_new_file_vma(map, vma); 2355 2386 else if (map->flags & VM_SHARED) 2356 2387 error = shmem_zero_setup(vma); 2357 2388 else ··· 2359 2390 2360 2391 if (error) 2361 2392 goto free_iter_vma; 2362 - 2363 - if (merged) 2364 - goto file_expanded; 2365 2393 2366 2394 #ifdef CONFIG_SPARC64 2367 2395 /* TODO: Fix SPARC ADI! */ ··· 2376 2410 * call covers the non-merge case. 2377 2411 */ 2378 2412 khugepaged_enter_vma(vma, map->flags); 2379 - 2380 - file_expanded: 2381 2413 ksm_add_vma(vma); 2382 2414 *vmap = vma; 2383 2415 return 0; ··· 2455 2491 error = __mmap_new_vma(&map, &vma); 2456 2492 if (error) 2457 2493 goto unacct_error; 2494 + } 2495 + 2496 + /* If flags changed, we might be able to merge, so try again. */ 2497 + if (map.retry_merge) { 2498 + VMG_MMAP_STATE(vmg, &map, vma); 2499 + 2500 + vma_iter_config(map.vmi, map.addr, map.end); 2501 + vma_merge_existing_range(&vmg); 2458 2502 } 2459 2503 2460 2504 __mmap_complete(&map, vma);

Configure Feed

Configure Feed