Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
VM: add "vm_mmap()" helper function
This continues the theme started with vm_brk() and vm_munmap():
vm_mmap() does the same thing as do_mmap(), but additionally does the
required VM locking.
This uninlines (and rewrites it to be clearer) do_mmap(), which sadly
duplicates it in mm/mmap.c and mm/nommu.c. But that way we don't have
to export our internal do_mmap_pgoff() function.
Some day we hopefully don't have to export do_mmap() either, if all
modular users can become the simpler vm_mmap() instead. We're actually
very close to that already, with the notable exception of the (broken)
use in i810, and a couple of stragglers in binfmt_elf.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
+87
-97
+1
-3
arch/tile/kernel/single_step.c
+1
-3
arch/tile/kernel/single_step.c
···
346
346
}
347
347
348
348
/* allocate a cache line of writable, executable memory */
349
-
down_write(¤t->mm->mmap_sem);
350
-
buffer = (void __user *) do_mmap(NULL, 0, 64,
349
+
buffer = (void __user *) vm_mmap(NULL, 0, 64,
351
350
PROT_EXEC | PROT_READ | PROT_WRITE,
352
351
MAP_PRIVATE | MAP_ANONYMOUS,
353
352
0);
354
-
up_write(¤t->mm->mmap_sem);
355
353
356
354
if (IS_ERR((void __force *)buffer)) {
357
355
kfree(state);
+3
-9
arch/x86/ia32/ia32_aout.c
+3
-9
arch/x86/ia32/ia32_aout.c
···
379
379
goto beyond_if;
380
380
}
381
381
382
-
down_write(¤t->mm->mmap_sem);
383
-
error = do_mmap(bprm->file, N_TXTADDR(ex), ex.a_text,
382
+
error = vm_mmap(bprm->file, N_TXTADDR(ex), ex.a_text,
384
383
PROT_READ | PROT_EXEC,
385
384
MAP_FIXED | MAP_PRIVATE | MAP_DENYWRITE |
386
385
MAP_EXECUTABLE | MAP_32BIT,
387
386
fd_offset);
388
-
up_write(¤t->mm->mmap_sem);
389
387
390
388
if (error != N_TXTADDR(ex)) {
391
389
send_sig(SIGKILL, current, 0);
392
390
return error;
393
391
}
394
392
395
-
down_write(¤t->mm->mmap_sem);
396
-
error = do_mmap(bprm->file, N_DATADDR(ex), ex.a_data,
393
+
error = vm_mmap(bprm->file, N_DATADDR(ex), ex.a_data,
397
394
PROT_READ | PROT_WRITE | PROT_EXEC,
398
395
MAP_FIXED | MAP_PRIVATE | MAP_DENYWRITE |
399
396
MAP_EXECUTABLE | MAP_32BIT,
400
397
fd_offset + ex.a_text);
401
-
up_write(¤t->mm->mmap_sem);
402
398
if (error != N_DATADDR(ex)) {
403
399
send_sig(SIGKILL, current, 0);
404
400
return error;
···
478
482
goto out;
479
483
}
480
484
/* Now use mmap to map the library into memory. */
481
-
down_write(¤t->mm->mmap_sem);
482
-
error = do_mmap(file, start_addr, ex.a_text + ex.a_data,
485
+
error = vm_mmap(file, start_addr, ex.a_text + ex.a_data,
483
486
PROT_READ | PROT_WRITE | PROT_EXEC,
484
487
MAP_FIXED | MAP_PRIVATE | MAP_DENYWRITE | MAP_32BIT,
485
488
N_TXTOFF(ex));
486
-
up_write(¤t->mm->mmap_sem);
487
489
retval = error;
488
490
if (error != start_addr)
489
491
goto out;
+1
-3
arch/x86/kvm/x86.c
+1
-3
arch/x86/kvm/x86.c
···
6336
6336
if (npages && !old.rmap) {
6337
6337
unsigned long userspace_addr;
6338
6338
6339
-
down_write(¤t->mm->mmap_sem);
6340
-
userspace_addr = do_mmap(NULL, 0,
6339
+
userspace_addr = vm_mmap(NULL, 0,
6341
6340
npages * PAGE_SIZE,
6342
6341
PROT_READ | PROT_WRITE,
6343
6342
map_flags,
6344
6343
0);
6345
-
up_write(¤t->mm->mmap_sem);
6346
6344
6347
6345
if (IS_ERR((void *)userspace_addr))
6348
6346
return PTR_ERR((void *)userspace_addr);
+4
-8
drivers/gpu/drm/drm_bufs.c
+4
-8
drivers/gpu/drm/drm_bufs.c
···
1510
1510
* \param arg pointer to a drm_buf_map structure.
1511
1511
* \return zero on success or a negative number on failure.
1512
1512
*
1513
-
* Maps the AGP, SG or PCI buffer region with do_mmap(), and copies information
1514
-
* about each buffer into user space. For PCI buffers, it calls do_mmap() with
1513
+
* Maps the AGP, SG or PCI buffer region with vm_mmap(), and copies information
1514
+
* about each buffer into user space. For PCI buffers, it calls vm_mmap() with
1515
1515
* offset equal to 0, which drm_mmap() interpretes as PCI buffers and calls
1516
1516
* drm_mmap_dma().
1517
1517
*/
···
1553
1553
retcode = -EINVAL;
1554
1554
goto done;
1555
1555
}
1556
-
down_write(¤t->mm->mmap_sem);
1557
-
virtual = do_mmap(file_priv->filp, 0, map->size,
1556
+
virtual = vm_mmap(file_priv->filp, 0, map->size,
1558
1557
PROT_READ | PROT_WRITE,
1559
1558
MAP_SHARED,
1560
1559
token);
1561
-
up_write(¤t->mm->mmap_sem);
1562
1560
} else {
1563
-
down_write(¤t->mm->mmap_sem);
1564
-
virtual = do_mmap(file_priv->filp, 0, dma->byte_count,
1561
+
virtual = vm_mmap(file_priv->filp, 0, dma->byte_count,
1565
1562
PROT_READ | PROT_WRITE,
1566
1563
MAP_SHARED, 0);
1567
-
up_write(¤t->mm->mmap_sem);
1568
1564
}
1569
1565
if (virtual > -1024UL) {
1570
1566
/* Real error */
+1
-3
drivers/gpu/drm/exynos/exynos_drm_gem.c
+1
-3
drivers/gpu/drm/exynos/exynos_drm_gem.c
···
581
581
obj->filp->f_op = &exynos_drm_gem_fops;
582
582
obj->filp->private_data = obj;
583
583
584
-
down_write(¤t->mm->mmap_sem);
585
-
addr = do_mmap(obj->filp, 0, args->size,
584
+
addr = vm_mmap(obj->filp, 0, args->size,
586
585
PROT_READ | PROT_WRITE, MAP_SHARED, 0);
587
-
up_write(¤t->mm->mmap_sem);
588
586
589
587
drm_gem_object_unreference_unlocked(obj);
590
588
+1
drivers/gpu/drm/i810/i810_dma.c
+1
drivers/gpu/drm/i810/i810_dma.c
+1
-3
drivers/gpu/drm/i915/i915_gem.c
+1
-3
drivers/gpu/drm/i915/i915_gem.c
···
1087
1087
if (obj == NULL)
1088
1088
return -ENOENT;
1089
1089
1090
-
down_write(¤t->mm->mmap_sem);
1091
-
addr = do_mmap(obj->filp, 0, args->size,
1090
+
addr = vm_mmap(obj->filp, 0, args->size,
1092
1091
PROT_READ | PROT_WRITE, MAP_SHARED,
1093
1092
args->offset);
1094
-
up_write(¤t->mm->mmap_sem);
1095
1093
drm_gem_object_unreference_unlocked(obj);
1096
1094
if (IS_ERR((void *)addr))
1097
1095
return addr;
+3
-9
fs/binfmt_aout.c
+3
-9
fs/binfmt_aout.c
···
319
319
goto beyond_if;
320
320
}
321
321
322
-
down_write(¤t->mm->mmap_sem);
323
-
error = do_mmap(bprm->file, N_TXTADDR(ex), ex.a_text,
322
+
error = vm_mmap(bprm->file, N_TXTADDR(ex), ex.a_text,
324
323
PROT_READ | PROT_EXEC,
325
324
MAP_FIXED | MAP_PRIVATE | MAP_DENYWRITE | MAP_EXECUTABLE,
326
325
fd_offset);
327
-
up_write(¤t->mm->mmap_sem);
328
326
329
327
if (error != N_TXTADDR(ex)) {
330
328
send_sig(SIGKILL, current, 0);
331
329
return error;
332
330
}
333
331
334
-
down_write(¤t->mm->mmap_sem);
335
-
error = do_mmap(bprm->file, N_DATADDR(ex), ex.a_data,
332
+
error = vm_mmap(bprm->file, N_DATADDR(ex), ex.a_data,
336
333
PROT_READ | PROT_WRITE | PROT_EXEC,
337
334
MAP_FIXED | MAP_PRIVATE | MAP_DENYWRITE | MAP_EXECUTABLE,
338
335
fd_offset + ex.a_text);
339
-
up_write(¤t->mm->mmap_sem);
340
336
if (error != N_DATADDR(ex)) {
341
337
send_sig(SIGKILL, current, 0);
342
338
return error;
···
413
417
goto out;
414
418
}
415
419
/* Now use mmap to map the library into memory. */
416
-
down_write(¤t->mm->mmap_sem);
417
-
error = do_mmap(file, start_addr, ex.a_text + ex.a_data,
420
+
error = vm_mmap(file, start_addr, ex.a_text + ex.a_data,
418
421
PROT_READ | PROT_WRITE | PROT_EXEC,
419
422
MAP_FIXED | MAP_PRIVATE | MAP_DENYWRITE,
420
423
N_TXTOFF(ex));
421
-
up_write(¤t->mm->mmap_sem);
422
424
retval = error;
423
425
if (error != start_addr)
424
426
goto out;
+2
-6
fs/binfmt_elf.c
+2
-6
fs/binfmt_elf.c
···
958
958
and some applications "depend" upon this behavior.
959
959
Since we do not have the power to recompile these, we
960
960
emulate the SVr4 behavior. Sigh. */
961
-
down_write(¤t->mm->mmap_sem);
962
-
error = do_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC,
961
+
error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC,
963
962
MAP_FIXED | MAP_PRIVATE, 0);
964
-
up_write(¤t->mm->mmap_sem);
965
963
}
966
964
967
965
#ifdef ELF_PLAT_INIT
···
1044
1046
eppnt++;
1045
1047
1046
1048
/* Now use mmap to map the library into memory. */
1047
-
down_write(¤t->mm->mmap_sem);
1048
-
error = do_mmap(file,
1049
+
error = vm_mmap(file,
1049
1050
ELF_PAGESTART(eppnt->p_vaddr),
1050
1051
(eppnt->p_filesz +
1051
1052
ELF_PAGEOFFSET(eppnt->p_vaddr)),
···
1052
1055
MAP_FIXED | MAP_PRIVATE | MAP_DENYWRITE,
1053
1056
(eppnt->p_offset -
1054
1057
ELF_PAGEOFFSET(eppnt->p_vaddr)));
1055
-
up_write(¤t->mm->mmap_sem);
1056
1058
if (error != ELF_PAGESTART(eppnt->p_vaddr))
1057
1059
goto out_free_ph;
1058
1060
+4
-14
fs/binfmt_elf_fdpic.c
+4
-14
fs/binfmt_elf_fdpic.c
···
390
390
(executable_stack == EXSTACK_DEFAULT && VM_STACK_FLAGS & VM_EXEC))
391
391
stack_prot |= PROT_EXEC;
392
392
393
-
down_write(¤t->mm->mmap_sem);
394
-
current->mm->start_brk = do_mmap(NULL, 0, stack_size, stack_prot,
393
+
current->mm->start_brk = vm_mmap(NULL, 0, stack_size, stack_prot,
395
394
MAP_PRIVATE | MAP_ANONYMOUS |
396
395
MAP_UNINITIALIZED | MAP_GROWSDOWN,
397
396
0);
398
397
399
398
if (IS_ERR_VALUE(current->mm->start_brk)) {
400
-
up_write(¤t->mm->mmap_sem);
401
399
retval = current->mm->start_brk;
402
400
current->mm->start_brk = 0;
403
401
goto error_kill;
404
402
}
405
-
406
-
up_write(¤t->mm->mmap_sem);
407
403
408
404
current->mm->brk = current->mm->start_brk;
409
405
current->mm->context.end_brk = current->mm->start_brk;
···
951
955
if (params->flags & ELF_FDPIC_FLAG_EXECUTABLE)
952
956
mflags |= MAP_EXECUTABLE;
953
957
954
-
down_write(&mm->mmap_sem);
955
-
maddr = do_mmap(NULL, load_addr, top - base,
958
+
maddr = vm_mmap(NULL, load_addr, top - base,
956
959
PROT_READ | PROT_WRITE | PROT_EXEC, mflags, 0);
957
-
up_write(&mm->mmap_sem);
958
960
if (IS_ERR_VALUE(maddr))
959
961
return (int) maddr;
960
962
···
1090
1096
1091
1097
/* create the mapping */
1092
1098
disp = phdr->p_vaddr & ~PAGE_MASK;
1093
-
down_write(&mm->mmap_sem);
1094
-
maddr = do_mmap(file, maddr, phdr->p_memsz + disp, prot, flags,
1099
+
maddr = vm_mmap(file, maddr, phdr->p_memsz + disp, prot, flags,
1095
1100
phdr->p_offset - disp);
1096
-
up_write(&mm->mmap_sem);
1097
1101
1098
1102
kdebug("mmap[%d] <file> sz=%lx pr=%x fl=%x of=%lx --> %08lx",
1099
1103
loop, phdr->p_memsz + disp, prot, flags,
···
1135
1143
unsigned long xmaddr;
1136
1144
1137
1145
flags |= MAP_FIXED | MAP_ANONYMOUS;
1138
-
down_write(&mm->mmap_sem);
1139
-
xmaddr = do_mmap(NULL, xaddr, excess - excess1,
1146
+
xmaddr = vm_mmap(NULL, xaddr, excess - excess1,
1140
1147
prot, flags, 0);
1141
-
up_write(&mm->mmap_sem);
1142
1148
1143
1149
kdebug("mmap[%d] <anon>"
1144
1150
" ad=%lx sz=%lx pr=%x fl=%x of=0 --> %08lx",
+3
-9
fs/binfmt_flat.c
+3
-9
fs/binfmt_flat.c
···
542
542
*/
543
543
DBG_FLT("BINFMT_FLAT: ROM mapping of file (we hope)\n");
544
544
545
-
down_write(¤t->mm->mmap_sem);
546
-
textpos = do_mmap(bprm->file, 0, text_len, PROT_READ|PROT_EXEC,
545
+
textpos = vm_mmap(bprm->file, 0, text_len, PROT_READ|PROT_EXEC,
547
546
MAP_PRIVATE|MAP_EXECUTABLE, 0);
548
-
up_write(¤t->mm->mmap_sem);
549
547
if (!textpos || IS_ERR_VALUE(textpos)) {
550
548
if (!textpos)
551
549
textpos = (unsigned long) -ENOMEM;
···
554
556
555
557
len = data_len + extra + MAX_SHARED_LIBS * sizeof(unsigned long);
556
558
len = PAGE_ALIGN(len);
557
-
down_write(¤t->mm->mmap_sem);
558
-
realdatastart = do_mmap(0, 0, len,
559
+
realdatastart = vm_mmap(0, 0, len,
559
560
PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE, 0);
560
-
up_write(¤t->mm->mmap_sem);
561
561
562
562
if (realdatastart == 0 || IS_ERR_VALUE(realdatastart)) {
563
563
if (!realdatastart)
···
599
603
600
604
len = text_len + data_len + extra + MAX_SHARED_LIBS * sizeof(unsigned long);
601
605
len = PAGE_ALIGN(len);
602
-
down_write(¤t->mm->mmap_sem);
603
-
textpos = do_mmap(0, 0, len,
606
+
textpos = vm_mmap(0, 0, len,
604
607
PROT_READ | PROT_EXEC | PROT_WRITE, MAP_PRIVATE, 0);
605
-
up_write(¤t->mm->mmap_sem);
606
608
607
609
if (!textpos || IS_ERR_VALUE(textpos)) {
608
610
if (!textpos)
+3
-9
fs/binfmt_som.c
+3
-9
fs/binfmt_som.c
···
147
147
code_size = SOM_PAGEALIGN(hpuxhdr->exec_tsize);
148
148
current->mm->start_code = code_start;
149
149
current->mm->end_code = code_start + code_size;
150
-
down_write(¤t->mm->mmap_sem);
151
-
retval = do_mmap(file, code_start, code_size, prot,
150
+
retval = vm_mmap(file, code_start, code_size, prot,
152
151
flags, SOM_PAGESTART(hpuxhdr->exec_tfile));
153
-
up_write(¤t->mm->mmap_sem);
154
152
if (retval < 0 && retval > -1024)
155
153
goto out;
156
154
···
156
158
data_size = SOM_PAGEALIGN(hpuxhdr->exec_dsize);
157
159
current->mm->start_data = data_start;
158
160
current->mm->end_data = bss_start = data_start + data_size;
159
-
down_write(¤t->mm->mmap_sem);
160
-
retval = do_mmap(file, data_start, data_size,
161
+
retval = vm_mmap(file, data_start, data_size,
161
162
prot | PROT_WRITE, flags,
162
163
SOM_PAGESTART(hpuxhdr->exec_dfile));
163
-
up_write(¤t->mm->mmap_sem);
164
164
if (retval < 0 && retval > -1024)
165
165
goto out;
166
166
167
167
som_brk = bss_start + SOM_PAGEALIGN(hpuxhdr->exec_bsize);
168
168
current->mm->start_brk = current->mm->brk = som_brk;
169
-
down_write(¤t->mm->mmap_sem);
170
-
retval = do_mmap(NULL, bss_start, som_brk - bss_start,
169
+
retval = vm_mmap(NULL, bss_start, som_brk - bss_start,
171
170
prot | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, 0);
172
-
up_write(¤t->mm->mmap_sem);
173
171
if (retval > 0 || retval < -1024)
174
172
retval = 0;
175
173
out:
+6
-17
include/linux/mm.h
+6
-17
include/linux/mm.h
···
1393
1393
1394
1394
extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1395
1395
1396
-
extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1397
-
unsigned long len, unsigned long prot,
1398
-
unsigned long flag, unsigned long pgoff);
1399
1396
extern unsigned long mmap_region(struct file *file, unsigned long addr,
1400
1397
unsigned long len, unsigned long flags,
1401
1398
vm_flags_t vm_flags, unsigned long pgoff);
1402
-
1403
-
static inline unsigned long do_mmap(struct file *file, unsigned long addr,
1404
-
unsigned long len, unsigned long prot,
1405
-
unsigned long flag, unsigned long offset)
1406
-
{
1407
-
unsigned long ret = -EINVAL;
1408
-
if ((offset + PAGE_ALIGN(len)) < offset)
1409
-
goto out;
1410
-
if (!(offset & ~PAGE_MASK))
1411
-
ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
1412
-
out:
1413
-
return ret;
1414
-
}
1415
-
1399
+
extern unsigned long do_mmap(struct file *, unsigned long,
1400
+
unsigned long, unsigned long,
1401
+
unsigned long, unsigned long);
1416
1402
extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1417
1403
1418
1404
/* These take the mm semaphore themselves */
1419
1405
extern unsigned long vm_brk(unsigned long, unsigned long);
1420
1406
extern int vm_munmap(struct mm_struct *, unsigned long, size_t);
1407
+
extern unsigned long vm_mmap(struct file *, unsigned long,
1408
+
unsigned long, unsigned long,
1409
+
unsigned long, unsigned long);
1421
1410
1422
1411
/* truncate.c */
1423
1412
extern void truncate_inode_pages(struct address_space *, loff_t);
+27
-2
mm/mmap.c
+27
-2
mm/mmap.c
···
953
953
* The caller must hold down_write(¤t->mm->mmap_sem).
954
954
*/
955
955
956
-
unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
956
+
static unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
957
957
unsigned long len, unsigned long prot,
958
958
unsigned long flags, unsigned long pgoff)
959
959
{
···
1089
1089
1090
1090
return mmap_region(file, addr, len, flags, vm_flags, pgoff);
1091
1091
}
1092
-
EXPORT_SYMBOL(do_mmap_pgoff);
1092
+
1093
+
unsigned long do_mmap(struct file *file, unsigned long addr,
1094
+
unsigned long len, unsigned long prot,
1095
+
unsigned long flag, unsigned long offset)
1096
+
{
1097
+
if (unlikely(offset + PAGE_ALIGN(len) < offset))
1098
+
return -EINVAL;
1099
+
if (unlikely(offset & ~PAGE_MASK))
1100
+
return -EINVAL;
1101
+
return do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
1102
+
}
1103
+
EXPORT_SYMBOL(do_mmap);
1104
+
1105
+
unsigned long vm_mmap(struct file *file, unsigned long addr,
1106
+
unsigned long len, unsigned long prot,
1107
+
unsigned long flag, unsigned long offset)
1108
+
{
1109
+
unsigned long ret;
1110
+
struct mm_struct *mm = current->mm;
1111
+
1112
+
down_write(&mm->mmap_sem);
1113
+
ret = do_mmap(file, addr, len, prot, flag, offset);
1114
+
up_write(&mm->mmap_sem);
1115
+
return ret;
1116
+
}
1117
+
EXPORT_SYMBOL(vm_mmap);
1093
1118
1094
1119
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1095
1120
unsigned long, prot, unsigned long, flags,
+27
-2
mm/nommu.c
+27
-2
mm/nommu.c
···
1233
1233
/*
1234
1234
* handle mapping creation for uClinux
1235
1235
*/
1236
-
unsigned long do_mmap_pgoff(struct file *file,
1236
+
static unsigned long do_mmap_pgoff(struct file *file,
1237
1237
unsigned long addr,
1238
1238
unsigned long len,
1239
1239
unsigned long prot,
···
1470
1470
show_free_areas(0);
1471
1471
return -ENOMEM;
1472
1472
}
1473
-
EXPORT_SYMBOL(do_mmap_pgoff);
1473
+
1474
+
unsigned long do_mmap(struct file *file, unsigned long addr,
1475
+
unsigned long len, unsigned long prot,
1476
+
unsigned long flag, unsigned long offset)
1477
+
{
1478
+
if (unlikely(offset + PAGE_ALIGN(len) < offset))
1479
+
return -EINVAL;
1480
+
if (unlikely(offset & ~PAGE_MASK))
1481
+
return -EINVAL;
1482
+
return do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
1483
+
}
1484
+
EXPORT_SYMBOL(do_mmap);
1485
+
1486
+
unsigned long vm_mmap(struct file *file, unsigned long addr,
1487
+
unsigned long len, unsigned long prot,
1488
+
unsigned long flag, unsigned long offset)
1489
+
{
1490
+
unsigned long ret;
1491
+
struct mm_struct *mm = current->mm;
1492
+
1493
+
down_write(&mm->mmap_sem);
1494
+
ret = do_mmap(file, addr, len, prot, flag, offset);
1495
+
up_write(&mm->mmap_sem);
1496
+
return ret;
1497
+
}
1498
+
EXPORT_SYMBOL(vm_mmap);
1474
1499
1475
1500
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1476
1501
unsigned long, prot, unsigned long, flags,