Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'akpm'
* akpm:
mm: madvise(MADV_DODUMP): allow hugetlbfs pages
ocfs2: fix locking for res->tracking and dlm->tracking_list
mm/vmscan.c: fix int overflow in callers of do_shrink_slab()
mm/vmstat.c: skip NR_TLB_REMOTE_FLUSH* properly
mm/vmstat.c: fix outdated vmstat_text
proc: restrict kernel stack dumps to root
mm/hugetlb: add mmap() encodings for 32MB and 512MB page sizes
mm/migrate.c: split only transparent huge pages when allocation fails
ipc/shm.c: use ERR_CAST() for shm_lock() error return
mm/gup_benchmark: fix unsigned comparison to zero in __gup_benchmark_ioctl
mm, thp: fix mlocking THP page with migration enabled
ocfs2: fix crash in ocfs2_duplicate_clusters_by_page()
hugetlb: take PMD sharing into account when flushing tlb/caches
mm: migration: fix migration of huge PMD shared pages
+189
-30
+2
-2
fs/ocfs2/dlm/dlmmaster.c
+2
-2
fs/ocfs2/dlm/dlmmaster.c
···
584
584
585
585
res->last_used = 0;
586
586
587
-
spin_lock(&dlm->spinlock);
587
+
spin_lock(&dlm->track_lock);
588
588
list_add_tail(&res->tracking, &dlm->tracking_list);
589
-
spin_unlock(&dlm->spinlock);
589
+
spin_unlock(&dlm->track_lock);
590
590
591
591
memset(res->lvb, 0, DLM_LVB_LEN);
592
592
memset(res->refmap, 0, sizeof(res->refmap));
+12
-4
fs/ocfs2/refcounttree.c
+12
-4
fs/ocfs2/refcounttree.c
···
2946
2946
if (map_end & (PAGE_SIZE - 1))
2947
2947
to = map_end & (PAGE_SIZE - 1);
2948
2948
2949
+
retry:
2949
2950
page = find_or_create_page(mapping, page_index, GFP_NOFS);
2950
2951
if (!page) {
2951
2952
ret = -ENOMEM;
···
2955
2954
}
2956
2955
2957
2956
/*
2958
-
* In case PAGE_SIZE <= CLUSTER_SIZE, This page
2959
-
* can't be dirtied before we CoW it out.
2957
+
* In case PAGE_SIZE <= CLUSTER_SIZE, we do not expect a dirty
2958
+
* page, so write it back.
2960
2959
*/
2961
-
if (PAGE_SIZE <= OCFS2_SB(sb)->s_clustersize)
2962
-
BUG_ON(PageDirty(page));
2960
+
if (PAGE_SIZE <= OCFS2_SB(sb)->s_clustersize) {
2961
+
if (PageDirty(page)) {
2962
+
/*
2963
+
* write_on_page will unlock the page on return
2964
+
*/
2965
+
ret = write_one_page(page);
2966
+
goto retry;
2967
+
}
2968
+
}
2963
2969
2964
2970
if (!PageUptodate(page)) {
2965
2971
ret = block_read_full_page(page, ocfs2_get_block);
+14
fs/proc/base.c
+14
fs/proc/base.c
···
407
407
unsigned long *entries;
408
408
int err;
409
409
410
+
/*
411
+
* The ability to racily run the kernel stack unwinder on a running task
412
+
* and then observe the unwinder output is scary; while it is useful for
413
+
* debugging kernel issues, it can also allow an attacker to leak kernel
414
+
* stack contents.
415
+
* Doing this in a manner that is at least safe from races would require
416
+
* some work to ensure that the remote task can not be scheduled; and
417
+
* even then, this would still expose the unwinder as local attack
418
+
* surface.
419
+
* Therefore, this interface is restricted to root.
420
+
*/
421
+
if (!file_ns_capable(m->file, &init_user_ns, CAP_SYS_ADMIN))
422
+
return -EACCES;
423
+
410
424
entries = kmalloc_array(MAX_STACK_TRACE_DEPTH, sizeof(*entries),
411
425
GFP_KERNEL);
412
426
if (!entries)
+14
include/linux/hugetlb.h
+14
include/linux/hugetlb.h
···
140
140
pte_t *huge_pte_offset(struct mm_struct *mm,
141
141
unsigned long addr, unsigned long sz);
142
142
int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep);
143
+
void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
144
+
unsigned long *start, unsigned long *end);
143
145
struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
144
146
int write);
145
147
struct page *follow_huge_pd(struct vm_area_struct *vma,
···
170
168
static inline unsigned long hugetlb_total_pages(void)
171
169
{
172
170
return 0;
171
+
}
172
+
173
+
static inline int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr,
174
+
pte_t *ptep)
175
+
{
176
+
return 0;
177
+
}
178
+
179
+
static inline void adjust_range_if_pmd_sharing_possible(
180
+
struct vm_area_struct *vma,
181
+
unsigned long *start, unsigned long *end)
182
+
{
173
183
}
174
184
175
185
#define follow_hugetlb_page(m,v,p,vs,a,b,i,w,n) ({ BUG(); 0; })
+6
include/linux/mm.h
+6
include/linux/mm.h
···
2455
2455
return vma;
2456
2456
}
2457
2457
2458
+
static inline bool range_in_vma(struct vm_area_struct *vma,
2459
+
unsigned long start, unsigned long end)
2460
+
{
2461
+
return (vma && vma->vm_start <= start && end <= vma->vm_end);
2462
+
}
2463
+
2458
2464
#ifdef CONFIG_MMU
2459
2465
pgprot_t vm_get_page_prot(unsigned long vm_flags);
2460
2466
void vma_set_page_prot(struct vm_area_struct *vma);
+2
include/uapi/asm-generic/hugetlb_encode.h
+2
include/uapi/asm-generic/hugetlb_encode.h
···
26
26
#define HUGETLB_FLAG_ENCODE_2MB (21 << HUGETLB_FLAG_ENCODE_SHIFT)
27
27
#define HUGETLB_FLAG_ENCODE_8MB (23 << HUGETLB_FLAG_ENCODE_SHIFT)
28
28
#define HUGETLB_FLAG_ENCODE_16MB (24 << HUGETLB_FLAG_ENCODE_SHIFT)
29
+
#define HUGETLB_FLAG_ENCODE_32MB (25 << HUGETLB_FLAG_ENCODE_SHIFT)
29
30
#define HUGETLB_FLAG_ENCODE_256MB (28 << HUGETLB_FLAG_ENCODE_SHIFT)
31
+
#define HUGETLB_FLAG_ENCODE_512MB (29 << HUGETLB_FLAG_ENCODE_SHIFT)
30
32
#define HUGETLB_FLAG_ENCODE_1GB (30 << HUGETLB_FLAG_ENCODE_SHIFT)
31
33
#define HUGETLB_FLAG_ENCODE_2GB (31 << HUGETLB_FLAG_ENCODE_SHIFT)
32
34
#define HUGETLB_FLAG_ENCODE_16GB (34 << HUGETLB_FLAG_ENCODE_SHIFT)
+2
include/uapi/linux/memfd.h
+2
include/uapi/linux/memfd.h
···
25
25
#define MFD_HUGE_2MB HUGETLB_FLAG_ENCODE_2MB
26
26
#define MFD_HUGE_8MB HUGETLB_FLAG_ENCODE_8MB
27
27
#define MFD_HUGE_16MB HUGETLB_FLAG_ENCODE_16MB
28
+
#define MFD_HUGE_32MB HUGETLB_FLAG_ENCODE_32MB
28
29
#define MFD_HUGE_256MB HUGETLB_FLAG_ENCODE_256MB
30
+
#define MFD_HUGE_512MB HUGETLB_FLAG_ENCODE_512MB
29
31
#define MFD_HUGE_1GB HUGETLB_FLAG_ENCODE_1GB
30
32
#define MFD_HUGE_2GB HUGETLB_FLAG_ENCODE_2GB
31
33
#define MFD_HUGE_16GB HUGETLB_FLAG_ENCODE_16GB
+2
include/uapi/linux/mman.h
+2
include/uapi/linux/mman.h
···
28
28
#define MAP_HUGE_2MB HUGETLB_FLAG_ENCODE_2MB
29
29
#define MAP_HUGE_8MB HUGETLB_FLAG_ENCODE_8MB
30
30
#define MAP_HUGE_16MB HUGETLB_FLAG_ENCODE_16MB
31
+
#define MAP_HUGE_32MB HUGETLB_FLAG_ENCODE_32MB
31
32
#define MAP_HUGE_256MB HUGETLB_FLAG_ENCODE_256MB
33
+
#define MAP_HUGE_512MB HUGETLB_FLAG_ENCODE_512MB
32
34
#define MAP_HUGE_1GB HUGETLB_FLAG_ENCODE_1GB
33
35
#define MAP_HUGE_2GB HUGETLB_FLAG_ENCODE_2GB
34
36
#define MAP_HUGE_16GB HUGETLB_FLAG_ENCODE_16GB
+2
include/uapi/linux/shm.h
+2
include/uapi/linux/shm.h
···
65
65
#define SHM_HUGE_2MB HUGETLB_FLAG_ENCODE_2MB
66
66
#define SHM_HUGE_8MB HUGETLB_FLAG_ENCODE_8MB
67
67
#define SHM_HUGE_16MB HUGETLB_FLAG_ENCODE_16MB
68
+
#define SHM_HUGE_32MB HUGETLB_FLAG_ENCODE_32MB
68
69
#define SHM_HUGE_256MB HUGETLB_FLAG_ENCODE_256MB
70
+
#define SHM_HUGE_512MB HUGETLB_FLAG_ENCODE_512MB
69
71
#define SHM_HUGE_1GB HUGETLB_FLAG_ENCODE_1GB
70
72
#define SHM_HUGE_2GB HUGETLB_FLAG_ENCODE_2GB
71
73
#define SHM_HUGE_16GB HUGETLB_FLAG_ENCODE_16GB
+1
-1
ipc/shm.c
+1
-1
ipc/shm.c
+2
-1
mm/gup_benchmark.c
+2
-1
mm/gup_benchmark.c
+1
-1
mm/huge_memory.c
+1
-1
mm/huge_memory.c
···
2931
2931
else
2932
2932
page_add_file_rmap(new, true);
2933
2933
set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);
2934
-
if (vma->vm_flags & VM_LOCKED)
2934
+
if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new))
2935
2935
mlock_vma_page(new);
2936
2936
update_mmu_cache_pmd(vma, address, pvmw->pmd);
2937
2937
}
+79
-11
mm/hugetlb.c
+79
-11
mm/hugetlb.c
···
3326
3326
struct page *page;
3327
3327
struct hstate *h = hstate_vma(vma);
3328
3328
unsigned long sz = huge_page_size(h);
3329
-
const unsigned long mmun_start = start; /* For mmu_notifiers */
3330
-
const unsigned long mmun_end = end; /* For mmu_notifiers */
3329
+
unsigned long mmun_start = start; /* For mmu_notifiers */
3330
+
unsigned long mmun_end = end; /* For mmu_notifiers */
3331
3331
3332
3332
WARN_ON(!is_vm_hugetlb_page(vma));
3333
3333
BUG_ON(start & ~huge_page_mask(h));
···
3339
3339
*/
3340
3340
tlb_remove_check_page_size_change(tlb, sz);
3341
3341
tlb_start_vma(tlb, vma);
3342
+
3343
+
/*
3344
+
* If sharing possible, alert mmu notifiers of worst case.
3345
+
*/
3346
+
adjust_range_if_pmd_sharing_possible(vma, &mmun_start, &mmun_end);
3342
3347
mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3343
3348
address = start;
3344
3349
for (; address < end; address += sz) {
···
3354
3349
ptl = huge_pte_lock(h, mm, ptep);
3355
3350
if (huge_pmd_unshare(mm, &address, ptep)) {
3356
3351
spin_unlock(ptl);
3352
+
/*
3353
+
* We just unmapped a page of PMDs by clearing a PUD.
3354
+
* The caller's TLB flush range should cover this area.
3355
+
*/
3357
3356
continue;
3358
3357
}
3359
3358
···
3440
3431
{
3441
3432
struct mm_struct *mm;
3442
3433
struct mmu_gather tlb;
3434
+
unsigned long tlb_start = start;
3435
+
unsigned long tlb_end = end;
3436
+
3437
+
/*
3438
+
* If shared PMDs were possibly used within this vma range, adjust
3439
+
* start/end for worst case tlb flushing.
3440
+
* Note that we can not be sure if PMDs are shared until we try to
3441
+
* unmap pages. However, we want to make sure TLB flushing covers
3442
+
* the largest possible range.
3443
+
*/
3444
+
adjust_range_if_pmd_sharing_possible(vma, &tlb_start, &tlb_end);
3443
3445
3444
3446
mm = vma->vm_mm;
3445
3447
3446
-
tlb_gather_mmu(&tlb, mm, start, end);
3448
+
tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end);
3447
3449
__unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3448
-
tlb_finish_mmu(&tlb, start, end);
3450
+
tlb_finish_mmu(&tlb, tlb_start, tlb_end);
3449
3451
}
3450
3452
3451
3453
/*
···
4318
4298
pte_t pte;
4319
4299
struct hstate *h = hstate_vma(vma);
4320
4300
unsigned long pages = 0;
4301
+
unsigned long f_start = start;
4302
+
unsigned long f_end = end;
4303
+
bool shared_pmd = false;
4304
+
4305
+
/*
4306
+
* In the case of shared PMDs, the area to flush could be beyond
4307
+
* start/end. Set f_start/f_end to cover the maximum possible
4308
+
* range if PMD sharing is possible.
4309
+
*/
4310
+
adjust_range_if_pmd_sharing_possible(vma, &f_start, &f_end);
4321
4311
4322
4312
BUG_ON(address >= end);
4323
-
flush_cache_range(vma, address, end);
4313
+
flush_cache_range(vma, f_start, f_end);
4324
4314
4325
-
mmu_notifier_invalidate_range_start(mm, start, end);
4315
+
mmu_notifier_invalidate_range_start(mm, f_start, f_end);
4326
4316
i_mmap_lock_write(vma->vm_file->f_mapping);
4327
4317
for (; address < end; address += huge_page_size(h)) {
4328
4318
spinlock_t *ptl;
···
4343
4313
if (huge_pmd_unshare(mm, &address, ptep)) {
4344
4314
pages++;
4345
4315
spin_unlock(ptl);
4316
+
shared_pmd = true;
4346
4317
continue;
4347
4318
}
4348
4319
pte = huge_ptep_get(ptep);
···
4379
4348
* Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4380
4349
* may have cleared our pud entry and done put_page on the page table:
4381
4350
* once we release i_mmap_rwsem, another task can do the final put_page
4382
-
* and that page table be reused and filled with junk.
4351
+
* and that page table be reused and filled with junk. If we actually
4352
+
* did unshare a page of pmds, flush the range corresponding to the pud.
4383
4353
*/
4384
-
flush_hugetlb_tlb_range(vma, start, end);
4354
+
if (shared_pmd)
4355
+
flush_hugetlb_tlb_range(vma, f_start, f_end);
4356
+
else
4357
+
flush_hugetlb_tlb_range(vma, start, end);
4385
4358
/*
4386
4359
* No need to call mmu_notifier_invalidate_range() we are downgrading
4387
4360
* page table protection not changing it to point to a new page.
···
4393
4358
* See Documentation/vm/mmu_notifier.rst
4394
4359
*/
4395
4360
i_mmap_unlock_write(vma->vm_file->f_mapping);
4396
-
mmu_notifier_invalidate_range_end(mm, start, end);
4361
+
mmu_notifier_invalidate_range_end(mm, f_start, f_end);
4397
4362
4398
4363
return pages << h->order;
4399
4364
}
···
4580
4545
/*
4581
4546
* check on proper vm_flags and page table alignment
4582
4547
*/
4583
-
if (vma->vm_flags & VM_MAYSHARE &&
4584
-
vma->vm_start <= base && end <= vma->vm_end)
4548
+
if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
4585
4549
return true;
4586
4550
return false;
4551
+
}
4552
+
4553
+
/*
4554
+
* Determine if start,end range within vma could be mapped by shared pmd.
4555
+
* If yes, adjust start and end to cover range associated with possible
4556
+
* shared pmd mappings.
4557
+
*/
4558
+
void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
4559
+
unsigned long *start, unsigned long *end)
4560
+
{
4561
+
unsigned long check_addr = *start;
4562
+
4563
+
if (!(vma->vm_flags & VM_MAYSHARE))
4564
+
return;
4565
+
4566
+
for (check_addr = *start; check_addr < *end; check_addr += PUD_SIZE) {
4567
+
unsigned long a_start = check_addr & PUD_MASK;
4568
+
unsigned long a_end = a_start + PUD_SIZE;
4569
+
4570
+
/*
4571
+
* If sharing is possible, adjust start/end if necessary.
4572
+
*/
4573
+
if (range_in_vma(vma, a_start, a_end)) {
4574
+
if (a_start < *start)
4575
+
*start = a_start;
4576
+
if (a_end > *end)
4577
+
*end = a_end;
4578
+
}
4579
+
}
4587
4580
}
4588
4581
4589
4582
/*
···
4710
4647
int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4711
4648
{
4712
4649
return 0;
4650
+
}
4651
+
4652
+
void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
4653
+
unsigned long *start, unsigned long *end)
4654
+
{
4713
4655
}
4714
4656
#define want_pmd_share() (0)
4715
4657
#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
+1
-1
mm/madvise.c
+1
-1
mm/madvise.c
+4
-1
mm/migrate.c
+4
-1
mm/migrate.c
···
275
275
if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
276
276
mlock_vma_page(new);
277
277
278
+
if (PageTransHuge(page) && PageMlocked(page))
279
+
clear_page_mlock(page);
280
+
278
281
/* No need to invalidate - it was non-present before */
279
282
update_mmu_cache(vma, pvmw.address, pvmw.pte);
280
283
}
···
1414
1411
* we encounter them after the rest of the list
1415
1412
* is processed.
1416
1413
*/
1417
-
if (PageTransHuge(page)) {
1414
+
if (PageTransHuge(page) && !PageHuge(page)) {
1418
1415
lock_page(page);
1419
1416
rc = split_huge_page_to_list(page, from);
1420
1417
unlock_page(page);
+39
-3
mm/rmap.c
+39
-3
mm/rmap.c
···
1362
1362
}
1363
1363
1364
1364
/*
1365
-
* We have to assume the worse case ie pmd for invalidation. Note that
1366
-
* the page can not be free in this function as call of try_to_unmap()
1367
-
* must hold a reference on the page.
1365
+
* For THP, we have to assume the worse case ie pmd for invalidation.
1366
+
* For hugetlb, it could be much worse if we need to do pud
1367
+
* invalidation in the case of pmd sharing.
1368
+
*
1369
+
* Note that the page can not be free in this function as call of
1370
+
* try_to_unmap() must hold a reference on the page.
1368
1371
*/
1369
1372
end = min(vma->vm_end, start + (PAGE_SIZE << compound_order(page)));
1373
+
if (PageHuge(page)) {
1374
+
/*
1375
+
* If sharing is possible, start and end will be adjusted
1376
+
* accordingly.
1377
+
*/
1378
+
adjust_range_if_pmd_sharing_possible(vma, &start, &end);
1379
+
}
1370
1380
mmu_notifier_invalidate_range_start(vma->vm_mm, start, end);
1371
1381
1372
1382
while (page_vma_mapped_walk(&pvmw)) {
···
1419
1409
subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
1420
1410
address = pvmw.address;
1421
1411
1412
+
if (PageHuge(page)) {
1413
+
if (huge_pmd_unshare(mm, &address, pvmw.pte)) {
1414
+
/*
1415
+
* huge_pmd_unshare unmapped an entire PMD
1416
+
* page. There is no way of knowing exactly
1417
+
* which PMDs may be cached for this mm, so
1418
+
* we must flush them all. start/end were
1419
+
* already adjusted above to cover this range.
1420
+
*/
1421
+
flush_cache_range(vma, start, end);
1422
+
flush_tlb_range(vma, start, end);
1423
+
mmu_notifier_invalidate_range(mm, start, end);
1424
+
1425
+
/*
1426
+
* The ref count of the PMD page was dropped
1427
+
* which is part of the way map counting
1428
+
* is done for shared PMDs. Return 'true'
1429
+
* here. When there is no other sharing,
1430
+
* huge_pmd_unshare returns false and we will
1431
+
* unmap the actual page and drop map count
1432
+
* to zero.
1433
+
*/
1434
+
page_vma_mapped_walk_done(&pvmw);
1435
+
break;
1436
+
}
1437
+
}
1422
1438
1423
1439
if (IS_ENABLED(CONFIG_MIGRATION) &&
1424
1440
(flags & TTU_MIGRATION) &&
+3
-4
mm/vmscan.c
+3
-4
mm/vmscan.c
···
580
580
struct mem_cgroup *memcg, int priority)
581
581
{
582
582
struct memcg_shrinker_map *map;
583
-
unsigned long freed = 0;
584
-
int ret, i;
583
+
unsigned long ret, freed = 0;
584
+
int i;
585
585
586
586
if (!memcg_kmem_enabled() || !mem_cgroup_online(memcg))
587
587
return 0;
···
677
677
struct mem_cgroup *memcg,
678
678
int priority)
679
679
{
680
+
unsigned long ret, freed = 0;
680
681
struct shrinker *shrinker;
681
-
unsigned long freed = 0;
682
-
int ret;
683
682
684
683
if (!mem_cgroup_is_root(memcg))
685
684
return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
+3
-1
mm/vmstat.c
+3
-1
mm/vmstat.c
···
1275
1275
#ifdef CONFIG_SMP
1276
1276
"nr_tlb_remote_flush",
1277
1277
"nr_tlb_remote_flush_received",
1278
+
#else
1279
+
"", /* nr_tlb_remote_flush */
1280
+
"", /* nr_tlb_remote_flush_received */
1278
1281
#endif /* CONFIG_SMP */
1279
1282
"nr_tlb_local_flush_all",
1280
1283
"nr_tlb_local_flush_one",
···
1286
1283
#ifdef CONFIG_DEBUG_VM_VMACACHE
1287
1284
"vmacache_find_calls",
1288
1285
"vmacache_find_hits",
1289
-
"vmacache_full_flushes",
1290
1286
#endif
1291
1287
#ifdef CONFIG_SWAP
1292
1288
"swap_ra",