Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'mm-hotfixes-stable-2024-08-17-19-34' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull misc fixes from Andrew Morton:
"16 hotfixes. All except one are for MM. 10 of these are cc:stable and
the others pertain to post-6.10 issues.
As usual with these merges, singletons and doubletons all over the
place, no identifiable-by-me theme. Please see the lovingly curated
changelogs to get the skinny"
* tag 'mm-hotfixes-stable-2024-08-17-19-34' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm:
mm/migrate: fix deadlock in migrate_pages_batch() on large folios
alloc_tag: mark pages reserved during CMA activation as not tagged
alloc_tag: introduce clear_page_tag_ref() helper function
crash: fix riscv64 crash memory reserve dead loop
selftests: memfd_secret: don't build memfd_secret test on unsupported arches
mm: fix endless reclaim on machines with unaccepted memory
selftests/mm: compaction_test: fix off by one in check_compaction()
mm/numa: no task_numa_fault() call if PMD is changed
mm/numa: no task_numa_fault() call if PTE is changed
mm/vmalloc: fix page mapping if vm_area_alloc_pages() with high order fallback to order 0
mm/memory-failure: use raw_spinlock_t in struct memory_failure_cpu
mm: don't account memmap per-node
mm: add system wide stats items category
mm: don't account memmap on failure
mm/hugetlb: fix hugetlb vs. core-mm PT locking
mseal: fix is_madv_discard()
+201
-182
+30
-3
include/linux/hugetlb.h
+30
-3
include/linux/hugetlb.h
···
944
944
static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
945
945
struct mm_struct *mm, pte_t *pte)
946
946
{
947
-
if (huge_page_size(h) == PMD_SIZE)
947
+
const unsigned long size = huge_page_size(h);
948
+
949
+
VM_WARN_ON(size == PAGE_SIZE);
950
+
951
+
/*
952
+
* hugetlb must use the exact same PT locks as core-mm page table
953
+
* walkers would. When modifying a PTE table, hugetlb must take the
954
+
* PTE PT lock, when modifying a PMD table, hugetlb must take the PMD
955
+
* PT lock etc.
956
+
*
957
+
* The expectation is that any hugetlb folio smaller than a PMD is
958
+
* always mapped into a single PTE table and that any hugetlb folio
959
+
* smaller than a PUD (but at least as big as a PMD) is always mapped
960
+
* into a single PMD table.
961
+
*
962
+
* If that does not hold for an architecture, then that architecture
963
+
* must disable split PT locks such that all *_lockptr() functions
964
+
* will give us the same result: the per-MM PT lock.
965
+
*
966
+
* Note that with e.g., CONFIG_PGTABLE_LEVELS=2 where
967
+
* PGDIR_SIZE==P4D_SIZE==PUD_SIZE==PMD_SIZE, we'd use pud_lockptr()
968
+
* and core-mm would use pmd_lockptr(). However, in such configurations
969
+
* split PMD locks are disabled -- they don't make sense on a single
970
+
* PGDIR page table -- and the end result is the same.
971
+
*/
972
+
if (size >= PUD_SIZE)
973
+
return pud_lockptr(mm, (pud_t *) pte);
974
+
else if (size >= PMD_SIZE || IS_ENABLED(CONFIG_HIGHPTE))
948
975
return pmd_lockptr(mm, (pmd_t *) pte);
949
-
VM_BUG_ON(huge_page_size(h) == PAGE_SIZE);
950
-
return &mm->page_table_lock;
976
+
/* pte_alloc_huge() only applies with !CONFIG_HIGHPTE */
977
+
return ptep_lockptr(mm, pte);
951
978
}
952
979
953
980
#ifndef hugepages_supported
+11
include/linux/mm.h
+11
include/linux/mm.h
···
2920
2920
return ptlock_ptr(page_ptdesc(pmd_page(*pmd)));
2921
2921
}
2922
2922
2923
+
static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte)
2924
+
{
2925
+
BUILD_BUG_ON(IS_ENABLED(CONFIG_HIGHPTE));
2926
+
BUILD_BUG_ON(MAX_PTRS_PER_PTE * sizeof(pte_t) > PAGE_SIZE);
2927
+
return ptlock_ptr(virt_to_ptdesc(pte));
2928
+
}
2929
+
2923
2930
static inline bool ptlock_init(struct ptdesc *ptdesc)
2924
2931
{
2925
2932
/*
···
2948
2941
* We use mm->page_table_lock to guard all pagetable pages of the mm.
2949
2942
*/
2950
2943
static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
2944
+
{
2945
+
return &mm->page_table_lock;
2946
+
}
2947
+
static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte)
2951
2948
{
2952
2949
return &mm->page_table_lock;
2953
2950
}
-2
include/linux/mmzone.h
-2
include/linux/mmzone.h
+13
include/linux/pgalloc_tag.h
+13
include/linux/pgalloc_tag.h
···
43
43
page_ext_put(page_ext_from_codetag_ref(ref));
44
44
}
45
45
46
+
static inline void clear_page_tag_ref(struct page *page)
47
+
{
48
+
if (mem_alloc_profiling_enabled()) {
49
+
union codetag_ref *ref = get_page_tag_ref(page);
50
+
51
+
if (ref) {
52
+
set_codetag_empty(ref);
53
+
put_page_tag_ref(ref);
54
+
}
55
+
}
56
+
}
57
+
46
58
static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
47
59
unsigned int nr)
48
60
{
···
138
126
139
127
static inline union codetag_ref *get_page_tag_ref(struct page *page) { return NULL; }
140
128
static inline void put_page_tag_ref(union codetag_ref *ref) {}
129
+
static inline void clear_page_tag_ref(struct page *page) {}
141
130
static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
142
131
unsigned int nr) {}
143
132
static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
+8
-14
include/linux/vmstat.h
+8
-14
include/linux/vmstat.h
···
34
34
unsigned nr_lazyfree_fail;
35
35
};
36
36
37
-
enum writeback_stat_item {
37
+
/* Stat data for system wide items */
38
+
enum vm_stat_item {
38
39
NR_DIRTY_THRESHOLD,
39
40
NR_DIRTY_BG_THRESHOLD,
40
-
NR_VM_WRITEBACK_STAT_ITEMS,
41
+
NR_MEMMAP_PAGES, /* page metadata allocated through buddy allocator */
42
+
NR_MEMMAP_BOOT_PAGES, /* page metadata allocated through boot allocator */
43
+
NR_VM_STAT_ITEMS,
41
44
};
42
45
43
46
#ifdef CONFIG_VM_EVENT_COUNTERS
···
517
514
return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
518
515
}
519
516
520
-
static inline const char *writeback_stat_name(enum writeback_stat_item item)
521
-
{
522
-
return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
523
-
NR_VM_NUMA_EVENT_ITEMS +
524
-
NR_VM_NODE_STAT_ITEMS +
525
-
item];
526
-
}
527
-
528
517
#if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
529
518
static inline const char *vm_event_name(enum vm_event_item item)
530
519
{
531
520
return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
532
521
NR_VM_NUMA_EVENT_ITEMS +
533
522
NR_VM_NODE_STAT_ITEMS +
534
-
NR_VM_WRITEBACK_STAT_ITEMS +
523
+
NR_VM_STAT_ITEMS +
535
524
item];
536
525
}
537
526
#endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
···
620
625
lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio));
621
626
}
622
627
623
-
void __meminit mod_node_early_perpage_metadata(int nid, long delta);
624
-
void __meminit store_early_perpage_metadata(void);
625
-
628
+
void memmap_boot_pages_add(long delta);
629
+
void memmap_pages_add(long delta);
626
630
#endif /* _LINUX_VMSTAT_H */
+2
-1
kernel/crash_reserve.c
+2
-1
kernel/crash_reserve.c
···
423
423
if (high && search_end == CRASH_ADDR_HIGH_MAX) {
424
424
search_end = CRASH_ADDR_LOW_MAX;
425
425
search_base = 0;
426
-
goto retry;
426
+
if (search_end != CRASH_ADDR_HIGH_MAX)
427
+
goto retry;
427
428
}
428
429
pr_warn("cannot allocate crashkernel (size:0x%llx)\n",
429
430
crash_size);
+13
-16
mm/huge_memory.c
+13
-16
mm/huge_memory.c
···
1685
1685
vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1686
1686
if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {
1687
1687
spin_unlock(vmf->ptl);
1688
-
goto out;
1688
+
return 0;
1689
1689
}
1690
1690
1691
1691
pmd = pmd_modify(oldpmd, vma->vm_page_prot);
···
1728
1728
if (!migrate_misplaced_folio(folio, vma, target_nid)) {
1729
1729
flags |= TNF_MIGRATED;
1730
1730
nid = target_nid;
1731
-
} else {
1732
-
flags |= TNF_MIGRATE_FAIL;
1733
-
vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1734
-
if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {
1735
-
spin_unlock(vmf->ptl);
1736
-
goto out;
1737
-
}
1738
-
goto out_map;
1731
+
task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags);
1732
+
return 0;
1739
1733
}
1740
1734
1741
-
out:
1742
-
if (nid != NUMA_NO_NODE)
1743
-
task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags);
1744
-
1745
-
return 0;
1746
-
1735
+
flags |= TNF_MIGRATE_FAIL;
1736
+
vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1737
+
if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {
1738
+
spin_unlock(vmf->ptl);
1739
+
return 0;
1740
+
}
1747
1741
out_map:
1748
1742
/* Restore the PMD */
1749
1743
pmd = pmd_modify(oldpmd, vma->vm_page_prot);
···
1747
1753
set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
1748
1754
update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1749
1755
spin_unlock(vmf->ptl);
1750
-
goto out;
1756
+
1757
+
if (nid != NUMA_NO_NODE)
1758
+
task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags);
1759
+
return 0;
1751
1760
}
1752
1761
1753
1762
/*
+5
-8
mm/hugetlb_vmemmap.c
+5
-8
mm/hugetlb_vmemmap.c
···
185
185
static inline void free_vmemmap_page(struct page *page)
186
186
{
187
187
if (PageReserved(page)) {
188
+
memmap_boot_pages_add(-1);
188
189
free_bootmem_page(page);
189
-
mod_node_page_state(page_pgdat(page), NR_MEMMAP_BOOT, -1);
190
190
} else {
191
+
memmap_pages_add(-1);
191
192
__free_page(page);
192
-
mod_node_page_state(page_pgdat(page), NR_MEMMAP, -1);
193
193
}
194
194
}
195
195
···
341
341
copy_page(page_to_virt(walk.reuse_page),
342
342
(void *)walk.reuse_addr);
343
343
list_add(&walk.reuse_page->lru, vmemmap_pages);
344
-
mod_node_page_state(NODE_DATA(nid), NR_MEMMAP, 1);
344
+
memmap_pages_add(1);
345
345
}
346
346
347
347
/*
···
392
392
393
393
for (i = 0; i < nr_pages; i++) {
394
394
page = alloc_pages_node(nid, gfp_mask, 0);
395
-
if (!page) {
396
-
mod_node_page_state(NODE_DATA(nid), NR_MEMMAP, i);
395
+
if (!page)
397
396
goto out;
398
-
}
399
397
list_add(&page->lru, list);
400
398
}
401
-
402
-
mod_node_page_state(NODE_DATA(nid), NR_MEMMAP, nr_pages);
399
+
memmap_pages_add(nr_pages);
403
400
404
401
return 0;
405
402
out:
+11
-9
mm/memory-failure.c
+11
-9
mm/memory-failure.c
···
2417
2417
struct memory_failure_cpu {
2418
2418
DECLARE_KFIFO(fifo, struct memory_failure_entry,
2419
2419
MEMORY_FAILURE_FIFO_SIZE);
2420
-
spinlock_t lock;
2420
+
raw_spinlock_t lock;
2421
2421
struct work_struct work;
2422
2422
};
2423
2423
···
2443
2443
{
2444
2444
struct memory_failure_cpu *mf_cpu;
2445
2445
unsigned long proc_flags;
2446
+
bool buffer_overflow;
2446
2447
struct memory_failure_entry entry = {
2447
2448
.pfn = pfn,
2448
2449
.flags = flags,
2449
2450
};
2450
2451
2451
2452
mf_cpu = &get_cpu_var(memory_failure_cpu);
2452
-
spin_lock_irqsave(&mf_cpu->lock, proc_flags);
2453
-
if (kfifo_put(&mf_cpu->fifo, entry))
2453
+
raw_spin_lock_irqsave(&mf_cpu->lock, proc_flags);
2454
+
buffer_overflow = !kfifo_put(&mf_cpu->fifo, entry);
2455
+
if (!buffer_overflow)
2454
2456
schedule_work_on(smp_processor_id(), &mf_cpu->work);
2455
-
else
2457
+
raw_spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
2458
+
put_cpu_var(memory_failure_cpu);
2459
+
if (buffer_overflow)
2456
2460
pr_err("buffer overflow when queuing memory failure at %#lx\n",
2457
2461
pfn);
2458
-
spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
2459
-
put_cpu_var(memory_failure_cpu);
2460
2462
}
2461
2463
EXPORT_SYMBOL_GPL(memory_failure_queue);
2462
2464
···
2471
2469
2472
2470
mf_cpu = container_of(work, struct memory_failure_cpu, work);
2473
2471
for (;;) {
2474
-
spin_lock_irqsave(&mf_cpu->lock, proc_flags);
2472
+
raw_spin_lock_irqsave(&mf_cpu->lock, proc_flags);
2475
2473
gotten = kfifo_get(&mf_cpu->fifo, &entry);
2476
-
spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
2474
+
raw_spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
2477
2475
if (!gotten)
2478
2476
break;
2479
2477
if (entry.flags & MF_SOFT_OFFLINE)
···
2503
2501
2504
2502
for_each_possible_cpu(cpu) {
2505
2503
mf_cpu = &per_cpu(memory_failure_cpu, cpu);
2506
-
spin_lock_init(&mf_cpu->lock);
2504
+
raw_spin_lock_init(&mf_cpu->lock);
2507
2505
INIT_KFIFO(mf_cpu->fifo);
2508
2506
INIT_WORK(&mf_cpu->work, memory_failure_work_func);
2509
2507
}
+16
-17
mm/memory.c
+16
-17
mm/memory.c
···
5295
5295
5296
5296
if (unlikely(!pte_same(old_pte, vmf->orig_pte))) {
5297
5297
pte_unmap_unlock(vmf->pte, vmf->ptl);
5298
-
goto out;
5298
+
return 0;
5299
5299
}
5300
5300
5301
5301
pte = pte_modify(old_pte, vma->vm_page_prot);
···
5358
5358
if (!migrate_misplaced_folio(folio, vma, target_nid)) {
5359
5359
nid = target_nid;
5360
5360
flags |= TNF_MIGRATED;
5361
-
} else {
5362
-
flags |= TNF_MIGRATE_FAIL;
5363
-
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
5364
-
vmf->address, &vmf->ptl);
5365
-
if (unlikely(!vmf->pte))
5366
-
goto out;
5367
-
if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
5368
-
pte_unmap_unlock(vmf->pte, vmf->ptl);
5369
-
goto out;
5370
-
}
5371
-
goto out_map;
5361
+
task_numa_fault(last_cpupid, nid, nr_pages, flags);
5362
+
return 0;
5372
5363
}
5373
5364
5374
-
out:
5375
-
if (nid != NUMA_NO_NODE)
5376
-
task_numa_fault(last_cpupid, nid, nr_pages, flags);
5377
-
return 0;
5365
+
flags |= TNF_MIGRATE_FAIL;
5366
+
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
5367
+
vmf->address, &vmf->ptl);
5368
+
if (unlikely(!vmf->pte))
5369
+
return 0;
5370
+
if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
5371
+
pte_unmap_unlock(vmf->pte, vmf->ptl);
5372
+
return 0;
5373
+
}
5378
5374
out_map:
5379
5375
/*
5380
5376
* Make it present again, depending on how arch implements
···
5383
5387
numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte,
5384
5388
writable);
5385
5389
pte_unmap_unlock(vmf->pte, vmf->ptl);
5386
-
goto out;
5390
+
5391
+
if (nid != NUMA_NO_NODE)
5392
+
task_numa_fault(last_cpupid, nid, nr_pages, flags);
5393
+
return 0;
5387
5394
}
5388
5395
5389
5396
static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
+11
-5
mm/migrate.c
+11
-5
mm/migrate.c
···
1479
1479
return rc;
1480
1480
}
1481
1481
1482
-
static inline int try_split_folio(struct folio *folio, struct list_head *split_folios)
1482
+
static inline int try_split_folio(struct folio *folio, struct list_head *split_folios,
1483
+
enum migrate_mode mode)
1483
1484
{
1484
1485
int rc;
1485
1486
1486
-
folio_lock(folio);
1487
+
if (mode == MIGRATE_ASYNC) {
1488
+
if (!folio_trylock(folio))
1489
+
return -EAGAIN;
1490
+
} else {
1491
+
folio_lock(folio);
1492
+
}
1487
1493
rc = split_folio_to_list(folio, split_folios);
1488
1494
folio_unlock(folio);
1489
1495
if (!rc)
···
1683
1677
*/
1684
1678
if (nr_pages > 2 &&
1685
1679
!list_empty(&folio->_deferred_list)) {
1686
-
if (try_split_folio(folio, split_folios) == 0) {
1680
+
if (!try_split_folio(folio, split_folios, mode)) {
1687
1681
nr_failed++;
1688
1682
stats->nr_thp_failed += is_thp;
1689
1683
stats->nr_thp_split += is_thp;
···
1705
1699
if (!thp_migration_supported() && is_thp) {
1706
1700
nr_failed++;
1707
1701
stats->nr_thp_failed++;
1708
-
if (!try_split_folio(folio, split_folios)) {
1702
+
if (!try_split_folio(folio, split_folios, mode)) {
1709
1703
stats->nr_thp_split++;
1710
1704
stats->nr_split++;
1711
1705
continue;
···
1737
1731
stats->nr_thp_failed += is_thp;
1738
1732
/* Large folio NUMA faulting doesn't split to retry. */
1739
1733
if (is_large && !nosplit) {
1740
-
int ret = try_split_folio(folio, split_folios);
1734
+
int ret = try_split_folio(folio, split_folios, mode);
1741
1735
1742
1736
if (!ret) {
1743
1737
stats->nr_thp_split += is_thp;
+4
-11
mm/mm_init.c
+4
-11
mm/mm_init.c
···
1623
1623
panic("Failed to allocate %ld bytes for node %d memory map\n",
1624
1624
size, pgdat->node_id);
1625
1625
pgdat->node_mem_map = map + offset;
1626
-
mod_node_early_perpage_metadata(pgdat->node_id,
1627
-
DIV_ROUND_UP(size, PAGE_SIZE));
1626
+
memmap_boot_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
1628
1627
pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
1629
1628
__func__, pgdat->node_id, (unsigned long)pgdat,
1630
1629
(unsigned long)pgdat->node_mem_map);
···
2244
2245
2245
2246
set_pageblock_migratetype(page, MIGRATE_CMA);
2246
2247
set_page_refcounted(page);
2248
+
/* pages were reserved and not allocated */
2249
+
clear_page_tag_ref(page);
2247
2250
__free_pages(page, pageblock_order);
2248
2251
2249
2252
adjust_managed_page_count(page, pageblock_nr_pages);
···
2461
2460
}
2462
2461
2463
2462
/* pages were reserved and not allocated */
2464
-
if (mem_alloc_profiling_enabled()) {
2465
-
union codetag_ref *ref = get_page_tag_ref(page);
2466
-
2467
-
if (ref) {
2468
-
set_codetag_empty(ref);
2469
-
put_page_tag_ref(ref);
2470
-
}
2471
-
}
2472
-
2463
+
clear_page_tag_ref(page);
2473
2464
__free_pages_core(page, order, MEMINIT_EARLY);
2474
2465
}
2475
2466
+11
-3
mm/mseal.c
+11
-3
mm/mseal.c
···
40
40
41
41
static bool is_madv_discard(int behavior)
42
42
{
43
-
return behavior &
44
-
(MADV_FREE | MADV_DONTNEED | MADV_DONTNEED_LOCKED |
45
-
MADV_REMOVE | MADV_DONTFORK | MADV_WIPEONFORK);
43
+
switch (behavior) {
44
+
case MADV_FREE:
45
+
case MADV_DONTNEED:
46
+
case MADV_DONTNEED_LOCKED:
47
+
case MADV_REMOVE:
48
+
case MADV_DONTFORK:
49
+
case MADV_WIPEONFORK:
50
+
return true;
51
+
}
52
+
53
+
return false;
46
54
}
47
55
48
56
static bool is_ro_anon(struct vm_area_struct *vma)
+21
-31
mm/page_alloc.c
+21
-31
mm/page_alloc.c
···
287
287
288
288
static bool page_contains_unaccepted(struct page *page, unsigned int order);
289
289
static void accept_page(struct page *page, unsigned int order);
290
-
static bool try_to_accept_memory(struct zone *zone, unsigned int order);
290
+
static bool cond_accept_memory(struct zone *zone, unsigned int order);
291
291
static inline bool has_unaccepted_memory(void);
292
292
static bool __free_unaccepted(struct page *page);
293
293
···
3072
3072
if (!(alloc_flags & ALLOC_CMA))
3073
3073
unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
3074
3074
#endif
3075
-
#ifdef CONFIG_UNACCEPTED_MEMORY
3076
-
unusable_free += zone_page_state(z, NR_UNACCEPTED);
3077
-
#endif
3078
3075
3079
3076
return unusable_free;
3080
3077
}
···
3365
3368
}
3366
3369
}
3367
3370
3371
+
cond_accept_memory(zone, order);
3372
+
3368
3373
/*
3369
3374
* Detect whether the number of free pages is below high
3370
3375
* watermark. If so, we will decrease pcp->high and free
···
3392
3393
gfp_mask)) {
3393
3394
int ret;
3394
3395
3395
-
if (has_unaccepted_memory()) {
3396
-
if (try_to_accept_memory(zone, order))
3397
-
goto try_this_zone;
3398
-
}
3396
+
if (cond_accept_memory(zone, order))
3397
+
goto try_this_zone;
3399
3398
3400
3399
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3401
3400
/*
···
3447
3450
3448
3451
return page;
3449
3452
} else {
3450
-
if (has_unaccepted_memory()) {
3451
-
if (try_to_accept_memory(zone, order))
3452
-
goto try_this_zone;
3453
-
}
3453
+
if (cond_accept_memory(zone, order))
3454
+
goto try_this_zone;
3454
3455
3455
3456
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3456
3457
/* Try again if zone has deferred pages */
···
5750
5755
for_each_online_pgdat(pgdat)
5751
5756
pgdat->per_cpu_nodestats =
5752
5757
alloc_percpu(struct per_cpu_nodestat);
5753
-
store_early_perpage_metadata();
5754
5758
}
5755
5759
5756
5760
__meminit void zone_pcp_init(struct zone *zone)
···
5815
5821
5816
5822
void free_reserved_page(struct page *page)
5817
5823
{
5818
-
if (mem_alloc_profiling_enabled()) {
5819
-
union codetag_ref *ref = get_page_tag_ref(page);
5820
-
5821
-
if (ref) {
5822
-
set_codetag_empty(ref);
5823
-
put_page_tag_ref(ref);
5824
-
}
5825
-
}
5824
+
clear_page_tag_ref(page);
5826
5825
ClearPageReserved(page);
5827
5826
init_page_count(page);
5828
5827
__free_page(page);
···
6938
6951
struct page *page;
6939
6952
bool last;
6940
6953
6941
-
if (list_empty(&zone->unaccepted_pages))
6942
-
return false;
6943
-
6944
6954
spin_lock_irqsave(&zone->lock, flags);
6945
6955
page = list_first_entry_or_null(&zone->unaccepted_pages,
6946
6956
struct page, lru);
···
6963
6979
return true;
6964
6980
}
6965
6981
6966
-
static bool try_to_accept_memory(struct zone *zone, unsigned int order)
6982
+
static bool cond_accept_memory(struct zone *zone, unsigned int order)
6967
6983
{
6968
6984
long to_accept;
6969
-
int ret = false;
6985
+
bool ret = false;
6986
+
6987
+
if (!has_unaccepted_memory())
6988
+
return false;
6989
+
6990
+
if (list_empty(&zone->unaccepted_pages))
6991
+
return false;
6970
6992
6971
6993
/* How much to accept to get to high watermark? */
6972
6994
to_accept = high_wmark_pages(zone) -
6973
6995
(zone_page_state(zone, NR_FREE_PAGES) -
6974
-
__zone_watermark_unusable_free(zone, order, 0));
6996
+
__zone_watermark_unusable_free(zone, order, 0) -
6997
+
zone_page_state(zone, NR_UNACCEPTED));
6975
6998
6976
-
/* Accept at least one page */
6977
-
do {
6999
+
while (to_accept > 0) {
6978
7000
if (!try_to_accept_memory_one(zone))
6979
7001
break;
6980
7002
ret = true;
6981
7003
to_accept -= MAX_ORDER_NR_PAGES;
6982
-
} while (to_accept > 0);
7004
+
}
6983
7005
6984
7006
return ret;
6985
7007
}
···
7028
7038
{
7029
7039
}
7030
7040
7031
-
static bool try_to_accept_memory(struct zone *zone, unsigned int order)
7041
+
static bool cond_accept_memory(struct zone *zone, unsigned int order)
7032
7042
{
7033
7043
return false;
7034
7044
}
+4
-14
mm/page_ext.c
+4
-14
mm/page_ext.c
···
214
214
return -ENOMEM;
215
215
NODE_DATA(nid)->node_page_ext = base;
216
216
total_usage += table_size;
217
-
mod_node_page_state(NODE_DATA(nid), NR_MEMMAP_BOOT,
218
-
DIV_ROUND_UP(table_size, PAGE_SIZE));
217
+
memmap_boot_pages_add(DIV_ROUND_UP(table_size, PAGE_SIZE));
219
218
return 0;
220
219
}
221
220
···
274
275
else
275
276
addr = vzalloc_node(size, nid);
276
277
277
-
if (addr) {
278
-
mod_node_page_state(NODE_DATA(nid), NR_MEMMAP,
279
-
DIV_ROUND_UP(size, PAGE_SIZE));
280
-
}
278
+
if (addr)
279
+
memmap_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
281
280
282
281
return addr;
283
282
}
···
320
323
{
321
324
size_t table_size;
322
325
struct page *page;
323
-
struct pglist_data *pgdat;
324
326
325
327
table_size = page_ext_size * PAGES_PER_SECTION;
328
+
memmap_pages_add(-1L * (DIV_ROUND_UP(table_size, PAGE_SIZE)));
326
329
327
330
if (is_vmalloc_addr(addr)) {
328
-
page = vmalloc_to_page(addr);
329
-
pgdat = page_pgdat(page);
330
331
vfree(addr);
331
332
} else {
332
333
page = virt_to_page(addr);
333
-
pgdat = page_pgdat(page);
334
334
BUG_ON(PageReserved(page));
335
335
kmemleak_free(addr);
336
336
free_pages_exact(addr, table_size);
337
337
}
338
-
339
-
mod_node_page_state(pgdat, NR_MEMMAP,
340
-
-1L * (DIV_ROUND_UP(table_size, PAGE_SIZE)));
341
-
342
338
}
343
339
344
340
static void __free_page_ext(unsigned long pfn)
+4
-7
mm/sparse-vmemmap.c
+4
-7
mm/sparse-vmemmap.c
···
469
469
if (r < 0)
470
470
return NULL;
471
471
472
-
if (system_state == SYSTEM_BOOTING) {
473
-
mod_node_early_perpage_metadata(nid, DIV_ROUND_UP(end - start,
474
-
PAGE_SIZE));
475
-
} else {
476
-
mod_node_page_state(NODE_DATA(nid), NR_MEMMAP,
477
-
DIV_ROUND_UP(end - start, PAGE_SIZE));
478
-
}
472
+
if (system_state == SYSTEM_BOOTING)
473
+
memmap_boot_pages_add(DIV_ROUND_UP(end - start, PAGE_SIZE));
474
+
else
475
+
memmap_pages_add(DIV_ROUND_UP(end - start, PAGE_SIZE));
479
476
480
477
return pfn_to_page(pfn);
481
478
}
+2
-3
mm/sparse.c
+2
-3
mm/sparse.c
···
463
463
sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true);
464
464
sparsemap_buf_end = sparsemap_buf + size;
465
465
#ifndef CONFIG_SPARSEMEM_VMEMMAP
466
-
mod_node_early_perpage_metadata(nid, DIV_ROUND_UP(size, PAGE_SIZE));
466
+
memmap_boot_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
467
467
#endif
468
468
}
469
469
···
643
643
unsigned long start = (unsigned long) pfn_to_page(pfn);
644
644
unsigned long end = start + nr_pages * sizeof(struct page);
645
645
646
-
mod_node_page_state(page_pgdat(pfn_to_page(pfn)), NR_MEMMAP,
647
-
-1L * (DIV_ROUND_UP(end - start, PAGE_SIZE)));
646
+
memmap_pages_add(-1L * (DIV_ROUND_UP(end - start, PAGE_SIZE)));
648
647
vmemmap_free(start, end, altmap);
649
648
}
650
649
static void free_map_bootmem(struct page *memmap)
+2
-9
mm/vmalloc.c
+2
-9
mm/vmalloc.c
···
3584
3584
page = alloc_pages_noprof(alloc_gfp, order);
3585
3585
else
3586
3586
page = alloc_pages_node_noprof(nid, alloc_gfp, order);
3587
-
if (unlikely(!page)) {
3588
-
if (!nofail)
3589
-
break;
3590
-
3591
-
/* fall back to the zero order allocations */
3592
-
alloc_gfp |= __GFP_NOFAIL;
3593
-
order = 0;
3594
-
continue;
3595
-
}
3587
+
if (unlikely(!page))
3588
+
break;
3596
3589
3597
3590
/*
3598
3591
* Higher order allocations must be able to be treated as
+25
-27
mm/vmstat.c
+25
-27
mm/vmstat.c
···
1033
1033
}
1034
1034
#endif
1035
1035
1036
+
/*
1037
+
* Count number of pages "struct page" and "struct page_ext" consume.
1038
+
* nr_memmap_boot_pages: # of pages allocated by boot allocator
1039
+
* nr_memmap_pages: # of pages that were allocated by buddy allocator
1040
+
*/
1041
+
static atomic_long_t nr_memmap_boot_pages = ATOMIC_LONG_INIT(0);
1042
+
static atomic_long_t nr_memmap_pages = ATOMIC_LONG_INIT(0);
1043
+
1044
+
void memmap_boot_pages_add(long delta)
1045
+
{
1046
+
atomic_long_add(delta, &nr_memmap_boot_pages);
1047
+
}
1048
+
1049
+
void memmap_pages_add(long delta)
1050
+
{
1051
+
atomic_long_add(delta, &nr_memmap_pages);
1052
+
}
1053
+
1036
1054
#ifdef CONFIG_COMPACTION
1037
1055
1038
1056
struct contig_page_info {
···
1273
1255
"pgdemote_kswapd",
1274
1256
"pgdemote_direct",
1275
1257
"pgdemote_khugepaged",
1276
-
"nr_memmap",
1277
-
"nr_memmap_boot",
1278
-
/* enum writeback_stat_item counters */
1258
+
/* system-wide enum vm_stat_item counters */
1279
1259
"nr_dirty_threshold",
1280
1260
"nr_dirty_background_threshold",
1261
+
"nr_memmap_pages",
1262
+
"nr_memmap_boot_pages",
1281
1263
1282
1264
#if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
1283
1265
/* enum vm_event_item counters */
···
1808
1790
#define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \
1809
1791
NR_VM_NUMA_EVENT_ITEMS + \
1810
1792
NR_VM_NODE_STAT_ITEMS + \
1811
-
NR_VM_WRITEBACK_STAT_ITEMS + \
1793
+
NR_VM_STAT_ITEMS + \
1812
1794
(IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \
1813
1795
NR_VM_EVENT_ITEMS : 0))
1814
1796
···
1845
1827
1846
1828
global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD,
1847
1829
v + NR_DIRTY_THRESHOLD);
1848
-
v += NR_VM_WRITEBACK_STAT_ITEMS;
1830
+
v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages);
1831
+
v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages);
1832
+
v += NR_VM_STAT_ITEMS;
1849
1833
1850
1834
#ifdef CONFIG_VM_EVENT_COUNTERS
1851
1835
all_vm_events(v);
···
2305
2285
module_init(extfrag_debug_init);
2306
2286
2307
2287
#endif
2308
-
2309
-
/*
2310
-
* Page metadata size (struct page and page_ext) in pages
2311
-
*/
2312
-
static unsigned long early_perpage_metadata[MAX_NUMNODES] __meminitdata;
2313
-
2314
-
void __meminit mod_node_early_perpage_metadata(int nid, long delta)
2315
-
{
2316
-
early_perpage_metadata[nid] += delta;
2317
-
}
2318
-
2319
-
void __meminit store_early_perpage_metadata(void)
2320
-
{
2321
-
int nid;
2322
-
struct pglist_data *pgdat;
2323
-
2324
-
for_each_online_pgdat(pgdat) {
2325
-
nid = pgdat->node_id;
2326
-
mod_node_page_state(NODE_DATA(nid), NR_MEMMAP_BOOT,
2327
-
early_perpage_metadata[nid]);
2328
-
}
2329
-
}
+2
tools/testing/selftests/mm/Makefile
+2
tools/testing/selftests/mm/Makefile
···
53
53
TEST_GEN_FILES += map_fixed_noreplace
54
54
TEST_GEN_FILES += map_hugetlb
55
55
TEST_GEN_FILES += map_populate
56
+
ifneq (,$(filter $(ARCH),arm64 riscv riscv64 x86 x86_64))
56
57
TEST_GEN_FILES += memfd_secret
58
+
endif
57
59
TEST_GEN_FILES += migration
58
60
TEST_GEN_FILES += mkdirty
59
61
TEST_GEN_FILES += mlock-random-test
+3
-2
tools/testing/selftests/mm/compaction_test.c
+3
-2
tools/testing/selftests/mm/compaction_test.c
···
89
89
int fd, ret = -1;
90
90
int compaction_index = 0;
91
91
char nr_hugepages[20] = {0};
92
-
char init_nr_hugepages[20] = {0};
92
+
char init_nr_hugepages[24] = {0};
93
93
94
-
sprintf(init_nr_hugepages, "%lu", initial_nr_hugepages);
94
+
snprintf(init_nr_hugepages, sizeof(init_nr_hugepages),
95
+
"%lu", initial_nr_hugepages);
95
96
96
97
/* We want to test with 80% of available memory. Else, OOM killer comes
97
98
in to play */
+3
tools/testing/selftests/mm/run_vmtests.sh
+3
tools/testing/selftests/mm/run_vmtests.sh
···
374
374
# MADV_POPULATE_READ and MADV_POPULATE_WRITE tests
375
375
CATEGORY="madv_populate" run_test ./madv_populate
376
376
377
+
if [ -x ./memfd_secret ]
378
+
then
377
379
(echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope 2>&1) | tap_prefix
378
380
CATEGORY="memfd_secret" run_test ./memfd_secret
381
+
fi
379
382
380
383
# KSM KSM_MERGE_TIME_HUGE_PAGES test with size of 100
381
384
CATEGORY="ksm" run_test ./ksm_tests -H -s 100