Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'akpm' (patches from Andrew)
Merge more updates from Andrew Morton:
"The post-linux-next material.
7 patches.
Subsystems affected by this patch series (all mm): debug,
slab-generic, migration, memcg, and kasan"
* emailed patches from Andrew Morton <akpm@linux-foundation.org>:
kasan: add kasan mode messages when kasan init
mm: unexport {,un}lock_page_memcg
mm: unexport folio_memcg_{,un}lock
mm/migrate.c: remove MIGRATE_PFN_LOCKED
mm: migrate: simplify the file-backed pages validation when migrating its mapping
mm: allow only SLUB on PREEMPT_RT
mm/page_owner.c: modify the type of argument "order" in some functions
+63
-152
+1
-1
Documentation/vm/hmm.rst
+1
-1
Documentation/vm/hmm.rst
···
360
360
system memory page, locks the page with ``lock_page()``, and fills in the
361
361
``dst`` array entry with::
362
362
363
-
dst[i] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
363
+
dst[i] = migrate_pfn(page_to_pfn(dpage));
364
364
365
365
Now that the driver knows that this page is being migrated, it can
366
366
invalidate device private MMU mappings and copy device private memory
+1
-1
arch/arm64/mm/kasan_init.c
+1
-1
arch/arm64/mm/kasan_init.c
+2
-2
arch/powerpc/kvm/book3s_hv_uvmem.c
+2
-2
arch/powerpc/kvm/book3s_hv_uvmem.c
···
560
560
gpa, 0, page_shift);
561
561
562
562
if (ret == U_SUCCESS)
563
-
*mig.dst = migrate_pfn(pfn) | MIGRATE_PFN_LOCKED;
563
+
*mig.dst = migrate_pfn(pfn);
564
564
else {
565
565
unlock_page(dpage);
566
566
__free_page(dpage);
···
774
774
}
775
775
}
776
776
777
-
*mig.dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
777
+
*mig.dst = migrate_pfn(page_to_pfn(dpage));
778
778
migrate_vma_pages(&mig);
779
779
out_finalize:
780
780
migrate_vma_finalize(&mig);
-2
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
-2
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
···
317
317
migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
318
318
svm_migrate_get_vram_page(prange, migrate->dst[i]);
319
319
migrate->dst[i] = migrate_pfn(migrate->dst[i]);
320
-
migrate->dst[i] |= MIGRATE_PFN_LOCKED;
321
320
src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
322
321
DMA_TO_DEVICE);
323
322
r = dma_mapping_error(dev, src[i]);
···
609
610
dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
610
611
611
612
migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
612
-
migrate->dst[i] |= MIGRATE_PFN_LOCKED;
613
613
j++;
614
614
}
615
615
+2
-2
drivers/gpu/drm/nouveau/nouveau_dmem.c
+2
-2
drivers/gpu/drm/nouveau/nouveau_dmem.c
···
166
166
goto error_dma_unmap;
167
167
mutex_unlock(&svmm->mutex);
168
168
169
-
args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
169
+
args->dst[0] = migrate_pfn(page_to_pfn(dpage));
170
170
return 0;
171
171
172
172
error_dma_unmap:
···
602
602
((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT);
603
603
if (src & MIGRATE_PFN_WRITE)
604
604
*pfn |= NVIF_VMM_PFNMAP_V0_W;
605
-
return migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
605
+
return migrate_pfn(page_to_pfn(dpage));
606
606
607
607
out_dma_unmap:
608
608
dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
-1
include/linux/migrate.h
-1
include/linux/migrate.h
+6
-6
include/linux/page_owner.h
+6
-6
include/linux/page_owner.h
···
8
8
extern struct static_key_false page_owner_inited;
9
9
extern struct page_ext_operations page_owner_ops;
10
10
11
-
extern void __reset_page_owner(struct page *page, unsigned int order);
11
+
extern void __reset_page_owner(struct page *page, unsigned short order);
12
12
extern void __set_page_owner(struct page *page,
13
-
unsigned int order, gfp_t gfp_mask);
13
+
unsigned short order, gfp_t gfp_mask);
14
14
extern void __split_page_owner(struct page *page, unsigned int nr);
15
15
extern void __folio_copy_owner(struct folio *newfolio, struct folio *old);
16
16
extern void __set_page_owner_migrate_reason(struct page *page, int reason);
···
18
18
extern void pagetypeinfo_showmixedcount_print(struct seq_file *m,
19
19
pg_data_t *pgdat, struct zone *zone);
20
20
21
-
static inline void reset_page_owner(struct page *page, unsigned int order)
21
+
static inline void reset_page_owner(struct page *page, unsigned short order)
22
22
{
23
23
if (static_branch_unlikely(&page_owner_inited))
24
24
__reset_page_owner(page, order);
25
25
}
26
26
27
27
static inline void set_page_owner(struct page *page,
28
-
unsigned int order, gfp_t gfp_mask)
28
+
unsigned short order, gfp_t gfp_mask)
29
29
{
30
30
if (static_branch_unlikely(&page_owner_inited))
31
31
__set_page_owner(page, order, gfp_mask);
···
52
52
__dump_page_owner(page);
53
53
}
54
54
#else
55
-
static inline void reset_page_owner(struct page *page, unsigned int order)
55
+
static inline void reset_page_owner(struct page *page, unsigned short order)
56
56
{
57
57
}
58
58
static inline void set_page_owner(struct page *page,
···
60
60
{
61
61
}
62
62
static inline void split_page_owner(struct page *page,
63
-
unsigned int order)
63
+
unsigned short order)
64
64
{
65
65
}
66
66
static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio)
+2
init/Kconfig
+2
init/Kconfig
···
1896
1896
1897
1897
config SLAB
1898
1898
bool "SLAB"
1899
+
depends on !PREEMPT_RT
1899
1900
select HAVE_HARDENED_USERCOPY_ALLOCATOR
1900
1901
help
1901
1902
The regular slab allocator that is established and known to work
···
1917
1916
config SLOB
1918
1917
depends on EXPERT
1919
1918
bool "SLOB (Simple Allocator)"
1919
+
depends on !PREEMPT_RT
1920
1920
help
1921
1921
SLOB replaces the stock allocator with a drastically simpler
1922
1922
allocator. SLOB is generally more space efficient but
+2
-3
lib/test_hmm.c
+2
-3
lib/test_hmm.c
···
613
613
*/
614
614
rpage->zone_device_data = dmirror;
615
615
616
-
*dst = migrate_pfn(page_to_pfn(dpage)) |
617
-
MIGRATE_PFN_LOCKED;
616
+
*dst = migrate_pfn(page_to_pfn(dpage));
618
617
if ((*src & MIGRATE_PFN_WRITE) ||
619
618
(!spage && args->vma->vm_flags & VM_WRITE))
620
619
*dst |= MIGRATE_PFN_WRITE;
···
1136
1137
lock_page(dpage);
1137
1138
xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
1138
1139
copy_highpage(dpage, spage);
1139
-
*dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
1140
+
*dst = migrate_pfn(page_to_pfn(dpage));
1140
1141
if (*src & MIGRATE_PFN_WRITE)
1141
1142
*dst |= MIGRATE_PFN_WRITE;
1142
1143
}
-4
mm/memcontrol.c
-4
mm/memcontrol.c
···
2058
2058
memcg->move_lock_task = current;
2059
2059
memcg->move_lock_flags = flags;
2060
2060
}
2061
-
EXPORT_SYMBOL(folio_memcg_lock);
2062
2061
2063
2062
void lock_page_memcg(struct page *page)
2064
2063
{
2065
2064
folio_memcg_lock(page_folio(page));
2066
2065
}
2067
-
EXPORT_SYMBOL(lock_page_memcg);
2068
2066
2069
2067
static void __folio_memcg_unlock(struct mem_cgroup *memcg)
2070
2068
{
···
2090
2092
{
2091
2093
__folio_memcg_unlock(folio_memcg(folio));
2092
2094
}
2093
-
EXPORT_SYMBOL(folio_memcg_unlock);
2094
2095
2095
2096
void unlock_page_memcg(struct page *page)
2096
2097
{
2097
2098
folio_memcg_unlock(page_folio(page));
2098
2099
}
2099
-
EXPORT_SYMBOL(unlock_page_memcg);
2100
2100
2101
2101
struct obj_stock {
2102
2102
#ifdef CONFIG_MEMCG_KMEM
+30
-125
mm/migrate.c
+30
-125
mm/migrate.c
···
404
404
newzone = folio_zone(newfolio);
405
405
406
406
xas_lock_irq(&xas);
407
-
if (folio_ref_count(folio) != expected_count ||
408
-
xas_load(&xas) != folio) {
409
-
xas_unlock_irq(&xas);
410
-
return -EAGAIN;
411
-
}
412
-
413
407
if (!folio_ref_freeze(folio, expected_count)) {
414
408
xas_unlock_irq(&xas);
415
409
return -EAGAIN;
···
2362
2368
* can't be dropped from it).
2363
2369
*/
2364
2370
get_page(page);
2365
-
migrate->cpages++;
2366
2371
2367
2372
/*
2368
2373
* Optimize for the common case where page is only mapped once
···
2371
2378
if (trylock_page(page)) {
2372
2379
pte_t swp_pte;
2373
2380
2374
-
mpfn |= MIGRATE_PFN_LOCKED;
2381
+
migrate->cpages++;
2375
2382
ptep_get_and_clear(mm, addr, ptep);
2376
2383
2377
2384
/* Setup special migration page table entry */
···
2405
2412
2406
2413
if (pte_present(pte))
2407
2414
unmapped++;
2415
+
} else {
2416
+
put_page(page);
2417
+
mpfn = 0;
2408
2418
}
2409
2419
2410
2420
next:
···
2512
2516
}
2513
2517
2514
2518
/*
2515
-
* migrate_vma_prepare() - lock pages and isolate them from the lru
2519
+
* migrate_vma_unmap() - replace page mapping with special migration pte entry
2516
2520
* @migrate: migrate struct containing all migration information
2517
2521
*
2518
-
* This locks pages that have been collected by migrate_vma_collect(). Once each
2519
-
* page is locked it is isolated from the lru (for non-device pages). Finally,
2520
-
* the ref taken by migrate_vma_collect() is dropped, as locked pages cannot be
2521
-
* migrated by concurrent kernel threads.
2522
+
* Isolate pages from the LRU and replace mappings (CPU page table pte) with a
2523
+
* special migration pte entry and check if it has been pinned. Pinned pages are
2524
+
* restored because we cannot migrate them.
2525
+
*
2526
+
* This is the last step before we call the device driver callback to allocate
2527
+
* destination memory and copy contents of original page over to new page.
2522
2528
*/
2523
-
static void migrate_vma_prepare(struct migrate_vma *migrate)
2529
+
static void migrate_vma_unmap(struct migrate_vma *migrate)
2524
2530
{
2525
2531
const unsigned long npages = migrate->npages;
2526
2532
const unsigned long start = migrate->start;
···
2531
2533
2532
2534
lru_add_drain();
2533
2535
2534
-
for (i = 0; (i < npages) && migrate->cpages; i++) {
2536
+
for (i = 0; i < npages; i++) {
2535
2537
struct page *page = migrate_pfn_to_page(migrate->src[i]);
2536
-
bool remap = true;
2537
2538
2538
2539
if (!page)
2539
2540
continue;
2540
-
2541
-
if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
2542
-
/*
2543
-
* Because we are migrating several pages there can be
2544
-
* a deadlock between 2 concurrent migration where each
2545
-
* are waiting on each other page lock.
2546
-
*
2547
-
* Make migrate_vma() a best effort thing and backoff
2548
-
* for any page we can not lock right away.
2549
-
*/
2550
-
if (!trylock_page(page)) {
2551
-
migrate->src[i] = 0;
2552
-
migrate->cpages--;
2553
-
put_page(page);
2554
-
continue;
2555
-
}
2556
-
remap = false;
2557
-
migrate->src[i] |= MIGRATE_PFN_LOCKED;
2558
-
}
2559
2541
2560
2542
/* ZONE_DEVICE pages are not on LRU */
2561
2543
if (!is_zone_device_page(page)) {
···
2546
2568
}
2547
2569
2548
2570
if (isolate_lru_page(page)) {
2549
-
if (remap) {
2550
-
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2551
-
migrate->cpages--;
2552
-
restore++;
2553
-
} else {
2554
-
migrate->src[i] = 0;
2555
-
unlock_page(page);
2556
-
migrate->cpages--;
2557
-
put_page(page);
2558
-
}
2571
+
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2572
+
migrate->cpages--;
2573
+
restore++;
2559
2574
continue;
2560
2575
}
2561
2576
···
2556
2585
put_page(page);
2557
2586
}
2558
2587
2559
-
if (!migrate_vma_check_page(page)) {
2560
-
if (remap) {
2561
-
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2562
-
migrate->cpages--;
2563
-
restore++;
2564
-
2565
-
if (!is_zone_device_page(page)) {
2566
-
get_page(page);
2567
-
putback_lru_page(page);
2568
-
}
2569
-
} else {
2570
-
migrate->src[i] = 0;
2571
-
unlock_page(page);
2572
-
migrate->cpages--;
2573
-
2574
-
if (!is_zone_device_page(page))
2575
-
putback_lru_page(page);
2576
-
else
2577
-
put_page(page);
2578
-
}
2579
-
}
2580
-
}
2581
-
2582
-
for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) {
2583
-
struct page *page = migrate_pfn_to_page(migrate->src[i]);
2584
-
2585
-
if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2586
-
continue;
2587
-
2588
-
remove_migration_pte(page, migrate->vma, addr, page);
2589
-
2590
-
migrate->src[i] = 0;
2591
-
unlock_page(page);
2592
-
put_page(page);
2593
-
restore--;
2594
-
}
2595
-
}
2596
-
2597
-
/*
2598
-
* migrate_vma_unmap() - replace page mapping with special migration pte entry
2599
-
* @migrate: migrate struct containing all migration information
2600
-
*
2601
-
* Replace page mapping (CPU page table pte) with a special migration pte entry
2602
-
* and check again if it has been pinned. Pinned pages are restored because we
2603
-
* cannot migrate them.
2604
-
*
2605
-
* This is the last step before we call the device driver callback to allocate
2606
-
* destination memory and copy contents of original page over to new page.
2607
-
*/
2608
-
static void migrate_vma_unmap(struct migrate_vma *migrate)
2609
-
{
2610
-
const unsigned long npages = migrate->npages;
2611
-
const unsigned long start = migrate->start;
2612
-
unsigned long addr, i, restore = 0;
2613
-
2614
-
for (i = 0; i < npages; i++) {
2615
-
struct page *page = migrate_pfn_to_page(migrate->src[i]);
2616
-
2617
-
if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2618
-
continue;
2619
-
2620
-
if (page_mapped(page)) {
2588
+
if (page_mapped(page))
2621
2589
try_to_migrate(page, 0);
2622
-
if (page_mapped(page))
2623
-
goto restore;
2624
-
}
2625
2590
2626
-
if (migrate_vma_check_page(page))
2591
+
if (page_mapped(page) || !migrate_vma_check_page(page)) {
2592
+
if (!is_zone_device_page(page)) {
2593
+
get_page(page);
2594
+
putback_lru_page(page);
2595
+
}
2596
+
2597
+
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2598
+
migrate->cpages--;
2599
+
restore++;
2627
2600
continue;
2628
-
2629
-
restore:
2630
-
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2631
-
migrate->cpages--;
2632
-
restore++;
2601
+
}
2633
2602
}
2634
2603
2635
2604
for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) {
···
2582
2671
2583
2672
migrate->src[i] = 0;
2584
2673
unlock_page(page);
2674
+
put_page(page);
2585
2675
restore--;
2586
-
2587
-
if (is_zone_device_page(page))
2588
-
put_page(page);
2589
-
else
2590
-
putback_lru_page(page);
2591
2676
}
2592
2677
}
2593
2678
···
2606
2699
* it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
2607
2700
* flag set). Once these are allocated and copied, the caller must update each
2608
2701
* corresponding entry in the dst array with the pfn value of the destination
2609
-
* page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_LOCKED flags set
2610
-
* (destination pages must have their struct pages locked, via lock_page()).
2702
+
* page and with MIGRATE_PFN_VALID. Destination pages must be locked via
2703
+
* lock_page().
2611
2704
*
2612
2705
* Note that the caller does not have to migrate all the pages that are marked
2613
2706
* with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
···
2676
2769
2677
2770
migrate_vma_collect(args);
2678
2771
2679
-
if (args->cpages)
2680
-
migrate_vma_prepare(args);
2681
2772
if (args->cpages)
2682
2773
migrate_vma_unmap(args);
2683
2774
+3
-3
mm/page_owner.c
+3
-3
mm/page_owner.c
···
125
125
return handle;
126
126
}
127
127
128
-
void __reset_page_owner(struct page *page, unsigned int order)
128
+
void __reset_page_owner(struct page *page, unsigned short order)
129
129
{
130
130
int i;
131
131
struct page_ext *page_ext;
···
149
149
150
150
static inline void __set_page_owner_handle(struct page_ext *page_ext,
151
151
depot_stack_handle_t handle,
152
-
unsigned int order, gfp_t gfp_mask)
152
+
unsigned short order, gfp_t gfp_mask)
153
153
{
154
154
struct page_owner *page_owner;
155
155
int i;
···
169
169
}
170
170
}
171
171
172
-
noinline void __set_page_owner(struct page *page, unsigned int order,
172
+
noinline void __set_page_owner(struct page *page, unsigned short order,
173
173
gfp_t gfp_mask)
174
174
{
175
175
struct page_ext *page_ext = lookup_page_ext(page);