Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm/migrate_device: implement THP migration of zone device pages
MIGRATE_VMA_SELECT_COMPOUND will be used to select THP pages during
migrate_vma_setup() and MIGRATE_PFN_COMPOUND will make migrating device
pages as compound pages during device pfn migration.
migrate_device code paths go through the collect, setup and finalize
phases of migration.
The entries in src and dst arrays passed to these functions still remain
at a PAGE_SIZE granularity. When a compound page is passed, the first
entry has the PFN along with MIGRATE_PFN_COMPOUND and other flags set
(MIGRATE_PFN_MIGRATE, MIGRATE_PFN_VALID), the remaining entries
(HPAGE_PMD_NR - 1) are filled with 0's. This representation allows for
the compound page to be split into smaller page sizes.
migrate_vma_collect_hole(), migrate_vma_collect_pmd() are now THP page
aware. Two new helper functions migrate_vma_collect_huge_pmd() and
migrate_vma_insert_huge_pmd_page() have been added.
migrate_vma_collect_huge_pmd() can collect THP pages, but if for some
reason this fails, there is fallback support to split the folio and
migrate it.
migrate_vma_insert_huge_pmd_page() closely follows the logic of
migrate_vma_insert_page()
Support for splitting pages as needed for migration will follow in later
patches in this series.
Link: https://lkml.kernel.org/r/20251001065707.920170-8-balbirs@nvidia.com
Signed-off-by: Balbir Singh <balbirs@nvidia.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: Ying Huang <ying.huang@linux.alibaba.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: "Liam R. Howlett" <Liam.Howlett@oracle.com>
Cc: Nico Pache <npache@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Lyude Paul <lyude@redhat.com>
Cc: Danilo Krummrich <dakr@kernel.org>
Cc: David Airlie <airlied@gmail.com>
Cc: Simona Vetter <simona@ffwll.ch>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Cc: Mika Penttilä <mpenttil@redhat.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Francois Dugast <francois.dugast@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Balbir Singh
and committed by
Andrew Morton
a30b48bf
022a12de
+409
-64
+2
include/linux/migrate.h
+2
include/linux/migrate.h
···
125
125
#define MIGRATE_PFN_VALID (1UL << 0)
126
126
#define MIGRATE_PFN_MIGRATE (1UL << 1)
127
127
#define MIGRATE_PFN_WRITE (1UL << 3)
128
+
#define MIGRATE_PFN_COMPOUND (1UL << 4)
128
129
#define MIGRATE_PFN_SHIFT 6
129
130
130
131
static inline struct page *migrate_pfn_to_page(unsigned long mpfn)
···
144
143
MIGRATE_VMA_SELECT_SYSTEM = 1 << 0,
145
144
MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1,
146
145
MIGRATE_VMA_SELECT_DEVICE_COHERENT = 1 << 2,
146
+
MIGRATE_VMA_SELECT_COMPOUND = 1 << 3,
147
147
};
148
148
149
149
struct migrate_vma {
+407
-64
mm/migrate_device.c
+407
-64
mm/migrate_device.c
···
14
14
#include <linux/pagewalk.h>
15
15
#include <linux/rmap.h>
16
16
#include <linux/swapops.h>
17
+
#include <linux/pgalloc.h>
17
18
#include <asm/tlbflush.h>
18
19
#include "internal.h"
19
20
···
44
43
/* Only allow populating anonymous memory. */
45
44
if (!vma_is_anonymous(walk->vma))
46
45
return migrate_vma_collect_skip(start, end, walk);
46
+
47
+
if (thp_migration_supported() &&
48
+
(migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) &&
49
+
(IS_ALIGNED(start, HPAGE_PMD_SIZE) &&
50
+
IS_ALIGNED(end, HPAGE_PMD_SIZE))) {
51
+
migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE |
52
+
MIGRATE_PFN_COMPOUND;
53
+
migrate->dst[migrate->npages] = 0;
54
+
migrate->npages++;
55
+
migrate->cpages++;
56
+
57
+
/*
58
+
* Collect the remaining entries as holes, in case we
59
+
* need to split later
60
+
*/
61
+
return migrate_vma_collect_skip(start + PAGE_SIZE, end, walk);
62
+
}
47
63
48
64
for (addr = start; addr < end; addr += PAGE_SIZE) {
49
65
migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
···
121
103
return 0;
122
104
}
123
105
106
+
/** migrate_vma_collect_huge_pmd - collect THP pages without splitting the
107
+
* folio for device private pages.
108
+
* @pmdp: pointer to pmd entry
109
+
* @start: start address of the range for migration
110
+
* @end: end address of the range for migration
111
+
* @walk: mm_walk callback structure
112
+
* @fault_folio: folio associated with the fault if any
113
+
*
114
+
* Collect the huge pmd entry at @pmdp for migration and set the
115
+
* MIGRATE_PFN_COMPOUND flag in the migrate src entry to indicate that
116
+
* migration will occur at HPAGE_PMD granularity
117
+
*/
118
+
static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,
119
+
unsigned long end, struct mm_walk *walk,
120
+
struct folio *fault_folio)
121
+
{
122
+
struct mm_struct *mm = walk->mm;
123
+
struct folio *folio;
124
+
struct migrate_vma *migrate = walk->private;
125
+
spinlock_t *ptl;
126
+
swp_entry_t entry;
127
+
int ret;
128
+
unsigned long write = 0;
129
+
130
+
ptl = pmd_lock(mm, pmdp);
131
+
if (pmd_none(*pmdp)) {
132
+
spin_unlock(ptl);
133
+
return migrate_vma_collect_hole(start, end, -1, walk);
134
+
}
135
+
136
+
if (pmd_trans_huge(*pmdp)) {
137
+
if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) {
138
+
spin_unlock(ptl);
139
+
return migrate_vma_collect_skip(start, end, walk);
140
+
}
141
+
142
+
folio = pmd_folio(*pmdp);
143
+
if (is_huge_zero_folio(folio)) {
144
+
spin_unlock(ptl);
145
+
return migrate_vma_collect_hole(start, end, -1, walk);
146
+
}
147
+
if (pmd_write(*pmdp))
148
+
write = MIGRATE_PFN_WRITE;
149
+
} else if (!pmd_present(*pmdp)) {
150
+
entry = pmd_to_swp_entry(*pmdp);
151
+
folio = pfn_swap_entry_folio(entry);
152
+
153
+
if (!is_device_private_entry(entry) ||
154
+
!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
155
+
(folio->pgmap->owner != migrate->pgmap_owner)) {
156
+
spin_unlock(ptl);
157
+
return migrate_vma_collect_skip(start, end, walk);
158
+
}
159
+
160
+
if (is_migration_entry(entry)) {
161
+
migration_entry_wait_on_locked(entry, ptl);
162
+
spin_unlock(ptl);
163
+
return -EAGAIN;
164
+
}
165
+
166
+
if (is_writable_device_private_entry(entry))
167
+
write = MIGRATE_PFN_WRITE;
168
+
} else {
169
+
spin_unlock(ptl);
170
+
return -EAGAIN;
171
+
}
172
+
173
+
folio_get(folio);
174
+
if (folio != fault_folio && unlikely(!folio_trylock(folio))) {
175
+
spin_unlock(ptl);
176
+
folio_put(folio);
177
+
return migrate_vma_collect_skip(start, end, walk);
178
+
}
179
+
180
+
if (thp_migration_supported() &&
181
+
(migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) &&
182
+
(IS_ALIGNED(start, HPAGE_PMD_SIZE) &&
183
+
IS_ALIGNED(end, HPAGE_PMD_SIZE))) {
184
+
185
+
struct page_vma_mapped_walk pvmw = {
186
+
.ptl = ptl,
187
+
.address = start,
188
+
.pmd = pmdp,
189
+
.vma = walk->vma,
190
+
};
191
+
192
+
unsigned long pfn = page_to_pfn(folio_page(folio, 0));
193
+
194
+
migrate->src[migrate->npages] = migrate_pfn(pfn) | write
195
+
| MIGRATE_PFN_MIGRATE
196
+
| MIGRATE_PFN_COMPOUND;
197
+
migrate->dst[migrate->npages++] = 0;
198
+
migrate->cpages++;
199
+
ret = set_pmd_migration_entry(&pvmw, folio_page(folio, 0));
200
+
if (ret) {
201
+
migrate->npages--;
202
+
migrate->cpages--;
203
+
migrate->src[migrate->npages] = 0;
204
+
migrate->dst[migrate->npages] = 0;
205
+
goto fallback;
206
+
}
207
+
migrate_vma_collect_skip(start + PAGE_SIZE, end, walk);
208
+
spin_unlock(ptl);
209
+
return 0;
210
+
}
211
+
212
+
fallback:
213
+
spin_unlock(ptl);
214
+
if (!folio_test_large(folio))
215
+
goto done;
216
+
ret = split_folio(folio);
217
+
if (fault_folio != folio)
218
+
folio_unlock(folio);
219
+
folio_put(folio);
220
+
if (ret)
221
+
return migrate_vma_collect_skip(start, end, walk);
222
+
if (pmd_none(pmdp_get_lockless(pmdp)))
223
+
return migrate_vma_collect_hole(start, end, -1, walk);
224
+
225
+
done:
226
+
return -ENOENT;
227
+
}
228
+
124
229
static int migrate_vma_collect_pmd(pmd_t *pmdp,
125
230
unsigned long start,
126
231
unsigned long end,
127
232
struct mm_walk *walk)
128
233
{
129
234
struct migrate_vma *migrate = walk->private;
130
-
struct folio *fault_folio = migrate->fault_page ?
131
-
page_folio(migrate->fault_page) : NULL;
132
235
struct vm_area_struct *vma = walk->vma;
133
236
struct mm_struct *mm = vma->vm_mm;
134
237
unsigned long addr = start, unmapped = 0;
135
238
spinlock_t *ptl;
239
+
struct folio *fault_folio = migrate->fault_page ?
240
+
page_folio(migrate->fault_page) : NULL;
136
241
pte_t *ptep;
137
242
138
243
again:
139
-
if (pmd_none(*pmdp))
140
-
return migrate_vma_collect_hole(start, end, -1, walk);
244
+
if (pmd_trans_huge(*pmdp) || !pmd_present(*pmdp)) {
245
+
int ret = migrate_vma_collect_huge_pmd(pmdp, start, end, walk, fault_folio);
141
246
142
-
if (pmd_trans_huge(*pmdp)) {
143
-
struct folio *folio;
144
-
145
-
ptl = pmd_lock(mm, pmdp);
146
-
if (unlikely(!pmd_trans_huge(*pmdp))) {
147
-
spin_unlock(ptl);
247
+
if (ret == -EAGAIN)
148
248
goto again;
149
-
}
150
-
151
-
folio = pmd_folio(*pmdp);
152
-
if (is_huge_zero_folio(folio)) {
153
-
spin_unlock(ptl);
154
-
split_huge_pmd(vma, pmdp, addr);
155
-
} else {
156
-
int ret;
157
-
158
-
folio_get(folio);
159
-
spin_unlock(ptl);
160
-
/* FIXME: we don't expect THP for fault_folio */
161
-
if (WARN_ON_ONCE(fault_folio == folio))
162
-
return migrate_vma_collect_skip(start, end,
163
-
walk);
164
-
if (unlikely(!folio_trylock(folio)))
165
-
return migrate_vma_collect_skip(start, end,
166
-
walk);
167
-
ret = split_folio(folio);
168
-
if (fault_folio != folio)
169
-
folio_unlock(folio);
170
-
folio_put(folio);
171
-
if (ret)
172
-
return migrate_vma_collect_skip(start, end,
173
-
walk);
174
-
}
249
+
if (ret == 0)
250
+
return 0;
175
251
}
176
252
177
253
ptep = pte_offset_map_lock(mm, pmdp, start, &ptl);
···
355
243
mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
356
244
}
357
245
358
-
/* FIXME support THP */
359
-
if (!page || !page->mapping || PageTransCompound(page)) {
246
+
if (!page || !page->mapping) {
360
247
mpfn = 0;
361
248
goto next;
362
249
}
···
526
415
*/
527
416
int extra = 1 + (page == fault_page);
528
417
529
-
/*
530
-
* FIXME support THP (transparent huge page), it is bit more complex to
531
-
* check them than regular pages, because they can be mapped with a pmd
532
-
* or with a pte (split pte mapping).
533
-
*/
534
-
if (folio_test_large(folio))
535
-
return false;
536
-
537
418
/* Page from ZONE_DEVICE have one extra reference */
538
419
if (folio_is_zone_device(folio))
539
420
extra++;
···
556
453
557
454
lru_add_drain();
558
455
559
-
for (i = 0; i < npages; i++) {
456
+
for (i = 0; i < npages; ) {
560
457
struct page *page = migrate_pfn_to_page(src_pfns[i]);
561
458
struct folio *folio;
459
+
unsigned int nr = 1;
562
460
563
461
if (!page) {
564
462
if (src_pfns[i] & MIGRATE_PFN_MIGRATE)
565
463
unmapped++;
566
-
continue;
464
+
goto next;
567
465
}
568
466
569
467
folio = page_folio(page);
468
+
nr = folio_nr_pages(folio);
469
+
470
+
if (nr > 1)
471
+
src_pfns[i] |= MIGRATE_PFN_COMPOUND;
472
+
473
+
570
474
/* ZONE_DEVICE folios are not on LRU */
571
475
if (!folio_is_zone_device(folio)) {
572
476
if (!folio_test_lru(folio) && allow_drain) {
···
585
475
if (!folio_isolate_lru(folio)) {
586
476
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
587
477
restore++;
588
-
continue;
478
+
goto next;
589
479
}
590
480
591
481
/* Drop the reference we took in collect */
···
604
494
605
495
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
606
496
restore++;
607
-
continue;
497
+
goto next;
608
498
}
609
499
610
500
unmapped++;
501
+
next:
502
+
i += nr;
611
503
}
612
504
613
505
for (i = 0; i < npages && restore; i++) {
···
755
643
}
756
644
EXPORT_SYMBOL(migrate_vma_setup);
757
645
646
+
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
647
+
/**
648
+
* migrate_vma_insert_huge_pmd_page: Insert a huge folio into @migrate->vma->vm_mm
649
+
* at @addr. folio is already allocated as a part of the migration process with
650
+
* large page.
651
+
*
652
+
* @page needs to be initialized and setup after it's allocated. The code bits
653
+
* here follow closely the code in __do_huge_pmd_anonymous_page(). This API does
654
+
* not support THP zero pages.
655
+
*
656
+
* @migrate: migrate_vma arguments
657
+
* @addr: address where the folio will be inserted
658
+
* @page: page to be inserted at @addr
659
+
* @src: src pfn which is being migrated
660
+
* @pmdp: pointer to the pmd
661
+
*/
662
+
static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
663
+
unsigned long addr,
664
+
struct page *page,
665
+
unsigned long *src,
666
+
pmd_t *pmdp)
667
+
{
668
+
struct vm_area_struct *vma = migrate->vma;
669
+
gfp_t gfp = vma_thp_gfp_mask(vma);
670
+
struct folio *folio = page_folio(page);
671
+
int ret;
672
+
vm_fault_t csa_ret;
673
+
spinlock_t *ptl;
674
+
pgtable_t pgtable;
675
+
pmd_t entry;
676
+
bool flush = false;
677
+
unsigned long i;
678
+
679
+
VM_WARN_ON_FOLIO(!folio, folio);
680
+
VM_WARN_ON_ONCE(!pmd_none(*pmdp) && !is_huge_zero_pmd(*pmdp));
681
+
682
+
if (!thp_vma_suitable_order(vma, addr, HPAGE_PMD_ORDER))
683
+
return -EINVAL;
684
+
685
+
ret = anon_vma_prepare(vma);
686
+
if (ret)
687
+
return ret;
688
+
689
+
folio_set_order(folio, HPAGE_PMD_ORDER);
690
+
folio_set_large_rmappable(folio);
691
+
692
+
if (mem_cgroup_charge(folio, migrate->vma->vm_mm, gfp)) {
693
+
count_vm_event(THP_FAULT_FALLBACK);
694
+
count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
695
+
ret = -ENOMEM;
696
+
goto abort;
697
+
}
698
+
699
+
__folio_mark_uptodate(folio);
700
+
701
+
pgtable = pte_alloc_one(vma->vm_mm);
702
+
if (unlikely(!pgtable))
703
+
goto abort;
704
+
705
+
if (folio_is_device_private(folio)) {
706
+
swp_entry_t swp_entry;
707
+
708
+
if (vma->vm_flags & VM_WRITE)
709
+
swp_entry = make_writable_device_private_entry(
710
+
page_to_pfn(page));
711
+
else
712
+
swp_entry = make_readable_device_private_entry(
713
+
page_to_pfn(page));
714
+
entry = swp_entry_to_pmd(swp_entry);
715
+
} else {
716
+
if (folio_is_zone_device(folio) &&
717
+
!folio_is_device_coherent(folio)) {
718
+
goto abort;
719
+
}
720
+
entry = folio_mk_pmd(folio, vma->vm_page_prot);
721
+
if (vma->vm_flags & VM_WRITE)
722
+
entry = pmd_mkwrite(pmd_mkdirty(entry), vma);
723
+
}
724
+
725
+
ptl = pmd_lock(vma->vm_mm, pmdp);
726
+
csa_ret = check_stable_address_space(vma->vm_mm);
727
+
if (csa_ret)
728
+
goto abort;
729
+
730
+
/*
731
+
* Check for userfaultfd but do not deliver the fault. Instead,
732
+
* just back off.
733
+
*/
734
+
if (userfaultfd_missing(vma))
735
+
goto unlock_abort;
736
+
737
+
if (!pmd_none(*pmdp)) {
738
+
if (!is_huge_zero_pmd(*pmdp))
739
+
goto unlock_abort;
740
+
flush = true;
741
+
} else if (!pmd_none(*pmdp))
742
+
goto unlock_abort;
743
+
744
+
add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
745
+
folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
746
+
if (!folio_is_zone_device(folio))
747
+
folio_add_lru_vma(folio, vma);
748
+
folio_get(folio);
749
+
750
+
if (flush) {
751
+
pte_free(vma->vm_mm, pgtable);
752
+
flush_cache_page(vma, addr, addr + HPAGE_PMD_SIZE);
753
+
pmdp_invalidate(vma, addr, pmdp);
754
+
} else {
755
+
pgtable_trans_huge_deposit(vma->vm_mm, pmdp, pgtable);
756
+
mm_inc_nr_ptes(vma->vm_mm);
757
+
}
758
+
set_pmd_at(vma->vm_mm, addr, pmdp, entry);
759
+
update_mmu_cache_pmd(vma, addr, pmdp);
760
+
761
+
spin_unlock(ptl);
762
+
763
+
count_vm_event(THP_FAULT_ALLOC);
764
+
count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC);
765
+
count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC);
766
+
767
+
return 0;
768
+
769
+
unlock_abort:
770
+
spin_unlock(ptl);
771
+
abort:
772
+
for (i = 0; i < HPAGE_PMD_NR; i++)
773
+
src[i] &= ~MIGRATE_PFN_MIGRATE;
774
+
return 0;
775
+
}
776
+
#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */
777
+
static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
778
+
unsigned long addr,
779
+
struct page *page,
780
+
unsigned long *src,
781
+
pmd_t *pmdp)
782
+
{
783
+
return 0;
784
+
}
785
+
#endif
786
+
787
+
static unsigned long migrate_vma_nr_pages(unsigned long *src)
788
+
{
789
+
unsigned long nr = 1;
790
+
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
791
+
if (*src & MIGRATE_PFN_COMPOUND)
792
+
nr = HPAGE_PMD_NR;
793
+
#else
794
+
if (*src & MIGRATE_PFN_COMPOUND)
795
+
VM_WARN_ON_ONCE(true);
796
+
#endif
797
+
return nr;
798
+
}
799
+
758
800
/*
759
801
* This code closely matches the code in:
760
802
* __handle_mm_fault()
···
919
653
*/
920
654
static void migrate_vma_insert_page(struct migrate_vma *migrate,
921
655
unsigned long addr,
922
-
struct page *page,
656
+
unsigned long *dst,
923
657
unsigned long *src)
924
658
{
659
+
struct page *page = migrate_pfn_to_page(*dst);
925
660
struct folio *folio = page_folio(page);
926
661
struct vm_area_struct *vma = migrate->vma;
927
662
struct mm_struct *mm = vma->vm_mm;
···
950
683
pmdp = pmd_alloc(mm, pudp, addr);
951
684
if (!pmdp)
952
685
goto abort;
953
-
if (pmd_trans_huge(*pmdp))
954
-
goto abort;
686
+
687
+
if (thp_migration_supported() && (*dst & MIGRATE_PFN_COMPOUND)) {
688
+
int ret = migrate_vma_insert_huge_pmd_page(migrate, addr, page,
689
+
src, pmdp);
690
+
if (ret)
691
+
goto abort;
692
+
return;
693
+
}
694
+
695
+
if (!pmd_none(*pmdp)) {
696
+
if (pmd_trans_huge(*pmdp)) {
697
+
if (!is_huge_zero_pmd(*pmdp))
698
+
goto abort;
699
+
split_huge_pmd(vma, pmdp, addr);
700
+
} else if (pmd_leaf(*pmdp))
701
+
goto abort;
702
+
}
703
+
955
704
if (pte_alloc(mm, pmdp))
956
705
goto abort;
957
706
if (unlikely(anon_vma_prepare(vma)))
···
1058
775
unsigned long i;
1059
776
bool notified = false;
1060
777
1061
-
for (i = 0; i < npages; i++) {
778
+
for (i = 0; i < npages; ) {
1062
779
struct page *newpage = migrate_pfn_to_page(dst_pfns[i]);
1063
780
struct page *page = migrate_pfn_to_page(src_pfns[i]);
1064
781
struct address_space *mapping;
1065
782
struct folio *newfolio, *folio;
1066
783
int r, extra_cnt = 0;
784
+
unsigned long nr = 1;
1067
785
1068
786
if (!newpage) {
1069
787
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
1070
-
continue;
788
+
goto next;
1071
789
}
1072
790
1073
791
if (!page) {
1074
792
unsigned long addr;
1075
793
1076
794
if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE))
1077
-
continue;
795
+
goto next;
1078
796
1079
797
/*
1080
798
* The only time there is no vma is when called from
···
1093
809
migrate->pgmap_owner);
1094
810
mmu_notifier_invalidate_range_start(&range);
1095
811
}
1096
-
migrate_vma_insert_page(migrate, addr, newpage,
812
+
813
+
if ((src_pfns[i] & MIGRATE_PFN_COMPOUND) &&
814
+
(!(dst_pfns[i] & MIGRATE_PFN_COMPOUND))) {
815
+
nr = migrate_vma_nr_pages(&src_pfns[i]);
816
+
src_pfns[i] &= ~MIGRATE_PFN_COMPOUND;
817
+
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
818
+
goto next;
819
+
}
820
+
821
+
migrate_vma_insert_page(migrate, addr, &dst_pfns[i],
1097
822
&src_pfns[i]);
1098
-
continue;
823
+
goto next;
1099
824
}
1100
825
1101
826
newfolio = page_folio(newpage);
1102
827
folio = page_folio(page);
1103
828
mapping = folio_mapping(folio);
829
+
830
+
/*
831
+
* If THP migration is enabled, check if both src and dst
832
+
* can migrate large pages
833
+
*/
834
+
if (thp_migration_supported()) {
835
+
if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) &&
836
+
(src_pfns[i] & MIGRATE_PFN_COMPOUND) &&
837
+
!(dst_pfns[i] & MIGRATE_PFN_COMPOUND)) {
838
+
839
+
if (!migrate) {
840
+
src_pfns[i] &= ~(MIGRATE_PFN_MIGRATE |
841
+
MIGRATE_PFN_COMPOUND);
842
+
goto next;
843
+
}
844
+
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
845
+
} else if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) &&
846
+
(dst_pfns[i] & MIGRATE_PFN_COMPOUND) &&
847
+
!(src_pfns[i] & MIGRATE_PFN_COMPOUND)) {
848
+
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
849
+
}
850
+
}
851
+
1104
852
1105
853
if (folio_is_device_private(newfolio) ||
1106
854
folio_is_device_coherent(newfolio)) {
···
1146
830
if (!folio_test_anon(folio) ||
1147
831
!folio_free_swap(folio)) {
1148
832
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
1149
-
continue;
833
+
goto next;
1150
834
}
1151
835
}
1152
836
} else if (folio_is_zone_device(newfolio)) {
···
1154
838
* Other types of ZONE_DEVICE page are not supported.
1155
839
*/
1156
840
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
1157
-
continue;
841
+
goto next;
1158
842
}
1159
843
1160
844
BUG_ON(folio_test_writeback(folio));
···
1166
850
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
1167
851
else
1168
852
folio_migrate_flags(newfolio, folio);
853
+
next:
854
+
i += nr;
1169
855
}
1170
856
1171
857
if (notified)
···
1329
1011
int migrate_device_range(unsigned long *src_pfns, unsigned long start,
1330
1012
unsigned long npages)
1331
1013
{
1332
-
unsigned long i, pfn;
1014
+
unsigned long i, j, pfn;
1333
1015
1334
-
for (pfn = start, i = 0; i < npages; pfn++, i++)
1016
+
for (pfn = start, i = 0; i < npages; pfn++, i++) {
1017
+
struct page *page = pfn_to_page(pfn);
1018
+
struct folio *folio = page_folio(page);
1019
+
unsigned int nr = 1;
1020
+
1335
1021
src_pfns[i] = migrate_device_pfn_lock(pfn);
1022
+
nr = folio_nr_pages(folio);
1023
+
if (nr > 1) {
1024
+
src_pfns[i] |= MIGRATE_PFN_COMPOUND;
1025
+
for (j = 1; j < nr; j++)
1026
+
src_pfns[i+j] = 0;
1027
+
i += j - 1;
1028
+
pfn += j - 1;
1029
+
}
1030
+
}
1336
1031
1337
1032
migrate_device_unmap(src_pfns, npages, NULL);
1338
1033
···
1363
1032
*/
1364
1033
int migrate_device_pfns(unsigned long *src_pfns, unsigned long npages)
1365
1034
{
1366
-
unsigned long i;
1035
+
unsigned long i, j;
1367
1036
1368
-
for (i = 0; i < npages; i++)
1037
+
for (i = 0; i < npages; i++) {
1038
+
struct page *page = pfn_to_page(src_pfns[i]);
1039
+
struct folio *folio = page_folio(page);
1040
+
unsigned int nr = 1;
1041
+
1369
1042
src_pfns[i] = migrate_device_pfn_lock(src_pfns[i]);
1043
+
nr = folio_nr_pages(folio);
1044
+
if (nr > 1) {
1045
+
src_pfns[i] |= MIGRATE_PFN_COMPOUND;
1046
+
for (j = 1; j < nr; j++)
1047
+
src_pfns[i+j] = 0;
1048
+
i += j - 1;
1049
+
}
1050
+
}
1370
1051
1371
1052
migrate_device_unmap(src_pfns, npages, NULL);
1372
1053