Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm/gup: consistently name GUP-fast functions
Patch series "mm/gup: consistently call it GUP-fast".
Some cleanups around function names, comments and the config option of
"GUP-fast" -- GUP without "lock" safety belts on.
With this cleanup it's easy to judge which functions are GUP-fast
specific. We now consistently call it "GUP-fast", avoiding mixing it with
"fast GUP", "lockless", or simply "gup" (which I always considered
confusing in the ode).
So the magic now happens in functions that contain "gup_fast", whereby
gup_fast() is the entry point into that magic. Comments consistently
reference either "GUP-fast" or "gup_fast()".
This patch (of 3):
Let's consistently call the "fast-only" part of GUP "GUP-fast" and rename
all relevant internal functions to start with "gup_fast", to make it
clearer that this is not ordinary GUP. The current mixture of "lockless",
"gup" and "gup_fast" is confusing.
Further, avoid the term "huge" when talking about a "leaf" -- for example,
we nowadays check pmd_leaf() because pmd_huge() is gone. For the
"hugepd"/"hugepte" stuff, it's part of the name ("is_hugepd"), so that
stays.
What remains is the "external" interface:
* get_user_pages_fast_only()
* get_user_pages_fast()
* pin_user_pages_fast()
The high-level internal functions for GUP-fast (+slow fallback) are now:
* internal_get_user_pages_fast() -> gup_fast_fallback()
* lockless_pages_from_mm() -> gup_fast()
The basic GUP-fast walker functions:
* gup_pgd_range() -> gup_fast_pgd_range()
* gup_p4d_range() -> gup_fast_p4d_range()
* gup_pud_range() -> gup_fast_pud_range()
* gup_pmd_range() -> gup_fast_pmd_range()
* gup_pte_range() -> gup_fast_pte_range()
* gup_huge_pgd() -> gup_fast_pgd_leaf()
* gup_huge_pud() -> gup_fast_pud_leaf()
* gup_huge_pmd() -> gup_fast_pmd_leaf()
The weird hugepd stuff:
* gup_huge_pd() -> gup_fast_hugepd()
* gup_hugepte() -> gup_fast_hugepte()
The weird devmap stuff:
* __gup_device_huge_pud() -> gup_fast_devmap_pud_leaf()
* __gup_device_huge_pmd -> gup_fast_devmap_pmd_leaf()
* __gup_device_huge() -> gup_fast_devmap_leaf()
* undo_dev_pagemap() -> gup_fast_undo_dev_pagemap()
Helper functions:
* unpin_user_pages_lockless() -> gup_fast_unpin_user_pages()
* gup_fast_folio_allowed() is already properly named
* gup_fast_permitted() is already properly named
With "gup_fast()", we now even have a function that is referred to in
comment in mm/mmu_gather.c.
Link: https://lkml.kernel.org/r/20240402125516.223131-1-david@redhat.com
Link: https://lkml.kernel.org/r/20240402125516.223131-2-david@redhat.com
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Mike Rapoport (IBM) <rppt@kernel.org>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
David Hildenbrand
and committed by
Andrew Morton
23babe19
f6a8dd98
+103
-102
+103
-102
mm/gup.c
+103
-102
mm/gup.c
···
440
440
}
441
441
EXPORT_SYMBOL(unpin_user_page_range_dirty_lock);
442
442
443
-
static void unpin_user_pages_lockless(struct page **pages, unsigned long npages)
443
+
static void gup_fast_unpin_user_pages(struct page **pages, unsigned long npages)
444
444
{
445
445
unsigned long i;
446
446
struct folio *folio;
···
525
525
return (__boundary - 1 < end - 1) ? __boundary : end;
526
526
}
527
527
528
-
static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
529
-
unsigned long end, unsigned int flags,
530
-
struct page **pages, int *nr)
528
+
static int gup_fast_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
529
+
unsigned long end, unsigned int flags, struct page **pages,
530
+
int *nr)
531
531
{
532
532
unsigned long pte_end;
533
533
struct page *page;
···
577
577
* of the other folios. See writable_file_mapping_allowed() and
578
578
* gup_fast_folio_allowed() for more information.
579
579
*/
580
-
static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
580
+
static int gup_fast_hugepd(hugepd_t hugepd, unsigned long addr,
581
581
unsigned int pdshift, unsigned long end, unsigned int flags,
582
582
struct page **pages, int *nr)
583
583
{
···
588
588
ptep = hugepte_offset(hugepd, addr, pdshift);
589
589
do {
590
590
next = hugepte_addr_end(addr, end, sz);
591
-
if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
591
+
if (!gup_fast_hugepte(ptep, sz, addr, end, flags, pages, nr))
592
592
return 0;
593
593
} while (ptep++, addr = next, addr != end);
594
594
···
613
613
h = hstate_vma(vma);
614
614
ptep = hugepte_offset(hugepd, addr, pdshift);
615
615
ptl = huge_pte_lock(h, vma->vm_mm, ptep);
616
-
ret = gup_huge_pd(hugepd, addr, pdshift, addr + PAGE_SIZE,
617
-
flags, &page, &nr);
616
+
ret = gup_fast_hugepd(hugepd, addr, pdshift, addr + PAGE_SIZE,
617
+
flags, &page, &nr);
618
618
spin_unlock(ptl);
619
619
620
620
if (ret) {
···
626
626
return NULL;
627
627
}
628
628
#else /* CONFIG_ARCH_HAS_HUGEPD */
629
-
static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
629
+
static inline int gup_fast_hugepd(hugepd_t hugepd, unsigned long addr,
630
630
unsigned int pdshift, unsigned long end, unsigned int flags,
631
631
struct page **pages, int *nr)
632
632
{
···
2750
2750
EXPORT_SYMBOL(get_user_pages_unlocked);
2751
2751
2752
2752
/*
2753
-
* Fast GUP
2753
+
* GUP-fast
2754
2754
*
2755
2755
* get_user_pages_fast attempts to pin user pages by walking the page
2756
2756
* tables directly and avoids taking locks. Thus the walker needs to be
···
2764
2764
*
2765
2765
* Another way to achieve this is to batch up page table containing pages
2766
2766
* belonging to more than one mm_user, then rcu_sched a callback to free those
2767
-
* pages. Disabling interrupts will allow the fast_gup walker to both block
2767
+
* pages. Disabling interrupts will allow the gup_fast() walker to both block
2768
2768
* the rcu_sched callback, and an IPI that we broadcast for splitting THPs
2769
2769
* (which is a relatively rare event). The code below adopts this strategy.
2770
2770
*
···
2873
2873
return !reject_file_backed || shmem_mapping(mapping);
2874
2874
}
2875
2875
2876
-
static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
2877
-
unsigned int flags,
2878
-
struct page **pages)
2876
+
static void __maybe_unused gup_fast_undo_dev_pagemap(int *nr, int nr_start,
2877
+
unsigned int flags, struct page **pages)
2879
2878
{
2880
2879
while ((*nr) - nr_start) {
2881
2880
struct page *page = pages[--(*nr)];
···
2889
2890
2890
2891
#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
2891
2892
/*
2892
-
* Fast-gup relies on pte change detection to avoid concurrent pgtable
2893
+
* GUP-fast relies on pte change detection to avoid concurrent pgtable
2893
2894
* operations.
2894
2895
*
2895
-
* To pin the page, fast-gup needs to do below in order:
2896
+
* To pin the page, GUP-fast needs to do below in order:
2896
2897
* (1) pin the page (by prefetching pte), then (2) check pte not changed.
2897
2898
*
2898
2899
* For the rest of pgtable operations where pgtable updates can be racy
2899
-
* with fast-gup, we need to do (1) clear pte, then (2) check whether page
2900
+
* with GUP-fast, we need to do (1) clear pte, then (2) check whether page
2900
2901
* is pinned.
2901
2902
*
2902
2903
* Above will work for all pte-level operations, including THP split.
2903
2904
*
2904
-
* For THP collapse, it's a bit more complicated because fast-gup may be
2905
+
* For THP collapse, it's a bit more complicated because GUP-fast may be
2905
2906
* walking a pgtable page that is being freed (pte is still valid but pmd
2906
2907
* can be cleared already). To avoid race in such condition, we need to
2907
2908
* also check pmd here to make sure pmd doesn't change (corresponds to
2908
2909
* pmdp_collapse_flush() in the THP collapse code path).
2909
2910
*/
2910
-
static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
2911
-
unsigned long end, unsigned int flags,
2912
-
struct page **pages, int *nr)
2911
+
static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
2912
+
unsigned long end, unsigned int flags, struct page **pages,
2913
+
int *nr)
2913
2914
{
2914
2915
struct dev_pagemap *pgmap = NULL;
2915
2916
int nr_start = *nr, ret = 0;
···
2942
2943
2943
2944
pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
2944
2945
if (unlikely(!pgmap)) {
2945
-
undo_dev_pagemap(nr, nr_start, flags, pages);
2946
+
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
2946
2947
goto pte_unmap;
2947
2948
}
2948
2949
} else if (pte_special(pte))
···
3006
3007
*
3007
3008
* For a futex to be placed on a THP tail page, get_futex_key requires a
3008
3009
* get_user_pages_fast_only implementation that can pin pages. Thus it's still
3009
-
* useful to have gup_huge_pmd even if we can't operate on ptes.
3010
+
* useful to have gup_fast_pmd_leaf even if we can't operate on ptes.
3010
3011
*/
3011
-
static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
3012
-
unsigned long end, unsigned int flags,
3013
-
struct page **pages, int *nr)
3012
+
static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
3013
+
unsigned long end, unsigned int flags, struct page **pages,
3014
+
int *nr)
3014
3015
{
3015
3016
return 0;
3016
3017
}
3017
3018
#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
3018
3019
3019
3020
#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
3020
-
static int __gup_device_huge(unsigned long pfn, unsigned long addr,
3021
-
unsigned long end, unsigned int flags,
3022
-
struct page **pages, int *nr)
3021
+
static int gup_fast_devmap_leaf(unsigned long pfn, unsigned long addr,
3022
+
unsigned long end, unsigned int flags, struct page **pages, int *nr)
3023
3023
{
3024
3024
int nr_start = *nr;
3025
3025
struct dev_pagemap *pgmap = NULL;
···
3028
3030
3029
3031
pgmap = get_dev_pagemap(pfn, pgmap);
3030
3032
if (unlikely(!pgmap)) {
3031
-
undo_dev_pagemap(nr, nr_start, flags, pages);
3033
+
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
3032
3034
break;
3033
3035
}
3034
3036
3035
3037
if (!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)) {
3036
-
undo_dev_pagemap(nr, nr_start, flags, pages);
3038
+
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
3037
3039
break;
3038
3040
}
3039
3041
3040
3042
SetPageReferenced(page);
3041
3043
pages[*nr] = page;
3042
3044
if (unlikely(try_grab_page(page, flags))) {
3043
-
undo_dev_pagemap(nr, nr_start, flags, pages);
3045
+
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
3044
3046
break;
3045
3047
}
3046
3048
(*nr)++;
···
3051
3053
return addr == end;
3052
3054
}
3053
3055
3054
-
static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3055
-
unsigned long end, unsigned int flags,
3056
-
struct page **pages, int *nr)
3056
+
static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3057
+
unsigned long end, unsigned int flags, struct page **pages,
3058
+
int *nr)
3057
3059
{
3058
3060
unsigned long fault_pfn;
3059
3061
int nr_start = *nr;
3060
3062
3061
3063
fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
3062
-
if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
3064
+
if (!gup_fast_devmap_leaf(fault_pfn, addr, end, flags, pages, nr))
3063
3065
return 0;
3064
3066
3065
3067
if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
3066
-
undo_dev_pagemap(nr, nr_start, flags, pages);
3068
+
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
3067
3069
return 0;
3068
3070
}
3069
3071
return 1;
3070
3072
}
3071
3073
3072
-
static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
3073
-
unsigned long end, unsigned int flags,
3074
-
struct page **pages, int *nr)
3074
+
static int gup_fast_devmap_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
3075
+
unsigned long end, unsigned int flags, struct page **pages,
3076
+
int *nr)
3075
3077
{
3076
3078
unsigned long fault_pfn;
3077
3079
int nr_start = *nr;
3078
3080
3079
3081
fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
3080
-
if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
3082
+
if (!gup_fast_devmap_leaf(fault_pfn, addr, end, flags, pages, nr))
3081
3083
return 0;
3082
3084
3083
3085
if (unlikely(pud_val(orig) != pud_val(*pudp))) {
3084
-
undo_dev_pagemap(nr, nr_start, flags, pages);
3086
+
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
3085
3087
return 0;
3086
3088
}
3087
3089
return 1;
3088
3090
}
3089
3091
#else
3090
-
static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3091
-
unsigned long end, unsigned int flags,
3092
-
struct page **pages, int *nr)
3092
+
static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3093
+
unsigned long end, unsigned int flags, struct page **pages,
3094
+
int *nr)
3093
3095
{
3094
3096
BUILD_BUG();
3095
3097
return 0;
3096
3098
}
3097
3099
3098
-
static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
3099
-
unsigned long end, unsigned int flags,
3100
-
struct page **pages, int *nr)
3100
+
static int gup_fast_devmap_pud_leaf(pud_t pud, pud_t *pudp, unsigned long addr,
3101
+
unsigned long end, unsigned int flags, struct page **pages,
3102
+
int *nr)
3101
3103
{
3102
3104
BUILD_BUG();
3103
3105
return 0;
3104
3106
}
3105
3107
#endif
3106
3108
3107
-
static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3108
-
unsigned long end, unsigned int flags,
3109
-
struct page **pages, int *nr)
3109
+
static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3110
+
unsigned long end, unsigned int flags, struct page **pages,
3111
+
int *nr)
3110
3112
{
3111
3113
struct page *page;
3112
3114
struct folio *folio;
···
3118
3120
if (pmd_devmap(orig)) {
3119
3121
if (unlikely(flags & FOLL_LONGTERM))
3120
3122
return 0;
3121
-
return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
3122
-
pages, nr);
3123
+
return gup_fast_devmap_pmd_leaf(orig, pmdp, addr, end, flags,
3124
+
pages, nr);
3123
3125
}
3124
3126
3125
3127
page = pmd_page(orig);
···
3148
3150
return 1;
3149
3151
}
3150
3152
3151
-
static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
3152
-
unsigned long end, unsigned int flags,
3153
-
struct page **pages, int *nr)
3153
+
static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
3154
+
unsigned long end, unsigned int flags, struct page **pages,
3155
+
int *nr)
3154
3156
{
3155
3157
struct page *page;
3156
3158
struct folio *folio;
···
3162
3164
if (pud_devmap(orig)) {
3163
3165
if (unlikely(flags & FOLL_LONGTERM))
3164
3166
return 0;
3165
-
return __gup_device_huge_pud(orig, pudp, addr, end, flags,
3166
-
pages, nr);
3167
+
return gup_fast_devmap_pud_leaf(orig, pudp, addr, end, flags,
3168
+
pages, nr);
3167
3169
}
3168
3170
3169
3171
page = pud_page(orig);
···
3193
3195
return 1;
3194
3196
}
3195
3197
3196
-
static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
3197
-
unsigned long end, unsigned int flags,
3198
-
struct page **pages, int *nr)
3198
+
static int gup_fast_pgd_leaf(pgd_t orig, pgd_t *pgdp, unsigned long addr,
3199
+
unsigned long end, unsigned int flags, struct page **pages,
3200
+
int *nr)
3199
3201
{
3200
3202
int refs;
3201
3203
struct page *page;
···
3233
3235
return 1;
3234
3236
}
3235
3237
3236
-
static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
3237
-
unsigned int flags, struct page **pages, int *nr)
3238
+
static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
3239
+
unsigned long end, unsigned int flags, struct page **pages,
3240
+
int *nr)
3238
3241
{
3239
3242
unsigned long next;
3240
3243
pmd_t *pmdp;
···
3249
3250
return 0;
3250
3251
3251
3252
if (unlikely(pmd_leaf(pmd))) {
3252
-
/* See gup_pte_range() */
3253
+
/* See gup_fast_pte_range() */
3253
3254
if (pmd_protnone(pmd))
3254
3255
return 0;
3255
3256
3256
-
if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
3257
+
if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags,
3257
3258
pages, nr))
3258
3259
return 0;
3259
3260
···
3262
3263
* architecture have different format for hugetlbfs
3263
3264
* pmd format and THP pmd format
3264
3265
*/
3265
-
if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
3266
-
PMD_SHIFT, next, flags, pages, nr))
3266
+
if (!gup_fast_hugepd(__hugepd(pmd_val(pmd)), addr,
3267
+
PMD_SHIFT, next, flags, pages, nr))
3267
3268
return 0;
3268
-
} else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))
3269
+
} else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags,
3270
+
pages, nr))
3269
3271
return 0;
3270
3272
} while (pmdp++, addr = next, addr != end);
3271
3273
3272
3274
return 1;
3273
3275
}
3274
3276
3275
-
static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
3276
-
unsigned int flags, struct page **pages, int *nr)
3277
+
static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr,
3278
+
unsigned long end, unsigned int flags, struct page **pages,
3279
+
int *nr)
3277
3280
{
3278
3281
unsigned long next;
3279
3282
pud_t *pudp;
···
3288
3287
if (unlikely(!pud_present(pud)))
3289
3288
return 0;
3290
3289
if (unlikely(pud_leaf(pud))) {
3291
-
if (!gup_huge_pud(pud, pudp, addr, next, flags,
3292
-
pages, nr))
3290
+
if (!gup_fast_pud_leaf(pud, pudp, addr, next, flags,
3291
+
pages, nr))
3293
3292
return 0;
3294
3293
} else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
3295
-
if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
3296
-
PUD_SHIFT, next, flags, pages, nr))
3294
+
if (!gup_fast_hugepd(__hugepd(pud_val(pud)), addr,
3295
+
PUD_SHIFT, next, flags, pages, nr))
3297
3296
return 0;
3298
-
} else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
3297
+
} else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags,
3298
+
pages, nr))
3299
3299
return 0;
3300
3300
} while (pudp++, addr = next, addr != end);
3301
3301
3302
3302
return 1;
3303
3303
}
3304
3304
3305
-
static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
3306
-
unsigned int flags, struct page **pages, int *nr)
3305
+
static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
3306
+
unsigned long end, unsigned int flags, struct page **pages,
3307
+
int *nr)
3307
3308
{
3308
3309
unsigned long next;
3309
3310
p4d_t *p4dp;
···
3319
3316
return 0;
3320
3317
BUILD_BUG_ON(p4d_leaf(p4d));
3321
3318
if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
3322
-
if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
3323
-
P4D_SHIFT, next, flags, pages, nr))
3319
+
if (!gup_fast_hugepd(__hugepd(p4d_val(p4d)), addr,
3320
+
P4D_SHIFT, next, flags, pages, nr))
3324
3321
return 0;
3325
-
} else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
3322
+
} else if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags,
3323
+
pages, nr))
3326
3324
return 0;
3327
3325
} while (p4dp++, addr = next, addr != end);
3328
3326
3329
3327
return 1;
3330
3328
}
3331
3329
3332
-
static void gup_pgd_range(unsigned long addr, unsigned long end,
3330
+
static void gup_fast_pgd_range(unsigned long addr, unsigned long end,
3333
3331
unsigned int flags, struct page **pages, int *nr)
3334
3332
{
3335
3333
unsigned long next;
···
3344
3340
if (pgd_none(pgd))
3345
3341
return;
3346
3342
if (unlikely(pgd_leaf(pgd))) {
3347
-
if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
3348
-
pages, nr))
3343
+
if (!gup_fast_pgd_leaf(pgd, pgdp, addr, next, flags,
3344
+
pages, nr))
3349
3345
return;
3350
3346
} else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
3351
-
if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
3352
-
PGDIR_SHIFT, next, flags, pages, nr))
3347
+
if (!gup_fast_hugepd(__hugepd(pgd_val(pgd)), addr,
3348
+
PGDIR_SHIFT, next, flags, pages, nr))
3353
3349
return;
3354
-
} else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
3350
+
} else if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags,
3351
+
pages, nr))
3355
3352
return;
3356
3353
} while (pgdp++, addr = next, addr != end);
3357
3354
}
3358
3355
#else
3359
-
static inline void gup_pgd_range(unsigned long addr, unsigned long end,
3356
+
static inline void gup_fast_pgd_range(unsigned long addr, unsigned long end,
3360
3357
unsigned int flags, struct page **pages, int *nr)
3361
3358
{
3362
3359
}
···
3374
3369
}
3375
3370
#endif
3376
3371
3377
-
static unsigned long lockless_pages_from_mm(unsigned long start,
3378
-
unsigned long end,
3379
-
unsigned int gup_flags,
3380
-
struct page **pages)
3372
+
static unsigned long gup_fast(unsigned long start, unsigned long end,
3373
+
unsigned int gup_flags, struct page **pages)
3381
3374
{
3382
3375
unsigned long flags;
3383
3376
int nr_pinned = 0;
···
3403
3400
* that come from THPs splitting.
3404
3401
*/
3405
3402
local_irq_save(flags);
3406
-
gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
3403
+
gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned);
3407
3404
local_irq_restore(flags);
3408
3405
3409
3406
/*
3410
3407
* When pinning pages for DMA there could be a concurrent write protect
3411
-
* from fork() via copy_page_range(), in this case always fail fast GUP.
3408
+
* from fork() via copy_page_range(), in this case always fail GUP-fast.
3412
3409
*/
3413
3410
if (gup_flags & FOLL_PIN) {
3414
3411
if (read_seqcount_retry(¤t->mm->write_protect_seq, seq)) {
3415
-
unpin_user_pages_lockless(pages, nr_pinned);
3412
+
gup_fast_unpin_user_pages(pages, nr_pinned);
3416
3413
return 0;
3417
3414
} else {
3418
3415
sanity_check_pinned_pages(pages, nr_pinned);
···
3421
3418
return nr_pinned;
3422
3419
}
3423
3420
3424
-
static int internal_get_user_pages_fast(unsigned long start,
3425
-
unsigned long nr_pages,
3426
-
unsigned int gup_flags,
3427
-
struct page **pages)
3421
+
static int gup_fast_fallback(unsigned long start, unsigned long nr_pages,
3422
+
unsigned int gup_flags, struct page **pages)
3428
3423
{
3429
3424
unsigned long len, end;
3430
3425
unsigned long nr_pinned;
···
3450
3449
if (unlikely(!access_ok((void __user *)start, len)))
3451
3450
return -EFAULT;
3452
3451
3453
-
nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
3452
+
nr_pinned = gup_fast(start, end, gup_flags, pages);
3454
3453
if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
3455
3454
return nr_pinned;
3456
3455
···
3504
3503
FOLL_GET | FOLL_FAST_ONLY))
3505
3504
return -EINVAL;
3506
3505
3507
-
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
3506
+
return gup_fast_fallback(start, nr_pages, gup_flags, pages);
3508
3507
}
3509
3508
EXPORT_SYMBOL_GPL(get_user_pages_fast_only);
3510
3509
···
3535
3534
*/
3536
3535
if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET))
3537
3536
return -EINVAL;
3538
-
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
3537
+
return gup_fast_fallback(start, nr_pages, gup_flags, pages);
3539
3538
}
3540
3539
EXPORT_SYMBOL_GPL(get_user_pages_fast);
3541
3540
···
3563
3562
{
3564
3563
if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))
3565
3564
return -EINVAL;
3566
-
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
3565
+
return gup_fast_fallback(start, nr_pages, gup_flags, pages);
3567
3566
}
3568
3567
EXPORT_SYMBOL_GPL(pin_user_pages_fast);
3569
3568