Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'prep-for-5level'
Merge 5-level page table prep from Kirill Shutemov:
"Here's relatively low-risk part of 5-level paging patchset. Merging it
now will make x86 5-level paging enabling in v4.12 easier.
The first patch is actually x86-specific: detect 5-level paging
support. It boils down to single define.
The rest of patchset converts Linux MMU abstraction from 4- to 5-level
paging.
Enabling of new abstraction in most cases requires adding single line
of code in arch-specific code. The rest is taken care by asm-generic/.
Changes to mm/ code are mostly mechanical: add support for new page
table level -- p4d_t -- where we deal with pud_t now.
v2:
- fix build on microblaze (Michal);
- comment for __ARCH_HAS_5LEVEL_HACK in kasan_populate_zero_shadow();
- acks from Michal"
* emailed patches from Kirill A Shutemov <kirill.shutemov@linux.intel.com>:
mm: introduce __p4d_alloc()
mm: convert generic code to 5-level paging
asm-generic: introduce <asm-generic/pgtable-nop4d.h>
arch, mm: convert all architectures to use 5level-fixup.h
asm-generic: introduce __ARCH_USE_5LEVEL_HACK
asm-generic: introduce 5level-fixup.h
x86/cpufeature: Add 5-level paging detection
+868
-147
+1
arch/arc/include/asm/hugepage.h
+1
arch/arc/include/asm/hugepage.h
+1
arch/arc/include/asm/pgtable.h
+1
arch/arc/include/asm/pgtable.h
+1
arch/arm/include/asm/pgtable.h
+1
arch/arm/include/asm/pgtable.h
+4
arch/arm64/include/asm/pgtable-types.h
+4
arch/arm64/include/asm/pgtable-types.h
···
55
55
#define __pgprot(x) ((pgprot_t) { (x) } )
56
56
57
57
#if CONFIG_PGTABLE_LEVELS == 2
58
+
#define __ARCH_USE_5LEVEL_HACK
58
59
#include <asm-generic/pgtable-nopmd.h>
59
60
#elif CONFIG_PGTABLE_LEVELS == 3
61
+
#define __ARCH_USE_5LEVEL_HACK
60
62
#include <asm-generic/pgtable-nopud.h>
63
+
#elif CONFIG_PGTABLE_LEVELS == 4
64
+
#include <asm-generic/5level-fixup.h>
61
65
#endif
62
66
63
67
#endif /* __ASM_PGTABLE_TYPES_H */
+1
arch/avr32/include/asm/pgtable-2level.h
+1
arch/avr32/include/asm/pgtable-2level.h
+1
arch/cris/include/asm/pgtable.h
+1
arch/cris/include/asm/pgtable.h
+1
arch/frv/include/asm/pgtable.h
+1
arch/frv/include/asm/pgtable.h
+1
arch/h8300/include/asm/pgtable.h
+1
arch/h8300/include/asm/pgtable.h
+1
arch/hexagon/include/asm/pgtable.h
+1
arch/hexagon/include/asm/pgtable.h
+2
arch/ia64/include/asm/pgtable.h
+2
arch/ia64/include/asm/pgtable.h
+1
arch/metag/include/asm/pgtable.h
+1
arch/metag/include/asm/pgtable.h
+2
-1
arch/microblaze/include/asm/page.h
+2
-1
arch/microblaze/include/asm/page.h
···
95
95
# else /* CONFIG_MMU */
96
96
typedef struct { unsigned long ste[64]; } pmd_t;
97
97
typedef struct { pmd_t pue[1]; } pud_t;
98
-
typedef struct { pud_t pge[1]; } pgd_t;
98
+
typedef struct { pud_t p4e[1]; } p4d_t;
99
+
typedef struct { p4d_t pge[1]; } pgd_t;
99
100
# endif /* CONFIG_MMU */
100
101
101
102
# define pte_val(x) ((x).pte)
+1
arch/mips/include/asm/pgtable-32.h
+1
arch/mips/include/asm/pgtable-32.h
+1
arch/mips/include/asm/pgtable-64.h
+1
arch/mips/include/asm/pgtable-64.h
+1
arch/mn10300/include/asm/page.h
+1
arch/mn10300/include/asm/page.h
+1
arch/nios2/include/asm/pgtable.h
+1
arch/nios2/include/asm/pgtable.h
+1
arch/openrisc/include/asm/pgtable.h
+1
arch/openrisc/include/asm/pgtable.h
+1
arch/powerpc/include/asm/book3s/32/pgtable.h
+1
arch/powerpc/include/asm/book3s/32/pgtable.h
+3
arch/powerpc/include/asm/book3s/64/pgtable.h
+3
arch/powerpc/include/asm/book3s/64/pgtable.h
+1
arch/powerpc/include/asm/nohash/32/pgtable.h
+1
arch/powerpc/include/asm/nohash/32/pgtable.h
+3
arch/powerpc/include/asm/nohash/64/pgtable-4k.h
+3
arch/powerpc/include/asm/nohash/64/pgtable-4k.h
···
1
1
#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H
2
2
#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H
3
+
4
+
#include <asm-generic/5level-fixup.h>
5
+
3
6
/*
4
7
* Entries per page directory level. The PTE level must use a 64b record
5
8
* for each page table entry. The PMD and PGD level use a 32b record for
+1
arch/powerpc/include/asm/nohash/64/pgtable-64k.h
+1
arch/powerpc/include/asm/nohash/64/pgtable-64k.h
+1
arch/s390/include/asm/pgtable.h
+1
arch/s390/include/asm/pgtable.h
+1
arch/score/include/asm/pgtable.h
+1
arch/score/include/asm/pgtable.h
+1
arch/sh/include/asm/pgtable-2level.h
+1
arch/sh/include/asm/pgtable-2level.h
+1
arch/sh/include/asm/pgtable-3level.h
+1
arch/sh/include/asm/pgtable-3level.h
+1
arch/sparc/include/asm/pgtable_64.h
+1
arch/sparc/include/asm/pgtable_64.h
+1
arch/tile/include/asm/pgtable_32.h
+1
arch/tile/include/asm/pgtable_32.h
+1
arch/tile/include/asm/pgtable_64.h
+1
arch/tile/include/asm/pgtable_64.h
+1
arch/um/include/asm/pgtable-2level.h
+1
arch/um/include/asm/pgtable-2level.h
+1
arch/um/include/asm/pgtable-3level.h
+1
arch/um/include/asm/pgtable-3level.h
+1
arch/unicore32/include/asm/pgtable.h
+1
arch/unicore32/include/asm/pgtable.h
+2
-1
arch/x86/include/asm/cpufeatures.h
+2
-1
arch/x86/include/asm/cpufeatures.h
···
289
289
#define X86_FEATURE_PKU (16*32+ 3) /* Protection Keys for Userspace */
290
290
#define X86_FEATURE_OSPKE (16*32+ 4) /* OS Protection Keys Enable */
291
291
#define X86_FEATURE_AVX512_VPOPCNTDQ (16*32+14) /* POPCNT for vectors of DW/QW */
292
-
#define X86_FEATURE_RDPID (16*32+ 22) /* RDPID instruction */
292
+
#define X86_FEATURE_LA57 (16*32+16) /* 5-level page tables */
293
+
#define X86_FEATURE_RDPID (16*32+22) /* RDPID instruction */
293
294
294
295
/* AMD-defined CPU features, CPUID level 0x80000007 (ebx), word 17 */
295
296
#define X86_FEATURE_OVERFLOW_RECOV (17*32+0) /* MCA overflow recovery support */
+4
arch/x86/include/asm/pgtable_types.h
+4
arch/x86/include/asm/pgtable_types.h
···
273
273
}
274
274
275
275
#if CONFIG_PGTABLE_LEVELS > 3
276
+
#include <asm-generic/5level-fixup.h>
277
+
276
278
typedef struct { pudval_t pud; } pud_t;
277
279
278
280
static inline pud_t native_make_pud(pmdval_t val)
···
287
285
return pud.pud;
288
286
}
289
287
#else
288
+
#define __ARCH_USE_5LEVEL_HACK
290
289
#include <asm-generic/pgtable-nopud.h>
291
290
292
291
static inline pudval_t native_pud_val(pud_t pud)
···
309
306
return pmd.pmd;
310
307
}
311
308
#else
309
+
#define __ARCH_USE_5LEVEL_HACK
312
310
#include <asm-generic/pgtable-nopmd.h>
313
311
314
312
static inline pmdval_t native_pmd_val(pmd_t pmd)
+1
arch/xtensa/include/asm/pgtable.h
+1
arch/xtensa/include/asm/pgtable.h
+7
-2
drivers/misc/sgi-gru/grufault.c
+7
-2
drivers/misc/sgi-gru/grufault.c
···
219
219
int write, unsigned long *paddr, int *pageshift)
220
220
{
221
221
pgd_t *pgdp;
222
-
pmd_t *pmdp;
222
+
p4d_t *p4dp;
223
223
pud_t *pudp;
224
+
pmd_t *pmdp;
224
225
pte_t pte;
225
226
226
227
pgdp = pgd_offset(vma->vm_mm, vaddr);
227
228
if (unlikely(pgd_none(*pgdp)))
228
229
goto err;
229
230
230
-
pudp = pud_offset(pgdp, vaddr);
231
+
p4dp = p4d_offset(pgdp, vaddr);
232
+
if (unlikely(p4d_none(*p4dp)))
233
+
goto err;
234
+
235
+
pudp = pud_offset(p4dp, vaddr);
231
236
if (unlikely(pud_none(*pudp)))
232
237
goto err;
233
238
+5
-1
fs/userfaultfd.c
+5
-1
fs/userfaultfd.c
···
265
265
{
266
266
struct mm_struct *mm = ctx->mm;
267
267
pgd_t *pgd;
268
+
p4d_t *p4d;
268
269
pud_t *pud;
269
270
pmd_t *pmd, _pmd;
270
271
pte_t *pte;
···
276
275
pgd = pgd_offset(mm, address);
277
276
if (!pgd_present(*pgd))
278
277
goto out;
279
-
pud = pud_offset(pgd, address);
278
+
p4d = p4d_offset(pgd, address);
279
+
if (!p4d_present(*p4d))
280
+
goto out;
281
+
pud = pud_offset(p4d, address);
280
282
if (!pud_present(*pud))
281
283
goto out;
282
284
pmd = pmd_offset(pud, address);
+2
-1
include/asm-generic/4level-fixup.h
+2
-1
include/asm-generic/4level-fixup.h
···
15
15
((unlikely(pgd_none(*(pud))) && __pmd_alloc(mm, pud, address))? \
16
16
NULL: pmd_offset(pud, address))
17
17
18
-
#define pud_alloc(mm, pgd, address) (pgd)
19
18
#define pud_offset(pgd, start) (pgd)
20
19
#define pud_none(pud) 0
21
20
#define pud_bad(pud) 0
···
33
34
34
35
#undef pud_addr_end
35
36
#define pud_addr_end(addr, end) (end)
37
+
38
+
#include <asm-generic/5level-fixup.h>
36
39
37
40
#endif
+41
include/asm-generic/5level-fixup.h
+41
include/asm-generic/5level-fixup.h
···
1
+
#ifndef _5LEVEL_FIXUP_H
2
+
#define _5LEVEL_FIXUP_H
3
+
4
+
#define __ARCH_HAS_5LEVEL_HACK
5
+
#define __PAGETABLE_P4D_FOLDED
6
+
7
+
#define P4D_SHIFT PGDIR_SHIFT
8
+
#define P4D_SIZE PGDIR_SIZE
9
+
#define P4D_MASK PGDIR_MASK
10
+
#define PTRS_PER_P4D 1
11
+
12
+
#define p4d_t pgd_t
13
+
14
+
#define pud_alloc(mm, p4d, address) \
15
+
((unlikely(pgd_none(*(p4d))) && __pud_alloc(mm, p4d, address)) ? \
16
+
NULL : pud_offset(p4d, address))
17
+
18
+
#define p4d_alloc(mm, pgd, address) (pgd)
19
+
#define p4d_offset(pgd, start) (pgd)
20
+
#define p4d_none(p4d) 0
21
+
#define p4d_bad(p4d) 0
22
+
#define p4d_present(p4d) 1
23
+
#define p4d_ERROR(p4d) do { } while (0)
24
+
#define p4d_clear(p4d) pgd_clear(p4d)
25
+
#define p4d_val(p4d) pgd_val(p4d)
26
+
#define p4d_populate(mm, p4d, pud) pgd_populate(mm, p4d, pud)
27
+
#define p4d_page(p4d) pgd_page(p4d)
28
+
#define p4d_page_vaddr(p4d) pgd_page_vaddr(p4d)
29
+
30
+
#define __p4d(x) __pgd(x)
31
+
#define set_p4d(p4dp, p4d) set_pgd(p4dp, p4d)
32
+
33
+
#undef p4d_free_tlb
34
+
#define p4d_free_tlb(tlb, x, addr) do { } while (0)
35
+
#define p4d_free(mm, x) do { } while (0)
36
+
#define __p4d_free_tlb(tlb, x, addr) do { } while (0)
37
+
38
+
#undef p4d_addr_end
39
+
#define p4d_addr_end(addr, end) (end)
40
+
41
+
#endif
+62
include/asm-generic/pgtable-nop4d-hack.h
+62
include/asm-generic/pgtable-nop4d-hack.h
···
1
+
#ifndef _PGTABLE_NOP4D_HACK_H
2
+
#define _PGTABLE_NOP4D_HACK_H
3
+
4
+
#ifndef __ASSEMBLY__
5
+
#include <asm-generic/5level-fixup.h>
6
+
7
+
#define __PAGETABLE_PUD_FOLDED
8
+
9
+
/*
10
+
* Having the pud type consist of a pgd gets the size right, and allows
11
+
* us to conceptually access the pgd entry that this pud is folded into
12
+
* without casting.
13
+
*/
14
+
typedef struct { pgd_t pgd; } pud_t;
15
+
16
+
#define PUD_SHIFT PGDIR_SHIFT
17
+
#define PTRS_PER_PUD 1
18
+
#define PUD_SIZE (1UL << PUD_SHIFT)
19
+
#define PUD_MASK (~(PUD_SIZE-1))
20
+
21
+
/*
22
+
* The "pgd_xxx()" functions here are trivial for a folded two-level
23
+
* setup: the pud is never bad, and a pud always exists (as it's folded
24
+
* into the pgd entry)
25
+
*/
26
+
static inline int pgd_none(pgd_t pgd) { return 0; }
27
+
static inline int pgd_bad(pgd_t pgd) { return 0; }
28
+
static inline int pgd_present(pgd_t pgd) { return 1; }
29
+
static inline void pgd_clear(pgd_t *pgd) { }
30
+
#define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
31
+
32
+
#define pgd_populate(mm, pgd, pud) do { } while (0)
33
+
/*
34
+
* (puds are folded into pgds so this doesn't get actually called,
35
+
* but the define is needed for a generic inline function.)
36
+
*/
37
+
#define set_pgd(pgdptr, pgdval) set_pud((pud_t *)(pgdptr), (pud_t) { pgdval })
38
+
39
+
static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
40
+
{
41
+
return (pud_t *)pgd;
42
+
}
43
+
44
+
#define pud_val(x) (pgd_val((x).pgd))
45
+
#define __pud(x) ((pud_t) { __pgd(x) })
46
+
47
+
#define pgd_page(pgd) (pud_page((pud_t){ pgd }))
48
+
#define pgd_page_vaddr(pgd) (pud_page_vaddr((pud_t){ pgd }))
49
+
50
+
/*
51
+
* allocating and freeing a pud is trivial: the 1-entry pud is
52
+
* inside the pgd, so has no extra memory associated with it.
53
+
*/
54
+
#define pud_alloc_one(mm, address) NULL
55
+
#define pud_free(mm, x) do { } while (0)
56
+
#define __pud_free_tlb(tlb, x, a) do { } while (0)
57
+
58
+
#undef pud_addr_end
59
+
#define pud_addr_end(addr, end) (end)
60
+
61
+
#endif /* __ASSEMBLY__ */
62
+
#endif /* _PGTABLE_NOP4D_HACK_H */
+56
include/asm-generic/pgtable-nop4d.h
+56
include/asm-generic/pgtable-nop4d.h
···
1
+
#ifndef _PGTABLE_NOP4D_H
2
+
#define _PGTABLE_NOP4D_H
3
+
4
+
#ifndef __ASSEMBLY__
5
+
6
+
#define __PAGETABLE_P4D_FOLDED
7
+
8
+
typedef struct { pgd_t pgd; } p4d_t;
9
+
10
+
#define P4D_SHIFT PGDIR_SHIFT
11
+
#define PTRS_PER_P4D 1
12
+
#define P4D_SIZE (1UL << P4D_SHIFT)
13
+
#define P4D_MASK (~(P4D_SIZE-1))
14
+
15
+
/*
16
+
* The "pgd_xxx()" functions here are trivial for a folded two-level
17
+
* setup: the p4d is never bad, and a p4d always exists (as it's folded
18
+
* into the pgd entry)
19
+
*/
20
+
static inline int pgd_none(pgd_t pgd) { return 0; }
21
+
static inline int pgd_bad(pgd_t pgd) { return 0; }
22
+
static inline int pgd_present(pgd_t pgd) { return 1; }
23
+
static inline void pgd_clear(pgd_t *pgd) { }
24
+
#define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd))
25
+
26
+
#define pgd_populate(mm, pgd, p4d) do { } while (0)
27
+
/*
28
+
* (p4ds are folded into pgds so this doesn't get actually called,
29
+
* but the define is needed for a generic inline function.)
30
+
*/
31
+
#define set_pgd(pgdptr, pgdval) set_p4d((p4d_t *)(pgdptr), (p4d_t) { pgdval })
32
+
33
+
static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
34
+
{
35
+
return (p4d_t *)pgd;
36
+
}
37
+
38
+
#define p4d_val(x) (pgd_val((x).pgd))
39
+
#define __p4d(x) ((p4d_t) { __pgd(x) })
40
+
41
+
#define pgd_page(pgd) (p4d_page((p4d_t){ pgd }))
42
+
#define pgd_page_vaddr(pgd) (p4d_page_vaddr((p4d_t){ pgd }))
43
+
44
+
/*
45
+
* allocating and freeing a p4d is trivial: the 1-entry p4d is
46
+
* inside the pgd, so has no extra memory associated with it.
47
+
*/
48
+
#define p4d_alloc_one(mm, address) NULL
49
+
#define p4d_free(mm, x) do { } while (0)
50
+
#define __p4d_free_tlb(tlb, x, a) do { } while (0)
51
+
52
+
#undef p4d_addr_end
53
+
#define p4d_addr_end(addr, end) (end)
54
+
55
+
#endif /* __ASSEMBLY__ */
56
+
#endif /* _PGTABLE_NOP4D_H */
+27
-21
include/asm-generic/pgtable-nopud.h
+27
-21
include/asm-generic/pgtable-nopud.h
···
3
3
4
4
#ifndef __ASSEMBLY__
5
5
6
+
#ifdef __ARCH_USE_5LEVEL_HACK
7
+
#include <asm-generic/pgtable-nop4d-hack.h>
8
+
#else
9
+
#include <asm-generic/pgtable-nop4d.h>
10
+
6
11
#define __PAGETABLE_PUD_FOLDED
7
12
8
13
/*
9
-
* Having the pud type consist of a pgd gets the size right, and allows
10
-
* us to conceptually access the pgd entry that this pud is folded into
14
+
* Having the pud type consist of a p4d gets the size right, and allows
15
+
* us to conceptually access the p4d entry that this pud is folded into
11
16
* without casting.
12
17
*/
13
-
typedef struct { pgd_t pgd; } pud_t;
18
+
typedef struct { p4d_t p4d; } pud_t;
14
19
15
-
#define PUD_SHIFT PGDIR_SHIFT
20
+
#define PUD_SHIFT P4D_SHIFT
16
21
#define PTRS_PER_PUD 1
17
22
#define PUD_SIZE (1UL << PUD_SHIFT)
18
23
#define PUD_MASK (~(PUD_SIZE-1))
19
24
20
25
/*
21
-
* The "pgd_xxx()" functions here are trivial for a folded two-level
26
+
* The "p4d_xxx()" functions here are trivial for a folded two-level
22
27
* setup: the pud is never bad, and a pud always exists (as it's folded
23
-
* into the pgd entry)
28
+
* into the p4d entry)
24
29
*/
25
-
static inline int pgd_none(pgd_t pgd) { return 0; }
26
-
static inline int pgd_bad(pgd_t pgd) { return 0; }
27
-
static inline int pgd_present(pgd_t pgd) { return 1; }
28
-
static inline void pgd_clear(pgd_t *pgd) { }
29
-
#define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
30
+
static inline int p4d_none(p4d_t p4d) { return 0; }
31
+
static inline int p4d_bad(p4d_t p4d) { return 0; }
32
+
static inline int p4d_present(p4d_t p4d) { return 1; }
33
+
static inline void p4d_clear(p4d_t *p4d) { }
34
+
#define pud_ERROR(pud) (p4d_ERROR((pud).p4d))
30
35
31
-
#define pgd_populate(mm, pgd, pud) do { } while (0)
36
+
#define p4d_populate(mm, p4d, pud) do { } while (0)
32
37
/*
33
-
* (puds are folded into pgds so this doesn't get actually called,
38
+
* (puds are folded into p4ds so this doesn't get actually called,
34
39
* but the define is needed for a generic inline function.)
35
40
*/
36
-
#define set_pgd(pgdptr, pgdval) set_pud((pud_t *)(pgdptr), (pud_t) { pgdval })
41
+
#define set_p4d(p4dptr, p4dval) set_pud((pud_t *)(p4dptr), (pud_t) { p4dval })
37
42
38
-
static inline pud_t * pud_offset(pgd_t * pgd, unsigned long address)
43
+
static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
39
44
{
40
-
return (pud_t *)pgd;
45
+
return (pud_t *)p4d;
41
46
}
42
47
43
-
#define pud_val(x) (pgd_val((x).pgd))
44
-
#define __pud(x) ((pud_t) { __pgd(x) } )
48
+
#define pud_val(x) (p4d_val((x).p4d))
49
+
#define __pud(x) ((pud_t) { __p4d(x) })
45
50
46
-
#define pgd_page(pgd) (pud_page((pud_t){ pgd }))
47
-
#define pgd_page_vaddr(pgd) (pud_page_vaddr((pud_t){ pgd }))
51
+
#define p4d_page(p4d) (pud_page((pud_t){ p4d }))
52
+
#define p4d_page_vaddr(p4d) (pud_page_vaddr((pud_t){ p4d }))
48
53
49
54
/*
50
55
* allocating and freeing a pud is trivial: the 1-entry pud is
51
-
* inside the pgd, so has no extra memory associated with it.
56
+
* inside the p4d, so has no extra memory associated with it.
52
57
*/
53
58
#define pud_alloc_one(mm, address) NULL
54
59
#define pud_free(mm, x) do { } while (0)
···
63
58
#define pud_addr_end(addr, end) (end)
64
59
65
60
#endif /* __ASSEMBLY__ */
61
+
#endif /* !__ARCH_USE_5LEVEL_HACK */
66
62
#endif /* _PGTABLE_NOPUD_H */
+45
-3
include/asm-generic/pgtable.h
+45
-3
include/asm-generic/pgtable.h
···
10
10
#include <linux/bug.h>
11
11
#include <linux/errno.h>
12
12
13
-
#if 4 - defined(__PAGETABLE_PUD_FOLDED) - defined(__PAGETABLE_PMD_FOLDED) != \
14
-
CONFIG_PGTABLE_LEVELS
15
-
#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{PUD,PMD}_FOLDED
13
+
#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \
14
+
defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS
15
+
#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED
16
16
#endif
17
17
18
18
/*
···
424
424
(__boundary - 1 < (end) - 1)? __boundary: (end); \
425
425
})
426
426
427
+
#ifndef p4d_addr_end
428
+
#define p4d_addr_end(addr, end) \
429
+
({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \
430
+
(__boundary - 1 < (end) - 1)? __boundary: (end); \
431
+
})
432
+
#endif
433
+
427
434
#ifndef pud_addr_end
428
435
#define pud_addr_end(addr, end) \
429
436
({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
···
451
444
* Do the tests inline, but report and clear the bad entry in mm/memory.c.
452
445
*/
453
446
void pgd_clear_bad(pgd_t *);
447
+
void p4d_clear_bad(p4d_t *);
454
448
void pud_clear_bad(pud_t *);
455
449
void pmd_clear_bad(pmd_t *);
456
450
···
461
453
return 1;
462
454
if (unlikely(pgd_bad(*pgd))) {
463
455
pgd_clear_bad(pgd);
456
+
return 1;
457
+
}
458
+
return 0;
459
+
}
460
+
461
+
static inline int p4d_none_or_clear_bad(p4d_t *p4d)
462
+
{
463
+
if (p4d_none(*p4d))
464
+
return 1;
465
+
if (unlikely(p4d_bad(*p4d))) {
466
+
p4d_clear_bad(p4d);
464
467
return 1;
465
468
}
466
469
return 0;
···
863
844
#endif /* CONFIG_MMU */
864
845
865
846
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
847
+
848
+
#ifndef __PAGETABLE_P4D_FOLDED
849
+
int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);
850
+
int p4d_clear_huge(p4d_t *p4d);
851
+
#else
852
+
static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
853
+
{
854
+
return 0;
855
+
}
856
+
static inline int p4d_clear_huge(p4d_t *p4d)
857
+
{
858
+
return 0;
859
+
}
860
+
#endif /* !__PAGETABLE_P4D_FOLDED */
861
+
866
862
int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);
867
863
int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
868
864
int pud_clear_huge(pud_t *pud);
869
865
int pmd_clear_huge(pmd_t *pmd);
870
866
#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
867
+
static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
868
+
{
869
+
return 0;
870
+
}
871
871
static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
872
872
{
873
873
return 0;
874
874
}
875
875
static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
876
+
{
877
+
return 0;
878
+
}
879
+
static inline int p4d_clear_huge(p4d_t *p4d)
876
880
{
877
881
return 0;
878
882
}
+11
-3
include/asm-generic/tlb.h
+11
-3
include/asm-generic/tlb.h
···
270
270
__pte_free_tlb(tlb, ptep, address); \
271
271
} while (0)
272
272
273
+
#define pmd_free_tlb(tlb, pmdp, address) \
274
+
do { \
275
+
__tlb_adjust_range(tlb, address, PAGE_SIZE); \
276
+
__pmd_free_tlb(tlb, pmdp, address); \
277
+
} while (0)
278
+
273
279
#ifndef __ARCH_HAS_4LEVEL_HACK
274
280
#define pud_free_tlb(tlb, pudp, address) \
275
281
do { \
···
284
278
} while (0)
285
279
#endif
286
280
287
-
#define pmd_free_tlb(tlb, pmdp, address) \
281
+
#ifndef __ARCH_HAS_5LEVEL_HACK
282
+
#define p4d_free_tlb(tlb, pudp, address) \
288
283
do { \
289
-
__tlb_adjust_range(tlb, address, PAGE_SIZE); \
290
-
__pmd_free_tlb(tlb, pmdp, address); \
284
+
__tlb_adjust_range(tlb, address, PAGE_SIZE); \
285
+
__p4d_free_tlb(tlb, pudp, address); \
291
286
} while (0)
287
+
#endif
292
288
293
289
#define tlb_migrate_finish(mm) do {} while (0)
294
290
+4
-1
include/linux/hugetlb.h
+4
-1
include/linux/hugetlb.h
···
122
122
struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
123
123
pud_t *pud, int flags);
124
124
int pmd_huge(pmd_t pmd);
125
-
int pud_huge(pud_t pmd);
125
+
int pud_huge(pud_t pud);
126
126
unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
127
127
unsigned long address, unsigned long end, pgprot_t newprot);
128
128
···
196
196
*/
197
197
#ifndef pgd_huge
198
198
#define pgd_huge(x) 0
199
+
#endif
200
+
#ifndef p4d_huge
201
+
#define p4d_huge(x) 0
199
202
#endif
200
203
201
204
#ifndef pgd_write
+1
include/linux/kasan.h
+1
include/linux/kasan.h
+29
-7
include/linux/mm.h
+29
-7
include/linux/mm.h
···
1560
1560
return ptep;
1561
1561
}
1562
1562
1563
-
#ifdef __PAGETABLE_PUD_FOLDED
1564
-
static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1563
+
#ifdef __PAGETABLE_P4D_FOLDED
1564
+
static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
1565
1565
unsigned long address)
1566
1566
{
1567
1567
return 0;
1568
1568
}
1569
1569
#else
1570
-
int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
1570
+
int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
1571
+
#endif
1572
+
1573
+
#ifdef __PAGETABLE_PUD_FOLDED
1574
+
static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
1575
+
unsigned long address)
1576
+
{
1577
+
return 0;
1578
+
}
1579
+
#else
1580
+
int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
1571
1581
#endif
1572
1582
1573
1583
#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
···
1629
1619
* Remove it when 4level-fixup.h has been removed.
1630
1620
*/
1631
1621
#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
1632
-
static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1622
+
1623
+
#ifndef __ARCH_HAS_5LEVEL_HACK
1624
+
static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
1625
+
unsigned long address)
1633
1626
{
1634
-
return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1635
-
NULL: pud_offset(pgd, address);
1627
+
return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
1628
+
NULL : p4d_offset(pgd, address);
1636
1629
}
1630
+
1631
+
static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
1632
+
unsigned long address)
1633
+
{
1634
+
return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ?
1635
+
NULL : pud_offset(p4d, address);
1636
+
}
1637
+
#endif /* !__ARCH_HAS_5LEVEL_HACK */
1637
1638
1638
1639
static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1639
1640
{
···
2406
2385
2407
2386
struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2408
2387
pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
2409
-
pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
2388
+
p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
2389
+
pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
2410
2390
pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
2411
2391
pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
2412
2392
void *vmemmap_alloc_block(unsigned long size, int node);
+36
-3
lib/ioremap.c
+36
-3
lib/ioremap.c
···
14
14
#include <asm/pgtable.h>
15
15
16
16
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
17
+
static int __read_mostly ioremap_p4d_capable;
17
18
static int __read_mostly ioremap_pud_capable;
18
19
static int __read_mostly ioremap_pmd_capable;
19
20
static int __read_mostly ioremap_huge_disabled;
···
36
35
}
37
36
}
38
37
38
+
static inline int ioremap_p4d_enabled(void)
39
+
{
40
+
return ioremap_p4d_capable;
41
+
}
42
+
39
43
static inline int ioremap_pud_enabled(void)
40
44
{
41
45
return ioremap_pud_capable;
···
52
46
}
53
47
54
48
#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
49
+
static inline int ioremap_p4d_enabled(void) { return 0; }
55
50
static inline int ioremap_pud_enabled(void) { return 0; }
56
51
static inline int ioremap_pmd_enabled(void) { return 0; }
57
52
#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
···
101
94
return 0;
102
95
}
103
96
104
-
static inline int ioremap_pud_range(pgd_t *pgd, unsigned long addr,
97
+
static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,
105
98
unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
106
99
{
107
100
pud_t *pud;
108
101
unsigned long next;
109
102
110
103
phys_addr -= addr;
111
-
pud = pud_alloc(&init_mm, pgd, addr);
104
+
pud = pud_alloc(&init_mm, p4d, addr);
112
105
if (!pud)
113
106
return -ENOMEM;
114
107
do {
···
127
120
return 0;
128
121
}
129
122
123
+
static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,
124
+
unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
125
+
{
126
+
p4d_t *p4d;
127
+
unsigned long next;
128
+
129
+
phys_addr -= addr;
130
+
p4d = p4d_alloc(&init_mm, pgd, addr);
131
+
if (!p4d)
132
+
return -ENOMEM;
133
+
do {
134
+
next = p4d_addr_end(addr, end);
135
+
136
+
if (ioremap_p4d_enabled() &&
137
+
((next - addr) == P4D_SIZE) &&
138
+
IS_ALIGNED(phys_addr + addr, P4D_SIZE)) {
139
+
if (p4d_set_huge(p4d, phys_addr + addr, prot))
140
+
continue;
141
+
}
142
+
143
+
if (ioremap_pud_range(p4d, addr, next, phys_addr + addr, prot))
144
+
return -ENOMEM;
145
+
} while (p4d++, addr = next, addr != end);
146
+
return 0;
147
+
}
148
+
130
149
int ioremap_page_range(unsigned long addr,
131
150
unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
132
151
{
···
168
135
pgd = pgd_offset_k(addr);
169
136
do {
170
137
next = pgd_addr_end(addr, end);
171
-
err = ioremap_pud_range(pgd, addr, next, phys_addr+addr, prot);
138
+
err = ioremap_p4d_range(pgd, addr, next, phys_addr+addr, prot);
172
139
if (err)
173
140
break;
174
141
} while (pgd++, addr = next, addr != end);
+40
-6
mm/gup.c
+40
-6
mm/gup.c
···
226
226
unsigned int *page_mask)
227
227
{
228
228
pgd_t *pgd;
229
+
p4d_t *p4d;
229
230
pud_t *pud;
230
231
pmd_t *pmd;
231
232
spinlock_t *ptl;
···
244
243
pgd = pgd_offset(mm, address);
245
244
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
246
245
return no_page_table(vma, flags);
247
-
248
-
pud = pud_offset(pgd, address);
246
+
p4d = p4d_offset(pgd, address);
247
+
if (p4d_none(*p4d))
248
+
return no_page_table(vma, flags);
249
+
BUILD_BUG_ON(p4d_huge(*p4d));
250
+
if (unlikely(p4d_bad(*p4d)))
251
+
return no_page_table(vma, flags);
252
+
pud = pud_offset(p4d, address);
249
253
if (pud_none(*pud))
250
254
return no_page_table(vma, flags);
251
255
if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) {
···
331
325
struct page **page)
332
326
{
333
327
pgd_t *pgd;
328
+
p4d_t *p4d;
334
329
pud_t *pud;
335
330
pmd_t *pmd;
336
331
pte_t *pte;
···
345
338
else
346
339
pgd = pgd_offset_gate(mm, address);
347
340
BUG_ON(pgd_none(*pgd));
348
-
pud = pud_offset(pgd, address);
341
+
p4d = p4d_offset(pgd, address);
342
+
BUG_ON(p4d_none(*p4d));
343
+
pud = pud_offset(p4d, address);
349
344
BUG_ON(pud_none(*pud));
350
345
pmd = pmd_offset(pud, address);
351
346
if (pmd_none(*pmd))
···
1409
1400
return 1;
1410
1401
}
1411
1402
1412
-
static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
1403
+
static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
1413
1404
int write, struct page **pages, int *nr)
1414
1405
{
1415
1406
unsigned long next;
1416
1407
pud_t *pudp;
1417
1408
1418
-
pudp = pud_offset(&pgd, addr);
1409
+
pudp = pud_offset(&p4d, addr);
1419
1410
do {
1420
1411
pud_t pud = READ_ONCE(*pudp);
1421
1412
···
1433
1424
} else if (!gup_pmd_range(pud, addr, next, write, pages, nr))
1434
1425
return 0;
1435
1426
} while (pudp++, addr = next, addr != end);
1427
+
1428
+
return 1;
1429
+
}
1430
+
1431
+
static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
1432
+
int write, struct page **pages, int *nr)
1433
+
{
1434
+
unsigned long next;
1435
+
p4d_t *p4dp;
1436
+
1437
+
p4dp = p4d_offset(&pgd, addr);
1438
+
do {
1439
+
p4d_t p4d = READ_ONCE(*p4dp);
1440
+
1441
+
next = p4d_addr_end(addr, end);
1442
+
if (p4d_none(p4d))
1443
+
return 0;
1444
+
BUILD_BUG_ON(p4d_huge(p4d));
1445
+
if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
1446
+
if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
1447
+
P4D_SHIFT, next, write, pages, nr))
1448
+
return 0;
1449
+
} else if (!gup_p4d_range(p4d, addr, next, write, pages, nr))
1450
+
return 0;
1451
+
} while (p4dp++, addr = next, addr != end);
1436
1452
1437
1453
return 1;
1438
1454
}
···
1512
1478
if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
1513
1479
PGDIR_SHIFT, next, write, pages, &nr))
1514
1480
break;
1515
-
} else if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
1481
+
} else if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
1516
1482
break;
1517
1483
} while (pgdp++, addr = next, addr != end);
1518
1484
local_irq_restore(flags);
+6
-1
mm/huge_memory.c
+6
-1
mm/huge_memory.c
···
2048
2048
bool freeze, struct page *page)
2049
2049
{
2050
2050
pgd_t *pgd;
2051
+
p4d_t *p4d;
2051
2052
pud_t *pud;
2052
2053
pmd_t *pmd;
2053
2054
···
2056
2055
if (!pgd_present(*pgd))
2057
2056
return;
2058
2057
2059
-
pud = pud_offset(pgd, address);
2058
+
p4d = p4d_offset(pgd, address);
2059
+
if (!p4d_present(*p4d))
2060
+
return;
2061
+
2062
+
pud = pud_offset(p4d, address);
2060
2063
if (!pud_present(*pud))
2061
2064
return;
2062
2065
+18
-11
mm/hugetlb.c
+18
-11
mm/hugetlb.c
···
4555
4555
int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4556
4556
{
4557
4557
pgd_t *pgd = pgd_offset(mm, *addr);
4558
-
pud_t *pud = pud_offset(pgd, *addr);
4558
+
p4d_t *p4d = p4d_offset(pgd, *addr);
4559
+
pud_t *pud = pud_offset(p4d, *addr);
4559
4560
4560
4561
BUG_ON(page_count(virt_to_page(ptep)) == 0);
4561
4562
if (page_count(virt_to_page(ptep)) == 1)
···
4587
4586
unsigned long addr, unsigned long sz)
4588
4587
{
4589
4588
pgd_t *pgd;
4589
+
p4d_t *p4d;
4590
4590
pud_t *pud;
4591
4591
pte_t *pte = NULL;
4592
4592
4593
4593
pgd = pgd_offset(mm, addr);
4594
-
pud = pud_alloc(mm, pgd, addr);
4594
+
p4d = p4d_offset(pgd, addr);
4595
+
pud = pud_alloc(mm, p4d, addr);
4595
4596
if (pud) {
4596
4597
if (sz == PUD_SIZE) {
4597
4598
pte = (pte_t *)pud;
···
4613
4610
pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
4614
4611
{
4615
4612
pgd_t *pgd;
4613
+
p4d_t *p4d;
4616
4614
pud_t *pud;
4617
-
pmd_t *pmd = NULL;
4615
+
pmd_t *pmd;
4618
4616
4619
4617
pgd = pgd_offset(mm, addr);
4620
-
if (pgd_present(*pgd)) {
4621
-
pud = pud_offset(pgd, addr);
4622
-
if (pud_present(*pud)) {
4623
-
if (pud_huge(*pud))
4624
-
return (pte_t *)pud;
4625
-
pmd = pmd_offset(pud, addr);
4626
-
}
4627
-
}
4618
+
if (!pgd_present(*pgd))
4619
+
return NULL;
4620
+
p4d = p4d_offset(pgd, addr);
4621
+
if (!p4d_present(*p4d))
4622
+
return NULL;
4623
+
pud = pud_offset(p4d, addr);
4624
+
if (!pud_present(*pud))
4625
+
return NULL;
4626
+
if (pud_huge(*pud))
4627
+
return (pte_t *)pud;
4628
+
pmd = pmd_offset(pud, addr);
4628
4629
return (pte_t *) pmd;
4629
4630
}
4630
4631
+39
-5
mm/kasan/kasan_init.c
+39
-5
mm/kasan/kasan_init.c
···
30
30
*/
31
31
unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
32
32
33
+
#if CONFIG_PGTABLE_LEVELS > 4
34
+
p4d_t kasan_zero_p4d[PTRS_PER_P4D] __page_aligned_bss;
35
+
#endif
33
36
#if CONFIG_PGTABLE_LEVELS > 3
34
37
pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
35
38
#endif
···
85
82
} while (pmd++, addr = next, addr != end);
86
83
}
87
84
88
-
static void __init zero_pud_populate(pgd_t *pgd, unsigned long addr,
85
+
static void __init zero_pud_populate(p4d_t *p4d, unsigned long addr,
89
86
unsigned long end)
90
87
{
91
-
pud_t *pud = pud_offset(pgd, addr);
88
+
pud_t *pud = pud_offset(p4d, addr);
92
89
unsigned long next;
93
90
94
91
do {
···
110
107
} while (pud++, addr = next, addr != end);
111
108
}
112
109
110
+
static void __init zero_p4d_populate(pgd_t *pgd, unsigned long addr,
111
+
unsigned long end)
112
+
{
113
+
p4d_t *p4d = p4d_offset(pgd, addr);
114
+
unsigned long next;
115
+
116
+
do {
117
+
next = p4d_addr_end(addr, end);
118
+
119
+
if (p4d_none(*p4d)) {
120
+
p4d_populate(&init_mm, p4d,
121
+
early_alloc(PAGE_SIZE, NUMA_NO_NODE));
122
+
}
123
+
zero_pud_populate(p4d, addr, next);
124
+
} while (p4d++, addr = next, addr != end);
125
+
}
126
+
113
127
/**
114
128
* kasan_populate_zero_shadow - populate shadow memory region with
115
129
* kasan_zero_page
···
145
125
next = pgd_addr_end(addr, end);
146
126
147
127
if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
128
+
p4d_t *p4d;
148
129
pud_t *pud;
149
130
pmd_t *pmd;
150
131
···
156
135
* 3,2 - level page tables where we don't have
157
136
* puds,pmds, so pgd_populate(), pud_populate()
158
137
* is noops.
138
+
*
139
+
* The ifndef is required to avoid build breakage.
140
+
*
141
+
* With 5level-fixup.h, pgd_populate() is not nop and
142
+
* we reference kasan_zero_p4d. It's not defined
143
+
* unless 5-level paging enabled.
144
+
*
145
+
* The ifndef can be dropped once all KASAN-enabled
146
+
* architectures will switch to pgtable-nop4d.h.
159
147
*/
160
-
pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_pud));
161
-
pud = pud_offset(pgd, addr);
148
+
#ifndef __ARCH_HAS_5LEVEL_HACK
149
+
pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_p4d));
150
+
#endif
151
+
p4d = p4d_offset(pgd, addr);
152
+
p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud));
153
+
pud = pud_offset(p4d, addr);
162
154
pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
163
155
pmd = pmd_offset(pud, addr);
164
156
pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
···
182
148
pgd_populate(&init_mm, pgd,
183
149
early_alloc(PAGE_SIZE, NUMA_NO_NODE));
184
150
}
185
-
zero_pud_populate(pgd, addr, next);
151
+
zero_p4d_populate(pgd, addr, next);
186
152
} while (pgd++, addr = next, addr != end);
187
153
}
+196
-34
mm/memory.c
+196
-34
mm/memory.c
···
445
445
mm_dec_nr_pmds(tlb->mm);
446
446
}
447
447
448
-
static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
448
+
static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
449
449
unsigned long addr, unsigned long end,
450
450
unsigned long floor, unsigned long ceiling)
451
451
{
···
454
454
unsigned long start;
455
455
456
456
start = addr;
457
-
pud = pud_offset(pgd, addr);
457
+
pud = pud_offset(p4d, addr);
458
458
do {
459
459
next = pud_addr_end(addr, end);
460
460
if (pud_none_or_clear_bad(pud))
461
461
continue;
462
462
free_pmd_range(tlb, pud, addr, next, floor, ceiling);
463
463
} while (pud++, addr = next, addr != end);
464
+
465
+
start &= P4D_MASK;
466
+
if (start < floor)
467
+
return;
468
+
if (ceiling) {
469
+
ceiling &= P4D_MASK;
470
+
if (!ceiling)
471
+
return;
472
+
}
473
+
if (end - 1 > ceiling - 1)
474
+
return;
475
+
476
+
pud = pud_offset(p4d, start);
477
+
p4d_clear(p4d);
478
+
pud_free_tlb(tlb, pud, start);
479
+
}
480
+
481
+
static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
482
+
unsigned long addr, unsigned long end,
483
+
unsigned long floor, unsigned long ceiling)
484
+
{
485
+
p4d_t *p4d;
486
+
unsigned long next;
487
+
unsigned long start;
488
+
489
+
start = addr;
490
+
p4d = p4d_offset(pgd, addr);
491
+
do {
492
+
next = p4d_addr_end(addr, end);
493
+
if (p4d_none_or_clear_bad(p4d))
494
+
continue;
495
+
free_pud_range(tlb, p4d, addr, next, floor, ceiling);
496
+
} while (p4d++, addr = next, addr != end);
464
497
465
498
start &= PGDIR_MASK;
466
499
if (start < floor)
···
506
473
if (end - 1 > ceiling - 1)
507
474
return;
508
475
509
-
pud = pud_offset(pgd, start);
476
+
p4d = p4d_offset(pgd, start);
510
477
pgd_clear(pgd);
511
-
pud_free_tlb(tlb, pud, start);
478
+
p4d_free_tlb(tlb, p4d, start);
512
479
}
513
480
514
481
/*
···
572
539
next = pgd_addr_end(addr, end);
573
540
if (pgd_none_or_clear_bad(pgd))
574
541
continue;
575
-
free_pud_range(tlb, pgd, addr, next, floor, ceiling);
542
+
free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
576
543
} while (pgd++, addr = next, addr != end);
577
544
}
578
545
···
691
658
pte_t pte, struct page *page)
692
659
{
693
660
pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
694
-
pud_t *pud = pud_offset(pgd, addr);
661
+
p4d_t *p4d = p4d_offset(pgd, addr);
662
+
pud_t *pud = pud_offset(p4d, addr);
695
663
pmd_t *pmd = pmd_offset(pud, addr);
696
664
struct address_space *mapping;
697
665
pgoff_t index;
···
1057
1023
}
1058
1024
1059
1025
static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1060
-
pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
1026
+
p4d_t *dst_p4d, p4d_t *src_p4d, struct vm_area_struct *vma,
1061
1027
unsigned long addr, unsigned long end)
1062
1028
{
1063
1029
pud_t *src_pud, *dst_pud;
1064
1030
unsigned long next;
1065
1031
1066
-
dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
1032
+
dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
1067
1033
if (!dst_pud)
1068
1034
return -ENOMEM;
1069
-
src_pud = pud_offset(src_pgd, addr);
1035
+
src_pud = pud_offset(src_p4d, addr);
1070
1036
do {
1071
1037
next = pud_addr_end(addr, end);
1072
1038
if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
···
1087
1053
vma, addr, next))
1088
1054
return -ENOMEM;
1089
1055
} while (dst_pud++, src_pud++, addr = next, addr != end);
1056
+
return 0;
1057
+
}
1058
+
1059
+
static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1060
+
pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
1061
+
unsigned long addr, unsigned long end)
1062
+
{
1063
+
p4d_t *src_p4d, *dst_p4d;
1064
+
unsigned long next;
1065
+
1066
+
dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
1067
+
if (!dst_p4d)
1068
+
return -ENOMEM;
1069
+
src_p4d = p4d_offset(src_pgd, addr);
1070
+
do {
1071
+
next = p4d_addr_end(addr, end);
1072
+
if (p4d_none_or_clear_bad(src_p4d))
1073
+
continue;
1074
+
if (copy_pud_range(dst_mm, src_mm, dst_p4d, src_p4d,
1075
+
vma, addr, next))
1076
+
return -ENOMEM;
1077
+
} while (dst_p4d++, src_p4d++, addr = next, addr != end);
1090
1078
return 0;
1091
1079
}
1092
1080
···
1167
1111
next = pgd_addr_end(addr, end);
1168
1112
if (pgd_none_or_clear_bad(src_pgd))
1169
1113
continue;
1170
-
if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
1114
+
if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd,
1171
1115
vma, addr, next))) {
1172
1116
ret = -ENOMEM;
1173
1117
break;
···
1323
1267
}
1324
1268
1325
1269
static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
1326
-
struct vm_area_struct *vma, pgd_t *pgd,
1270
+
struct vm_area_struct *vma, p4d_t *p4d,
1327
1271
unsigned long addr, unsigned long end,
1328
1272
struct zap_details *details)
1329
1273
{
1330
1274
pud_t *pud;
1331
1275
unsigned long next;
1332
1276
1333
-
pud = pud_offset(pgd, addr);
1277
+
pud = pud_offset(p4d, addr);
1334
1278
do {
1335
1279
next = pud_addr_end(addr, end);
1336
1280
if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
···
1351
1295
return addr;
1352
1296
}
1353
1297
1298
+
static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
1299
+
struct vm_area_struct *vma, pgd_t *pgd,
1300
+
unsigned long addr, unsigned long end,
1301
+
struct zap_details *details)
1302
+
{
1303
+
p4d_t *p4d;
1304
+
unsigned long next;
1305
+
1306
+
p4d = p4d_offset(pgd, addr);
1307
+
do {
1308
+
next = p4d_addr_end(addr, end);
1309
+
if (p4d_none_or_clear_bad(p4d))
1310
+
continue;
1311
+
next = zap_pud_range(tlb, vma, p4d, addr, next, details);
1312
+
} while (p4d++, addr = next, addr != end);
1313
+
1314
+
return addr;
1315
+
}
1316
+
1354
1317
void unmap_page_range(struct mmu_gather *tlb,
1355
1318
struct vm_area_struct *vma,
1356
1319
unsigned long addr, unsigned long end,
···
1385
1310
next = pgd_addr_end(addr, end);
1386
1311
if (pgd_none_or_clear_bad(pgd))
1387
1312
continue;
1388
-
next = zap_pud_range(tlb, vma, pgd, addr, next, details);
1313
+
next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
1389
1314
} while (pgd++, addr = next, addr != end);
1390
1315
tlb_end_vma(tlb, vma);
1391
1316
}
···
1540
1465
pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1541
1466
spinlock_t **ptl)
1542
1467
{
1543
-
pgd_t *pgd = pgd_offset(mm, addr);
1544
-
pud_t *pud = pud_alloc(mm, pgd, addr);
1545
-
if (pud) {
1546
-
pmd_t *pmd = pmd_alloc(mm, pud, addr);
1547
-
if (pmd) {
1548
-
VM_BUG_ON(pmd_trans_huge(*pmd));
1549
-
return pte_alloc_map_lock(mm, pmd, addr, ptl);
1550
-
}
1551
-
}
1552
-
return NULL;
1468
+
pgd_t *pgd;
1469
+
p4d_t *p4d;
1470
+
pud_t *pud;
1471
+
pmd_t *pmd;
1472
+
1473
+
pgd = pgd_offset(mm, addr);
1474
+
p4d = p4d_alloc(mm, pgd, addr);
1475
+
if (!p4d)
1476
+
return NULL;
1477
+
pud = pud_alloc(mm, p4d, addr);
1478
+
if (!pud)
1479
+
return NULL;
1480
+
pmd = pmd_alloc(mm, pud, addr);
1481
+
if (!pmd)
1482
+
return NULL;
1483
+
1484
+
VM_BUG_ON(pmd_trans_huge(*pmd));
1485
+
return pte_alloc_map_lock(mm, pmd, addr, ptl);
1553
1486
}
1554
1487
1555
1488
/*
···
1823
1740
return 0;
1824
1741
}
1825
1742
1826
-
static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
1743
+
static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
1827
1744
unsigned long addr, unsigned long end,
1828
1745
unsigned long pfn, pgprot_t prot)
1829
1746
{
···
1831
1748
unsigned long next;
1832
1749
1833
1750
pfn -= addr >> PAGE_SHIFT;
1834
-
pud = pud_alloc(mm, pgd, addr);
1751
+
pud = pud_alloc(mm, p4d, addr);
1835
1752
if (!pud)
1836
1753
return -ENOMEM;
1837
1754
do {
···
1840
1757
pfn + (addr >> PAGE_SHIFT), prot))
1841
1758
return -ENOMEM;
1842
1759
} while (pud++, addr = next, addr != end);
1760
+
return 0;
1761
+
}
1762
+
1763
+
static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
1764
+
unsigned long addr, unsigned long end,
1765
+
unsigned long pfn, pgprot_t prot)
1766
+
{
1767
+
p4d_t *p4d;
1768
+
unsigned long next;
1769
+
1770
+
pfn -= addr >> PAGE_SHIFT;
1771
+
p4d = p4d_alloc(mm, pgd, addr);
1772
+
if (!p4d)
1773
+
return -ENOMEM;
1774
+
do {
1775
+
next = p4d_addr_end(addr, end);
1776
+
if (remap_pud_range(mm, p4d, addr, next,
1777
+
pfn + (addr >> PAGE_SHIFT), prot))
1778
+
return -ENOMEM;
1779
+
} while (p4d++, addr = next, addr != end);
1843
1780
return 0;
1844
1781
}
1845
1782
···
1919
1816
flush_cache_range(vma, addr, end);
1920
1817
do {
1921
1818
next = pgd_addr_end(addr, end);
1922
-
err = remap_pud_range(mm, pgd, addr, next,
1819
+
err = remap_p4d_range(mm, pgd, addr, next,
1923
1820
pfn + (addr >> PAGE_SHIFT), prot);
1924
1821
if (err)
1925
1822
break;
···
2035
1932
return err;
2036
1933
}
2037
1934
2038
-
static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
1935
+
static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
2039
1936
unsigned long addr, unsigned long end,
2040
1937
pte_fn_t fn, void *data)
2041
1938
{
···
2043
1940
unsigned long next;
2044
1941
int err;
2045
1942
2046
-
pud = pud_alloc(mm, pgd, addr);
1943
+
pud = pud_alloc(mm, p4d, addr);
2047
1944
if (!pud)
2048
1945
return -ENOMEM;
2049
1946
do {
···
2052
1949
if (err)
2053
1950
break;
2054
1951
} while (pud++, addr = next, addr != end);
1952
+
return err;
1953
+
}
1954
+
1955
+
static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
1956
+
unsigned long addr, unsigned long end,
1957
+
pte_fn_t fn, void *data)
1958
+
{
1959
+
p4d_t *p4d;
1960
+
unsigned long next;
1961
+
int err;
1962
+
1963
+
p4d = p4d_alloc(mm, pgd, addr);
1964
+
if (!p4d)
1965
+
return -ENOMEM;
1966
+
do {
1967
+
next = p4d_addr_end(addr, end);
1968
+
err = apply_to_pud_range(mm, p4d, addr, next, fn, data);
1969
+
if (err)
1970
+
break;
1971
+
} while (p4d++, addr = next, addr != end);
2055
1972
return err;
2056
1973
}
2057
1974
···
2093
1970
pgd = pgd_offset(mm, addr);
2094
1971
do {
2095
1972
next = pgd_addr_end(addr, end);
2096
-
err = apply_to_pud_range(mm, pgd, addr, next, fn, data);
1973
+
err = apply_to_p4d_range(mm, pgd, addr, next, fn, data);
2097
1974
if (err)
2098
1975
break;
2099
1976
} while (pgd++, addr = next, addr != end);
···
3776
3653
};
3777
3654
struct mm_struct *mm = vma->vm_mm;
3778
3655
pgd_t *pgd;
3656
+
p4d_t *p4d;
3779
3657
int ret;
3780
3658
3781
3659
pgd = pgd_offset(mm, address);
3660
+
p4d = p4d_alloc(mm, pgd, address);
3661
+
if (!p4d)
3662
+
return VM_FAULT_OOM;
3782
3663
3783
-
vmf.pud = pud_alloc(mm, pgd, address);
3664
+
vmf.pud = pud_alloc(mm, p4d, address);
3784
3665
if (!vmf.pud)
3785
3666
return VM_FAULT_OOM;
3786
3667
if (pud_none(*vmf.pud) && transparent_hugepage_enabled(vma)) {
···
3906
3779
}
3907
3780
EXPORT_SYMBOL_GPL(handle_mm_fault);
3908
3781
3782
+
#ifndef __PAGETABLE_P4D_FOLDED
3783
+
/*
3784
+
* Allocate p4d page table.
3785
+
* We've already handled the fast-path in-line.
3786
+
*/
3787
+
int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
3788
+
{
3789
+
p4d_t *new = p4d_alloc_one(mm, address);
3790
+
if (!new)
3791
+
return -ENOMEM;
3792
+
3793
+
smp_wmb(); /* See comment in __pte_alloc */
3794
+
3795
+
spin_lock(&mm->page_table_lock);
3796
+
if (pgd_present(*pgd)) /* Another has populated it */
3797
+
p4d_free(mm, new);
3798
+
else
3799
+
pgd_populate(mm, pgd, new);
3800
+
spin_unlock(&mm->page_table_lock);
3801
+
return 0;
3802
+
}
3803
+
#endif /* __PAGETABLE_P4D_FOLDED */
3804
+
3909
3805
#ifndef __PAGETABLE_PUD_FOLDED
3910
3806
/*
3911
3807
* Allocate page upper directory.
3912
3808
* We've already handled the fast-path in-line.
3913
3809
*/
3914
-
int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
3810
+
int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
3915
3811
{
3916
3812
pud_t *new = pud_alloc_one(mm, address);
3917
3813
if (!new)
···
3943
3793
smp_wmb(); /* See comment in __pte_alloc */
3944
3794
3945
3795
spin_lock(&mm->page_table_lock);
3946
-
if (pgd_present(*pgd)) /* Another has populated it */
3796
+
#ifndef __ARCH_HAS_5LEVEL_HACK
3797
+
if (p4d_present(*p4d)) /* Another has populated it */
3947
3798
pud_free(mm, new);
3948
3799
else
3949
-
pgd_populate(mm, pgd, new);
3800
+
p4d_populate(mm, p4d, new);
3801
+
#else
3802
+
if (pgd_present(*p4d)) /* Another has populated it */
3803
+
pud_free(mm, new);
3804
+
else
3805
+
pgd_populate(mm, p4d, new);
3806
+
#endif /* __ARCH_HAS_5LEVEL_HACK */
3950
3807
spin_unlock(&mm->page_table_lock);
3951
3808
return 0;
3952
3809
}
···
3996
3839
pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
3997
3840
{
3998
3841
pgd_t *pgd;
3842
+
p4d_t *p4d;
3999
3843
pud_t *pud;
4000
3844
pmd_t *pmd;
4001
3845
pte_t *ptep;
···
4005
3847
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
4006
3848
goto out;
4007
3849
4008
-
pud = pud_offset(pgd, address);
3850
+
p4d = p4d_offset(pgd, address);
3851
+
if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
3852
+
goto out;
3853
+
3854
+
pud = pud_offset(p4d, address);
4009
3855
if (pud_none(*pud) || unlikely(pud_bad(*pud)))
4010
3856
goto out;
4011
3857
+1
mm/mlock.c
+1
mm/mlock.c
+23
-3
mm/mprotect.c
+23
-3
mm/mprotect.c
···
193
193
}
194
194
195
195
static inline unsigned long change_pud_range(struct vm_area_struct *vma,
196
-
pgd_t *pgd, unsigned long addr, unsigned long end,
196
+
p4d_t *p4d, unsigned long addr, unsigned long end,
197
197
pgprot_t newprot, int dirty_accountable, int prot_numa)
198
198
{
199
199
pud_t *pud;
200
200
unsigned long next;
201
201
unsigned long pages = 0;
202
202
203
-
pud = pud_offset(pgd, addr);
203
+
pud = pud_offset(p4d, addr);
204
204
do {
205
205
next = pud_addr_end(addr, end);
206
206
if (pud_none_or_clear_bad(pud))
···
208
208
pages += change_pmd_range(vma, pud, addr, next, newprot,
209
209
dirty_accountable, prot_numa);
210
210
} while (pud++, addr = next, addr != end);
211
+
212
+
return pages;
213
+
}
214
+
215
+
static inline unsigned long change_p4d_range(struct vm_area_struct *vma,
216
+
pgd_t *pgd, unsigned long addr, unsigned long end,
217
+
pgprot_t newprot, int dirty_accountable, int prot_numa)
218
+
{
219
+
p4d_t *p4d;
220
+
unsigned long next;
221
+
unsigned long pages = 0;
222
+
223
+
p4d = p4d_offset(pgd, addr);
224
+
do {
225
+
next = p4d_addr_end(addr, end);
226
+
if (p4d_none_or_clear_bad(p4d))
227
+
continue;
228
+
pages += change_pud_range(vma, p4d, addr, next, newprot,
229
+
dirty_accountable, prot_numa);
230
+
} while (p4d++, addr = next, addr != end);
211
231
212
232
return pages;
213
233
}
···
250
230
next = pgd_addr_end(addr, end);
251
231
if (pgd_none_or_clear_bad(pgd))
252
232
continue;
253
-
pages += change_pud_range(vma, pgd, addr, next, newprot,
233
+
pages += change_p4d_range(vma, pgd, addr, next, newprot,
254
234
dirty_accountable, prot_numa);
255
235
} while (pgd++, addr = next, addr != end);
256
236
+11
-2
mm/mremap.c
+11
-2
mm/mremap.c
···
32
32
static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
33
33
{
34
34
pgd_t *pgd;
35
+
p4d_t *p4d;
35
36
pud_t *pud;
36
37
pmd_t *pmd;
37
38
···
40
39
if (pgd_none_or_clear_bad(pgd))
41
40
return NULL;
42
41
43
-
pud = pud_offset(pgd, addr);
42
+
p4d = p4d_offset(pgd, addr);
43
+
if (p4d_none_or_clear_bad(p4d))
44
+
return NULL;
45
+
46
+
pud = pud_offset(p4d, addr);
44
47
if (pud_none_or_clear_bad(pud))
45
48
return NULL;
46
49
···
59
54
unsigned long addr)
60
55
{
61
56
pgd_t *pgd;
57
+
p4d_t *p4d;
62
58
pud_t *pud;
63
59
pmd_t *pmd;
64
60
65
61
pgd = pgd_offset(mm, addr);
66
-
pud = pud_alloc(mm, pgd, addr);
62
+
p4d = p4d_alloc(mm, pgd, addr);
63
+
if (!p4d)
64
+
return NULL;
65
+
pud = pud_alloc(mm, p4d, addr);
67
66
if (!pud)
68
67
return NULL;
69
68
+5
-1
mm/page_vma_mapped.c
+5
-1
mm/page_vma_mapped.c
···
104
104
struct mm_struct *mm = pvmw->vma->vm_mm;
105
105
struct page *page = pvmw->page;
106
106
pgd_t *pgd;
107
+
p4d_t *p4d;
107
108
pud_t *pud;
108
109
109
110
/* The only possible pmd mapping has been handled on last iteration */
···
134
133
pgd = pgd_offset(mm, pvmw->address);
135
134
if (!pgd_present(*pgd))
136
135
return false;
137
-
pud = pud_offset(pgd, pvmw->address);
136
+
p4d = p4d_offset(pgd, pvmw->address);
137
+
if (!p4d_present(*p4d))
138
+
return false;
139
+
pud = pud_offset(p4d, pvmw->address);
138
140
if (!pud_present(*pud))
139
141
return false;
140
142
pvmw->pmd = pmd_offset(pud, pvmw->address);
+29
-3
mm/pagewalk.c
+29
-3
mm/pagewalk.c
···
69
69
return err;
70
70
}
71
71
72
-
static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
72
+
static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
73
73
struct mm_walk *walk)
74
74
{
75
75
pud_t *pud;
76
76
unsigned long next;
77
77
int err = 0;
78
78
79
-
pud = pud_offset(pgd, addr);
79
+
pud = pud_offset(p4d, addr);
80
80
do {
81
81
again:
82
82
next = pud_addr_end(addr, end);
···
113
113
return err;
114
114
}
115
115
116
+
static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
117
+
struct mm_walk *walk)
118
+
{
119
+
p4d_t *p4d;
120
+
unsigned long next;
121
+
int err = 0;
122
+
123
+
p4d = p4d_offset(pgd, addr);
124
+
do {
125
+
next = p4d_addr_end(addr, end);
126
+
if (p4d_none_or_clear_bad(p4d)) {
127
+
if (walk->pte_hole)
128
+
err = walk->pte_hole(addr, next, walk);
129
+
if (err)
130
+
break;
131
+
continue;
132
+
}
133
+
if (walk->pmd_entry || walk->pte_entry)
134
+
err = walk_pud_range(p4d, addr, next, walk);
135
+
if (err)
136
+
break;
137
+
} while (p4d++, addr = next, addr != end);
138
+
139
+
return err;
140
+
}
141
+
116
142
static int walk_pgd_range(unsigned long addr, unsigned long end,
117
143
struct mm_walk *walk)
118
144
{
···
157
131
continue;
158
132
}
159
133
if (walk->pmd_entry || walk->pte_entry)
160
-
err = walk_pud_range(pgd, addr, next, walk);
134
+
err = walk_p4d_range(pgd, addr, next, walk);
161
135
if (err)
162
136
break;
163
137
} while (pgd++, addr = next, addr != end);
+6
mm/pgtable-generic.c
+6
mm/pgtable-generic.c
+6
-1
mm/rmap.c
+6
-1
mm/rmap.c
···
684
684
pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
685
685
{
686
686
pgd_t *pgd;
687
+
p4d_t *p4d;
687
688
pud_t *pud;
688
689
pmd_t *pmd = NULL;
689
690
pmd_t pmde;
···
693
692
if (!pgd_present(*pgd))
694
693
goto out;
695
694
696
-
pud = pud_offset(pgd, address);
695
+
p4d = p4d_offset(pgd, address);
696
+
if (!p4d_present(*p4d))
697
+
goto out;
698
+
699
+
pud = pud_offset(p4d, address);
697
700
if (!pud_present(*pud))
698
701
goto out;
699
702
+19
-3
mm/sparse-vmemmap.c
+19
-3
mm/sparse-vmemmap.c
···
196
196
return pmd;
197
197
}
198
198
199
-
pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node)
199
+
pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
200
200
{
201
-
pud_t *pud = pud_offset(pgd, addr);
201
+
pud_t *pud = pud_offset(p4d, addr);
202
202
if (pud_none(*pud)) {
203
203
void *p = vmemmap_alloc_block(PAGE_SIZE, node);
204
204
if (!p)
···
206
206
pud_populate(&init_mm, pud, p);
207
207
}
208
208
return pud;
209
+
}
210
+
211
+
p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
212
+
{
213
+
p4d_t *p4d = p4d_offset(pgd, addr);
214
+
if (p4d_none(*p4d)) {
215
+
void *p = vmemmap_alloc_block(PAGE_SIZE, node);
216
+
if (!p)
217
+
return NULL;
218
+
p4d_populate(&init_mm, p4d, p);
219
+
}
220
+
return p4d;
209
221
}
210
222
211
223
pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
···
237
225
{
238
226
unsigned long addr = start;
239
227
pgd_t *pgd;
228
+
p4d_t *p4d;
240
229
pud_t *pud;
241
230
pmd_t *pmd;
242
231
pte_t *pte;
···
246
233
pgd = vmemmap_pgd_populate(addr, node);
247
234
if (!pgd)
248
235
return -ENOMEM;
249
-
pud = vmemmap_pud_populate(pgd, addr, node);
236
+
p4d = vmemmap_p4d_populate(pgd, addr, node);
237
+
if (!p4d)
238
+
return -ENOMEM;
239
+
pud = vmemmap_pud_populate(p4d, addr, node);
250
240
if (!pud)
251
241
return -ENOMEM;
252
242
pmd = vmemmap_pmd_populate(pud, addr, node);
+23
-3
mm/swapfile.c
+23
-3
mm/swapfile.c
···
1517
1517
return 0;
1518
1518
}
1519
1519
1520
-
static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
1520
+
static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
1521
1521
unsigned long addr, unsigned long end,
1522
1522
swp_entry_t entry, struct page *page)
1523
1523
{
···
1525
1525
unsigned long next;
1526
1526
int ret;
1527
1527
1528
-
pud = pud_offset(pgd, addr);
1528
+
pud = pud_offset(p4d, addr);
1529
1529
do {
1530
1530
next = pud_addr_end(addr, end);
1531
1531
if (pud_none_or_clear_bad(pud))
···
1534
1534
if (ret)
1535
1535
return ret;
1536
1536
} while (pud++, addr = next, addr != end);
1537
+
return 0;
1538
+
}
1539
+
1540
+
static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
1541
+
unsigned long addr, unsigned long end,
1542
+
swp_entry_t entry, struct page *page)
1543
+
{
1544
+
p4d_t *p4d;
1545
+
unsigned long next;
1546
+
int ret;
1547
+
1548
+
p4d = p4d_offset(pgd, addr);
1549
+
do {
1550
+
next = p4d_addr_end(addr, end);
1551
+
if (p4d_none_or_clear_bad(p4d))
1552
+
continue;
1553
+
ret = unuse_pud_range(vma, p4d, addr, next, entry, page);
1554
+
if (ret)
1555
+
return ret;
1556
+
} while (p4d++, addr = next, addr != end);
1537
1557
return 0;
1538
1558
}
1539
1559
···
1580
1560
next = pgd_addr_end(addr, end);
1581
1561
if (pgd_none_or_clear_bad(pgd))
1582
1562
continue;
1583
-
ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
1563
+
ret = unuse_p4d_range(vma, pgd, addr, next, entry, page);
1584
1564
if (ret)
1585
1565
return ret;
1586
1566
} while (pgd++, addr = next, addr != end);
+13
-10
mm/userfaultfd.c
+13
-10
mm/userfaultfd.c
···
128
128
static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
129
129
{
130
130
pgd_t *pgd;
131
+
p4d_t *p4d;
131
132
pud_t *pud;
132
-
pmd_t *pmd = NULL;
133
133
134
134
pgd = pgd_offset(mm, address);
135
-
pud = pud_alloc(mm, pgd, address);
136
-
if (pud)
137
-
/*
138
-
* Note that we didn't run this because the pmd was
139
-
* missing, the *pmd may be already established and in
140
-
* turn it may also be a trans_huge_pmd.
141
-
*/
142
-
pmd = pmd_alloc(mm, pud, address);
143
-
return pmd;
135
+
p4d = p4d_alloc(mm, pgd, address);
136
+
if (!p4d)
137
+
return NULL;
138
+
pud = pud_alloc(mm, p4d, address);
139
+
if (!pud)
140
+
return NULL;
141
+
/*
142
+
* Note that we didn't run this because the pmd was
143
+
* missing, the *pmd may be already established and in
144
+
* turn it may also be a trans_huge_pmd.
145
+
*/
146
+
return pmd_alloc(mm, pud, address);
144
147
}
145
148
146
149
#ifdef CONFIG_HUGETLB_PAGE
+59
-20
mm/vmalloc.c
+59
-20
mm/vmalloc.c
···
86
86
} while (pmd++, addr = next, addr != end);
87
87
}
88
88
89
-
static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
89
+
static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end)
90
90
{
91
91
pud_t *pud;
92
92
unsigned long next;
93
93
94
-
pud = pud_offset(pgd, addr);
94
+
pud = pud_offset(p4d, addr);
95
95
do {
96
96
next = pud_addr_end(addr, end);
97
97
if (pud_clear_huge(pud))
···
100
100
continue;
101
101
vunmap_pmd_range(pud, addr, next);
102
102
} while (pud++, addr = next, addr != end);
103
+
}
104
+
105
+
static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end)
106
+
{
107
+
p4d_t *p4d;
108
+
unsigned long next;
109
+
110
+
p4d = p4d_offset(pgd, addr);
111
+
do {
112
+
next = p4d_addr_end(addr, end);
113
+
if (p4d_clear_huge(p4d))
114
+
continue;
115
+
if (p4d_none_or_clear_bad(p4d))
116
+
continue;
117
+
vunmap_pud_range(p4d, addr, next);
118
+
} while (p4d++, addr = next, addr != end);
103
119
}
104
120
105
121
static void vunmap_page_range(unsigned long addr, unsigned long end)
···
129
113
next = pgd_addr_end(addr, end);
130
114
if (pgd_none_or_clear_bad(pgd))
131
115
continue;
132
-
vunmap_pud_range(pgd, addr, next);
116
+
vunmap_p4d_range(pgd, addr, next);
133
117
} while (pgd++, addr = next, addr != end);
134
118
}
135
119
···
176
160
return 0;
177
161
}
178
162
179
-
static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
163
+
static int vmap_pud_range(p4d_t *p4d, unsigned long addr,
180
164
unsigned long end, pgprot_t prot, struct page **pages, int *nr)
181
165
{
182
166
pud_t *pud;
183
167
unsigned long next;
184
168
185
-
pud = pud_alloc(&init_mm, pgd, addr);
169
+
pud = pud_alloc(&init_mm, p4d, addr);
186
170
if (!pud)
187
171
return -ENOMEM;
188
172
do {
···
190
174
if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
191
175
return -ENOMEM;
192
176
} while (pud++, addr = next, addr != end);
177
+
return 0;
178
+
}
179
+
180
+
static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
181
+
unsigned long end, pgprot_t prot, struct page **pages, int *nr)
182
+
{
183
+
p4d_t *p4d;
184
+
unsigned long next;
185
+
186
+
p4d = p4d_alloc(&init_mm, pgd, addr);
187
+
if (!p4d)
188
+
return -ENOMEM;
189
+
do {
190
+
next = p4d_addr_end(addr, end);
191
+
if (vmap_pud_range(p4d, addr, next, prot, pages, nr))
192
+
return -ENOMEM;
193
+
} while (p4d++, addr = next, addr != end);
193
194
return 0;
194
195
}
195
196
···
229
196
pgd = pgd_offset_k(addr);
230
197
do {
231
198
next = pgd_addr_end(addr, end);
232
-
err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
199
+
err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr);
233
200
if (err)
234
201
return err;
235
202
} while (pgd++, addr = next, addr != end);
···
270
237
unsigned long addr = (unsigned long) vmalloc_addr;
271
238
struct page *page = NULL;
272
239
pgd_t *pgd = pgd_offset_k(addr);
240
+
p4d_t *p4d;
241
+
pud_t *pud;
242
+
pmd_t *pmd;
243
+
pte_t *ptep, pte;
273
244
274
245
/*
275
246
* XXX we might need to change this if we add VIRTUAL_BUG_ON for
···
281
244
*/
282
245
VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
283
246
284
-
if (!pgd_none(*pgd)) {
285
-
pud_t *pud = pud_offset(pgd, addr);
286
-
if (!pud_none(*pud)) {
287
-
pmd_t *pmd = pmd_offset(pud, addr);
288
-
if (!pmd_none(*pmd)) {
289
-
pte_t *ptep, pte;
247
+
if (pgd_none(*pgd))
248
+
return NULL;
249
+
p4d = p4d_offset(pgd, addr);
250
+
if (p4d_none(*p4d))
251
+
return NULL;
252
+
pud = pud_offset(p4d, addr);
253
+
if (pud_none(*pud))
254
+
return NULL;
255
+
pmd = pmd_offset(pud, addr);
256
+
if (pmd_none(*pmd))
257
+
return NULL;
290
258
291
-
ptep = pte_offset_map(pmd, addr);
292
-
pte = *ptep;
293
-
if (pte_present(pte))
294
-
page = pte_page(pte);
295
-
pte_unmap(ptep);
296
-
}
297
-
}
298
-
}
259
+
ptep = pte_offset_map(pmd, addr);
260
+
pte = *ptep;
261
+
if (pte_present(pte))
262
+
page = pte_page(pte);
263
+
pte_unmap(ptep);
299
264
return page;
300
265
}
301
266
EXPORT_SYMBOL(vmalloc_to_page);