Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 fixes from Will Deacon:
"Ryan's been hard at work finding and fixing mm bugs in the arm64 code,
so here's a small crop of fixes for -rc5.
The main changes are to fix our zapping of non-present PTEs for
hugetlb entries created using the contiguous bit in the page-table
rather than a block entry at the level above. Prior to these fixes, we
were pulling the contiguous bit back out of the PTE in order to
determine the size of the hugetlb page but this is clearly bogus if
the thing isn't present and consequently both the clearing of the
PTE(s) and the TLB invalidation were unreliable.
Although the problem was found by code inspection, we really don't
want this sitting around waiting to trigger and the changes are CC'd
to stable accordingly.
Note that the diffstat looks a lot worse than it really is;
huge_ptep_get_and_clear() now takes a size argument from the core code
and so all the arch implementations of that have been updated in a
pretty mechanical fashion.
- Fix a sporadic boot failure due to incorrect randomization of the
linear map on systems that support it
- Fix the zapping (both clearing the entries *and* invalidating the
TLB) of hugetlb PTEs constructed using the contiguous bit"
* tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux:
arm64: hugetlb: Fix flush_hugetlb_tlb_range() invalidation level
arm64: hugetlb: Fix huge_ptep_get_and_clear() for non-present ptes
mm: hugetlb: Add huge page size param to huge_ptep_get_and_clear()
arm64/mm: Fix Boot panic on Ampere Altra
+82
-71
+18
-8
arch/arm64/include/asm/hugetlb.h
+18
-8
arch/arm64/include/asm/hugetlb.h
···
42
42
unsigned long addr, pte_t *ptep,
43
43
pte_t pte, int dirty);
44
44
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
45
-
extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
46
-
unsigned long addr, pte_t *ptep);
45
+
extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
46
+
pte_t *ptep, unsigned long sz);
47
47
#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT
48
48
extern void huge_ptep_set_wrprotect(struct mm_struct *mm,
49
49
unsigned long addr, pte_t *ptep);
···
76
76
{
77
77
unsigned long stride = huge_page_size(hstate_vma(vma));
78
78
79
-
if (stride == PMD_SIZE)
80
-
__flush_tlb_range(vma, start, end, stride, false, 2);
81
-
else if (stride == PUD_SIZE)
82
-
__flush_tlb_range(vma, start, end, stride, false, 1);
83
-
else
84
-
__flush_tlb_range(vma, start, end, PAGE_SIZE, false, 0);
79
+
switch (stride) {
80
+
#ifndef __PAGETABLE_PMD_FOLDED
81
+
case PUD_SIZE:
82
+
__flush_tlb_range(vma, start, end, PUD_SIZE, false, 1);
83
+
break;
84
+
#endif
85
+
case CONT_PMD_SIZE:
86
+
case PMD_SIZE:
87
+
__flush_tlb_range(vma, start, end, PMD_SIZE, false, 2);
88
+
break;
89
+
case CONT_PTE_SIZE:
90
+
__flush_tlb_range(vma, start, end, PAGE_SIZE, false, 3);
91
+
break;
92
+
default:
93
+
__flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);
94
+
}
85
95
}
86
96
87
97
#endif /* __ASM_HUGETLB_H */
+24
-35
arch/arm64/mm/hugetlbpage.c
+24
-35
arch/arm64/mm/hugetlbpage.c
···
100
100
101
101
static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
102
102
{
103
-
int contig_ptes = 0;
103
+
int contig_ptes = 1;
104
104
105
105
*pgsize = size;
106
106
107
107
switch (size) {
108
-
#ifndef __PAGETABLE_PMD_FOLDED
109
-
case PUD_SIZE:
110
-
if (pud_sect_supported())
111
-
contig_ptes = 1;
112
-
break;
113
-
#endif
114
-
case PMD_SIZE:
115
-
contig_ptes = 1;
116
-
break;
117
108
case CONT_PMD_SIZE:
118
109
*pgsize = PMD_SIZE;
119
110
contig_ptes = CONT_PMDS;
···
113
122
*pgsize = PAGE_SIZE;
114
123
contig_ptes = CONT_PTES;
115
124
break;
125
+
default:
126
+
WARN_ON(!__hugetlb_valid_size(size));
116
127
}
117
128
118
129
return contig_ptes;
···
156
163
unsigned long pgsize,
157
164
unsigned long ncontig)
158
165
{
159
-
pte_t orig_pte = __ptep_get(ptep);
160
-
unsigned long i;
166
+
pte_t pte, tmp_pte;
167
+
bool present;
161
168
162
-
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
163
-
pte_t pte = __ptep_get_and_clear(mm, addr, ptep);
164
-
165
-
/*
166
-
* If HW_AFDBM is enabled, then the HW could turn on
167
-
* the dirty or accessed bit for any page in the set,
168
-
* so check them all.
169
-
*/
170
-
if (pte_dirty(pte))
171
-
orig_pte = pte_mkdirty(orig_pte);
172
-
173
-
if (pte_young(pte))
174
-
orig_pte = pte_mkyoung(orig_pte);
169
+
pte = __ptep_get_and_clear(mm, addr, ptep);
170
+
present = pte_present(pte);
171
+
while (--ncontig) {
172
+
ptep++;
173
+
addr += pgsize;
174
+
tmp_pte = __ptep_get_and_clear(mm, addr, ptep);
175
+
if (present) {
176
+
if (pte_dirty(tmp_pte))
177
+
pte = pte_mkdirty(pte);
178
+
if (pte_young(tmp_pte))
179
+
pte = pte_mkyoung(pte);
180
+
}
175
181
}
176
-
return orig_pte;
182
+
return pte;
177
183
}
178
184
179
185
static pte_t get_clear_contig_flush(struct mm_struct *mm,
···
388
396
__pte_clear(mm, addr, ptep);
389
397
}
390
398
391
-
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
392
-
unsigned long addr, pte_t *ptep)
399
+
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
400
+
pte_t *ptep, unsigned long sz)
393
401
{
394
402
int ncontig;
395
403
size_t pgsize;
396
-
pte_t orig_pte = __ptep_get(ptep);
397
404
398
-
if (!pte_cont(orig_pte))
399
-
return __ptep_get_and_clear(mm, addr, ptep);
400
-
401
-
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
402
-
405
+
ncontig = num_contig_ptes(sz, &pgsize);
403
406
return get_clear_contig(mm, addr, ptep, pgsize, ncontig);
404
407
}
405
408
···
536
549
537
550
pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
538
551
{
552
+
unsigned long psize = huge_page_size(hstate_vma(vma));
553
+
539
554
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
540
555
/*
541
556
* Break-before-make (BBM) is required for all user space mappings
···
547
558
if (pte_user_exec(__ptep_get(ptep)))
548
559
return huge_ptep_clear_flush(vma, addr, ptep);
549
560
}
550
-
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
561
+
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
551
562
}
552
563
553
564
void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+1
-6
arch/arm64/mm/init.c
+1
-6
arch/arm64/mm/init.c
···
279
279
280
280
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
281
281
extern u16 memstart_offset_seed;
282
-
283
-
/*
284
-
* Use the sanitised version of id_aa64mmfr0_el1 so that linear
285
-
* map randomization can be enabled by shrinking the IPA space.
286
-
*/
287
-
u64 mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
282
+
u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
288
283
int parange = cpuid_feature_extract_unsigned_field(
289
284
mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
290
285
s64 range = linear_region_size -
+4
-2
arch/loongarch/include/asm/hugetlb.h
+4
-2
arch/loongarch/include/asm/hugetlb.h
···
36
36
37
37
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
38
38
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
39
-
unsigned long addr, pte_t *ptep)
39
+
unsigned long addr, pte_t *ptep,
40
+
unsigned long sz)
40
41
{
41
42
pte_t clear;
42
43
pte_t pte = ptep_get(ptep);
···
52
51
unsigned long addr, pte_t *ptep)
53
52
{
54
53
pte_t pte;
54
+
unsigned long sz = huge_page_size(hstate_vma(vma));
55
55
56
-
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
56
+
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
57
57
flush_tlb_page(vma, addr);
58
58
return pte;
59
59
}
+4
-2
arch/mips/include/asm/hugetlb.h
+4
-2
arch/mips/include/asm/hugetlb.h
···
27
27
28
28
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
29
29
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
30
-
unsigned long addr, pte_t *ptep)
30
+
unsigned long addr, pte_t *ptep,
31
+
unsigned long sz)
31
32
{
32
33
pte_t clear;
33
34
pte_t pte = *ptep;
···
43
42
unsigned long addr, pte_t *ptep)
44
43
{
45
44
pte_t pte;
45
+
unsigned long sz = huge_page_size(hstate_vma(vma));
46
46
47
47
/*
48
48
* clear the huge pte entry firstly, so that the other smp threads will
49
49
* not get old pte entry after finishing flush_tlb_page and before
50
50
* setting new huge pte entry
51
51
*/
52
-
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
52
+
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
53
53
flush_tlb_page(vma, addr);
54
54
return pte;
55
55
}
+1
-1
arch/parisc/include/asm/hugetlb.h
+1
-1
arch/parisc/include/asm/hugetlb.h
···
10
10
11
11
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
12
12
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
13
-
pte_t *ptep);
13
+
pte_t *ptep, unsigned long sz);
14
14
15
15
#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
16
16
static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
+1
-1
arch/parisc/mm/hugetlbpage.c
+1
-1
arch/parisc/mm/hugetlbpage.c
+4
-2
arch/powerpc/include/asm/hugetlb.h
+4
-2
arch/powerpc/include/asm/hugetlb.h
···
45
45
46
46
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
47
47
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
48
-
unsigned long addr, pte_t *ptep)
48
+
unsigned long addr, pte_t *ptep,
49
+
unsigned long sz)
49
50
{
50
51
return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1));
51
52
}
···
56
55
unsigned long addr, pte_t *ptep)
57
56
{
58
57
pte_t pte;
58
+
unsigned long sz = huge_page_size(hstate_vma(vma));
59
59
60
-
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
60
+
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
61
61
flush_hugetlb_page(vma, addr);
62
62
return pte;
63
63
}
+2
-1
arch/riscv/include/asm/hugetlb.h
+2
-1
arch/riscv/include/asm/hugetlb.h
···
28
28
29
29
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
30
30
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
31
-
unsigned long addr, pte_t *ptep);
31
+
unsigned long addr, pte_t *ptep,
32
+
unsigned long sz);
32
33
33
34
#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
34
35
pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
+1
-1
arch/riscv/mm/hugetlbpage.c
+1
-1
arch/riscv/mm/hugetlbpage.c
+12
-4
arch/s390/include/asm/hugetlb.h
+12
-4
arch/s390/include/asm/hugetlb.h
···
25
25
#define __HAVE_ARCH_HUGE_PTEP_GET
26
26
pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
27
27
28
+
pte_t __huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
29
+
pte_t *ptep);
30
+
28
31
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
29
-
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
32
+
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
33
+
unsigned long addr, pte_t *ptep,
34
+
unsigned long sz)
35
+
{
36
+
return __huge_ptep_get_and_clear(mm, addr, ptep);
37
+
}
30
38
31
39
static inline void arch_clear_hugetlb_flags(struct folio *folio)
32
40
{
···
56
48
static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
57
49
unsigned long address, pte_t *ptep)
58
50
{
59
-
return huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
51
+
return __huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
60
52
}
61
53
62
54
#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS
···
67
59
int changed = !pte_same(huge_ptep_get(vma->vm_mm, addr, ptep), pte);
68
60
69
61
if (changed) {
70
-
huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
62
+
__huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
71
63
__set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
72
64
}
73
65
return changed;
···
77
69
static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
78
70
unsigned long addr, pte_t *ptep)
79
71
{
80
-
pte_t pte = huge_ptep_get_and_clear(mm, addr, ptep);
72
+
pte_t pte = __huge_ptep_get_and_clear(mm, addr, ptep);
81
73
82
74
__set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte));
83
75
}
+2
-2
arch/s390/mm/hugetlbpage.c
+2
-2
arch/s390/mm/hugetlbpage.c
···
188
188
return __rste_to_pte(pte_val(*ptep));
189
189
}
190
190
191
-
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
192
-
unsigned long addr, pte_t *ptep)
191
+
pte_t __huge_ptep_get_and_clear(struct mm_struct *mm,
192
+
unsigned long addr, pte_t *ptep)
193
193
{
194
194
pte_t pte = huge_ptep_get(mm, addr, ptep);
195
195
pmd_t *pmdp = (pmd_t *) ptep;
+1
-1
arch/sparc/include/asm/hugetlb.h
+1
-1
arch/sparc/include/asm/hugetlb.h
···
20
20
21
21
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
22
22
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
23
-
pte_t *ptep);
23
+
pte_t *ptep, unsigned long sz);
24
24
25
25
#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
26
26
static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
+1
-1
arch/sparc/mm/hugetlbpage.c
+1
-1
arch/sparc/mm/hugetlbpage.c
+1
-1
include/asm-generic/hugetlb.h
+1
-1
include/asm-generic/hugetlb.h
+3
-1
include/linux/hugetlb.h
+3
-1
include/linux/hugetlb.h
···
1004
1004
static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
1005
1005
unsigned long addr, pte_t *ptep)
1006
1006
{
1007
-
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
1007
+
unsigned long psize = huge_page_size(hstate_vma(vma));
1008
+
1009
+
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
1008
1010
}
1009
1011
#endif
1010
1012
+2
-2
mm/hugetlb.c
+2
-2
mm/hugetlb.c
···
5447
5447
if (src_ptl != dst_ptl)
5448
5448
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5449
5449
5450
-
pte = huge_ptep_get_and_clear(mm, old_addr, src_pte);
5450
+
pte = huge_ptep_get_and_clear(mm, old_addr, src_pte, sz);
5451
5451
5452
5452
if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
5453
5453
huge_pte_clear(mm, new_addr, dst_pte, sz);
···
5622
5622
set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
5623
5623
}
5624
5624
5625
-
pte = huge_ptep_get_and_clear(mm, address, ptep);
5625
+
pte = huge_ptep_get_and_clear(mm, address, ptep, sz);
5626
5626
tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
5627
5627
if (huge_pte_dirty(pte))
5628
5628
set_page_dirty(page);