Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'riscv-for-linus-6.8-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux
Pull RISC-V fixes from Palmer Dabbelt:
- fix missing TLB flush during early boot on SPARSEMEM_VMEMMAP
configurations
- fixes to correctly implement the break-before-make behavior requried
by the ISA for NAPOT mappings
- fix a missing TLB flush on intermediate mapping changes
- fix build warning about a missing declaration of overflow_stack
- fix performace regression related to incorrect tracking of completed
batch TLB flushes
* tag 'riscv-for-linus-6.8-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux:
riscv: Fix arch_tlbbatch_flush() by clearing the batch cpumask
riscv: declare overflow_stack as exported from traps.c
riscv: Fix arch_hugetlb_migration_supported() for NAPOT
riscv: Flush the tlb when a page directory is freed
riscv: Fix hugetlb_mask_last_page() when NAPOT is enabled
riscv: Fix set_huge_pte_at() for NAPOT mapping
riscv: mm: execute local TLB flush after populating vmemmap
+89
-6
+3
arch/riscv/include/asm/hugetlb.h
+3
arch/riscv/include/asm/hugetlb.h
···
11
11
}
12
12
#define arch_clear_hugepage_flags arch_clear_hugepage_flags
13
13
14
+
bool arch_hugetlb_migration_supported(struct hstate *h);
15
+
#define arch_hugetlb_migration_supported arch_hugetlb_migration_supported
16
+
14
17
#ifdef CONFIG_RISCV_ISA_SVNAPOT
15
18
#define __HAVE_ARCH_HUGE_PTE_CLEAR
16
19
void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
+5
arch/riscv/include/asm/stacktrace.h
+5
arch/riscv/include/asm/stacktrace.h
···
21
21
return !(((unsigned long)(current->stack) ^ current_stack_pointer) & ~(THREAD_SIZE - 1));
22
22
}
23
23
24
+
25
+
#ifdef CONFIG_VMAP_STACK
26
+
DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
27
+
#endif /* CONFIG_VMAP_STACK */
28
+
24
29
#endif /* _ASM_RISCV_STACKTRACE_H */
+1
-1
arch/riscv/include/asm/tlb.h
+1
-1
arch/riscv/include/asm/tlb.h
···
16
16
static inline void tlb_flush(struct mmu_gather *tlb)
17
17
{
18
18
#ifdef CONFIG_MMU
19
-
if (tlb->fullmm || tlb->need_flush_all)
19
+
if (tlb->fullmm || tlb->need_flush_all || tlb->freed_tables)
20
20
flush_tlb_mm(tlb->mm);
21
21
else
22
22
flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end,
+1
arch/riscv/include/asm/tlbflush.h
+1
arch/riscv/include/asm/tlbflush.h
+73
-5
arch/riscv/mm/hugetlbpage.c
+73
-5
arch/riscv/mm/hugetlbpage.c
···
125
125
return pte;
126
126
}
127
127
128
+
unsigned long hugetlb_mask_last_page(struct hstate *h)
129
+
{
130
+
unsigned long hp_size = huge_page_size(h);
131
+
132
+
switch (hp_size) {
133
+
#ifndef __PAGETABLE_PMD_FOLDED
134
+
case PUD_SIZE:
135
+
return P4D_SIZE - PUD_SIZE;
136
+
#endif
137
+
case PMD_SIZE:
138
+
return PUD_SIZE - PMD_SIZE;
139
+
case napot_cont_size(NAPOT_CONT64KB_ORDER):
140
+
return PMD_SIZE - napot_cont_size(NAPOT_CONT64KB_ORDER);
141
+
default:
142
+
break;
143
+
}
144
+
145
+
return 0UL;
146
+
}
147
+
128
148
static pte_t get_clear_contig(struct mm_struct *mm,
129
149
unsigned long addr,
130
150
pte_t *ptep,
···
197
177
return entry;
198
178
}
199
179
180
+
static void clear_flush(struct mm_struct *mm,
181
+
unsigned long addr,
182
+
pte_t *ptep,
183
+
unsigned long pgsize,
184
+
unsigned long ncontig)
185
+
{
186
+
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
187
+
unsigned long i, saddr = addr;
188
+
189
+
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
190
+
ptep_get_and_clear(mm, addr, ptep);
191
+
192
+
flush_tlb_range(&vma, saddr, addr);
193
+
}
194
+
195
+
/*
196
+
* When dealing with NAPOT mappings, the privileged specification indicates that
197
+
* "if an update needs to be made, the OS generally should first mark all of the
198
+
* PTEs invalid, then issue SFENCE.VMA instruction(s) covering all 4 KiB regions
199
+
* within the range, [...] then update the PTE(s), as described in Section
200
+
* 4.2.1.". That's the equivalent of the Break-Before-Make approach used by
201
+
* arm64.
202
+
*/
200
203
void set_huge_pte_at(struct mm_struct *mm,
201
204
unsigned long addr,
202
205
pte_t *ptep,
203
206
pte_t pte,
204
207
unsigned long sz)
205
208
{
206
-
unsigned long hugepage_shift;
209
+
unsigned long hugepage_shift, pgsize;
207
210
int i, pte_num;
208
211
209
212
if (sz >= PGDIR_SIZE)
···
241
198
hugepage_shift = PAGE_SHIFT;
242
199
243
200
pte_num = sz >> hugepage_shift;
244
-
for (i = 0; i < pte_num; i++, ptep++, addr += (1 << hugepage_shift))
201
+
pgsize = 1 << hugepage_shift;
202
+
203
+
if (!pte_present(pte)) {
204
+
for (i = 0; i < pte_num; i++, ptep++, addr += pgsize)
205
+
set_ptes(mm, addr, ptep, pte, 1);
206
+
return;
207
+
}
208
+
209
+
if (!pte_napot(pte)) {
210
+
set_ptes(mm, addr, ptep, pte, 1);
211
+
return;
212
+
}
213
+
214
+
clear_flush(mm, addr, ptep, pgsize, pte_num);
215
+
216
+
for (i = 0; i < pte_num; i++, ptep++, addr += pgsize)
245
217
set_pte_at(mm, addr, ptep, pte);
246
218
}
247
219
···
364
306
pte_clear(mm, addr, ptep);
365
307
}
366
308
367
-
static __init bool is_napot_size(unsigned long size)
309
+
static bool is_napot_size(unsigned long size)
368
310
{
369
311
unsigned long order;
370
312
···
392
334
393
335
#else
394
336
395
-
static __init bool is_napot_size(unsigned long size)
337
+
static bool is_napot_size(unsigned long size)
396
338
{
397
339
return false;
398
340
}
···
409
351
return pmd_leaf(pmd);
410
352
}
411
353
412
-
bool __init arch_hugetlb_valid_size(unsigned long size)
354
+
static bool __hugetlb_valid_size(unsigned long size)
413
355
{
414
356
if (size == HPAGE_SIZE)
415
357
return true;
···
419
361
return true;
420
362
else
421
363
return false;
364
+
}
365
+
366
+
bool __init arch_hugetlb_valid_size(unsigned long size)
367
+
{
368
+
return __hugetlb_valid_size(size);
369
+
}
370
+
371
+
bool arch_hugetlb_migration_supported(struct hstate *h)
372
+
{
373
+
return __hugetlb_valid_size(huge_page_size(h));
422
374
}
423
375
424
376
#ifdef CONFIG_CONTIG_ALLOC
+4
arch/riscv/mm/init.c
+4
arch/riscv/mm/init.c
···
1385
1385
early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT);
1386
1386
arch_numa_init();
1387
1387
sparse_init();
1388
+
#ifdef CONFIG_SPARSEMEM_VMEMMAP
1389
+
/* The entire VMEMMAP region has been populated. Flush TLB for this region */
1390
+
local_flush_tlb_kernel_range(VMEMMAP_START, VMEMMAP_END);
1391
+
#endif
1388
1392
zone_sizes_init();
1389
1393
arch_reserve_crashkernel();
1390
1394
memblock_dump_all();
+2
arch/riscv/mm/tlbflush.c
+2
arch/riscv/mm/tlbflush.c
···
66
66
local_flush_tlb_range_threshold_asid(start, size, stride, asid);
67
67
}
68
68
69
+
/* Flush a range of kernel pages without broadcasting */
69
70
void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
70
71
{
71
72
local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID);
···
234
233
{
235
234
__flush_tlb_range(&batch->cpumask, FLUSH_TLB_NO_ASID, 0,
236
235
FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
236
+
cpumask_clear(&batch->cpumask);
237
237
}