Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
riscv: implement the new page table range API
Add set_ptes(), update_mmu_cache_range() and flush_dcache_folio(). Change
the PG_dcache_clean flag from being per-page to per-folio.
Link: https://lkml.kernel.org/r/20230802151406.3735276-23-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Acked-by: Mike Rapoport (IBM) <rppt@kernel.org>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Matthew Wilcox (Oracle)
and committed by
Andrew Morton
864609c6
9fee28ba
+36
-33
+9
-10
arch/riscv/include/asm/cacheflush.h
+9
-10
arch/riscv/include/asm/cacheflush.h
···
15
15
16
16
#define PG_dcache_clean PG_arch_1
17
17
18
+
static inline void flush_dcache_folio(struct folio *folio)
19
+
{
20
+
if (test_bit(PG_dcache_clean, &folio->flags))
21
+
clear_bit(PG_dcache_clean, &folio->flags);
22
+
}
23
+
#define flush_dcache_folio flush_dcache_folio
24
+
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
25
+
18
26
static inline void flush_dcache_page(struct page *page)
19
27
{
20
-
/*
21
-
* HugeTLB pages are always fully mapped and only head page will be
22
-
* set PG_dcache_clean (see comments in flush_icache_pte()).
23
-
*/
24
-
if (PageHuge(page))
25
-
page = compound_head(page);
26
-
27
-
if (test_bit(PG_dcache_clean, &page->flags))
28
-
clear_bit(PG_dcache_clean, &page->flags);
28
+
flush_dcache_folio(page_folio(page));
29
29
}
30
-
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
31
30
32
31
/*
33
32
* RISC-V doesn't have an instruction to flush parts of the instruction cache,
+24
-13
arch/riscv/include/asm/pgtable.h
+24
-13
arch/riscv/include/asm/pgtable.h
···
445
445
446
446
447
447
/* Commit new configuration to MMU hardware */
448
-
static inline void update_mmu_cache(struct vm_area_struct *vma,
449
-
unsigned long address, pte_t *ptep)
448
+
static inline void update_mmu_cache_range(struct vm_fault *vmf,
449
+
struct vm_area_struct *vma, unsigned long address,
450
+
pte_t *ptep, unsigned int nr)
450
451
{
451
452
/*
452
453
* The kernel assumes that TLBs don't cache invalid entries, but
···
456
455
* Relying on flush_tlb_fix_spurious_fault would suffice, but
457
456
* the extra traps reduce performance. So, eagerly SFENCE.VMA.
458
457
*/
459
-
local_flush_tlb_page(address);
458
+
while (nr--)
459
+
local_flush_tlb_page(address + nr * PAGE_SIZE);
460
460
}
461
+
#define update_mmu_cache(vma, addr, ptep) \
462
+
update_mmu_cache_range(NULL, vma, addr, ptep, 1)
461
463
462
464
#define __HAVE_ARCH_UPDATE_MMU_TLB
463
465
#define update_mmu_tlb update_mmu_cache
···
491
487
492
488
void flush_icache_pte(pte_t pte);
493
489
494
-
static inline void __set_pte_at(struct mm_struct *mm,
495
-
unsigned long addr, pte_t *ptep, pte_t pteval)
490
+
static inline void __set_pte_at(pte_t *ptep, pte_t pteval)
496
491
{
497
492
if (pte_present(pteval) && pte_exec(pteval))
498
493
flush_icache_pte(pteval);
···
499
496
set_pte(ptep, pteval);
500
497
}
501
498
502
-
static inline void set_pte_at(struct mm_struct *mm,
503
-
unsigned long addr, pte_t *ptep, pte_t pteval)
499
+
static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
500
+
pte_t *ptep, pte_t pteval, unsigned int nr)
504
501
{
505
-
page_table_check_ptes_set(mm, ptep, pteval, 1);
506
-
__set_pte_at(mm, addr, ptep, pteval);
502
+
page_table_check_ptes_set(mm, ptep, pteval, nr);
503
+
504
+
for (;;) {
505
+
__set_pte_at(ptep, pteval);
506
+
if (--nr == 0)
507
+
break;
508
+
ptep++;
509
+
pte_val(pteval) += 1 << _PAGE_PFN_SHIFT;
510
+
}
507
511
}
512
+
#define set_ptes set_ptes
508
513
509
514
static inline void pte_clear(struct mm_struct *mm,
510
515
unsigned long addr, pte_t *ptep)
511
516
{
512
-
__set_pte_at(mm, addr, ptep, __pte(0));
517
+
__set_pte_at(ptep, __pte(0));
513
518
}
514
519
515
520
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
···
526
515
pte_t entry, int dirty)
527
516
{
528
517
if (!pte_same(*ptep, entry))
529
-
set_pte_at(vma->vm_mm, address, ptep, entry);
518
+
__set_pte_at(ptep, entry);
530
519
/*
531
520
* update_mmu_cache will unconditionally execute, handling both
532
521
* the case that the PTE changed and the spurious fault case.
···
699
688
pmd_t *pmdp, pmd_t pmd)
700
689
{
701
690
page_table_check_pmd_set(mm, pmdp, pmd);
702
-
return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd));
691
+
return __set_pte_at((pte_t *)pmdp, pmd_pte(pmd));
703
692
}
704
693
705
694
static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
706
695
pud_t *pudp, pud_t pud)
707
696
{
708
697
page_table_check_pud_set(mm, pudp, pud);
709
-
return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud));
698
+
return __set_pte_at((pte_t *)pudp, pud_pte(pud));
710
699
}
711
700
712
701
#ifdef CONFIG_PAGE_TABLE_CHECK
+3
-10
arch/riscv/mm/cacheflush.c
+3
-10
arch/riscv/mm/cacheflush.c
···
82
82
#ifdef CONFIG_MMU
83
83
void flush_icache_pte(pte_t pte)
84
84
{
85
-
struct page *page = pte_page(pte);
85
+
struct folio *folio = page_folio(pte_page(pte));
86
86
87
-
/*
88
-
* HugeTLB pages are always fully mapped, so only setting head page's
89
-
* PG_dcache_clean flag is enough.
90
-
*/
91
-
if (PageHuge(page))
92
-
page = compound_head(page);
93
-
94
-
if (!test_bit(PG_dcache_clean, &page->flags)) {
87
+
if (!test_bit(PG_dcache_clean, &folio->flags)) {
95
88
flush_icache_all();
96
-
set_bit(PG_dcache_clean, &page->flags);
89
+
set_bit(PG_dcache_clean, &folio->flags);
97
90
}
98
91
}
99
92
#endif /* CONFIG_MMU */