riscv: pgtable: move pagetable_dtor() to __tlb_remove_table()

Move pagetable_dtor() to __tlb_remove_table(), so that ptlock and page
table pages can be freed together (regardless of whether RCU is used).
This prevents the use-after-free problem where the ptlock is freed
immediately but the page table pages is freed later via RCU.

Page tables shouldn't have swap cache, so use pagetable_free() instead of
free_page_and_swap_cache() to free page table pages.

By the way, move the comment above __tlb_remove_table() to
riscv_tlb_remove_ptdesc(), it will be more appropriate.

Link: https://lkml.kernel.org/r/b89d77c965507b1b102cbabe988e69365cb288b6.1736317725.git.zhengqi.arch@bytedance.com
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: Andreas Larsson <andreas@gaisler.com>
Cc: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Qi Zheng and committed by

Andrew Morton 1 year ago deab5a35 12359c03

+21 -31

2 changed files

expand all

arch

riscv

include

asm

pgalloc.h

tlb.h

+17 -21

arch/riscv/include/asm/pgalloc.h

··· 15 15 #define __HAVE_ARCH_PUD_FREE 16 16 #include <asm-generic/pgalloc.h> 17 17 18 + /* 19 + * While riscv platforms with riscv_ipi_for_rfence as true require an IPI to 20 + * perform TLB shootdown, some platforms with riscv_ipi_for_rfence as false use 21 + * SBI to perform TLB shootdown. To keep software pagetable walkers safe in this 22 + * case we switch to RCU based table free (MMU_GATHER_RCU_TABLE_FREE). See the 23 + * comment below 'ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE' in include/asm-generic/tlb.h 24 + * for more details. 25 + */ 18 26 static inline void riscv_tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt) 19 27 { 20 - if (riscv_use_sbi_for_rfence()) 28 + if (riscv_use_sbi_for_rfence()) { 21 29 tlb_remove_ptdesc(tlb, pt); 22 - else 30 + } else { 31 + pagetable_dtor(pt); 23 32 tlb_remove_page_ptdesc(tlb, pt); 33 + } 24 34 } 25 35 26 36 static inline void pmd_populate_kernel(struct mm_struct *mm, ··· 107 97 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 108 98 unsigned long addr) 109 99 { 110 - if (pgtable_l4_enabled) { 111 - struct ptdesc *ptdesc = virt_to_ptdesc(pud); 112 - 113 - pagetable_dtor(ptdesc); 114 - riscv_tlb_remove_ptdesc(tlb, ptdesc); 115 - } 100 + if (pgtable_l4_enabled) 101 + riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(pud)); 116 102 } 117 103 118 104 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, 119 105 unsigned long addr) 120 106 { 121 - if (pgtable_l5_enabled) { 122 - struct ptdesc *ptdesc = virt_to_ptdesc(p4d); 123 - 124 - pagetable_dtor(ptdesc); 107 + if (pgtable_l5_enabled) 125 108 riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d)); 126 - } 127 109 } 128 110 #endif /* __PAGETABLE_PMD_FOLDED */ 129 111 ··· 144 142 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 145 143 unsigned long addr) 146 144 { 147 - struct ptdesc *ptdesc = virt_to_ptdesc(pmd); 148 - 149 - pagetable_dtor(ptdesc); 150 - riscv_tlb_remove_ptdesc(tlb, ptdesc); 145 + riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(pmd)); 151 146 } 152 147 153 148 #endif /* __PAGETABLE_PMD_FOLDED */ ··· 152 153 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 153 154 unsigned long addr) 154 155 { 155 - struct ptdesc *ptdesc = page_ptdesc(pte); 156 - 157 - pagetable_dtor(ptdesc); 158 - riscv_tlb_remove_ptdesc(tlb, ptdesc); 156 + riscv_tlb_remove_ptdesc(tlb, page_ptdesc(pte)); 159 157 } 160 158 #endif /* CONFIG_MMU */ 161 159

+4 -10

arch/riscv/include/asm/tlb.h

··· 11 11 static void tlb_flush(struct mmu_gather *tlb); 12 12 13 13 #ifdef CONFIG_MMU 14 - #include <linux/swap.h> 15 14 16 - /* 17 - * While riscv platforms with riscv_ipi_for_rfence as true require an IPI to 18 - * perform TLB shootdown, some platforms with riscv_ipi_for_rfence as false use 19 - * SBI to perform TLB shootdown. To keep software pagetable walkers safe in this 20 - * case we switch to RCU based table free (MMU_GATHER_RCU_TABLE_FREE). See the 21 - * comment below 'ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE' in include/asm-generic/tlb.h 22 - * for more details. 23 - */ 24 15 static inline void __tlb_remove_table(void *table) 25 16 { 26 - free_page_and_swap_cache(table); 17 + struct ptdesc *ptdesc = (struct ptdesc *)table; 18 + 19 + pagetable_dtor(ptdesc); 20 + pagetable_free(ptdesc); 27 21 } 28 22 29 23 #endif /* CONFIG_MMU */

Configure Feed

Configure Feed