Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

+1 -1

arch/arm64/include/asm/cputype.h

··· 20 20 21 21 #define MPIDR_UP_BITMASK (0x1 << 30) 22 22 #define MPIDR_MT_BITMASK (0x1 << 24) 23 - #define MPIDR_HWID_BITMASK 0xff00ffffff 23 + #define MPIDR_HWID_BITMASK 0xff00ffffffUL 24 24 25 25 #define MPIDR_LEVEL_BITS_SHIFT 3 26 26 #define MPIDR_LEVEL_BITS (1 << MPIDR_LEVEL_BITS_SHIFT)

+1 -1

arch/arm64/include/asm/hugetlb.h

··· 22 22 23 23 static inline pte_t huge_ptep_get(pte_t *ptep) 24 24 { 25 - return *ptep; 25 + return READ_ONCE(*ptep); 26 26 } 27 27 28 28

+13 -13

arch/arm64/include/asm/kvm_mmu.h

··· 185 185 return pmd; 186 186 } 187 187 188 - static inline void kvm_set_s2pte_readonly(pte_t *pte) 188 + static inline void kvm_set_s2pte_readonly(pte_t *ptep) 189 189 { 190 190 pteval_t old_pteval, pteval; 191 191 192 - pteval = READ_ONCE(pte_val(*pte)); 192 + pteval = READ_ONCE(pte_val(*ptep)); 193 193 do { 194 194 old_pteval = pteval; 195 195 pteval &= ~PTE_S2_RDWR; 196 196 pteval |= PTE_S2_RDONLY; 197 - pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval); 197 + pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); 198 198 } while (pteval != old_pteval); 199 199 } 200 200 201 - static inline bool kvm_s2pte_readonly(pte_t *pte) 201 + static inline bool kvm_s2pte_readonly(pte_t *ptep) 202 202 { 203 - return (pte_val(*pte) & PTE_S2_RDWR) == PTE_S2_RDONLY; 203 + return (READ_ONCE(pte_val(*ptep)) & PTE_S2_RDWR) == PTE_S2_RDONLY; 204 204 } 205 205 206 - static inline bool kvm_s2pte_exec(pte_t *pte) 206 + static inline bool kvm_s2pte_exec(pte_t *ptep) 207 207 { 208 - return !(pte_val(*pte) & PTE_S2_XN); 208 + return !(READ_ONCE(pte_val(*ptep)) & PTE_S2_XN); 209 209 } 210 210 211 - static inline void kvm_set_s2pmd_readonly(pmd_t *pmd) 211 + static inline void kvm_set_s2pmd_readonly(pmd_t *pmdp) 212 212 { 213 - kvm_set_s2pte_readonly((pte_t *)pmd); 213 + kvm_set_s2pte_readonly((pte_t *)pmdp); 214 214 } 215 215 216 - static inline bool kvm_s2pmd_readonly(pmd_t *pmd) 216 + static inline bool kvm_s2pmd_readonly(pmd_t *pmdp) 217 217 { 218 - return kvm_s2pte_readonly((pte_t *)pmd); 218 + return kvm_s2pte_readonly((pte_t *)pmdp); 219 219 } 220 220 221 - static inline bool kvm_s2pmd_exec(pmd_t *pmd) 221 + static inline bool kvm_s2pmd_exec(pmd_t *pmdp) 222 222 { 223 - return !(pmd_val(*pmd) & PMD_S2_XN); 223 + return !(READ_ONCE(pmd_val(*pmdp)) & PMD_S2_XN); 224 224 } 225 225 226 226 static inline bool kvm_page_empty(void *ptr)

+2 -2

arch/arm64/include/asm/mmu_context.h

··· 141 141 * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD, 142 142 * avoiding the possibility of conflicting TLB entries being allocated. 143 143 */ 144 - static inline void cpu_replace_ttbr1(pgd_t *pgd) 144 + static inline void cpu_replace_ttbr1(pgd_t *pgdp) 145 145 { 146 146 typedef void (ttbr_replace_func)(phys_addr_t); 147 147 extern ttbr_replace_func idmap_cpu_replace_ttbr1; 148 148 ttbr_replace_func *replace_phys; 149 149 150 - phys_addr_t pgd_phys = virt_to_phys(pgd); 150 + phys_addr_t pgd_phys = virt_to_phys(pgdp); 151 151 152 152 replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1); 153 153

+22 -22

arch/arm64/include/asm/pgalloc.h

··· 36 36 return (pmd_t *)__get_free_page(PGALLOC_GFP); 37 37 } 38 38 39 - static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) 39 + static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp) 40 40 { 41 - BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); 42 - free_page((unsigned long)pmd); 41 + BUG_ON((unsigned long)pmdp & (PAGE_SIZE-1)); 42 + free_page((unsigned long)pmdp); 43 43 } 44 44 45 - static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot) 45 + static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) 46 46 { 47 - set_pud(pud, __pud(__phys_to_pud_val(pmd) | prot)); 47 + set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); 48 48 } 49 49 50 - static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 50 + static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) 51 51 { 52 - __pud_populate(pud, __pa(pmd), PMD_TYPE_TABLE); 52 + __pud_populate(pudp, __pa(pmdp), PMD_TYPE_TABLE); 53 53 } 54 54 #else 55 - static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot) 55 + static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) 56 56 { 57 57 BUILD_BUG(); 58 58 } ··· 65 65 return (pud_t *)__get_free_page(PGALLOC_GFP); 66 66 } 67 67 68 - static inline void pud_free(struct mm_struct *mm, pud_t *pud) 68 + static inline void pud_free(struct mm_struct *mm, pud_t *pudp) 69 69 { 70 - BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); 71 - free_page((unsigned long)pud); 70 + BUG_ON((unsigned long)pudp & (PAGE_SIZE-1)); 71 + free_page((unsigned long)pudp); 72 72 } 73 73 74 - static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot) 74 + static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot) 75 75 { 76 - set_pgd(pgdp, __pgd(__phys_to_pgd_val(pud) | prot)); 76 + set_pgd(pgdp, __pgd(__phys_to_pgd_val(pudp) | prot)); 77 77 } 78 78 79 - static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) 79 + static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, pud_t *pudp) 80 80 { 81 - __pgd_populate(pgd, __pa(pud), PUD_TYPE_TABLE); 81 + __pgd_populate(pgdp, __pa(pudp), PUD_TYPE_TABLE); 82 82 } 83 83 #else 84 - static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot) 84 + static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot) 85 85 { 86 86 BUILD_BUG(); 87 87 } 88 88 #endif /* CONFIG_PGTABLE_LEVELS > 3 */ 89 89 90 90 extern pgd_t *pgd_alloc(struct mm_struct *mm); 91 - extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 91 + extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp); 92 92 93 93 static inline pte_t * 94 94 pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) ··· 114 114 /* 115 115 * Free a PTE table. 116 116 */ 117 - static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 117 + static inline void pte_free_kernel(struct mm_struct *mm, pte_t *ptep) 118 118 { 119 - if (pte) 120 - free_page((unsigned long)pte); 119 + if (ptep) 120 + free_page((unsigned long)ptep); 121 121 } 122 122 123 123 static inline void pte_free(struct mm_struct *mm, pgtable_t pte) ··· 126 126 __free_page(pte); 127 127 } 128 128 129 - static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, 129 + static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep, 130 130 pmdval_t prot) 131 131 { 132 - set_pmd(pmdp, __pmd(__phys_to_pmd_val(pte) | prot)); 132 + set_pmd(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); 133 133 } 134 134 135 135 /*

+13 -10

arch/arm64/include/asm/pgtable.h

··· 218 218 219 219 static inline void set_pte(pte_t *ptep, pte_t pte) 220 220 { 221 - *ptep = pte; 221 + WRITE_ONCE(*ptep, pte); 222 222 223 223 /* 224 224 * Only if the new pte is valid and kernel, otherwise TLB maintenance ··· 250 250 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 251 251 pte_t *ptep, pte_t pte) 252 252 { 253 + pte_t old_pte; 254 + 253 255 if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) 254 256 __sync_icache_dcache(pte, addr); 255 257 ··· 260 258 * hardware updates of the pte (ptep_set_access_flags safely changes 261 259 * valid ptes without going through an invalid entry). 262 260 */ 263 - if (IS_ENABLED(CONFIG_DEBUG_VM) && pte_valid(*ptep) && pte_valid(pte) && 261 + old_pte = READ_ONCE(*ptep); 262 + if (IS_ENABLED(CONFIG_DEBUG_VM) && pte_valid(old_pte) && pte_valid(pte) && 264 263 (mm == current->active_mm || atomic_read(&mm->mm_users) > 1)) { 265 264 VM_WARN_ONCE(!pte_young(pte), 266 265 "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", 267 - __func__, pte_val(*ptep), pte_val(pte)); 268 - VM_WARN_ONCE(pte_write(*ptep) && !pte_dirty(pte), 266 + __func__, pte_val(old_pte), pte_val(pte)); 267 + VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte), 269 268 "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx", 270 - __func__, pte_val(*ptep), pte_val(pte)); 269 + __func__, pte_val(old_pte), pte_val(pte)); 271 270 } 272 271 273 272 set_pte(ptep, pte); ··· 434 431 435 432 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 436 433 { 437 - *pmdp = pmd; 434 + WRITE_ONCE(*pmdp, pmd); 438 435 dsb(ishst); 439 436 isb(); 440 437 } ··· 485 482 486 483 static inline void set_pud(pud_t *pudp, pud_t pud) 487 484 { 488 - *pudp = pud; 485 + WRITE_ONCE(*pudp, pud); 489 486 dsb(ishst); 490 487 isb(); 491 488 } ··· 503 500 /* Find an entry in the second-level page table. */ 504 501 #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) 505 502 506 - #define pmd_offset_phys(dir, addr) (pud_page_paddr(*(dir)) + pmd_index(addr) * sizeof(pmd_t)) 503 + #define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t)) 507 504 #define pmd_offset(dir, addr) ((pmd_t *)__va(pmd_offset_phys((dir), (addr)))) 508 505 509 506 #define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) ··· 538 535 539 536 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 540 537 { 541 - *pgdp = pgd; 538 + WRITE_ONCE(*pgdp, pgd); 542 539 dsb(ishst); 543 540 } 544 541 ··· 555 552 /* Find an entry in the frst-level page table. */ 556 553 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) 557 554 558 - #define pud_offset_phys(dir, addr) (pgd_page_paddr(*(dir)) + pud_index(addr) * sizeof(pud_t)) 555 + #define pud_offset_phys(dir, addr) (pgd_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t)) 559 556 #define pud_offset(dir, addr) ((pud_t *)__va(pud_offset_phys((dir), (addr)))) 560 557 561 558 #define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr))

+9

arch/arm64/kernel/cpu_errata.c

··· 408 408 }, 409 409 { 410 410 .capability = ARM64_HARDEN_BRANCH_PREDICTOR, 411 + MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR), 412 + .enable = qcom_enable_link_stack_sanitization, 413 + }, 414 + { 415 + .capability = ARM64_HARDEN_BP_POST_GUEST_EXIT, 416 + MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR), 417 + }, 418 + { 419 + .capability = ARM64_HARDEN_BRANCH_PREDICTOR, 411 420 MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN), 412 421 .enable = enable_smccc_arch_workaround_1, 413 422 },

+1 -1

arch/arm64/kernel/efi.c

··· 90 90 unsigned long addr, void *data) 91 91 { 92 92 efi_memory_desc_t *md = data; 93 - pte_t pte = *ptep; 93 + pte_t pte = READ_ONCE(*ptep); 94 94 95 95 if (md->attribute & EFI_MEMORY_RO) 96 96 pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));

+76 -72

arch/arm64/kernel/hibernate.c

··· 202 202 gfp_t mask) 203 203 { 204 204 int rc = 0; 205 - pgd_t *pgd; 206 - pud_t *pud; 207 - pmd_t *pmd; 208 - pte_t *pte; 205 + pgd_t *pgdp; 206 + pud_t *pudp; 207 + pmd_t *pmdp; 208 + pte_t *ptep; 209 209 unsigned long dst = (unsigned long)allocator(mask); 210 210 211 211 if (!dst) { ··· 216 216 memcpy((void *)dst, src_start, length); 217 217 flush_icache_range(dst, dst + length); 218 218 219 - pgd = pgd_offset_raw(allocator(mask), dst_addr); 220 - if (pgd_none(*pgd)) { 221 - pud = allocator(mask); 222 - if (!pud) { 219 + pgdp = pgd_offset_raw(allocator(mask), dst_addr); 220 + if (pgd_none(READ_ONCE(*pgdp))) { 221 + pudp = allocator(mask); 222 + if (!pudp) { 223 223 rc = -ENOMEM; 224 224 goto out; 225 225 } 226 - pgd_populate(&init_mm, pgd, pud); 226 + pgd_populate(&init_mm, pgdp, pudp); 227 227 } 228 228 229 - pud = pud_offset(pgd, dst_addr); 230 - if (pud_none(*pud)) { 231 - pmd = allocator(mask); 232 - if (!pmd) { 229 + pudp = pud_offset(pgdp, dst_addr); 230 + if (pud_none(READ_ONCE(*pudp))) { 231 + pmdp = allocator(mask); 232 + if (!pmdp) { 233 233 rc = -ENOMEM; 234 234 goto out; 235 235 } 236 - pud_populate(&init_mm, pud, pmd); 236 + pud_populate(&init_mm, pudp, pmdp); 237 237 } 238 238 239 - pmd = pmd_offset(pud, dst_addr); 240 - if (pmd_none(*pmd)) { 241 - pte = allocator(mask); 242 - if (!pte) { 239 + pmdp = pmd_offset(pudp, dst_addr); 240 + if (pmd_none(READ_ONCE(*pmdp))) { 241 + ptep = allocator(mask); 242 + if (!ptep) { 243 243 rc = -ENOMEM; 244 244 goto out; 245 245 } 246 - pmd_populate_kernel(&init_mm, pmd, pte); 246 + pmd_populate_kernel(&init_mm, pmdp, ptep); 247 247 } 248 248 249 - pte = pte_offset_kernel(pmd, dst_addr); 250 - set_pte(pte, pfn_pte(virt_to_pfn(dst), PAGE_KERNEL_EXEC)); 249 + ptep = pte_offset_kernel(pmdp, dst_addr); 250 + set_pte(ptep, pfn_pte(virt_to_pfn(dst), PAGE_KERNEL_EXEC)); 251 251 252 252 /* 253 253 * Load our new page tables. A strict BBM approach requires that we ··· 263 263 */ 264 264 cpu_set_reserved_ttbr0(); 265 265 local_flush_tlb_all(); 266 - write_sysreg(phys_to_ttbr(virt_to_phys(pgd)), ttbr0_el1); 266 + write_sysreg(phys_to_ttbr(virt_to_phys(pgdp)), ttbr0_el1); 267 267 isb(); 268 268 269 269 *phys_dst_addr = virt_to_phys((void *)dst); ··· 320 320 return ret; 321 321 } 322 322 323 - static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) 323 + static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) 324 324 { 325 - pte_t pte = *src_pte; 325 + pte_t pte = READ_ONCE(*src_ptep); 326 326 327 327 if (pte_valid(pte)) { 328 328 /* ··· 330 330 * read only (code, rodata). Clear the RDONLY bit from 331 331 * the temporary mappings we use during restore. 332 332 */ 333 - set_pte(dst_pte, pte_mkwrite(pte)); 333 + set_pte(dst_ptep, pte_mkwrite(pte)); 334 334 } else if (debug_pagealloc_enabled() && !pte_none(pte)) { 335 335 /* 336 336 * debug_pagealloc will removed the PTE_VALID bit if ··· 343 343 */ 344 344 BUG_ON(!pfn_valid(pte_pfn(pte))); 345 345 346 - set_pte(dst_pte, pte_mkpresent(pte_mkwrite(pte))); 346 + set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte))); 347 347 } 348 348 } 349 349 350 - static int copy_pte(pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long start, 350 + static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, 351 351 unsigned long end) 352 352 { 353 - pte_t *src_pte; 354 - pte_t *dst_pte; 353 + pte_t *src_ptep; 354 + pte_t *dst_ptep; 355 355 unsigned long addr = start; 356 356 357 - dst_pte = (pte_t *)get_safe_page(GFP_ATOMIC); 358 - if (!dst_pte) 357 + dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC); 358 + if (!dst_ptep) 359 359 return -ENOMEM; 360 - pmd_populate_kernel(&init_mm, dst_pmd, dst_pte); 361 - dst_pte = pte_offset_kernel(dst_pmd, start); 360 + pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep); 361 + dst_ptep = pte_offset_kernel(dst_pmdp, start); 362 362 363 - src_pte = pte_offset_kernel(src_pmd, start); 363 + src_ptep = pte_offset_kernel(src_pmdp, start); 364 364 do { 365 - _copy_pte(dst_pte, src_pte, addr); 366 - } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 365 + _copy_pte(dst_ptep, src_ptep, addr); 366 + } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); 367 367 368 368 return 0; 369 369 } 370 370 371 - static int copy_pmd(pud_t *dst_pud, pud_t *src_pud, unsigned long start, 371 + static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start, 372 372 unsigned long end) 373 373 { 374 - pmd_t *src_pmd; 375 - pmd_t *dst_pmd; 374 + pmd_t *src_pmdp; 375 + pmd_t *dst_pmdp; 376 376 unsigned long next; 377 377 unsigned long addr = start; 378 378 379 - if (pud_none(*dst_pud)) { 380 - dst_pmd = (pmd_t *)get_safe_page(GFP_ATOMIC); 381 - if (!dst_pmd) 379 + if (pud_none(READ_ONCE(*dst_pudp))) { 380 + dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC); 381 + if (!dst_pmdp) 382 382 return -ENOMEM; 383 - pud_populate(&init_mm, dst_pud, dst_pmd); 383 + pud_populate(&init_mm, dst_pudp, dst_pmdp); 384 384 } 385 - dst_pmd = pmd_offset(dst_pud, start); 385 + dst_pmdp = pmd_offset(dst_pudp, start); 386 386 387 - src_pmd = pmd_offset(src_pud, start); 387 + src_pmdp = pmd_offset(src_pudp, start); 388 388 do { 389 + pmd_t pmd = READ_ONCE(*src_pmdp); 390 + 389 391 next = pmd_addr_end(addr, end); 390 - if (pmd_none(*src_pmd)) 392 + if (pmd_none(pmd)) 391 393 continue; 392 - if (pmd_table(*src_pmd)) { 393 - if (copy_pte(dst_pmd, src_pmd, addr, next)) 394 + if (pmd_table(pmd)) { 395 + if (copy_pte(dst_pmdp, src_pmdp, addr, next)) 394 396 return -ENOMEM; 395 397 } else { 396 - set_pmd(dst_pmd, 397 - __pmd(pmd_val(*src_pmd) & ~PMD_SECT_RDONLY)); 398 + set_pmd(dst_pmdp, 399 + __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); 398 400 } 399 - } while (dst_pmd++, src_pmd++, addr = next, addr != end); 401 + } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); 400 402 401 403 return 0; 402 404 } 403 405 404 - static int copy_pud(pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long start, 406 + static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start, 405 407 unsigned long end) 406 408 { 407 - pud_t *dst_pud; 408 - pud_t *src_pud; 409 + pud_t *dst_pudp; 410 + pud_t *src_pudp; 409 411 unsigned long next; 410 412 unsigned long addr = start; 411 413 412 - if (pgd_none(*dst_pgd)) { 413 - dst_pud = (pud_t *)get_safe_page(GFP_ATOMIC); 414 - if (!dst_pud) 414 + if (pgd_none(READ_ONCE(*dst_pgdp))) { 415 + dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC); 416 + if (!dst_pudp) 415 417 return -ENOMEM; 416 - pgd_populate(&init_mm, dst_pgd, dst_pud); 418 + pgd_populate(&init_mm, dst_pgdp, dst_pudp); 417 419 } 418 - dst_pud = pud_offset(dst_pgd, start); 420 + dst_pudp = pud_offset(dst_pgdp, start); 419 421 420 - src_pud = pud_offset(src_pgd, start); 422 + src_pudp = pud_offset(src_pgdp, start); 421 423 do { 424 + pud_t pud = READ_ONCE(*src_pudp); 425 + 422 426 next = pud_addr_end(addr, end); 423 - if (pud_none(*src_pud)) 427 + if (pud_none(pud)) 424 428 continue; 425 - if (pud_table(*(src_pud))) { 426 - if (copy_pmd(dst_pud, src_pud, addr, next)) 429 + if (pud_table(pud)) { 430 + if (copy_pmd(dst_pudp, src_pudp, addr, next)) 427 431 return -ENOMEM; 428 432 } else { 429 - set_pud(dst_pud, 430 - __pud(pud_val(*src_pud) & ~PMD_SECT_RDONLY)); 433 + set_pud(dst_pudp, 434 + __pud(pud_val(pud) & ~PMD_SECT_RDONLY)); 431 435 } 432 - } while (dst_pud++, src_pud++, addr = next, addr != end); 436 + } while (dst_pudp++, src_pudp++, addr = next, addr != end); 433 437 434 438 return 0; 435 439 } 436 440 437 - static int copy_page_tables(pgd_t *dst_pgd, unsigned long start, 441 + static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start, 438 442 unsigned long end) 439 443 { 440 444 unsigned long next; 441 445 unsigned long addr = start; 442 - pgd_t *src_pgd = pgd_offset_k(start); 446 + pgd_t *src_pgdp = pgd_offset_k(start); 443 447 444 - dst_pgd = pgd_offset_raw(dst_pgd, start); 448 + dst_pgdp = pgd_offset_raw(dst_pgdp, start); 445 449 do { 446 450 next = pgd_addr_end(addr, end); 447 - if (pgd_none(*src_pgd)) 451 + if (pgd_none(READ_ONCE(*src_pgdp))) 448 452 continue; 449 - if (copy_pud(dst_pgd, src_pgd, addr, next)) 453 + if (copy_pud(dst_pgdp, src_pgdp, addr, next)) 450 454 return -ENOMEM; 451 - } while (dst_pgd++, src_pgd++, addr = next, addr != end); 455 + } while (dst_pgdp++, src_pgdp++, addr = next, addr != end); 452 456 453 457 return 0; 454 458 }

+3 -1

arch/arm64/kvm/hyp/switch.c

··· 407 407 u32 midr = read_cpuid_id(); 408 408 409 409 /* Apply BTAC predictors mitigation to all Falkor chips */ 410 - if ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1) 410 + if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) || 411 + ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1)) { 411 412 __qcom_hyp_sanitize_btac_predictors(); 413 + } 412 414 } 413 415 414 416 fp_enabled = __fpsimd_enabled();

+30 -24

arch/arm64/mm/dump.c

··· 286 286 287 287 } 288 288 289 - static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start) 289 + static void walk_pte(struct pg_state *st, pmd_t *pmdp, unsigned long start) 290 290 { 291 - pte_t *pte = pte_offset_kernel(pmd, 0UL); 291 + pte_t *ptep = pte_offset_kernel(pmdp, 0UL); 292 292 unsigned long addr; 293 293 unsigned i; 294 294 295 - for (i = 0; i < PTRS_PER_PTE; i++, pte++) { 295 + for (i = 0; i < PTRS_PER_PTE; i++, ptep++) { 296 296 addr = start + i * PAGE_SIZE; 297 - note_page(st, addr, 4, pte_val(*pte)); 297 + note_page(st, addr, 4, READ_ONCE(pte_val(*ptep))); 298 298 } 299 299 } 300 300 301 - static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) 301 + static void walk_pmd(struct pg_state *st, pud_t *pudp, unsigned long start) 302 302 { 303 - pmd_t *pmd = pmd_offset(pud, 0UL); 303 + pmd_t *pmdp = pmd_offset(pudp, 0UL); 304 304 unsigned long addr; 305 305 unsigned i; 306 306 307 - for (i = 0; i < PTRS_PER_PMD; i++, pmd++) { 307 + for (i = 0; i < PTRS_PER_PMD; i++, pmdp++) { 308 + pmd_t pmd = READ_ONCE(*pmdp); 309 + 308 310 addr = start + i * PMD_SIZE; 309 - if (pmd_none(*pmd) || pmd_sect(*pmd)) { 310 - note_page(st, addr, 3, pmd_val(*pmd)); 311 + if (pmd_none(pmd) || pmd_sect(pmd)) { 312 + note_page(st, addr, 3, pmd_val(pmd)); 311 313 } else { 312 - BUG_ON(pmd_bad(*pmd)); 313 - walk_pte(st, pmd, addr); 314 + BUG_ON(pmd_bad(pmd)); 315 + walk_pte(st, pmdp, addr); 314 316 } 315 317 } 316 318 } 317 319 318 - static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) 320 + static void walk_pud(struct pg_state *st, pgd_t *pgdp, unsigned long start) 319 321 { 320 - pud_t *pud = pud_offset(pgd, 0UL); 322 + pud_t *pudp = pud_offset(pgdp, 0UL); 321 323 unsigned long addr; 322 324 unsigned i; 323 325 324 - for (i = 0; i < PTRS_PER_PUD; i++, pud++) { 326 + for (i = 0; i < PTRS_PER_PUD; i++, pudp++) { 327 + pud_t pud = READ_ONCE(*pudp); 328 + 325 329 addr = start + i * PUD_SIZE; 326 - if (pud_none(*pud) || pud_sect(*pud)) { 327 - note_page(st, addr, 2, pud_val(*pud)); 330 + if (pud_none(pud) || pud_sect(pud)) { 331 + note_page(st, addr, 2, pud_val(pud)); 328 332 } else { 329 - BUG_ON(pud_bad(*pud)); 330 - walk_pmd(st, pud, addr); 333 + BUG_ON(pud_bad(pud)); 334 + walk_pmd(st, pudp, addr); 331 335 } 332 336 } 333 337 } ··· 339 335 static void walk_pgd(struct pg_state *st, struct mm_struct *mm, 340 336 unsigned long start) 341 337 { 342 - pgd_t *pgd = pgd_offset(mm, 0UL); 338 + pgd_t *pgdp = pgd_offset(mm, 0UL); 343 339 unsigned i; 344 340 unsigned long addr; 345 341 346 - for (i = 0; i < PTRS_PER_PGD; i++, pgd++) { 342 + for (i = 0; i < PTRS_PER_PGD; i++, pgdp++) { 343 + pgd_t pgd = READ_ONCE(*pgdp); 344 + 347 345 addr = start + i * PGDIR_SIZE; 348 - if (pgd_none(*pgd)) { 349 - note_page(st, addr, 1, pgd_val(*pgd)); 346 + if (pgd_none(pgd)) { 347 + note_page(st, addr, 1, pgd_val(pgd)); 350 348 } else { 351 - BUG_ON(pgd_bad(*pgd)); 352 - walk_pud(st, pgd, addr); 349 + BUG_ON(pgd_bad(pgd)); 350 + walk_pud(st, pgdp, addr); 353 351 } 354 352 } 355 353 }

+25 -19

arch/arm64/mm/fault.c

··· 130 130 void show_pte(unsigned long addr) 131 131 { 132 132 struct mm_struct *mm; 133 - pgd_t *pgd; 133 + pgd_t *pgdp; 134 + pgd_t pgd; 134 135 135 136 if (addr < TASK_SIZE) { 136 137 /* TTBR0 */ ··· 150 149 return; 151 150 } 152 151 153 - pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgd = %p\n", 152 + pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgdp = %p\n", 154 153 mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, 155 154 VA_BITS, mm->pgd); 156 - pgd = pgd_offset(mm, addr); 157 - pr_alert("[%016lx] *pgd=%016llx", addr, pgd_val(*pgd)); 155 + pgdp = pgd_offset(mm, addr); 156 + pgd = READ_ONCE(*pgdp); 157 + pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd)); 158 158 159 159 do { 160 - pud_t *pud; 161 - pmd_t *pmd; 162 - pte_t *pte; 160 + pud_t *pudp, pud; 161 + pmd_t *pmdp, pmd; 162 + pte_t *ptep, pte; 163 163 164 - if (pgd_none(*pgd) || pgd_bad(*pgd)) 164 + if (pgd_none(pgd) || pgd_bad(pgd)) 165 165 break; 166 166 167 - pud = pud_offset(pgd, addr); 168 - pr_cont(", *pud=%016llx", pud_val(*pud)); 169 - if (pud_none(*pud) || pud_bad(*pud)) 167 + pudp = pud_offset(pgdp, addr); 168 + pud = READ_ONCE(*pudp); 169 + pr_cont(", pud=%016llx", pud_val(pud)); 170 + if (pud_none(pud) || pud_bad(pud)) 170 171 break; 171 172 172 - pmd = pmd_offset(pud, addr); 173 - pr_cont(", *pmd=%016llx", pmd_val(*pmd)); 174 - if (pmd_none(*pmd) || pmd_bad(*pmd)) 173 + pmdp = pmd_offset(pudp, addr); 174 + pmd = READ_ONCE(*pmdp); 175 + pr_cont(", pmd=%016llx", pmd_val(pmd)); 176 + if (pmd_none(pmd) || pmd_bad(pmd)) 175 177 break; 176 178 177 - pte = pte_offset_map(pmd, addr); 178 - pr_cont(", *pte=%016llx", pte_val(*pte)); 179 - pte_unmap(pte); 179 + ptep = pte_offset_map(pmdp, addr); 180 + pte = READ_ONCE(*ptep); 181 + pr_cont(", pte=%016llx", pte_val(pte)); 182 + pte_unmap(ptep); 180 183 } while(0); 181 184 182 185 pr_cont("\n"); ··· 201 196 pte_t entry, int dirty) 202 197 { 203 198 pteval_t old_pteval, pteval; 199 + pte_t pte = READ_ONCE(*ptep); 204 200 205 - if (pte_same(*ptep, entry)) 201 + if (pte_same(pte, entry)) 206 202 return 0; 207 203 208 204 /* only preserve the access flags and write permission */ ··· 216 210 * (calculated as: a & b == ~(~a | ~b)). 217 211 */ 218 212 pte_val(entry) ^= PTE_RDONLY; 219 - pteval = READ_ONCE(pte_val(*ptep)); 213 + pteval = pte_val(pte); 220 214 do { 221 215 old_pteval = pteval; 222 216 pteval ^= PTE_RDONLY;

+42 -50

arch/arm64/mm/hugetlbpage.c

··· 54 54 static int find_num_contig(struct mm_struct *mm, unsigned long addr, 55 55 pte_t *ptep, size_t *pgsize) 56 56 { 57 - pgd_t *pgd = pgd_offset(mm, addr); 58 - pud_t *pud; 59 - pmd_t *pmd; 57 + pgd_t *pgdp = pgd_offset(mm, addr); 58 + pud_t *pudp; 59 + pmd_t *pmdp; 60 60 61 61 *pgsize = PAGE_SIZE; 62 - pud = pud_offset(pgd, addr); 63 - pmd = pmd_offset(pud, addr); 64 - if ((pte_t *)pmd == ptep) { 62 + pudp = pud_offset(pgdp, addr); 63 + pmdp = pmd_offset(pudp, addr); 64 + if ((pte_t *)pmdp == ptep) { 65 65 *pgsize = PMD_SIZE; 66 66 return CONT_PMDS; 67 67 } ··· 181 181 182 182 clear_flush(mm, addr, ptep, pgsize, ncontig); 183 183 184 - for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) { 185 - pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, 186 - pte_val(pfn_pte(pfn, hugeprot))); 184 + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) 187 185 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); 188 - } 189 186 } 190 187 191 188 void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, ··· 200 203 pte_t *huge_pte_alloc(struct mm_struct *mm, 201 204 unsigned long addr, unsigned long sz) 202 205 { 203 - pgd_t *pgd; 204 - pud_t *pud; 205 - pte_t *pte = NULL; 206 + pgd_t *pgdp; 207 + pud_t *pudp; 208 + pmd_t *pmdp; 209 + pte_t *ptep = NULL; 206 210 207 - pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz); 208 - pgd = pgd_offset(mm, addr); 209 - pud = pud_alloc(mm, pgd, addr); 210 - if (!pud) 211 + pgdp = pgd_offset(mm, addr); 212 + pudp = pud_alloc(mm, pgdp, addr); 213 + if (!pudp) 211 214 return NULL; 212 215 213 216 if (sz == PUD_SIZE) { 214 - pte = (pte_t *)pud; 217 + ptep = (pte_t *)pudp; 215 218 } else if (sz == (PAGE_SIZE * CONT_PTES)) { 216 - pmd_t *pmd = pmd_alloc(mm, pud, addr); 219 + pmdp = pmd_alloc(mm, pudp, addr); 217 220 218 221 WARN_ON(addr & (sz - 1)); 219 222 /* ··· 223 226 * will be no pte_unmap() to correspond with this 224 227 * pte_alloc_map(). 225 228 */ 226 - pte = pte_alloc_map(mm, pmd, addr); 229 + ptep = pte_alloc_map(mm, pmdp, addr); 227 230 } else if (sz == PMD_SIZE) { 228 231 if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && 229 - pud_none(*pud)) 230 - pte = huge_pmd_share(mm, addr, pud); 232 + pud_none(READ_ONCE(*pudp))) 233 + ptep = huge_pmd_share(mm, addr, pudp); 231 234 else 232 - pte = (pte_t *)pmd_alloc(mm, pud, addr); 235 + ptep = (pte_t *)pmd_alloc(mm, pudp, addr); 233 236 } else if (sz == (PMD_SIZE * CONT_PMDS)) { 234 - pmd_t *pmd; 235 - 236 - pmd = pmd_alloc(mm, pud, addr); 237 + pmdp = pmd_alloc(mm, pudp, addr); 237 238 WARN_ON(addr & (sz - 1)); 238 - return (pte_t *)pmd; 239 + return (pte_t *)pmdp; 239 240 } 240 241 241 - pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr, 242 - sz, pte, pte_val(*pte)); 243 - return pte; 242 + return ptep; 244 243 } 245 244 246 245 pte_t *huge_pte_offset(struct mm_struct *mm, 247 246 unsigned long addr, unsigned long sz) 248 247 { 249 - pgd_t *pgd; 250 - pud_t *pud; 251 - pmd_t *pmd; 248 + pgd_t *pgdp; 249 + pud_t *pudp, pud; 250 + pmd_t *pmdp, pmd; 252 251 253 - pgd = pgd_offset(mm, addr); 254 - pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd); 255 - if (!pgd_present(*pgd)) 252 + pgdp = pgd_offset(mm, addr); 253 + if (!pgd_present(READ_ONCE(*pgdp))) 256 254 return NULL; 257 255 258 - pud = pud_offset(pgd, addr); 259 - if (sz != PUD_SIZE && pud_none(*pud)) 256 + pudp = pud_offset(pgdp, addr); 257 + pud = READ_ONCE(*pudp); 258 + if (sz != PUD_SIZE && pud_none(pud)) 260 259 return NULL; 261 260 /* hugepage or swap? */ 262 - if (pud_huge(*pud) || !pud_present(*pud)) 263 - return (pte_t *)pud; 261 + if (pud_huge(pud) || !pud_present(pud)) 262 + return (pte_t *)pudp; 264 263 /* table; check the next level */ 265 264 266 265 if (sz == CONT_PMD_SIZE) 267 266 addr &= CONT_PMD_MASK; 268 267 269 - pmd = pmd_offset(pud, addr); 268 + pmdp = pmd_offset(pudp, addr); 269 + pmd = READ_ONCE(*pmdp); 270 270 if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) && 271 - pmd_none(*pmd)) 271 + pmd_none(pmd)) 272 272 return NULL; 273 - if (pmd_huge(*pmd) || !pmd_present(*pmd)) 274 - return (pte_t *)pmd; 273 + if (pmd_huge(pmd) || !pmd_present(pmd)) 274 + return (pte_t *)pmdp; 275 275 276 - if (sz == CONT_PTE_SIZE) { 277 - pte_t *pte = pte_offset_kernel(pmd, (addr & CONT_PTE_MASK)); 278 - return pte; 279 - } 276 + if (sz == CONT_PTE_SIZE) 277 + return pte_offset_kernel(pmdp, (addr & CONT_PTE_MASK)); 280 278 281 279 return NULL; 282 280 } ··· 359 367 size_t pgsize; 360 368 pte_t pte; 361 369 362 - if (!pte_cont(*ptep)) { 370 + if (!pte_cont(READ_ONCE(*ptep))) { 363 371 ptep_set_wrprotect(mm, addr, ptep); 364 372 return; 365 373 } ··· 383 391 size_t pgsize; 384 392 int ncontig; 385 393 386 - if (!pte_cont(*ptep)) { 394 + if (!pte_cont(READ_ONCE(*ptep))) { 387 395 ptep_clear_flush(vma, addr, ptep); 388 396 return; 389 397 }

+35 -35

arch/arm64/mm/kasan_init.c

··· 44 44 return __pa(p); 45 45 } 46 46 47 - static pte_t *__init kasan_pte_offset(pmd_t *pmd, unsigned long addr, int node, 47 + static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, 48 48 bool early) 49 49 { 50 - if (pmd_none(*pmd)) { 50 + if (pmd_none(READ_ONCE(*pmdp))) { 51 51 phys_addr_t pte_phys = early ? __pa_symbol(kasan_zero_pte) 52 52 : kasan_alloc_zeroed_page(node); 53 - __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE); 53 + __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE); 54 54 } 55 55 56 - return early ? pte_offset_kimg(pmd, addr) 57 - : pte_offset_kernel(pmd, addr); 56 + return early ? pte_offset_kimg(pmdp, addr) 57 + : pte_offset_kernel(pmdp, addr); 58 58 } 59 59 60 - static pmd_t *__init kasan_pmd_offset(pud_t *pud, unsigned long addr, int node, 60 + static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, 61 61 bool early) 62 62 { 63 - if (pud_none(*pud)) { 63 + if (pud_none(READ_ONCE(*pudp))) { 64 64 phys_addr_t pmd_phys = early ? __pa_symbol(kasan_zero_pmd) 65 65 : kasan_alloc_zeroed_page(node); 66 - __pud_populate(pud, pmd_phys, PMD_TYPE_TABLE); 66 + __pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE); 67 67 } 68 68 69 - return early ? pmd_offset_kimg(pud, addr) : pmd_offset(pud, addr); 69 + return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr); 70 70 } 71 71 72 - static pud_t *__init kasan_pud_offset(pgd_t *pgd, unsigned long addr, int node, 72 + static pud_t *__init kasan_pud_offset(pgd_t *pgdp, unsigned long addr, int node, 73 73 bool early) 74 74 { 75 - if (pgd_none(*pgd)) { 75 + if (pgd_none(READ_ONCE(*pgdp))) { 76 76 phys_addr_t pud_phys = early ? __pa_symbol(kasan_zero_pud) 77 77 : kasan_alloc_zeroed_page(node); 78 - __pgd_populate(pgd, pud_phys, PMD_TYPE_TABLE); 78 + __pgd_populate(pgdp, pud_phys, PMD_TYPE_TABLE); 79 79 } 80 80 81 - return early ? pud_offset_kimg(pgd, addr) : pud_offset(pgd, addr); 81 + return early ? pud_offset_kimg(pgdp, addr) : pud_offset(pgdp, addr); 82 82 } 83 83 84 - static void __init kasan_pte_populate(pmd_t *pmd, unsigned long addr, 84 + static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, 85 85 unsigned long end, int node, bool early) 86 86 { 87 87 unsigned long next; 88 - pte_t *pte = kasan_pte_offset(pmd, addr, node, early); 88 + pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early); 89 89 90 90 do { 91 91 phys_addr_t page_phys = early ? __pa_symbol(kasan_zero_page) 92 92 : kasan_alloc_zeroed_page(node); 93 93 next = addr + PAGE_SIZE; 94 - set_pte(pte, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL)); 95 - } while (pte++, addr = next, addr != end && pte_none(*pte)); 94 + set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL)); 95 + } while (ptep++, addr = next, addr != end && pte_none(READ_ONCE(*ptep))); 96 96 } 97 97 98 - static void __init kasan_pmd_populate(pud_t *pud, unsigned long addr, 98 + static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, 99 99 unsigned long end, int node, bool early) 100 100 { 101 101 unsigned long next; 102 - pmd_t *pmd = kasan_pmd_offset(pud, addr, node, early); 102 + pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); 103 103 104 104 do { 105 105 next = pmd_addr_end(addr, end); 106 - kasan_pte_populate(pmd, addr, next, node, early); 107 - } while (pmd++, addr = next, addr != end && pmd_none(*pmd)); 106 + kasan_pte_populate(pmdp, addr, next, node, early); 107 + } while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp))); 108 108 } 109 109 110 - static void __init kasan_pud_populate(pgd_t *pgd, unsigned long addr, 110 + static void __init kasan_pud_populate(pgd_t *pgdp, unsigned long addr, 111 111 unsigned long end, int node, bool early) 112 112 { 113 113 unsigned long next; 114 - pud_t *pud = kasan_pud_offset(pgd, addr, node, early); 114 + pud_t *pudp = kasan_pud_offset(pgdp, addr, node, early); 115 115 116 116 do { 117 117 next = pud_addr_end(addr, end); 118 - kasan_pmd_populate(pud, addr, next, node, early); 119 - } while (pud++, addr = next, addr != end && pud_none(*pud)); 118 + kasan_pmd_populate(pudp, addr, next, node, early); 119 + } while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp))); 120 120 } 121 121 122 122 static void __init kasan_pgd_populate(unsigned long addr, unsigned long end, 123 123 int node, bool early) 124 124 { 125 125 unsigned long next; 126 - pgd_t *pgd; 126 + pgd_t *pgdp; 127 127 128 - pgd = pgd_offset_k(addr); 128 + pgdp = pgd_offset_k(addr); 129 129 do { 130 130 next = pgd_addr_end(addr, end); 131 - kasan_pud_populate(pgd, addr, next, node, early); 132 - } while (pgd++, addr = next, addr != end); 131 + kasan_pud_populate(pgdp, addr, next, node, early); 132 + } while (pgdp++, addr = next, addr != end); 133 133 } 134 134 135 135 /* The early shadow maps everything to a single page of zeroes */ ··· 155 155 */ 156 156 void __init kasan_copy_shadow(pgd_t *pgdir) 157 157 { 158 - pgd_t *pgd, *pgd_new, *pgd_end; 158 + pgd_t *pgdp, *pgdp_new, *pgdp_end; 159 159 160 - pgd = pgd_offset_k(KASAN_SHADOW_START); 161 - pgd_end = pgd_offset_k(KASAN_SHADOW_END); 162 - pgd_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START); 160 + pgdp = pgd_offset_k(KASAN_SHADOW_START); 161 + pgdp_end = pgd_offset_k(KASAN_SHADOW_END); 162 + pgdp_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START); 163 163 do { 164 - set_pgd(pgd_new, *pgd); 165 - } while (pgd++, pgd_new++, pgd != pgd_end); 164 + set_pgd(pgdp_new, READ_ONCE(*pgdp)); 165 + } while (pgdp++, pgdp_new++, pgdp != pgdp_end); 166 166 } 167 167 168 168 static void __init clear_pgds(unsigned long start,

+148 -134

arch/arm64/mm/mmu.c

··· 125 125 return ((old ^ new) & ~mask) == 0; 126 126 } 127 127 128 - static void init_pte(pmd_t *pmd, unsigned long addr, unsigned long end, 128 + static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, 129 129 phys_addr_t phys, pgprot_t prot) 130 130 { 131 - pte_t *pte; 131 + pte_t *ptep; 132 132 133 - pte = pte_set_fixmap_offset(pmd, addr); 133 + ptep = pte_set_fixmap_offset(pmdp, addr); 134 134 do { 135 - pte_t old_pte = *pte; 135 + pte_t old_pte = READ_ONCE(*ptep); 136 136 137 - set_pte(pte, pfn_pte(__phys_to_pfn(phys), prot)); 137 + set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot)); 138 138 139 139 /* 140 140 * After the PTE entry has been populated once, we 141 141 * only allow updates to the permission attributes. 142 142 */ 143 - BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte))); 143 + BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), 144 + READ_ONCE(pte_val(*ptep)))); 144 145 145 146 phys += PAGE_SIZE; 146 - } while (pte++, addr += PAGE_SIZE, addr != end); 147 + } while (ptep++, addr += PAGE_SIZE, addr != end); 147 148 148 149 pte_clear_fixmap(); 149 150 } 150 151 151 - static void alloc_init_cont_pte(pmd_t *pmd, unsigned long addr, 152 + static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, 152 153 unsigned long end, phys_addr_t phys, 153 154 pgprot_t prot, 154 155 phys_addr_t (*pgtable_alloc)(void), 155 156 int flags) 156 157 { 157 158 unsigned long next; 159 + pmd_t pmd = READ_ONCE(*pmdp); 158 160 159 - BUG_ON(pmd_sect(*pmd)); 160 - if (pmd_none(*pmd)) { 161 + BUG_ON(pmd_sect(pmd)); 162 + if (pmd_none(pmd)) { 161 163 phys_addr_t pte_phys; 162 164 BUG_ON(!pgtable_alloc); 163 165 pte_phys = pgtable_alloc(); 164 - __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE); 166 + __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE); 167 + pmd = READ_ONCE(*pmdp); 165 168 } 166 - BUG_ON(pmd_bad(*pmd)); 169 + BUG_ON(pmd_bad(pmd)); 167 170 168 171 do { 169 172 pgprot_t __prot = prot; ··· 178 175 (flags & NO_CONT_MAPPINGS) == 0) 179 176 __prot = __pgprot(pgprot_val(prot) | PTE_CONT); 180 177 181 - init_pte(pmd, addr, next, phys, __prot); 178 + init_pte(pmdp, addr, next, phys, __prot); 182 179 183 180 phys += next - addr; 184 181 } while (addr = next, addr != end); 185 182 } 186 183 187 - static void init_pmd(pud_t *pud, unsigned long addr, unsigned long end, 184 + static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, 188 185 phys_addr_t phys, pgprot_t prot, 189 186 phys_addr_t (*pgtable_alloc)(void), int flags) 190 187 { 191 188 unsigned long next; 192 - pmd_t *pmd; 189 + pmd_t *pmdp; 193 190 194 - pmd = pmd_set_fixmap_offset(pud, addr); 191 + pmdp = pmd_set_fixmap_offset(pudp, addr); 195 192 do { 196 - pmd_t old_pmd = *pmd; 193 + pmd_t old_pmd = READ_ONCE(*pmdp); 197 194 198 195 next = pmd_addr_end(addr, end); 199 196 200 197 /* try section mapping first */ 201 198 if (((addr | next | phys) & ~SECTION_MASK) == 0 && 202 199 (flags & NO_BLOCK_MAPPINGS) == 0) { 203 - pmd_set_huge(pmd, phys, prot); 200 + pmd_set_huge(pmdp, phys, prot); 204 201 205 202 /* 206 203 * After the PMD entry has been populated once, we 207 204 * only allow updates to the permission attributes. 208 205 */ 209 206 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd), 210 - pmd_val(*pmd))); 207 + READ_ONCE(pmd_val(*pmdp)))); 211 208 } else { 212 - alloc_init_cont_pte(pmd, addr, next, phys, prot, 209 + alloc_init_cont_pte(pmdp, addr, next, phys, prot, 213 210 pgtable_alloc, flags); 214 211 215 212 BUG_ON(pmd_val(old_pmd) != 0 && 216 - pmd_val(old_pmd) != pmd_val(*pmd)); 213 + pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); 217 214 } 218 215 phys += next - addr; 219 - } while (pmd++, addr = next, addr != end); 216 + } while (pmdp++, addr = next, addr != end); 220 217 221 218 pmd_clear_fixmap(); 222 219 } 223 220 224 - static void alloc_init_cont_pmd(pud_t *pud, unsigned long addr, 221 + static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, 225 222 unsigned long end, phys_addr_t phys, 226 223 pgprot_t prot, 227 224 phys_addr_t (*pgtable_alloc)(void), int flags) 228 225 { 229 226 unsigned long next; 227 + pud_t pud = READ_ONCE(*pudp); 230 228 231 229 /* 232 230 * Check for initial section mappings in the pgd/pud. 233 231 */ 234 - BUG_ON(pud_sect(*pud)); 235 - if (pud_none(*pud)) { 232 + BUG_ON(pud_sect(pud)); 233 + if (pud_none(pud)) { 236 234 phys_addr_t pmd_phys; 237 235 BUG_ON(!pgtable_alloc); 238 236 pmd_phys = pgtable_alloc(); 239 - __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE); 237 + __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE); 238 + pud = READ_ONCE(*pudp); 240 239 } 241 - BUG_ON(pud_bad(*pud)); 240 + BUG_ON(pud_bad(pud)); 242 241 243 242 do { 244 243 pgprot_t __prot = prot; ··· 252 247 (flags & NO_CONT_MAPPINGS) == 0) 253 248 __prot = __pgprot(pgprot_val(prot) | PTE_CONT); 254 249 255 - init_pmd(pud, addr, next, phys, __prot, pgtable_alloc, flags); 250 + init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags); 256 251 257 252 phys += next - addr; 258 253 } while (addr = next, addr != end); ··· 270 265 return true; 271 266 } 272 267 273 - static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, 274 - phys_addr_t phys, pgprot_t prot, 275 - phys_addr_t (*pgtable_alloc)(void), 276 - int flags) 268 + static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, 269 + phys_addr_t phys, pgprot_t prot, 270 + phys_addr_t (*pgtable_alloc)(void), 271 + int flags) 277 272 { 278 - pud_t *pud; 279 273 unsigned long next; 274 + pud_t *pudp; 275 + pgd_t pgd = READ_ONCE(*pgdp); 280 276 281 - if (pgd_none(*pgd)) { 277 + if (pgd_none(pgd)) { 282 278 phys_addr_t pud_phys; 283 279 BUG_ON(!pgtable_alloc); 284 280 pud_phys = pgtable_alloc(); 285 - __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); 281 + __pgd_populate(pgdp, pud_phys, PUD_TYPE_TABLE); 282 + pgd = READ_ONCE(*pgdp); 286 283 } 287 - BUG_ON(pgd_bad(*pgd)); 284 + BUG_ON(pgd_bad(pgd)); 288 285 289 - pud = pud_set_fixmap_offset(pgd, addr); 286 + pudp = pud_set_fixmap_offset(pgdp, addr); 290 287 do { 291 - pud_t old_pud = *pud; 288 + pud_t old_pud = READ_ONCE(*pudp); 292 289 293 290 next = pud_addr_end(addr, end); 294 291 ··· 299 292 */ 300 293 if (use_1G_block(addr, next, phys) && 301 294 (flags & NO_BLOCK_MAPPINGS) == 0) { 302 - pud_set_huge(pud, phys, prot); 295 + pud_set_huge(pudp, phys, prot); 303 296 304 297 /* 305 298 * After the PUD entry has been populated once, we 306 299 * only allow updates to the permission attributes. 307 300 */ 308 301 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud), 309 - pud_val(*pud))); 302 + READ_ONCE(pud_val(*pudp)))); 310 303 } else { 311 - alloc_init_cont_pmd(pud, addr, next, phys, prot, 304 + alloc_init_cont_pmd(pudp, addr, next, phys, prot, 312 305 pgtable_alloc, flags); 313 306 314 307 BUG_ON(pud_val(old_pud) != 0 && 315 - pud_val(old_pud) != pud_val(*pud)); 308 + pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); 316 309 } 317 310 phys += next - addr; 318 - } while (pud++, addr = next, addr != end); 311 + } while (pudp++, addr = next, addr != end); 319 312 320 313 pud_clear_fixmap(); 321 314 } ··· 327 320 int flags) 328 321 { 329 322 unsigned long addr, length, end, next; 330 - pgd_t *pgd = pgd_offset_raw(pgdir, virt); 323 + pgd_t *pgdp = pgd_offset_raw(pgdir, virt); 331 324 332 325 /* 333 326 * If the virtual and physical address don't have the same offset ··· 343 336 end = addr + length; 344 337 do { 345 338 next = pgd_addr_end(addr, end); 346 - alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc, 339 + alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, 347 340 flags); 348 341 phys += next - addr; 349 - } while (pgd++, addr = next, addr != end); 342 + } while (pgdp++, addr = next, addr != end); 350 343 } 351 344 352 345 static phys_addr_t pgd_pgtable_alloc(void) ··· 408 401 flush_tlb_kernel_range(virt, virt + size); 409 402 } 410 403 411 - static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, 404 + static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, 412 405 phys_addr_t end, pgprot_t prot, int flags) 413 406 { 414 - __create_pgd_mapping(pgd, start, __phys_to_virt(start), end - start, 407 + __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, 415 408 prot, early_pgtable_alloc, flags); 416 409 } 417 410 ··· 425 418 PAGE_KERNEL_RO); 426 419 } 427 420 428 - static void __init map_mem(pgd_t *pgd) 421 + static void __init map_mem(pgd_t *pgdp) 429 422 { 430 423 phys_addr_t kernel_start = __pa_symbol(_text); 431 424 phys_addr_t kernel_end = __pa_symbol(__init_begin); ··· 458 451 if (memblock_is_nomap(reg)) 459 452 continue; 460 453 461 - __map_memblock(pgd, start, end, PAGE_KERNEL, flags); 454 + __map_memblock(pgdp, start, end, PAGE_KERNEL, flags); 462 455 } 463 456 464 457 /* ··· 471 464 * Note that contiguous mappings cannot be remapped in this way, 472 465 * so we should avoid them here. 473 466 */ 474 - __map_memblock(pgd, kernel_start, kernel_end, 467 + __map_memblock(pgdp, kernel_start, kernel_end, 475 468 PAGE_KERNEL, NO_CONT_MAPPINGS); 476 469 memblock_clear_nomap(kernel_start, kernel_end - kernel_start); 477 470 ··· 482 475 * through /sys/kernel/kexec_crash_size interface. 483 476 */ 484 477 if (crashk_res.end) { 485 - __map_memblock(pgd, crashk_res.start, crashk_res.end + 1, 478 + __map_memblock(pgdp, crashk_res.start, crashk_res.end + 1, 486 479 PAGE_KERNEL, 487 480 NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); 488 481 memblock_clear_nomap(crashk_res.start, ··· 506 499 debug_checkwx(); 507 500 } 508 501 509 - static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, 502 + static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, 510 503 pgprot_t prot, struct vm_struct *vma, 511 504 int flags, unsigned long vm_flags) 512 505 { ··· 516 509 BUG_ON(!PAGE_ALIGNED(pa_start)); 517 510 BUG_ON(!PAGE_ALIGNED(size)); 518 511 519 - __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, 512 + __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot, 520 513 early_pgtable_alloc, flags); 521 514 522 515 if (!(vm_flags & VM_NO_GUARD)) ··· 569 562 /* 570 563 * Create fine-grained mappings for the kernel. 571 564 */ 572 - static void __init map_kernel(pgd_t *pgd) 565 + static void __init map_kernel(pgd_t *pgdp) 573 566 { 574 567 static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext, 575 568 vmlinux_initdata, vmlinux_data; ··· 585 578 * Only rodata will be remapped with different permissions later on, 586 579 * all other segments are allowed to use contiguous mappings. 587 580 */ 588 - map_kernel_segment(pgd, _text, _etext, text_prot, &vmlinux_text, 0, 581 + map_kernel_segment(pgdp, _text, _etext, text_prot, &vmlinux_text, 0, 589 582 VM_NO_GUARD); 590 - map_kernel_segment(pgd, __start_rodata, __inittext_begin, PAGE_KERNEL, 583 + map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, 591 584 &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD); 592 - map_kernel_segment(pgd, __inittext_begin, __inittext_end, text_prot, 585 + map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, 593 586 &vmlinux_inittext, 0, VM_NO_GUARD); 594 - map_kernel_segment(pgd, __initdata_begin, __initdata_end, PAGE_KERNEL, 587 + map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, 595 588 &vmlinux_initdata, 0, VM_NO_GUARD); 596 - map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); 589 + map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); 597 590 598 - if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) { 591 + if (!READ_ONCE(pgd_val(*pgd_offset_raw(pgdp, FIXADDR_START)))) { 599 592 /* 600 593 * The fixmap falls in a separate pgd to the kernel, and doesn't 601 594 * live in the carveout for the swapper_pg_dir. We can simply 602 595 * re-use the existing dir for the fixmap. 603 596 */ 604 - set_pgd(pgd_offset_raw(pgd, FIXADDR_START), 605 - *pgd_offset_k(FIXADDR_START)); 597 + set_pgd(pgd_offset_raw(pgdp, FIXADDR_START), 598 + READ_ONCE(*pgd_offset_k(FIXADDR_START))); 606 599 } else if (CONFIG_PGTABLE_LEVELS > 3) { 607 600 /* 608 601 * The fixmap shares its top level pgd entry with the kernel ··· 611 604 * entry instead. 612 605 */ 613 606 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 614 - pud_populate(&init_mm, pud_set_fixmap_offset(pgd, FIXADDR_START), 607 + pud_populate(&init_mm, 608 + pud_set_fixmap_offset(pgdp, FIXADDR_START), 615 609 lm_alias(bm_pmd)); 616 610 pud_clear_fixmap(); 617 611 } else { 618 612 BUG(); 619 613 } 620 614 621 - kasan_copy_shadow(pgd); 615 + kasan_copy_shadow(pgdp); 622 616 } 623 617 624 618 /* ··· 629 621 void __init paging_init(void) 630 622 { 631 623 phys_addr_t pgd_phys = early_pgtable_alloc(); 632 - pgd_t *pgd = pgd_set_fixmap(pgd_phys); 624 + pgd_t *pgdp = pgd_set_fixmap(pgd_phys); 633 625 634 - map_kernel(pgd); 635 - map_mem(pgd); 626 + map_kernel(pgdp); 627 + map_mem(pgdp); 636 628 637 629 /* 638 630 * We want to reuse the original swapper_pg_dir so we don't have to ··· 643 635 * To do this we need to go via a temporary pgd. 644 636 */ 645 637 cpu_replace_ttbr1(__va(pgd_phys)); 646 - memcpy(swapper_pg_dir, pgd, PGD_SIZE); 638 + memcpy(swapper_pg_dir, pgdp, PGD_SIZE); 647 639 cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); 648 640 649 641 pgd_clear_fixmap(); ··· 663 655 */ 664 656 int kern_addr_valid(unsigned long addr) 665 657 { 666 - pgd_t *pgd; 667 - pud_t *pud; 668 - pmd_t *pmd; 669 - pte_t *pte; 658 + pgd_t *pgdp; 659 + pud_t *pudp, pud; 660 + pmd_t *pmdp, pmd; 661 + pte_t *ptep, pte; 670 662 671 663 if ((((long)addr) >> VA_BITS) != -1UL) 672 664 return 0; 673 665 674 - pgd = pgd_offset_k(addr); 675 - if (pgd_none(*pgd)) 666 + pgdp = pgd_offset_k(addr); 667 + if (pgd_none(READ_ONCE(*pgdp))) 676 668 return 0; 677 669 678 - pud = pud_offset(pgd, addr); 679 - if (pud_none(*pud)) 670 + pudp = pud_offset(pgdp, addr); 671 + pud = READ_ONCE(*pudp); 672 + if (pud_none(pud)) 680 673 return 0; 681 674 682 - if (pud_sect(*pud)) 683 - return pfn_valid(pud_pfn(*pud)); 675 + if (pud_sect(pud)) 676 + return pfn_valid(pud_pfn(pud)); 684 677 685 - pmd = pmd_offset(pud, addr); 686 - if (pmd_none(*pmd)) 678 + pmdp = pmd_offset(pudp, addr); 679 + pmd = READ_ONCE(*pmdp); 680 + if (pmd_none(pmd)) 687 681 return 0; 688 682 689 - if (pmd_sect(*pmd)) 690 - return pfn_valid(pmd_pfn(*pmd)); 683 + if (pmd_sect(pmd)) 684 + return pfn_valid(pmd_pfn(pmd)); 691 685 692 - pte = pte_offset_kernel(pmd, addr); 693 - if (pte_none(*pte)) 686 + ptep = pte_offset_kernel(pmdp, addr); 687 + pte = READ_ONCE(*ptep); 688 + if (pte_none(pte)) 694 689 return 0; 695 690 696 - return pfn_valid(pte_pfn(*pte)); 691 + return pfn_valid(pte_pfn(pte)); 697 692 } 698 693 #ifdef CONFIG_SPARSEMEM_VMEMMAP 699 694 #if !ARM64_SWAPPER_USES_SECTION_MAPS ··· 711 700 { 712 701 unsigned long addr = start; 713 702 unsigned long next; 714 - pgd_t *pgd; 715 - pud_t *pud; 716 - pmd_t *pmd; 703 + pgd_t *pgdp; 704 + pud_t *pudp; 705 + pmd_t *pmdp; 717 706 718 707 do { 719 708 next = pmd_addr_end(addr, end); 720 709 721 - pgd = vmemmap_pgd_populate(addr, node); 722 - if (!pgd) 710 + pgdp = vmemmap_pgd_populate(addr, node); 711 + if (!pgdp) 723 712 return -ENOMEM; 724 713 725 - pud = vmemmap_pud_populate(pgd, addr, node); 726 - if (!pud) 714 + pudp = vmemmap_pud_populate(pgdp, addr, node); 715 + if (!pudp) 727 716 return -ENOMEM; 728 717 729 - pmd = pmd_offset(pud, addr); 730 - if (pmd_none(*pmd)) { 718 + pmdp = pmd_offset(pudp, addr); 719 + if (pmd_none(READ_ONCE(*pmdp))) { 731 720 void *p = NULL; 732 721 733 722 p = vmemmap_alloc_block_buf(PMD_SIZE, node); 734 723 if (!p) 735 724 return -ENOMEM; 736 725 737 - pmd_set_huge(pmd, __pa(p), __pgprot(PROT_SECT_NORMAL)); 726 + pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL)); 738 727 } else 739 - vmemmap_verify((pte_t *)pmd, node, addr, next); 728 + vmemmap_verify((pte_t *)pmdp, node, addr, next); 740 729 } while (addr = next, addr != end); 741 730 742 731 return 0; ··· 750 739 751 740 static inline pud_t * fixmap_pud(unsigned long addr) 752 741 { 753 - pgd_t *pgd = pgd_offset_k(addr); 742 + pgd_t *pgdp = pgd_offset_k(addr); 743 + pgd_t pgd = READ_ONCE(*pgdp); 754 744 755 - BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); 745 + BUG_ON(pgd_none(pgd) || pgd_bad(pgd)); 756 746 757 - return pud_offset_kimg(pgd, addr); 747 + return pud_offset_kimg(pgdp, addr); 758 748 } 759 749 760 750 static inline pmd_t * fixmap_pmd(unsigned long addr) 761 751 { 762 - pud_t *pud = fixmap_pud(addr); 752 + pud_t *pudp = fixmap_pud(addr); 753 + pud_t pud = READ_ONCE(*pudp); 763 754 764 - BUG_ON(pud_none(*pud) || pud_bad(*pud)); 755 + BUG_ON(pud_none(pud) || pud_bad(pud)); 765 756 766 - return pmd_offset_kimg(pud, addr); 757 + return pmd_offset_kimg(pudp, addr); 767 758 } 768 759 769 760 static inline pte_t * fixmap_pte(unsigned long addr) ··· 781 768 */ 782 769 void __init early_fixmap_init(void) 783 770 { 784 - pgd_t *pgd; 785 - pud_t *pud; 786 - pmd_t *pmd; 771 + pgd_t *pgdp, pgd; 772 + pud_t *pudp; 773 + pmd_t *pmdp; 787 774 unsigned long addr = FIXADDR_START; 788 775 789 - pgd = pgd_offset_k(addr); 776 + pgdp = pgd_offset_k(addr); 777 + pgd = READ_ONCE(*pgdp); 790 778 if (CONFIG_PGTABLE_LEVELS > 3 && 791 - !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa_symbol(bm_pud))) { 779 + !(pgd_none(pgd) || pgd_page_paddr(pgd) == __pa_symbol(bm_pud))) { 792 780 /* 793 781 * We only end up here if the kernel mapping and the fixmap 794 782 * share the top level pgd entry, which should only happen on 795 783 * 16k/4 levels configurations. 796 784 */ 797 785 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 798 - pud = pud_offset_kimg(pgd, addr); 786 + pudp = pud_offset_kimg(pgdp, addr); 799 787 } else { 800 - if (pgd_none(*pgd)) 801 - __pgd_populate(pgd, __pa_symbol(bm_pud), PUD_TYPE_TABLE); 802 - pud = fixmap_pud(addr); 788 + if (pgd_none(pgd)) 789 + __pgd_populate(pgdp, __pa_symbol(bm_pud), PUD_TYPE_TABLE); 790 + pudp = fixmap_pud(addr); 803 791 } 804 - if (pud_none(*pud)) 805 - __pud_populate(pud, __pa_symbol(bm_pmd), PMD_TYPE_TABLE); 806 - pmd = fixmap_pmd(addr); 807 - __pmd_populate(pmd, __pa_symbol(bm_pte), PMD_TYPE_TABLE); 792 + if (pud_none(READ_ONCE(*pudp))) 793 + __pud_populate(pudp, __pa_symbol(bm_pmd), PMD_TYPE_TABLE); 794 + pmdp = fixmap_pmd(addr); 795 + __pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE); 808 796 809 797 /* 810 798 * The boot-ioremap range spans multiple pmds, for which ··· 814 800 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 815 801 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 816 802 817 - if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 818 - || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 803 + if ((pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 804 + || pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 819 805 WARN_ON(1); 820 - pr_warn("pmd %p != %p, %p\n", 821 - pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 806 + pr_warn("pmdp %p != %p, %p\n", 807 + pmdp, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 822 808 fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 823 809 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 824 810 fix_to_virt(FIX_BTMAP_BEGIN)); ··· 838 824 phys_addr_t phys, pgprot_t flags) 839 825 { 840 826 unsigned long addr = __fix_to_virt(idx); 841 - pte_t *pte; 827 + pte_t *ptep; 842 828 843 829 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 844 830 845 - pte = fixmap_pte(addr); 831 + ptep = fixmap_pte(addr); 846 832 847 833 if (pgprot_val(flags)) { 848 - set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 834 + set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); 849 835 } else { 850 - pte_clear(&init_mm, addr, pte); 836 + pte_clear(&init_mm, addr, ptep); 851 837 flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 852 838 } 853 839 } ··· 929 915 return 1; 930 916 } 931 917 932 - int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot) 918 + int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot) 933 919 { 934 920 pgprot_t sect_prot = __pgprot(PUD_TYPE_SECT | 935 921 pgprot_val(mk_sect_prot(prot))); 936 922 BUG_ON(phys & ~PUD_MASK); 937 - set_pud(pud, pfn_pud(__phys_to_pfn(phys), sect_prot)); 923 + set_pud(pudp, pfn_pud(__phys_to_pfn(phys), sect_prot)); 938 924 return 1; 939 925 } 940 926 941 - int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot) 927 + int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) 942 928 { 943 929 pgprot_t sect_prot = __pgprot(PMD_TYPE_SECT | 944 930 pgprot_val(mk_sect_prot(prot))); 945 931 BUG_ON(phys & ~PMD_MASK); 946 - set_pmd(pmd, pfn_pmd(__phys_to_pfn(phys), sect_prot)); 932 + set_pmd(pmdp, pfn_pmd(__phys_to_pfn(phys), sect_prot)); 947 933 return 1; 948 934 } 949 935 950 - int pud_clear_huge(pud_t *pud) 936 + int pud_clear_huge(pud_t *pudp) 951 937 { 952 - if (!pud_sect(*pud)) 938 + if (!pud_sect(READ_ONCE(*pudp))) 953 939 return 0; 954 - pud_clear(pud); 940 + pud_clear(pudp); 955 941 return 1; 956 942 } 957 943 958 - int pmd_clear_huge(pmd_t *pmd) 944 + int pmd_clear_huge(pmd_t *pmdp) 959 945 { 960 - if (!pmd_sect(*pmd)) 946 + if (!pmd_sect(READ_ONCE(*pmdp))) 961 947 return 0; 962 - pmd_clear(pmd); 948 + pmd_clear(pmdp); 963 949 return 1; 964 950 }

+17 -15

arch/arm64/mm/pageattr.c

··· 29 29 void *data) 30 30 { 31 31 struct page_change_data *cdata = data; 32 - pte_t pte = *ptep; 32 + pte_t pte = READ_ONCE(*ptep); 33 33 34 34 pte = clear_pte_bit(pte, cdata->clear_mask); 35 35 pte = set_pte_bit(pte, cdata->set_mask); ··· 156 156 */ 157 157 bool kernel_page_present(struct page *page) 158 158 { 159 - pgd_t *pgd; 160 - pud_t *pud; 161 - pmd_t *pmd; 162 - pte_t *pte; 159 + pgd_t *pgdp; 160 + pud_t *pudp, pud; 161 + pmd_t *pmdp, pmd; 162 + pte_t *ptep; 163 163 unsigned long addr = (unsigned long)page_address(page); 164 164 165 - pgd = pgd_offset_k(addr); 166 - if (pgd_none(*pgd)) 165 + pgdp = pgd_offset_k(addr); 166 + if (pgd_none(READ_ONCE(*pgdp))) 167 167 return false; 168 168 169 - pud = pud_offset(pgd, addr); 170 - if (pud_none(*pud)) 169 + pudp = pud_offset(pgdp, addr); 170 + pud = READ_ONCE(*pudp); 171 + if (pud_none(pud)) 171 172 return false; 172 - if (pud_sect(*pud)) 173 + if (pud_sect(pud)) 173 174 return true; 174 175 175 - pmd = pmd_offset(pud, addr); 176 - if (pmd_none(*pmd)) 176 + pmdp = pmd_offset(pudp, addr); 177 + pmd = READ_ONCE(*pmdp); 178 + if (pmd_none(pmd)) 177 179 return false; 178 - if (pmd_sect(*pmd)) 180 + if (pmd_sect(pmd)) 179 181 return true; 180 182 181 - pte = pte_offset_kernel(pmd, addr); 182 - return pte_valid(*pte); 183 + ptep = pte_offset_kernel(pmdp, addr); 184 + return pte_valid(READ_ONCE(*ptep)); 183 185 } 184 186 #endif /* CONFIG_HIBERNATION */ 185 187 #endif /* CONFIG_DEBUG_PAGEALLOC */

+9 -5

arch/arm64/mm/proc.S

··· 205 205 dc cvac, cur_\()\type\()p // Ensure any existing dirty 206 206 dmb sy // lines are written back before 207 207 ldr \type, [cur_\()\type\()p] // loading the entry 208 - tbz \type, #0, next_\()\type // Skip invalid entries 208 + tbz \type, #0, skip_\()\type // Skip invalid and 209 + tbnz \type, #11, skip_\()\type // non-global entries 209 210 .endm 210 211 211 212 .macro __idmap_kpti_put_pgtable_ent_ng, type ··· 266 265 add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8) 267 266 do_pgd: __idmap_kpti_get_pgtable_ent pgd 268 267 tbnz pgd, #1, walk_puds 269 - __idmap_kpti_put_pgtable_ent_ng pgd 270 268 next_pgd: 269 + __idmap_kpti_put_pgtable_ent_ng pgd 270 + skip_pgd: 271 271 add cur_pgdp, cur_pgdp, #8 272 272 cmp cur_pgdp, end_pgdp 273 273 b.ne do_pgd ··· 296 294 add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8) 297 295 do_pud: __idmap_kpti_get_pgtable_ent pud 298 296 tbnz pud, #1, walk_pmds 299 - __idmap_kpti_put_pgtable_ent_ng pud 300 297 next_pud: 298 + __idmap_kpti_put_pgtable_ent_ng pud 299 + skip_pud: 301 300 add cur_pudp, cur_pudp, 8 302 301 cmp cur_pudp, end_pudp 303 302 b.ne do_pud ··· 317 314 add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8) 318 315 do_pmd: __idmap_kpti_get_pgtable_ent pmd 319 316 tbnz pmd, #1, walk_ptes 320 - __idmap_kpti_put_pgtable_ent_ng pmd 321 317 next_pmd: 318 + __idmap_kpti_put_pgtable_ent_ng pmd 319 + skip_pmd: 322 320 add cur_pmdp, cur_pmdp, #8 323 321 cmp cur_pmdp, end_pmdp 324 322 b.ne do_pmd ··· 337 333 add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8) 338 334 do_pte: __idmap_kpti_get_pgtable_ent pte 339 335 __idmap_kpti_put_pgtable_ent_ng pte 340 - next_pte: 336 + skip_pte: 341 337 add cur_ptep, cur_ptep, #8 342 338 cmp cur_ptep, end_ptep 343 339 b.ne do_pte

Configure Feed

Configure Feed