sparc64: implement the new page table range API

+12 -6

arch/sparc/include/asm/cacheflush_64.h

··· 35 35 void __flush_icache_page(unsigned long); 36 36 37 37 void __flush_dcache_page(void *addr, int flush_icache); 38 - void flush_dcache_page_impl(struct page *page); 38 + void flush_dcache_folio_impl(struct folio *folio); 39 39 #ifdef CONFIG_SMP 40 - void smp_flush_dcache_page_impl(struct page *page, int cpu); 41 - void flush_dcache_page_all(struct mm_struct *mm, struct page *page); 40 + void smp_flush_dcache_folio_impl(struct folio *folio, int cpu); 41 + void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio); 42 42 #else 43 - #define smp_flush_dcache_page_impl(page,cpu) flush_dcache_page_impl(page) 44 - #define flush_dcache_page_all(mm,page) flush_dcache_page_impl(page) 43 + #define smp_flush_dcache_folio_impl(folio, cpu) flush_dcache_folio_impl(folio) 44 + #define flush_dcache_folio_all(mm, folio) flush_dcache_folio_impl(folio) 45 45 #endif 46 46 47 47 void __flush_dcache_range(unsigned long start, unsigned long end); 48 48 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 49 - void flush_dcache_page(struct page *page); 49 + void flush_dcache_folio(struct folio *folio); 50 + #define flush_dcache_folio flush_dcache_folio 51 + static inline void flush_dcache_page(struct page *page) 52 + { 53 + flush_dcache_folio(page_folio(page)); 54 + } 50 55 51 56 #define flush_icache_page(vma, pg) do { } while(0) 57 + #define flush_icache_pages(vma, pg, nr) do { } while(0) 52 58 53 59 void flush_ptrace_access(struct vm_area_struct *, struct page *, 54 60 unsigned long uaddr, void *kaddr,

+22 -7

arch/sparc/include/asm/pgtable_64.h

··· 86 86 #define vmemmap ((struct page *)VMEMMAP_BASE) 87 87 88 88 #include <linux/sched.h> 89 + #include <asm/tlbflush.h> 89 90 90 91 bool kern_addr_valid(unsigned long addr); 91 92 ··· 928 927 maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT); 929 928 } 930 929 931 - #define set_pte_at(mm,addr,ptep,pte) \ 932 - __set_pte_at((mm), (addr), (ptep), (pte), 0) 930 + static inline void set_ptes(struct mm_struct *mm, unsigned long addr, 931 + pte_t *ptep, pte_t pte, unsigned int nr) 932 + { 933 + arch_enter_lazy_mmu_mode(); 934 + for (;;) { 935 + __set_pte_at(mm, addr, ptep, pte, 0); 936 + if (--nr == 0) 937 + break; 938 + ptep++; 939 + pte_val(pte) += PAGE_SIZE; 940 + addr += PAGE_SIZE; 941 + } 942 + arch_leave_lazy_mmu_mode(); 943 + } 944 + #define set_ptes set_ptes 933 945 934 946 #define pte_clear(mm,addr,ptep) \ 935 947 set_pte_at((mm), (addr), (ptep), __pte(0UL)) ··· 961 947 \ 962 948 if (pfn_valid(this_pfn) && \ 963 949 (((old_addr) ^ (new_addr)) & (1 << 13))) \ 964 - flush_dcache_page_all(current->mm, \ 965 - pfn_to_page(this_pfn)); \ 950 + flush_dcache_folio_all(current->mm, \ 951 + page_folio(pfn_to_page(this_pfn))); \ 966 952 } \ 967 953 newpte; \ 968 954 }) ··· 977 963 void mmu_info(struct seq_file *); 978 964 979 965 struct vm_area_struct; 980 - void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); 966 + void update_mmu_cache_range(struct vm_fault *, struct vm_area_struct *, 967 + unsigned long addr, pte_t *ptep, unsigned int nr); 968 + #define update_mmu_cache(vma, addr, ptep) \ 969 + update_mmu_cache_range(NULL, vma, addr, ptep, 1) 981 970 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 982 971 void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, 983 972 pmd_t *pmd); ··· 1137 1120 return (pte_val(pte) & (prot | _PAGE_SPECIAL)) == prot; 1138 1121 } 1139 1122 #define pte_access_permitted pte_access_permitted 1140 - 1141 - #include <asm/tlbflush.h> 1142 1123 1143 1124 /* We provide our own get_unmapped_area to cope with VA holes and 1144 1125 * SHM area cache aliasing for userland.

+36 -20

arch/sparc/kernel/smp_64.c

··· 921 921 #endif 922 922 extern unsigned long xcall_flush_dcache_page_spitfire; 923 923 924 - static inline void __local_flush_dcache_page(struct page *page) 924 + static inline void __local_flush_dcache_folio(struct folio *folio) 925 925 { 926 + unsigned int i, nr = folio_nr_pages(folio); 927 + 926 928 #ifdef DCACHE_ALIASING_POSSIBLE 927 - __flush_dcache_page(page_address(page), 929 + for (i = 0; i < nr; i++) 930 + __flush_dcache_page(folio_address(folio) + i * PAGE_SIZE, 928 931 ((tlb_type == spitfire) && 929 - page_mapping_file(page) != NULL)); 932 + folio_flush_mapping(folio) != NULL)); 930 933 #else 931 - if (page_mapping_file(page) != NULL && 932 - tlb_type == spitfire) 933 - __flush_icache_page(__pa(page_address(page))); 934 + if (folio_flush_mapping(folio) != NULL && 935 + tlb_type == spitfire) { 936 + unsigned long pfn = folio_pfn(folio) 937 + for (i = 0; i < nr; i++) 938 + __flush_icache_page((pfn + i) * PAGE_SIZE); 939 + } 934 940 #endif 935 941 } 936 942 937 - void smp_flush_dcache_page_impl(struct page *page, int cpu) 943 + void smp_flush_dcache_folio_impl(struct folio *folio, int cpu) 938 944 { 939 945 int this_cpu; 940 946 ··· 954 948 this_cpu = get_cpu(); 955 949 956 950 if (cpu == this_cpu) { 957 - __local_flush_dcache_page(page); 951 + __local_flush_dcache_folio(folio); 958 952 } else if (cpu_online(cpu)) { 959 - void *pg_addr = page_address(page); 953 + void *pg_addr = folio_address(folio); 960 954 u64 data0 = 0; 961 955 962 956 if (tlb_type == spitfire) { 963 957 data0 = ((u64)&xcall_flush_dcache_page_spitfire); 964 - if (page_mapping_file(page) != NULL) 958 + if (folio_flush_mapping(folio) != NULL) 965 959 data0 |= ((u64)1 << 32); 966 960 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 967 961 #ifdef DCACHE_ALIASING_POSSIBLE ··· 969 963 #endif 970 964 } 971 965 if (data0) { 972 - xcall_deliver(data0, __pa(pg_addr), 973 - (u64) pg_addr, cpumask_of(cpu)); 966 + unsigned int i, nr = folio_nr_pages(folio); 967 + 968 + for (i = 0; i < nr; i++) { 969 + xcall_deliver(data0, __pa(pg_addr), 970 + (u64) pg_addr, cpumask_of(cpu)); 974 971 #ifdef CONFIG_DEBUG_DCFLUSH 975 - atomic_inc(&dcpage_flushes_xcall); 972 + atomic_inc(&dcpage_flushes_xcall); 976 973 #endif 974 + pg_addr += PAGE_SIZE; 975 + } 977 976 } 978 977 } 979 978 980 979 put_cpu(); 981 980 } 982 981 983 - void flush_dcache_page_all(struct mm_struct *mm, struct page *page) 982 + void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio) 984 983 { 985 984 void *pg_addr; 986 985 u64 data0; ··· 999 988 atomic_inc(&dcpage_flushes); 1000 989 #endif 1001 990 data0 = 0; 1002 - pg_addr = page_address(page); 991 + pg_addr = folio_address(folio); 1003 992 if (tlb_type == spitfire) { 1004 993 data0 = ((u64)&xcall_flush_dcache_page_spitfire); 1005 - if (page_mapping_file(page) != NULL) 994 + if (folio_flush_mapping(folio) != NULL) 1006 995 data0 |= ((u64)1 << 32); 1007 996 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 1008 997 #ifdef DCACHE_ALIASING_POSSIBLE ··· 1010 999 #endif 1011 1000 } 1012 1001 if (data0) { 1013 - xcall_deliver(data0, __pa(pg_addr), 1014 - (u64) pg_addr, cpu_online_mask); 1002 + unsigned int i, nr = folio_nr_pages(folio); 1003 + 1004 + for (i = 0; i < nr; i++) { 1005 + xcall_deliver(data0, __pa(pg_addr), 1006 + (u64) pg_addr, cpu_online_mask); 1015 1007 #ifdef CONFIG_DEBUG_DCFLUSH 1016 - atomic_inc(&dcpage_flushes_xcall); 1008 + atomic_inc(&dcpage_flushes_xcall); 1017 1009 #endif 1010 + pg_addr += PAGE_SIZE; 1011 + } 1018 1012 } 1019 - __local_flush_dcache_page(page); 1013 + __local_flush_dcache_folio(folio); 1020 1014 1021 1015 preempt_enable(); 1022 1016 }

+46 -32

arch/sparc/mm/init_64.c

··· 195 195 #endif 196 196 #endif 197 197 198 - inline void flush_dcache_page_impl(struct page *page) 198 + inline void flush_dcache_folio_impl(struct folio *folio) 199 199 { 200 + unsigned int i, nr = folio_nr_pages(folio); 201 + 200 202 BUG_ON(tlb_type == hypervisor); 201 203 #ifdef CONFIG_DEBUG_DCFLUSH 202 204 atomic_inc(&dcpage_flushes); 203 205 #endif 204 206 205 207 #ifdef DCACHE_ALIASING_POSSIBLE 206 - __flush_dcache_page(page_address(page), 207 - ((tlb_type == spitfire) && 208 - page_mapping_file(page) != NULL)); 208 + for (i = 0; i < nr; i++) 209 + __flush_dcache_page(folio_address(folio) + i * PAGE_SIZE, 210 + ((tlb_type == spitfire) && 211 + folio_flush_mapping(folio) != NULL)); 209 212 #else 210 - if (page_mapping_file(page) != NULL && 211 - tlb_type == spitfire) 212 - __flush_icache_page(__pa(page_address(page))); 213 + if (folio_flush_mapping(folio) != NULL && 214 + tlb_type == spitfire) { 215 + for (i = 0; i < nr; i++) 216 + __flush_icache_page((pfn + i) * PAGE_SIZE); 217 + } 213 218 #endif 214 219 } 215 220 ··· 223 218 #define PG_dcache_cpu_mask \ 224 219 ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL) 225 220 226 - #define dcache_dirty_cpu(page) \ 227 - (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask) 221 + #define dcache_dirty_cpu(folio) \ 222 + (((folio)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask) 228 223 229 - static inline void set_dcache_dirty(struct page *page, int this_cpu) 224 + static inline void set_dcache_dirty(struct folio *folio, int this_cpu) 230 225 { 231 226 unsigned long mask = this_cpu; 232 227 unsigned long non_cpu_bits; ··· 243 238 "bne,pn %%xcc, 1b\n\t" 244 239 " nop" 245 240 : /* no outputs */ 246 - : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags) 241 + : "r" (mask), "r" (non_cpu_bits), "r" (&folio->flags) 247 242 : "g1", "g7"); 248 243 } 249 244 250 - static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu) 245 + static inline void clear_dcache_dirty_cpu(struct folio *folio, unsigned long cpu) 251 246 { 252 247 unsigned long mask = (1UL << PG_dcache_dirty); 253 248 ··· 265 260 " nop\n" 266 261 "2:" 267 262 : /* no outputs */ 268 - : "r" (cpu), "r" (mask), "r" (&page->flags), 263 + : "r" (cpu), "r" (mask), "r" (&folio->flags), 269 264 "i" (PG_dcache_cpu_mask), 270 265 "i" (PG_dcache_cpu_shift) 271 266 : "g1", "g7"); ··· 289 284 290 285 page = pfn_to_page(pfn); 291 286 if (page) { 287 + struct folio *folio = page_folio(page); 292 288 unsigned long pg_flags; 293 289 294 - pg_flags = page->flags; 290 + pg_flags = folio->flags; 295 291 if (pg_flags & (1UL << PG_dcache_dirty)) { 296 292 int cpu = ((pg_flags >> PG_dcache_cpu_shift) & 297 293 PG_dcache_cpu_mask); ··· 302 296 * in the SMP case. 303 297 */ 304 298 if (cpu == this_cpu) 305 - flush_dcache_page_impl(page); 299 + flush_dcache_folio_impl(folio); 306 300 else 307 - smp_flush_dcache_page_impl(page, cpu); 301 + smp_flush_dcache_folio_impl(folio, cpu); 308 302 309 - clear_dcache_dirty_cpu(page, cpu); 303 + clear_dcache_dirty_cpu(folio, cpu); 310 304 311 305 put_cpu(); 312 306 } ··· 394 388 } 395 389 #endif /* CONFIG_HUGETLB_PAGE */ 396 390 397 - void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) 391 + void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, 392 + unsigned long address, pte_t *ptep, unsigned int nr) 398 393 { 399 394 struct mm_struct *mm; 400 395 unsigned long flags; 401 396 bool is_huge_tsb; 402 397 pte_t pte = *ptep; 398 + unsigned int i; 403 399 404 400 if (tlb_type != hypervisor) { 405 401 unsigned long pfn = pte_pfn(pte); ··· 448 440 } 449 441 } 450 442 #endif 451 - if (!is_huge_tsb) 452 - __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, 453 - address, pte_val(pte)); 443 + if (!is_huge_tsb) { 444 + for (i = 0; i < nr; i++) { 445 + __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, 446 + address, pte_val(pte)); 447 + address += PAGE_SIZE; 448 + pte_val(pte) += PAGE_SIZE; 449 + } 450 + } 454 451 455 452 spin_unlock_irqrestore(&mm->context.lock, flags); 456 453 } 457 454 458 - void flush_dcache_page(struct page *page) 455 + void flush_dcache_folio(struct folio *folio) 459 456 { 457 + unsigned long pfn = folio_pfn(folio); 460 458 struct address_space *mapping; 461 459 int this_cpu; 462 460 ··· 473 459 * is merely the zero page. The 'bigcore' testcase in GDB 474 460 * causes this case to run millions of times. 475 461 */ 476 - if (page == ZERO_PAGE(0)) 462 + if (is_zero_pfn(pfn)) 477 463 return; 478 464 479 465 this_cpu = get_cpu(); 480 466 481 - mapping = page_mapping_file(page); 467 + mapping = folio_flush_mapping(folio); 482 468 if (mapping && !mapping_mapped(mapping)) { 483 - int dirty = test_bit(PG_dcache_dirty, &page->flags); 469 + bool dirty = test_bit(PG_dcache_dirty, &folio->flags); 484 470 if (dirty) { 485 - int dirty_cpu = dcache_dirty_cpu(page); 471 + int dirty_cpu = dcache_dirty_cpu(folio); 486 472 487 473 if (dirty_cpu == this_cpu) 488 474 goto out; 489 - smp_flush_dcache_page_impl(page, dirty_cpu); 475 + smp_flush_dcache_folio_impl(folio, dirty_cpu); 490 476 } 491 - set_dcache_dirty(page, this_cpu); 477 + set_dcache_dirty(folio, this_cpu); 492 478 } else { 493 479 /* We could delay the flush for the !page_mapping 494 480 * case too. But that case is for exec env/arg 495 481 * pages and those are %99 certainly going to get 496 482 * faulted into the tlb (and thus flushed) anyways. 497 483 */ 498 - flush_dcache_page_impl(page); 484 + flush_dcache_folio_impl(folio); 499 485 } 500 486 501 487 out: 502 488 put_cpu(); 503 489 } 504 - EXPORT_SYMBOL(flush_dcache_page); 490 + EXPORT_SYMBOL(flush_dcache_folio); 505 491 506 492 void __kprobes flush_icache_range(unsigned long start, unsigned long end) 507 493 { ··· 2294 2280 setup_page_offset(); 2295 2281 2296 2282 /* These build time checkes make sure that the dcache_dirty_cpu() 2297 - * page->flags usage will work. 2283 + * folio->flags usage will work. 2298 2284 * 2299 2285 * When a page gets marked as dcache-dirty, we store the 2300 - * cpu number starting at bit 32 in the page->flags. Also, 2286 + * cpu number starting at bit 32 in the folio->flags. Also, 2301 2287 * functions like clear_dcache_dirty_cpu use the cpu mask 2302 2288 * in 13-bit signed-immediate instruction fields. 2303 2289 */

+3 -2

arch/sparc/mm/tlb.c

··· 118 118 unsigned long paddr, pfn = pte_pfn(orig); 119 119 struct address_space *mapping; 120 120 struct page *page; 121 + struct folio *folio; 121 122 122 123 if (!pfn_valid(pfn)) 123 124 goto no_cache_flush; ··· 128 127 goto no_cache_flush; 129 128 130 129 /* A real file page? */ 131 - mapping = page_mapping_file(page); 130 + mapping = folio_flush_mapping(folio); 132 131 if (!mapping) 133 132 goto no_cache_flush; 134 133 135 134 paddr = (unsigned long) page_address(page); 136 135 if ((paddr ^ vaddr) & (1 << 13)) 137 - flush_dcache_page_all(mm, page); 136 + flush_dcache_folio_all(mm, folio); 138 137 } 139 138 140 139 no_cache_flush:

Configure Feed

Configure Feed