tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm: support tlbbatch flush for a range of PTEs

This patch lays the groundwork for supporting batch PTE unmapping in
try_to_unmap_one(). It introduces range handling for TLB batch flushing,
with the range currently set to the size of PAGE_SIZE.

The function __flush_tlb_range_nosync() is architecture-specific and is
only used within arch/arm64. This function requires the mm structure
instead of the vma structure. To allow its reuse by
arch_tlbbatch_add_pending(), which operates with mm but not vma, this
patch modifies the argument of __flush_tlb_range_nosync() to take mm as
its parameter.

Link: https://lkml.kernel.org/r/20250214093015.51024-3-21cnbao@gmail.com
Signed-off-by: Barry Song <v-songbaohua@oppo.com>
Acked-by: Will Deacon <will@kernel.org>
Reviewed-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Shaoqin Huang <shahuang@redhat.com>
Cc: Gavin Shan <gshan@redhat.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Lance Yang <ioworker0@gmail.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Yicong Yang <yangyicong@hisilicon.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Chis Li <chrisl@kernel.org>
Cc: "Huang, Ying" <ying.huang@intel.com>
Cc: Kairui Song <kasong@tencent.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mauricio Faria de Oliveira <mfo@canonical.com>
Cc: Tangquan Zheng <zhengtangquan@oppo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Barry Song and committed by
Andrew Morton 2f4ab3ac faeb2831

+20 -24
+11 -12
arch/arm64/include/asm/tlbflush.h
··· 322 322 return true; 323 323 } 324 324 325 - static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 326 - struct mm_struct *mm, 327 - unsigned long uaddr) 328 - { 329 - __flush_tlb_page_nosync(mm, uaddr); 330 - } 331 - 332 325 /* 333 326 * If mprotect/munmap/etc occurs during TLB batched flushing, we need to 334 327 * synchronise all the TLBI issued with a DSB to avoid the race mentioned in ··· 441 448 return false; 442 449 } 443 450 444 - static inline void __flush_tlb_range_nosync(struct vm_area_struct *vma, 451 + static inline void __flush_tlb_range_nosync(struct mm_struct *mm, 445 452 unsigned long start, unsigned long end, 446 453 unsigned long stride, bool last_level, 447 454 int tlb_level) ··· 453 460 pages = (end - start) >> PAGE_SHIFT; 454 461 455 462 if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { 456 - flush_tlb_mm(vma->vm_mm); 463 + flush_tlb_mm(mm); 457 464 return; 458 465 } 459 466 460 467 dsb(ishst); 461 - asid = ASID(vma->vm_mm); 468 + asid = ASID(mm); 462 469 463 470 if (last_level) 464 471 __flush_tlb_range_op(vale1is, start, pages, stride, asid, ··· 467 474 __flush_tlb_range_op(vae1is, start, pages, stride, asid, 468 475 tlb_level, true, lpa2_is_enabled()); 469 476 470 - mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, start, end); 477 + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); 471 478 } 472 479 473 480 static inline void __flush_tlb_range(struct vm_area_struct *vma, ··· 475 482 unsigned long stride, bool last_level, 476 483 int tlb_level) 477 484 { 478 - __flush_tlb_range_nosync(vma, start, end, stride, 485 + __flush_tlb_range_nosync(vma->vm_mm, start, end, stride, 479 486 last_level, tlb_level); 480 487 dsb(ish); 481 488 } ··· 525 532 __tlbi(vaae1is, addr); 526 533 dsb(ish); 527 534 isb(); 535 + } 536 + 537 + static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 538 + struct mm_struct *mm, unsigned long start, unsigned long end) 539 + { 540 + __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3); 528 541 } 529 542 #endif 530 543
+1 -1
arch/arm64/mm/contpte.c
··· 335 335 * eliding the trailing DSB applies here. 336 336 */ 337 337 addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); 338 - __flush_tlb_range_nosync(vma, addr, addr + CONT_PTE_SIZE, 338 + __flush_tlb_range_nosync(vma->vm_mm, addr, addr + CONT_PTE_SIZE, 339 339 PAGE_SIZE, true, 3); 340 340 } 341 341
+1 -2
arch/riscv/include/asm/tlbflush.h
··· 60 60 61 61 bool arch_tlbbatch_should_defer(struct mm_struct *mm); 62 62 void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 63 - struct mm_struct *mm, 64 - unsigned long uaddr); 63 + struct mm_struct *mm, unsigned long start, unsigned long end); 65 64 void arch_flush_tlb_batched_pending(struct mm_struct *mm); 66 65 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch); 67 66
+1 -2
arch/riscv/mm/tlbflush.c
··· 186 186 } 187 187 188 188 void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 189 - struct mm_struct *mm, 190 - unsigned long uaddr) 189 + struct mm_struct *mm, unsigned long start, unsigned long end) 191 190 { 192 191 cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); 193 192 }
+1 -2
arch/x86/include/asm/tlbflush.h
··· 279 279 } 280 280 281 281 static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 282 - struct mm_struct *mm, 283 - unsigned long uaddr) 282 + struct mm_struct *mm, unsigned long start, unsigned long end) 284 283 { 285 284 inc_mm_tlb_gen(mm); 286 285 cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
+5 -5
mm/rmap.c
··· 672 672 (TLB_FLUSH_BATCH_PENDING_MASK / 2) 673 673 674 674 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval, 675 - unsigned long uaddr) 675 + unsigned long start, unsigned long end) 676 676 { 677 677 struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc; 678 678 int batch; ··· 681 681 if (!pte_accessible(mm, pteval)) 682 682 return; 683 683 684 - arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, uaddr); 684 + arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, start, end); 685 685 tlb_ubc->flush_required = true; 686 686 687 687 /* ··· 757 757 } 758 758 #else 759 759 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval, 760 - unsigned long uaddr) 760 + unsigned long start, unsigned long end) 761 761 { 762 762 } 763 763 ··· 1887 1887 */ 1888 1888 pteval = ptep_get_and_clear(mm, address, pvmw.pte); 1889 1889 1890 - set_tlb_ubc_flush_pending(mm, pteval, address); 1890 + set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE); 1891 1891 } else { 1892 1892 pteval = ptep_clear_flush(vma, address, pvmw.pte); 1893 1893 } ··· 2270 2270 */ 2271 2271 pteval = ptep_get_and_clear(mm, address, pvmw.pte); 2272 2272 2273 - set_tlb_ubc_flush_pending(mm, pteval, address); 2273 + set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE); 2274 2274 } else { 2275 2275 pteval = ptep_clear_flush(vma, address, pvmw.pte); 2276 2276 }