arm64: mm: support batch clearing of the young flag for large folios

Currently, contpte_ptep_test_and_clear_young() and
contpte_ptep_clear_flush_young() only clear the young flag and flush TLBs
for PTEs within the contiguous range. To support batch PTE operations for
other sized large folios in the following patches, adding a new parameter
to specify the number of PTEs that map consecutive pages of the same large
folio in a single VMA and a single page table.

While we are at it, rename the functions to maintain consistency with
other contpte_*() functions.

Link: https://lkml.kernel.org/r/5644250dcc0417278c266ad37118d27f541fd052.1770645603.git.baolin.wang@linux.alibaba.com
Signed-off-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Reviewed-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: David Hildenbrand (Arm) <david@kernel.org>
Cc: Barry Song <baohua@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Harry Yoo <harry.yoo@oracle.com>
Cc: Jann Horn <jannh@google.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Rik van Riel <riel@surriel.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Baolin Wang and committed by

Andrew Morton 4 months ago 6f0e1142 67d59bdd

+25 -20

2 changed files

expand all

arch

arm64

include

asm

pgtable.h

contpte.c

+6 -6

arch/arm64/include/asm/pgtable.h

··· 1648 1648 extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm, 1649 1649 unsigned long addr, pte_t *ptep, 1650 1650 unsigned int nr, int full); 1651 - extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma, 1652 - unsigned long addr, pte_t *ptep); 1653 - extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma, 1654 - unsigned long addr, pte_t *ptep); 1651 + int contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, 1652 + unsigned long addr, pte_t *ptep, unsigned int nr); 1653 + int contpte_clear_flush_young_ptes(struct vm_area_struct *vma, 1654 + unsigned long addr, pte_t *ptep, unsigned int nr); 1655 1655 extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr, 1656 1656 pte_t *ptep, unsigned int nr); 1657 1657 extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma, ··· 1823 1823 if (likely(!pte_valid_cont(orig_pte))) 1824 1824 return __ptep_test_and_clear_young(vma, addr, ptep); 1825 1825 1826 - return contpte_ptep_test_and_clear_young(vma, addr, ptep); 1826 + return contpte_test_and_clear_young_ptes(vma, addr, ptep, 1); 1827 1827 } 1828 1828 1829 1829 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH ··· 1835 1835 if (likely(!pte_valid_cont(orig_pte))) 1836 1836 return __ptep_clear_flush_young(vma, addr, ptep); 1837 1837 1838 - return contpte_ptep_clear_flush_young(vma, addr, ptep); 1838 + return contpte_clear_flush_young_ptes(vma, addr, ptep, 1); 1839 1839 } 1840 1840 1841 1841 #define wrprotect_ptes wrprotect_ptes

+19 -14

arch/arm64/mm/contpte.c

··· 508 508 } 509 509 EXPORT_SYMBOL_GPL(contpte_get_and_clear_full_ptes); 510 510 511 - int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma, 512 - unsigned long addr, pte_t *ptep) 511 + int contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, 512 + unsigned long addr, pte_t *ptep, 513 + unsigned int nr) 513 514 { 514 515 /* 515 516 * ptep_clear_flush_young() technically requires us to clear the access ··· 519 518 * contig range when the range is covered by a single folio, we can get 520 519 * away with clearing young for the whole contig range here, so we avoid 521 520 * having to unfold. 521 + * 522 + * The 'nr' means consecutive (present) PTEs that map consecutive pages 523 + * of the same large folio in a single VMA and a single page table. 522 524 */ 523 525 526 + unsigned long end = addr + nr * PAGE_SIZE; 524 527 int young = 0; 525 - int i; 526 528 527 - ptep = contpte_align_down(ptep); 528 - addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); 529 - 530 - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) 529 + ptep = contpte_align_addr_ptep(&addr, &end, ptep, nr); 530 + for (; addr != end; ptep++, addr += PAGE_SIZE) 531 531 young |= __ptep_test_and_clear_young(vma, addr, ptep); 532 532 533 533 return young; 534 534 } 535 - EXPORT_SYMBOL_GPL(contpte_ptep_test_and_clear_young); 535 + EXPORT_SYMBOL_GPL(contpte_test_and_clear_young_ptes); 536 536 537 - int contpte_ptep_clear_flush_young(struct vm_area_struct *vma, 538 - unsigned long addr, pte_t *ptep) 537 + int contpte_clear_flush_young_ptes(struct vm_area_struct *vma, 538 + unsigned long addr, pte_t *ptep, 539 + unsigned int nr) 539 540 { 540 541 int young; 541 542 542 - young = contpte_ptep_test_and_clear_young(vma, addr, ptep); 543 + young = contpte_test_and_clear_young_ptes(vma, addr, ptep, nr); 543 544 544 545 if (young) { 546 + unsigned long end = addr + nr * PAGE_SIZE; 547 + 548 + contpte_align_addr_ptep(&addr, &end, ptep, nr); 545 549 /* 546 550 * See comment in __ptep_clear_flush_young(); same rationale for 547 551 * eliding the trailing DSB applies here. 548 552 */ 549 - addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); 550 - __flush_tlb_range_nosync(vma->vm_mm, addr, addr + CONT_PTE_SIZE, 553 + __flush_tlb_range_nosync(vma->vm_mm, addr, end, 551 554 PAGE_SIZE, true, 3); 552 555 } 553 556 554 557 return young; 555 558 } 556 - EXPORT_SYMBOL_GPL(contpte_ptep_clear_flush_young); 559 + EXPORT_SYMBOL_GPL(contpte_clear_flush_young_ptes); 557 560 558 561 void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr, 559 562 pte_t *ptep, unsigned int nr)

Configure Feed

Configure Feed