mm: use inline helper functions instead of ugly macros

Patch series "support batched checking of the young flag for MGLRU", v3.

This is a follow-up to the previous work [1], to support batched checking
of the young flag for MGLRU.

Similarly, batched checking of young flag for large folios can improve
performance during large-folio reclamation when MGLRU is enabled. I
observed noticeable performance improvements (see patch 5) on an Arm64
machine that supports contiguous PTEs. All mm-selftests are passed.

Patch 1 - 3: cleanup patches.
Patch 4: add a new generic batched PTE helper: test_and_clear_young_ptes().
Patch 5: support batched young flag checking for MGLRU.
Patch 6: implement the Arm64 arch-specific test_and_clear_young_ptes().

This patch (of 6):

People have already complained that these *_clear_young_notify() related
macros are very ugly, so let's use inline helpers to make them more
readable.

In addition, we cannot implement these inline helper functions in the
mmu_notifier.h file, because some arch-specific files will include the
mmu_notifier.h, which introduces header compilation dependencies and
causes build errors (e.g., arch/arm64/include/asm/tlbflush.h). Moreover,
since these functions are only used in the mm, implementing these inline
helpers in the mm/internal.h header seems reasonable.

Link: https://lkml.kernel.org/r/cover.1772778858.git.baolin.wang@linux.alibaba.com
Link: https://lkml.kernel.org/r/ea14af84e7967ccebb25082c28a8669d6da8fe57.1772778858.git.baolin.wang@linux.alibaba.com
Link: https://lore.kernel.org/all/cover.1770645603.git.baolin.wang@linux.alibaba.com/ [1]
Signed-off-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Reviewed-by: Rik van Riel <riel@surriel.com>
Reviewed-by: Barry Song <baohua@kernel.org>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wei Xu <weixugc@google.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Alistair Popple <apopple@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Baolin Wang and committed by

Andrew Morton 2 months ago 5a970006 cf2124a9

+52 -54

2 changed files

expand all

include

linux

mmu_notifier.h

internal.h

-54

include/linux/mmu_notifier.h

··· 516 516 range->owner = owner; 517 517 } 518 518 519 - #define clear_flush_young_ptes_notify(__vma, __address, __ptep, __nr) \ 520 - ({ \ 521 - int __young; \ 522 - struct vm_area_struct *___vma = __vma; \ 523 - unsigned long ___address = __address; \ 524 - unsigned int ___nr = __nr; \ 525 - __young = clear_flush_young_ptes(___vma, ___address, __ptep, ___nr); \ 526 - __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \ 527 - ___address, \ 528 - ___address + \ 529 - ___nr * PAGE_SIZE); \ 530 - __young; \ 531 - }) 532 - 533 - #define pmdp_clear_flush_young_notify(__vma, __address, __pmdp) \ 534 - ({ \ 535 - int __young; \ 536 - struct vm_area_struct *___vma = __vma; \ 537 - unsigned long ___address = __address; \ 538 - __young = pmdp_clear_flush_young(___vma, ___address, __pmdp); \ 539 - __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \ 540 - ___address, \ 541 - ___address + \ 542 - PMD_SIZE); \ 543 - __young; \ 544 - }) 545 - 546 - #define ptep_clear_young_notify(__vma, __address, __ptep) \ 547 - ({ \ 548 - int __young; \ 549 - struct vm_area_struct *___vma = __vma; \ 550 - unsigned long ___address = __address; \ 551 - __young = ptep_test_and_clear_young(___vma, ___address, __ptep);\ 552 - __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \ 553 - ___address + PAGE_SIZE); \ 554 - __young; \ 555 - }) 556 - 557 - #define pmdp_clear_young_notify(__vma, __address, __pmdp) \ 558 - ({ \ 559 - int __young; \ 560 - struct vm_area_struct *___vma = __vma; \ 561 - unsigned long ___address = __address; \ 562 - __young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\ 563 - __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \ 564 - ___address + PMD_SIZE); \ 565 - __young; \ 566 - }) 567 - 568 519 #else /* CONFIG_MMU_NOTIFIER */ 569 520 570 521 struct mmu_notifier_range { ··· 602 651 } 603 652 604 653 #define mmu_notifier_range_update_to_read_only(r) false 605 - 606 - #define clear_flush_young_ptes_notify clear_flush_young_ptes 607 - #define pmdp_clear_flush_young_notify pmdp_clear_flush_young 608 - #define ptep_clear_young_notify ptep_test_and_clear_young 609 - #define pmdp_clear_young_notify pmdp_test_and_clear_young 610 654 611 655 static inline void mmu_notifier_synchronize(void) 612 656 {

+52

mm/internal.h

··· 11 11 #include <linux/khugepaged.h> 12 12 #include <linux/mm.h> 13 13 #include <linux/mm_inline.h> 14 + #include <linux/mmu_notifier.h> 14 15 #include <linux/pagemap.h> 15 16 #include <linux/pagewalk.h> 16 17 #include <linux/rmap.h> ··· 1796 1795 1797 1796 return remap_pfn_range_complete(vma, addr, pfn, size, prot); 1798 1797 } 1798 + 1799 + #ifdef CONFIG_MMU_NOTIFIER 1800 + static inline int clear_flush_young_ptes_notify(struct vm_area_struct *vma, 1801 + unsigned long addr, pte_t *ptep, unsigned int nr) 1802 + { 1803 + int young; 1804 + 1805 + young = clear_flush_young_ptes(vma, addr, ptep, nr); 1806 + young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, 1807 + addr + nr * PAGE_SIZE); 1808 + return young; 1809 + } 1810 + 1811 + static inline int pmdp_clear_flush_young_notify(struct vm_area_struct *vma, 1812 + unsigned long addr, pmd_t *pmdp) 1813 + { 1814 + int young; 1815 + 1816 + young = pmdp_clear_flush_young(vma, addr, pmdp); 1817 + young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, addr + PMD_SIZE); 1818 + return young; 1819 + } 1820 + 1821 + static inline int ptep_clear_young_notify(struct vm_area_struct *vma, 1822 + unsigned long addr, pte_t *ptep) 1823 + { 1824 + int young; 1825 + 1826 + young = ptep_test_and_clear_young(vma, addr, ptep); 1827 + young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); 1828 + return young; 1829 + } 1830 + 1831 + static inline int pmdp_clear_young_notify(struct vm_area_struct *vma, 1832 + unsigned long addr, pmd_t *pmdp) 1833 + { 1834 + int young; 1835 + 1836 + young = pmdp_test_and_clear_young(vma, addr, pmdp); 1837 + young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PMD_SIZE); 1838 + return young; 1839 + } 1840 + 1841 + #else /* CONFIG_MMU_NOTIFIER */ 1842 + 1843 + #define clear_flush_young_ptes_notify clear_flush_young_ptes 1844 + #define pmdp_clear_flush_young_notify pmdp_clear_flush_young 1845 + #define ptep_clear_young_notify ptep_test_and_clear_young 1846 + #define pmdp_clear_young_notify pmdp_test_and_clear_young 1847 + 1848 + #endif /* CONFIG_MMU_NOTIFIER */ 1799 1849 1800 1850 #endif /* __MM_INTERNAL_H */

Configure Feed

Configure Feed