ksm: perform a range-walk in break_ksm

Make break_ksm() receive an address range and change break_ksm_pmd_entry()
to perform a range-walk and return the address of the first ksm page
found.

This change allows break_ksm() to skip unmapped regions instead of
iterating every page address. When unmerging large sparse VMAs, this
significantly reduces runtime.

In a benchmark unmerging a 32 TiB sparse virtual address space where only
one page was populated, the runtime dropped from 9 minutes to less then 5
seconds.

Link: https://lkml.kernel.org/r/20251105184912.186329-3-pedrodemargomes@gmail.com
Signed-off-by: Pedro Demarchi Gomes <pedrodemargomes@gmail.com>
Suggested-by: David Hildenbrand (Red Hat) <david@kernel.org>
Acked-by: David Hildenbrand (Red Hat) <david@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Pedro Demarchi Gomes and committed by

Andrew Morton 7 months ago 5d4939fc 912aa825

+43 -40

1 changed file

expand all

ksm.c

+43 -40

mm/ksm.c

··· 607 607 return atomic_read(&mm->mm_users) == 0; 608 608 } 609 609 610 - static int break_ksm_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long next, 610 + static int break_ksm_pmd_entry(pmd_t *pmdp, unsigned long addr, unsigned long end, 611 611 struct mm_walk *walk) 612 612 { 613 - struct folio *folio = NULL; 613 + unsigned long *found_addr = (unsigned long *) walk->private; 614 + struct mm_struct *mm = walk->mm; 615 + pte_t *start_ptep, *ptep; 614 616 spinlock_t *ptl; 615 - pte_t *pte; 616 - pte_t ptent; 617 - int found; 617 + int found = 0; 618 618 619 - pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); 620 - if (!pte) 619 + if (ksm_test_exit(walk->mm)) 621 620 return 0; 622 - ptent = ptep_get(pte); 623 - if (pte_present(ptent)) { 624 - folio = vm_normal_folio(walk->vma, addr, ptent); 625 - } else if (!pte_none(ptent)) { 626 - swp_entry_t entry = pte_to_swp_entry(ptent); 621 + if (signal_pending(current)) 622 + return -ERESTARTSYS; 627 623 628 - /* 629 - * As KSM pages remain KSM pages until freed, no need to wait 630 - * here for migration to end. 631 - */ 632 - if (is_migration_entry(entry)) 633 - folio = pfn_swap_entry_folio(entry); 624 + start_ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); 625 + if (!start_ptep) 626 + return 0; 627 + 628 + for (ptep = start_ptep; addr < end; ptep++, addr += PAGE_SIZE) { 629 + pte_t pte = ptep_get(ptep); 630 + struct folio *folio = NULL; 631 + 632 + if (pte_present(pte)) { 633 + folio = vm_normal_folio(walk->vma, addr, pte); 634 + } else if (!pte_none(pte)) { 635 + swp_entry_t entry = pte_to_swp_entry(pte); 636 + 637 + /* 638 + * As KSM pages remain KSM pages until freed, no need to wait 639 + * here for migration to end. 640 + */ 641 + if (is_migration_entry(entry)) 642 + folio = pfn_swap_entry_folio(entry); 643 + } 644 + /* return 1 if the page is an normal ksm page or KSM-placed zero page */ 645 + found = (folio && folio_test_ksm(folio)) || 646 + (pte_present(pte) && is_ksm_zero_pte(pte)); 647 + if (found) { 648 + *found_addr = addr; 649 + goto out_unlock; 650 + } 634 651 } 635 - /* return 1 if the page is an normal ksm page or KSM-placed zero page */ 636 - found = (folio && folio_test_ksm(folio)) || 637 - (pte_present(ptent) && is_ksm_zero_pte(ptent)); 638 - pte_unmap_unlock(pte, ptl); 652 + out_unlock: 653 + pte_unmap_unlock(ptep, ptl); 639 654 return found; 640 655 } 641 656 ··· 677 662 * of the process that owns 'vma'. We also do not want to enforce 678 663 * protection keys here anyway. 679 664 */ 680 - static int break_ksm(struct vm_area_struct *vma, unsigned long addr, bool lock_vma) 665 + static int break_ksm(struct vm_area_struct *vma, unsigned long addr, 666 + unsigned long end, bool lock_vma) 681 667 { 682 668 vm_fault_t ret = 0; 683 669 const struct mm_walk_ops *ops = lock_vma ? ··· 688 672 int ksm_page; 689 673 690 674 cond_resched(); 691 - ksm_page = walk_page_range_vma(vma, addr, addr + 1, ops, NULL); 692 - if (WARN_ON_ONCE(ksm_page < 0)) 675 + ksm_page = walk_page_range_vma(vma, addr, end, ops, &addr); 676 + if (ksm_page <= 0) 693 677 return ksm_page; 694 - if (!ksm_page) 695 - return 0; 696 678 ret = handle_mm_fault(vma, addr, 697 679 FAULT_FLAG_UNSHARE | FAULT_FLAG_REMOTE, 698 680 NULL); ··· 776 762 mmap_read_lock(mm); 777 763 vma = find_mergeable_vma(mm, addr); 778 764 if (vma) 779 - break_ksm(vma, addr, false); 765 + break_ksm(vma, addr, addr + PAGE_SIZE, false); 780 766 mmap_read_unlock(mm); 781 767 } 782 768 ··· 1087 1073 static int unmerge_ksm_pages(struct vm_area_struct *vma, 1088 1074 unsigned long start, unsigned long end, bool lock_vma) 1089 1075 { 1090 - unsigned long addr; 1091 - int err = 0; 1092 - 1093 - for (addr = start; addr < end && !err; addr += PAGE_SIZE) { 1094 - if (ksm_test_exit(vma->vm_mm)) 1095 - break; 1096 - if (signal_pending(current)) 1097 - err = -ERESTARTSYS; 1098 - else 1099 - err = break_ksm(vma, addr, lock_vma); 1100 - } 1101 - return err; 1076 + return break_ksm(vma, start, end, lock_vma); 1102 1077 } 1103 1078 1104 1079 static inline

Configure Feed

Configure Feed