mm: memory: improve copy_user_large_folio()

Use nr_pages instead of pages_per_huge_page and move the address alignment
from copy_user_large_folio() into the callers since it is only needed when
we don't know which address will be accessed.

Link: https://lkml.kernel.org/r/20240618091242.2140164-4-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: "Huang, Ying" <ying.huang@intel.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Kefeng Wang and committed by

Andrew Morton 2 years ago 530dd992 5132633e

+12 -17

2 changed files

expand all

hugetlb.c

memory.c

+8 -10

mm/hugetlb.c

··· 5492 5492 ret = PTR_ERR(new_folio); 5493 5493 break; 5494 5494 } 5495 - ret = copy_user_large_folio(new_folio, 5496 - pte_folio, 5497 - addr, dst_vma); 5495 + ret = copy_user_large_folio(new_folio, pte_folio, 5496 + ALIGN_DOWN(addr, sz), dst_vma); 5498 5497 folio_put(pte_folio); 5499 5498 if (ret) { 5500 5499 folio_put(new_folio); ··· 6683 6684 struct hstate *h = hstate_vma(dst_vma); 6684 6685 struct address_space *mapping = dst_vma->vm_file->f_mapping; 6685 6686 pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr); 6686 - unsigned long size; 6687 + unsigned long size = huge_page_size(h); 6687 6688 int vm_shared = dst_vma->vm_flags & VM_SHARED; 6688 6689 pte_t _dst_pte; 6689 6690 spinlock_t *ptl; ··· 6702 6703 } 6703 6704 6704 6705 _dst_pte = make_pte_marker(PTE_MARKER_POISONED); 6705 - set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, 6706 - huge_page_size(h)); 6706 + set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size); 6707 6707 6708 6708 /* No need to invalidate - it was non-present before */ 6709 6709 update_mmu_cache(dst_vma, dst_addr, dst_pte); ··· 6776 6778 *foliop = NULL; 6777 6779 goto out; 6778 6780 } 6779 - ret = copy_user_large_folio(folio, *foliop, dst_addr, dst_vma); 6781 + ret = copy_user_large_folio(folio, *foliop, 6782 + ALIGN_DOWN(dst_addr, size), dst_vma); 6780 6783 folio_put(*foliop); 6781 6784 *foliop = NULL; 6782 6785 if (ret) { ··· 6804 6805 6805 6806 /* Add shared, newly allocated pages to the page cache. */ 6806 6807 if (vm_shared && !is_continue) { 6807 - size = i_size_read(mapping->host) >> huge_page_shift(h); 6808 6808 ret = -EFAULT; 6809 - if (idx >= size) 6809 + if (idx >= (i_size_read(mapping->host) >> huge_page_shift(h))) 6810 6810 goto out_release_nounlock; 6811 6811 6812 6812 /* ··· 6862 6864 if (wp_enabled) 6863 6865 _dst_pte = huge_pte_mkuffd_wp(_dst_pte); 6864 6866 6865 - set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, huge_page_size(h)); 6867 + set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size); 6866 6868 6867 6869 hugetlb_count_add(pages_per_huge_page(h), dst_mm); 6868 6870

+4 -7

mm/memory.c

··· 6521 6521 int copy_user_large_folio(struct folio *dst, struct folio *src, 6522 6522 unsigned long addr_hint, struct vm_area_struct *vma) 6523 6523 { 6524 - unsigned int pages_per_huge_page = folio_nr_pages(dst); 6525 - unsigned long addr = addr_hint & 6526 - ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1); 6524 + unsigned int nr_pages = folio_nr_pages(dst); 6527 6525 struct copy_subpage_arg arg = { 6528 6526 .dst = dst, 6529 6527 .src = src, 6530 6528 .vma = vma, 6531 6529 }; 6532 6530 6533 - if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) 6534 - return copy_user_gigantic_page(dst, src, addr, vma, 6535 - pages_per_huge_page); 6531 + if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) 6532 + return copy_user_gigantic_page(dst, src, addr_hint, vma, nr_pages); 6536 6533 6537 - return process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg); 6534 + return process_huge_page(addr_hint, nr_pages, copy_subpage, &arg); 6538 6535 } 6539 6536 6540 6537 long copy_folio_from_user(struct folio *dst_folio,

Configure Feed

Configure Feed