mm/rmap: convert "enum rmap_level" to "enum pgtable_level"

+8

include/linux/pgtable.h

··· 1975 1975 /* Page-Table Modification Mask */ 1976 1976 typedef unsigned int pgtbl_mod_mask; 1977 1977 1978 + enum pgtable_level { 1979 + PGTABLE_LEVEL_PTE = 0, 1980 + PGTABLE_LEVEL_PMD, 1981 + PGTABLE_LEVEL_PUD, 1982 + PGTABLE_LEVEL_P4D, 1983 + PGTABLE_LEVEL_PGD, 1984 + }; 1985 + 1978 1986 #endif /* !__ASSEMBLY__ */ 1979 1987 1980 1988 #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)

+28 -34

include/linux/rmap.h

··· 394 394 /* The anonymous (sub)page is exclusive to a single process. */ 395 395 #define RMAP_EXCLUSIVE ((__force rmap_t)BIT(0)) 396 396 397 - /* 398 - * Internally, we're using an enum to specify the granularity. We make the 399 - * compiler emit specialized code for each granularity. 400 - */ 401 - enum rmap_level { 402 - RMAP_LEVEL_PTE = 0, 403 - RMAP_LEVEL_PMD, 404 - RMAP_LEVEL_PUD, 405 - }; 406 - 407 - static inline void __folio_rmap_sanity_checks(const struct folio *folio, 408 - const struct page *page, int nr_pages, enum rmap_level level) 397 + static __always_inline void __folio_rmap_sanity_checks(const struct folio *folio, 398 + const struct page *page, int nr_pages, enum pgtable_level level) 409 399 { 410 400 /* hugetlb folios are handled separately. */ 411 401 VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); ··· 417 427 VM_WARN_ON_FOLIO(page_folio(page + nr_pages - 1) != folio, folio); 418 428 419 429 switch (level) { 420 - case RMAP_LEVEL_PTE: 430 + case PGTABLE_LEVEL_PTE: 421 431 break; 422 - case RMAP_LEVEL_PMD: 432 + case PGTABLE_LEVEL_PMD: 423 433 /* 424 434 * We don't support folios larger than a single PMD yet. So 425 - * when RMAP_LEVEL_PMD is set, we assume that we are creating 435 + * when PGTABLE_LEVEL_PMD is set, we assume that we are creating 426 436 * a single "entire" mapping of the folio. 427 437 */ 428 438 VM_WARN_ON_FOLIO(folio_nr_pages(folio) != HPAGE_PMD_NR, folio); 429 439 VM_WARN_ON_FOLIO(nr_pages != HPAGE_PMD_NR, folio); 430 440 break; 431 - case RMAP_LEVEL_PUD: 441 + case PGTABLE_LEVEL_PUD: 432 442 /* 433 443 * Assume that we are creating a single "entire" mapping of the 434 444 * folio. ··· 437 447 VM_WARN_ON_FOLIO(nr_pages != HPAGE_PUD_NR, folio); 438 448 break; 439 449 default: 440 - VM_WARN_ON_ONCE(true); 450 + BUILD_BUG(); 441 451 } 442 452 443 453 /* ··· 557 567 558 568 static __always_inline void __folio_dup_file_rmap(struct folio *folio, 559 569 struct page *page, int nr_pages, struct vm_area_struct *dst_vma, 560 - enum rmap_level level) 570 + enum pgtable_level level) 561 571 { 562 572 const int orig_nr_pages = nr_pages; 563 573 564 574 __folio_rmap_sanity_checks(folio, page, nr_pages, level); 565 575 566 576 switch (level) { 567 - case RMAP_LEVEL_PTE: 577 + case PGTABLE_LEVEL_PTE: 568 578 if (!folio_test_large(folio)) { 569 579 atomic_inc(&folio->_mapcount); 570 580 break; ··· 577 587 } 578 588 folio_add_large_mapcount(folio, orig_nr_pages, dst_vma); 579 589 break; 580 - case RMAP_LEVEL_PMD: 581 - case RMAP_LEVEL_PUD: 590 + case PGTABLE_LEVEL_PMD: 591 + case PGTABLE_LEVEL_PUD: 582 592 atomic_inc(&folio->_entire_mapcount); 583 593 folio_inc_large_mapcount(folio, dst_vma); 584 594 break; 595 + default: 596 + BUILD_BUG(); 585 597 } 586 598 } 587 599 ··· 601 609 static inline void folio_dup_file_rmap_ptes(struct folio *folio, 602 610 struct page *page, int nr_pages, struct vm_area_struct *dst_vma) 603 611 { 604 - __folio_dup_file_rmap(folio, page, nr_pages, dst_vma, RMAP_LEVEL_PTE); 612 + __folio_dup_file_rmap(folio, page, nr_pages, dst_vma, PGTABLE_LEVEL_PTE); 605 613 } 606 614 607 615 static __always_inline void folio_dup_file_rmap_pte(struct folio *folio, 608 616 struct page *page, struct vm_area_struct *dst_vma) 609 617 { 610 - __folio_dup_file_rmap(folio, page, 1, dst_vma, RMAP_LEVEL_PTE); 618 + __folio_dup_file_rmap(folio, page, 1, dst_vma, PGTABLE_LEVEL_PTE); 611 619 } 612 620 613 621 /** ··· 624 632 struct page *page, struct vm_area_struct *dst_vma) 625 633 { 626 634 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 627 - __folio_dup_file_rmap(folio, page, HPAGE_PMD_NR, dst_vma, RMAP_LEVEL_PTE); 635 + __folio_dup_file_rmap(folio, page, HPAGE_PMD_NR, dst_vma, PGTABLE_LEVEL_PTE); 628 636 #else 629 637 WARN_ON_ONCE(true); 630 638 #endif ··· 632 640 633 641 static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio, 634 642 struct page *page, int nr_pages, struct vm_area_struct *dst_vma, 635 - struct vm_area_struct *src_vma, enum rmap_level level) 643 + struct vm_area_struct *src_vma, enum pgtable_level level) 636 644 { 637 645 const int orig_nr_pages = nr_pages; 638 646 bool maybe_pinned; ··· 657 665 * copying if the folio maybe pinned. 658 666 */ 659 667 switch (level) { 660 - case RMAP_LEVEL_PTE: 668 + case PGTABLE_LEVEL_PTE: 661 669 if (unlikely(maybe_pinned)) { 662 670 for (i = 0; i < nr_pages; i++) 663 671 if (PageAnonExclusive(page + i)) ··· 679 687 } while (page++, --nr_pages > 0); 680 688 folio_add_large_mapcount(folio, orig_nr_pages, dst_vma); 681 689 break; 682 - case RMAP_LEVEL_PMD: 683 - case RMAP_LEVEL_PUD: 690 + case PGTABLE_LEVEL_PMD: 691 + case PGTABLE_LEVEL_PUD: 684 692 if (PageAnonExclusive(page)) { 685 693 if (unlikely(maybe_pinned)) 686 694 return -EBUSY; ··· 689 697 atomic_inc(&folio->_entire_mapcount); 690 698 folio_inc_large_mapcount(folio, dst_vma); 691 699 break; 700 + default: 701 + BUILD_BUG(); 692 702 } 693 703 return 0; 694 704 } ··· 724 730 struct vm_area_struct *src_vma) 725 731 { 726 732 return __folio_try_dup_anon_rmap(folio, page, nr_pages, dst_vma, 727 - src_vma, RMAP_LEVEL_PTE); 733 + src_vma, PGTABLE_LEVEL_PTE); 728 734 } 729 735 730 736 static __always_inline int folio_try_dup_anon_rmap_pte(struct folio *folio, ··· 732 738 struct vm_area_struct *src_vma) 733 739 { 734 740 return __folio_try_dup_anon_rmap(folio, page, 1, dst_vma, src_vma, 735 - RMAP_LEVEL_PTE); 741 + PGTABLE_LEVEL_PTE); 736 742 } 737 743 738 744 /** ··· 764 770 { 765 771 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 766 772 return __folio_try_dup_anon_rmap(folio, page, HPAGE_PMD_NR, dst_vma, 767 - src_vma, RMAP_LEVEL_PMD); 773 + src_vma, PGTABLE_LEVEL_PMD); 768 774 #else 769 775 WARN_ON_ONCE(true); 770 776 return -EBUSY; ··· 772 778 } 773 779 774 780 static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, 775 - struct page *page, int nr_pages, enum rmap_level level) 781 + struct page *page, int nr_pages, enum pgtable_level level) 776 782 { 777 783 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 778 784 VM_WARN_ON_FOLIO(!PageAnonExclusive(page), folio); ··· 867 873 static inline int folio_try_share_anon_rmap_pte(struct folio *folio, 868 874 struct page *page) 869 875 { 870 - return __folio_try_share_anon_rmap(folio, page, 1, RMAP_LEVEL_PTE); 876 + return __folio_try_share_anon_rmap(folio, page, 1, PGTABLE_LEVEL_PTE); 871 877 } 872 878 873 879 /** ··· 898 904 { 899 905 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 900 906 return __folio_try_share_anon_rmap(folio, page, HPAGE_PMD_NR, 901 - RMAP_LEVEL_PMD); 907 + PGTABLE_LEVEL_PMD); 902 908 #else 903 909 WARN_ON_ONCE(true); 904 910 return -EBUSY;

+31 -25

mm/rmap.c

··· 1265 1265 1266 1266 static __always_inline void __folio_add_rmap(struct folio *folio, 1267 1267 struct page *page, int nr_pages, struct vm_area_struct *vma, 1268 - enum rmap_level level) 1268 + enum pgtable_level level) 1269 1269 { 1270 1270 atomic_t *mapped = &folio->_nr_pages_mapped; 1271 1271 const int orig_nr_pages = nr_pages; ··· 1274 1274 __folio_rmap_sanity_checks(folio, page, nr_pages, level); 1275 1275 1276 1276 switch (level) { 1277 - case RMAP_LEVEL_PTE: 1277 + case PGTABLE_LEVEL_PTE: 1278 1278 if (!folio_test_large(folio)) { 1279 1279 nr = atomic_inc_and_test(&folio->_mapcount); 1280 1280 break; ··· 1300 1300 1301 1301 folio_add_large_mapcount(folio, orig_nr_pages, vma); 1302 1302 break; 1303 - case RMAP_LEVEL_PMD: 1304 - case RMAP_LEVEL_PUD: 1303 + case PGTABLE_LEVEL_PMD: 1304 + case PGTABLE_LEVEL_PUD: 1305 1305 first = atomic_inc_and_test(&folio->_entire_mapcount); 1306 1306 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 1307 - if (level == RMAP_LEVEL_PMD && first) 1307 + if (level == PGTABLE_LEVEL_PMD && first) 1308 1308 nr_pmdmapped = folio_large_nr_pages(folio); 1309 1309 nr = folio_inc_return_large_mapcount(folio, vma); 1310 1310 if (nr == 1) ··· 1323 1323 * We only track PMD mappings of PMD-sized 1324 1324 * folios separately. 1325 1325 */ 1326 - if (level == RMAP_LEVEL_PMD) 1326 + if (level == PGTABLE_LEVEL_PMD) 1327 1327 nr_pmdmapped = nr_pages; 1328 1328 nr = nr_pages - (nr & FOLIO_PAGES_MAPPED); 1329 1329 /* Raced ahead of a remove and another add? */ ··· 1336 1336 } 1337 1337 folio_inc_large_mapcount(folio, vma); 1338 1338 break; 1339 + default: 1340 + BUILD_BUG(); 1339 1341 } 1340 1342 __folio_mod_stat(folio, nr, nr_pmdmapped); 1341 1343 } ··· 1429 1427 1430 1428 static __always_inline void __folio_add_anon_rmap(struct folio *folio, 1431 1429 struct page *page, int nr_pages, struct vm_area_struct *vma, 1432 - unsigned long address, rmap_t flags, enum rmap_level level) 1430 + unsigned long address, rmap_t flags, enum pgtable_level level) 1433 1431 { 1434 1432 int i; 1435 1433 ··· 1442 1440 1443 1441 if (flags & RMAP_EXCLUSIVE) { 1444 1442 switch (level) { 1445 - case RMAP_LEVEL_PTE: 1443 + case PGTABLE_LEVEL_PTE: 1446 1444 for (i = 0; i < nr_pages; i++) 1447 1445 SetPageAnonExclusive(page + i); 1448 1446 break; 1449 - case RMAP_LEVEL_PMD: 1447 + case PGTABLE_LEVEL_PMD: 1450 1448 SetPageAnonExclusive(page); 1451 1449 break; 1452 - case RMAP_LEVEL_PUD: 1450 + case PGTABLE_LEVEL_PUD: 1453 1451 /* 1454 1452 * Keep the compiler happy, we don't support anonymous 1455 1453 * PUD mappings. 1456 1454 */ 1457 1455 WARN_ON_ONCE(1); 1458 1456 break; 1457 + default: 1458 + BUILD_BUG(); 1459 1459 } 1460 1460 } 1461 1461 ··· 1511 1507 rmap_t flags) 1512 1508 { 1513 1509 __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, 1514 - RMAP_LEVEL_PTE); 1510 + PGTABLE_LEVEL_PTE); 1515 1511 } 1516 1512 1517 1513 /** ··· 1532 1528 { 1533 1529 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1534 1530 __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, 1535 - RMAP_LEVEL_PMD); 1531 + PGTABLE_LEVEL_PMD); 1536 1532 #else 1537 1533 WARN_ON_ONCE(true); 1538 1534 #endif ··· 1613 1609 1614 1610 static __always_inline void __folio_add_file_rmap(struct folio *folio, 1615 1611 struct page *page, int nr_pages, struct vm_area_struct *vma, 1616 - enum rmap_level level) 1612 + enum pgtable_level level) 1617 1613 { 1618 1614 VM_WARN_ON_FOLIO(folio_test_anon(folio), folio); 1619 1615 ··· 1638 1634 void folio_add_file_rmap_ptes(struct folio *folio, struct page *page, 1639 1635 int nr_pages, struct vm_area_struct *vma) 1640 1636 { 1641 - __folio_add_file_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE); 1637 + __folio_add_file_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE); 1642 1638 } 1643 1639 1644 1640 /** ··· 1655 1651 struct vm_area_struct *vma) 1656 1652 { 1657 1653 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1658 - __folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD); 1654 + __folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD); 1659 1655 #else 1660 1656 WARN_ON_ONCE(true); 1661 1657 #endif ··· 1676 1672 { 1677 1673 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1678 1674 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 1679 - __folio_add_file_rmap(folio, page, HPAGE_PUD_NR, vma, RMAP_LEVEL_PUD); 1675 + __folio_add_file_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD); 1680 1676 #else 1681 1677 WARN_ON_ONCE(true); 1682 1678 #endif ··· 1684 1680 1685 1681 static __always_inline void __folio_remove_rmap(struct folio *folio, 1686 1682 struct page *page, int nr_pages, struct vm_area_struct *vma, 1687 - enum rmap_level level) 1683 + enum pgtable_level level) 1688 1684 { 1689 1685 atomic_t *mapped = &folio->_nr_pages_mapped; 1690 1686 int last = 0, nr = 0, nr_pmdmapped = 0; ··· 1693 1689 __folio_rmap_sanity_checks(folio, page, nr_pages, level); 1694 1690 1695 1691 switch (level) { 1696 - case RMAP_LEVEL_PTE: 1692 + case PGTABLE_LEVEL_PTE: 1697 1693 if (!folio_test_large(folio)) { 1698 1694 nr = atomic_add_negative(-1, &folio->_mapcount); 1699 1695 break; ··· 1723 1719 1724 1720 partially_mapped = nr && atomic_read(mapped); 1725 1721 break; 1726 - case RMAP_LEVEL_PMD: 1727 - case RMAP_LEVEL_PUD: 1722 + case PGTABLE_LEVEL_PMD: 1723 + case PGTABLE_LEVEL_PUD: 1728 1724 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 1729 1725 last = atomic_add_negative(-1, &folio->_entire_mapcount); 1730 - if (level == RMAP_LEVEL_PMD && last) 1726 + if (level == PGTABLE_LEVEL_PMD && last) 1731 1727 nr_pmdmapped = folio_large_nr_pages(folio); 1732 1728 nr = folio_dec_return_large_mapcount(folio, vma); 1733 1729 if (!nr) { ··· 1747 1743 nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped); 1748 1744 if (likely(nr < ENTIRELY_MAPPED)) { 1749 1745 nr_pages = folio_large_nr_pages(folio); 1750 - if (level == RMAP_LEVEL_PMD) 1746 + if (level == PGTABLE_LEVEL_PMD) 1751 1747 nr_pmdmapped = nr_pages; 1752 1748 nr = nr_pages - (nr & FOLIO_PAGES_MAPPED); 1753 1749 /* Raced ahead of another remove and an add? */ ··· 1761 1757 1762 1758 partially_mapped = nr && nr < nr_pmdmapped; 1763 1759 break; 1760 + default: 1761 + BUILD_BUG(); 1764 1762 } 1765 1763 1766 1764 /* ··· 1802 1796 void folio_remove_rmap_ptes(struct folio *folio, struct page *page, 1803 1797 int nr_pages, struct vm_area_struct *vma) 1804 1798 { 1805 - __folio_remove_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE); 1799 + __folio_remove_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE); 1806 1800 } 1807 1801 1808 1802 /** ··· 1819 1813 struct vm_area_struct *vma) 1820 1814 { 1821 1815 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1822 - __folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD); 1816 + __folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD); 1823 1817 #else 1824 1818 WARN_ON_ONCE(true); 1825 1819 #endif ··· 1840 1834 { 1841 1835 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1842 1836 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 1843 - __folio_remove_rmap(folio, page, HPAGE_PUD_NR, vma, RMAP_LEVEL_PUD); 1837 + __folio_remove_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD); 1844 1838 #else 1845 1839 WARN_ON_ONCE(true); 1846 1840 #endif

Configure Feed

Configure Feed