Merge branch 'slab/for-6.19/memdesc_prep' into slab/for-next

+2 -14

include/linux/page-flags.h

··· 1048 1048 */ 1049 1049 PAGE_TYPE_OPS(Guard, guard, guard) 1050 1050 1051 - FOLIO_TYPE_OPS(slab, slab) 1052 - 1053 - /** 1054 - * PageSlab - Determine if the page belongs to the slab allocator 1055 - * @page: The page to test. 1056 - * 1057 - * Context: Any context. 1058 - * Return: True for slab pages, false for any other kind of page. 1059 - */ 1060 - static inline bool PageSlab(const struct page *page) 1061 - { 1062 - return folio_test_slab(page_folio(page)); 1063 - } 1051 + PAGE_TYPE_OPS(Slab, slab, slab) 1064 1052 1065 1053 #ifdef CONFIG_HUGETLB_PAGE 1066 1054 FOLIO_TYPE_OPS(hugetlb, hugetlb) ··· 1064 1076 * Serialized with zone lock. 1065 1077 */ 1066 1078 PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted) 1067 - FOLIO_TYPE_OPS(large_kmalloc, large_kmalloc) 1079 + PAGE_TYPE_OPS(LargeKmalloc, large_kmalloc, large_kmalloc) 1068 1080 1069 1081 /** 1070 1082 * PageHuge - Determine if the page belongs to hugetlbfs

+4 -8

mm/kasan/common.c

··· 520 520 521 521 bool __kasan_mempool_poison_object(void *ptr, unsigned long ip) 522 522 { 523 - struct folio *folio = virt_to_folio(ptr); 523 + struct page *page = virt_to_page(ptr); 524 524 struct slab *slab; 525 525 526 - /* 527 - * This function can be called for large kmalloc allocation that get 528 - * their memory from page_alloc. Thus, the folio might not be a slab. 529 - */ 530 - if (unlikely(!folio_test_slab(folio))) { 526 + if (unlikely(PageLargeKmalloc(page))) { 531 527 if (check_page_allocation(ptr, ip)) 532 528 return false; 533 - kasan_poison(ptr, folio_size(folio), KASAN_PAGE_FREE, false); 529 + kasan_poison(ptr, page_size(page), KASAN_PAGE_FREE, false); 534 530 return true; 535 531 } 536 532 537 533 if (is_kfence_address(ptr)) 538 534 return true; 539 535 540 - slab = folio_slab(folio); 536 + slab = page_slab(page); 541 537 542 538 if (check_slab_allocation(slab->slab_cache, ptr, ip)) 543 539 return false;

+8 -6

mm/kfence/core.c

··· 612 612 * enters __slab_free() slow-path. 613 613 */ 614 614 for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) { 615 - struct slab *slab; 615 + struct page *page; 616 616 617 617 if (!i || (i % 2)) 618 618 continue; 619 619 620 - slab = page_slab(pfn_to_page(start_pfn + i)); 621 - __folio_set_slab(slab_folio(slab)); 620 + page = pfn_to_page(start_pfn + i); 621 + __SetPageSlab(page); 622 622 #ifdef CONFIG_MEMCG 623 + struct slab *slab = page_slab(page); 623 624 slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts | 624 625 MEMCG_DATA_OBJEXTS; 625 626 #endif ··· 666 665 667 666 reset_slab: 668 667 for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) { 669 - struct slab *slab; 668 + struct page *page; 670 669 671 670 if (!i || (i % 2)) 672 671 continue; 673 672 674 - slab = page_slab(pfn_to_page(start_pfn + i)); 673 + page = pfn_to_page(start_pfn + i); 675 674 #ifdef CONFIG_MEMCG 675 + struct slab *slab = page_slab(page); 676 676 slab->obj_exts = 0; 677 677 #endif 678 - __folio_clear_slab(slab_folio(slab)); 678 + __ClearPageSlab(page); 679 679 } 680 680 681 681 return addr;

+16 -24

mm/memcontrol.c

··· 2557 2557 } 2558 2558 2559 2559 static __always_inline 2560 - struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p) 2560 + struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p) 2561 2561 { 2562 2562 /* 2563 2563 * Slab objects are accounted individually, not per-page. 2564 2564 * Memcg membership data for each individual object is saved in 2565 2565 * slab->obj_exts. 2566 2566 */ 2567 - if (folio_test_slab(folio)) { 2568 - struct slabobj_ext *obj_exts; 2569 - struct slab *slab; 2570 - unsigned int off; 2567 + struct slabobj_ext *obj_exts; 2568 + unsigned int off; 2571 2569 2572 - slab = folio_slab(folio); 2573 - obj_exts = slab_obj_exts(slab); 2574 - if (!obj_exts) 2575 - return NULL; 2576 - 2577 - off = obj_to_index(slab->slab_cache, slab, p); 2578 - if (obj_exts[off].objcg) 2579 - return obj_cgroup_memcg(obj_exts[off].objcg); 2580 - 2570 + obj_exts = slab_obj_exts(slab); 2571 + if (!obj_exts) 2581 2572 return NULL; 2582 - } 2583 2573 2584 - /* 2585 - * folio_memcg_check() is used here, because in theory we can encounter 2586 - * a folio where the slab flag has been cleared already, but 2587 - * slab->obj_exts has not been freed yet 2588 - * folio_memcg_check() will guarantee that a proper memory 2589 - * cgroup pointer or NULL will be returned. 2590 - */ 2591 - return folio_memcg_check(folio); 2574 + off = obj_to_index(slab->slab_cache, slab, p); 2575 + if (obj_exts[off].objcg) 2576 + return obj_cgroup_memcg(obj_exts[off].objcg); 2577 + 2578 + return NULL; 2592 2579 } 2593 2580 2594 2581 /* ··· 2589 2602 */ 2590 2603 struct mem_cgroup *mem_cgroup_from_slab_obj(void *p) 2591 2604 { 2605 + struct slab *slab; 2606 + 2592 2607 if (mem_cgroup_disabled()) 2593 2608 return NULL; 2594 2609 2595 - return mem_cgroup_from_obj_folio(virt_to_folio(p), p); 2610 + slab = virt_to_slab(p); 2611 + if (slab) 2612 + return mem_cgroup_from_obj_slab(slab, p); 2613 + return folio_memcg_check(virt_to_folio(p)); 2596 2614 } 2597 2615 2598 2616 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)

+27 -31

mm/slab.h

··· 118 118 #endif 119 119 120 120 /** 121 - * folio_slab - Converts from folio to slab. 122 - * @folio: The folio. 123 - * 124 - * Currently struct slab is a different representation of a folio where 125 - * folio_test_slab() is true. 126 - * 127 - * Return: The slab which contains this folio. 128 - */ 129 - #define folio_slab(folio) (_Generic((folio), \ 130 - const struct folio *: (const struct slab *)(folio), \ 131 - struct folio *: (struct slab *)(folio))) 132 - 133 - /** 134 121 * slab_folio - The folio allocated for a slab 135 122 * @s: The slab. 136 123 * ··· 133 146 struct slab *: (struct folio *)s)) 134 147 135 148 /** 136 - * page_slab - Converts from first struct page to slab. 137 - * @p: The first (either head of compound or single) page of slab. 149 + * page_slab - Converts from struct page to its slab. 150 + * @page: A page which may or may not belong to a slab. 138 151 * 139 - * A temporary wrapper to convert struct page to struct slab in situations where 140 - * we know the page is the compound head, or single order-0 page. 141 - * 142 - * Long-term ideally everything would work with struct slab directly or go 143 - * through folio to struct slab. 144 - * 145 - * Return: The slab which contains this page 152 + * Return: The slab which contains this page or NULL if the page does 153 + * not belong to a slab. This includes pages returned from large kmalloc. 146 154 */ 147 - #define page_slab(p) (_Generic((p), \ 148 - const struct page *: (const struct slab *)(p), \ 149 - struct page *: (struct slab *)(p))) 155 + static inline struct slab *page_slab(const struct page *page) 156 + { 157 + unsigned long head; 158 + 159 + head = READ_ONCE(page->compound_head); 160 + if (head & 1) 161 + page = (struct page *)(head - 1); 162 + if (data_race(page->page_type >> 24) != PGTY_slab) 163 + page = NULL; 164 + 165 + return (struct slab *)page; 166 + } 150 167 151 168 /** 152 169 * slab_page - The first struct page allocated for a slab ··· 179 188 180 189 static inline struct slab *virt_to_slab(const void *addr) 181 190 { 182 - struct folio *folio = virt_to_folio(addr); 183 - 184 - if (!folio_test_slab(folio)) 185 - return NULL; 186 - 187 - return folio_slab(folio); 191 + return page_slab(virt_to_page(addr)); 188 192 } 189 193 190 194 static inline int slab_order(const struct slab *slab) ··· 583 597 * Else we can use all the padding etc for the allocation 584 598 */ 585 599 return s->size; 600 + } 601 + 602 + static inline unsigned int large_kmalloc_order(const struct page *page) 603 + { 604 + return page[1].flags.f & 0xff; 605 + } 606 + 607 + static inline size_t large_kmalloc_size(const struct page *page) 608 + { 609 + return PAGE_SIZE << large_kmalloc_order(page); 586 610 } 587 611 588 612 #ifdef CONFIG_SLUB_DEBUG

+14 -15

mm/slab_common.c

··· 997 997 */ 998 998 size_t __ksize(const void *object) 999 999 { 1000 - struct folio *folio; 1000 + const struct page *page; 1001 + const struct slab *slab; 1001 1002 1002 1003 if (unlikely(object == ZERO_SIZE_PTR)) 1003 1004 return 0; 1004 1005 1005 - folio = virt_to_folio(object); 1006 + page = virt_to_page(object); 1006 1007 1007 - if (unlikely(!folio_test_slab(folio))) { 1008 - if (WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE)) 1009 - return 0; 1010 - if (WARN_ON(object != folio_address(folio))) 1011 - return 0; 1012 - return folio_size(folio); 1013 - } 1008 + if (unlikely(PageLargeKmalloc(page))) 1009 + return large_kmalloc_size(page); 1010 + 1011 + slab = page_slab(page); 1012 + /* Delete this after we're sure there are no users */ 1013 + if (WARN_ON(!slab)) 1014 + return page_size(page); 1014 1015 1015 1016 #ifdef CONFIG_SLUB_DEBUG 1016 - skip_orig_size_check(folio_slab(folio)->slab_cache, object); 1017 + skip_orig_size_check(slab->slab_cache, object); 1017 1018 #endif 1018 1019 1019 - return slab_ksize(folio_slab(folio)->slab_cache); 1020 + return slab_ksize(slab->slab_cache); 1020 1021 } 1021 1022 1022 1023 gfp_t kmalloc_fix_flags(gfp_t flags) ··· 1615 1614 static bool kfree_rcu_sheaf(void *obj) 1616 1615 { 1617 1616 struct kmem_cache *s; 1618 - struct folio *folio; 1619 1617 struct slab *slab; 1620 1618 1621 1619 if (is_vmalloc_addr(obj)) 1622 1620 return false; 1623 1621 1624 - folio = virt_to_folio(obj); 1625 - if (unlikely(!folio_test_slab(folio))) 1622 + slab = virt_to_slab(obj); 1623 + if (unlikely(!slab)) 1626 1624 return false; 1627 1625 1628 - slab = folio_slab(folio); 1629 1626 s = slab->slab_cache; 1630 1627 if (s->cpu_sheaves) { 1631 1628 if (likely(!IS_ENABLED(CONFIG_NUMA) ||

+70 -69

mm/slub.c

··· 2372 2372 { 2373 2373 struct slabobj_ext *slab_exts; 2374 2374 struct kmem_cache *s; 2375 - struct folio *folio; 2375 + struct page *page; 2376 2376 struct slab *slab; 2377 2377 unsigned long off; 2378 2378 2379 - folio = virt_to_folio(p); 2380 - if (!folio_test_slab(folio)) { 2379 + page = virt_to_page(p); 2380 + if (PageLargeKmalloc(page)) { 2381 + unsigned int order; 2381 2382 int size; 2382 2383 2383 - if (folio_memcg_kmem(folio)) 2384 + if (PageMemcgKmem(page)) 2384 2385 return true; 2385 2386 2386 - if (__memcg_kmem_charge_page(folio_page(folio, 0), flags, 2387 - folio_order(folio))) 2387 + order = large_kmalloc_order(page); 2388 + if (__memcg_kmem_charge_page(page, flags, order)) 2388 2389 return false; 2389 2390 2390 2391 /* 2391 - * This folio has already been accounted in the global stats but 2392 + * This page has already been accounted in the global stats but 2392 2393 * not in the memcg stats. So, subtract from the global and use 2393 2394 * the interface which adds to both global and memcg stats. 2394 2395 */ 2395 - size = folio_size(folio); 2396 - node_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, -size); 2397 - lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, size); 2396 + size = PAGE_SIZE << order; 2397 + mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B, -size); 2398 + mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, size); 2398 2399 return true; 2399 2400 } 2400 2401 2401 - slab = folio_slab(folio); 2402 + slab = page_slab(page); 2402 2403 s = slab->slab_cache; 2403 2404 2404 2405 /* ··· 3067 3066 struct kmem_cache_order_objects oo, 3068 3067 bool allow_spin) 3069 3068 { 3070 - struct folio *folio; 3069 + struct page *page; 3071 3070 struct slab *slab; 3072 3071 unsigned int order = oo_order(oo); 3073 3072 3074 3073 if (unlikely(!allow_spin)) 3075 - folio = (struct folio *)alloc_frozen_pages_nolock(0/* __GFP_COMP is implied */, 3074 + page = alloc_frozen_pages_nolock(0/* __GFP_COMP is implied */, 3076 3075 node, order); 3077 3076 else if (node == NUMA_NO_NODE) 3078 - folio = (struct folio *)alloc_frozen_pages(flags, order); 3077 + page = alloc_frozen_pages(flags, order); 3079 3078 else 3080 - folio = (struct folio *)__alloc_frozen_pages(flags, order, node, NULL); 3079 + page = __alloc_frozen_pages(flags, order, node, NULL); 3081 3080 3082 - if (!folio) 3081 + if (!page) 3083 3082 return NULL; 3084 3083 3085 - slab = folio_slab(folio); 3086 - __folio_set_slab(folio); 3087 - if (folio_is_pfmemalloc(folio)) 3084 + __SetPageSlab(page); 3085 + slab = page_slab(page); 3086 + if (page_is_pfmemalloc(page)) 3088 3087 slab_set_pfmemalloc(slab); 3089 3088 3090 3089 return slab; ··· 3308 3307 3309 3308 static void __free_slab(struct kmem_cache *s, struct slab *slab) 3310 3309 { 3311 - struct folio *folio = slab_folio(slab); 3312 - int order = folio_order(folio); 3310 + struct page *page = slab_page(slab); 3311 + int order = compound_order(page); 3313 3312 int pages = 1 << order; 3314 3313 3315 3314 __slab_clear_pfmemalloc(slab); 3316 - folio->mapping = NULL; 3317 - __folio_clear_slab(folio); 3315 + page->mapping = NULL; 3316 + __ClearPageSlab(page); 3318 3317 mm_account_reclaimed_pages(pages); 3319 3318 unaccount_slab(slab, order, s); 3320 - free_frozen_pages(&folio->page, order); 3319 + free_frozen_pages(page, order); 3321 3320 } 3322 3321 3323 3322 static void rcu_free_slab(struct rcu_head *h) ··· 5140 5139 * be false because of cpu migration during an unlocked part of 5141 5140 * the current allocation or previous freeing process. 5142 5141 */ 5143 - if (folio_nid(virt_to_folio(object)) != node) { 5142 + if (page_to_nid(virt_to_page(object)) != node) { 5144 5143 local_unlock(&s->cpu_sheaves->lock); 5145 5144 return NULL; 5146 5145 } ··· 5594 5593 */ 5595 5594 static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) 5596 5595 { 5597 - struct folio *folio; 5596 + struct page *page; 5598 5597 void *ptr = NULL; 5599 5598 unsigned int order = get_order(size); 5600 5599 ··· 5604 5603 flags |= __GFP_COMP; 5605 5604 5606 5605 if (node == NUMA_NO_NODE) 5607 - folio = (struct folio *)alloc_frozen_pages_noprof(flags, order); 5606 + page = alloc_frozen_pages_noprof(flags, order); 5608 5607 else 5609 - folio = (struct folio *)__alloc_frozen_pages_noprof(flags, order, node, NULL); 5608 + page = __alloc_frozen_pages_noprof(flags, order, node, NULL); 5610 5609 5611 - if (folio) { 5612 - ptr = folio_address(folio); 5613 - lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, 5610 + if (page) { 5611 + ptr = page_address(page); 5612 + mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, 5614 5613 PAGE_SIZE << order); 5615 - __folio_set_large_kmalloc(folio); 5614 + __SetPageLargeKmalloc(page); 5616 5615 } 5617 5616 5618 5617 ptr = kasan_kmalloc_large(ptr, size, flags); ··· 6784 6783 } 6785 6784 EXPORT_SYMBOL(kmem_cache_free); 6786 6785 6787 - static void free_large_kmalloc(struct folio *folio, void *object) 6786 + static void free_large_kmalloc(struct page *page, void *object) 6788 6787 { 6789 - unsigned int order = folio_order(folio); 6788 + unsigned int order = compound_order(page); 6790 6789 6791 - if (WARN_ON_ONCE(!folio_test_large_kmalloc(folio))) { 6792 - dump_page(&folio->page, "Not a kmalloc allocation"); 6790 + if (WARN_ON_ONCE(!PageLargeKmalloc(page))) { 6791 + dump_page(page, "Not a kmalloc allocation"); 6793 6792 return; 6794 6793 } 6795 6794 ··· 6800 6799 kasan_kfree_large(object); 6801 6800 kmsan_kfree_large(object); 6802 6801 6803 - lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, 6802 + mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, 6804 6803 -(PAGE_SIZE << order)); 6805 - __folio_clear_large_kmalloc(folio); 6806 - free_frozen_pages(&folio->page, order); 6804 + __ClearPageLargeKmalloc(page); 6805 + free_frozen_pages(page, order); 6807 6806 } 6808 6807 6809 6808 /* ··· 6813 6812 void kvfree_rcu_cb(struct rcu_head *head) 6814 6813 { 6815 6814 void *obj = head; 6816 - struct folio *folio; 6815 + struct page *page; 6817 6816 struct slab *slab; 6818 6817 struct kmem_cache *s; 6819 6818 void *slab_addr; ··· 6824 6823 return; 6825 6824 } 6826 6825 6827 - folio = virt_to_folio(obj); 6828 - if (!folio_test_slab(folio)) { 6826 + page = virt_to_page(obj); 6827 + slab = page_slab(page); 6828 + if (!slab) { 6829 6829 /* 6830 6830 * rcu_head offset can be only less than page size so no need to 6831 - * consider folio order 6831 + * consider allocation order 6832 6832 */ 6833 6833 obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); 6834 - free_large_kmalloc(folio, obj); 6834 + free_large_kmalloc(page, obj); 6835 6835 return; 6836 6836 } 6837 6837 6838 - slab = folio_slab(folio); 6839 6838 s = slab->slab_cache; 6840 - slab_addr = folio_address(folio); 6839 + slab_addr = slab_address(slab); 6841 6840 6842 6841 if (is_kfence_address(obj)) { 6843 6842 obj = kfence_object_start(obj); ··· 6859 6858 */ 6860 6859 void kfree(const void *object) 6861 6860 { 6862 - struct folio *folio; 6861 + struct page *page; 6863 6862 struct slab *slab; 6864 6863 struct kmem_cache *s; 6865 6864 void *x = (void *)object; ··· 6869 6868 if (unlikely(ZERO_OR_NULL_PTR(object))) 6870 6869 return; 6871 6870 6872 - folio = virt_to_folio(object); 6873 - if (unlikely(!folio_test_slab(folio))) { 6874 - free_large_kmalloc(folio, (void *)object); 6871 + page = virt_to_page(object); 6872 + slab = page_slab(page); 6873 + if (!slab) { 6874 + free_large_kmalloc(page, (void *)object); 6875 6875 return; 6876 6876 } 6877 6877 6878 - slab = folio_slab(folio); 6879 6878 s = slab->slab_cache; 6880 6879 slab_free(s, slab, x, _RET_IP_); 6881 6880 } ··· 6892 6891 */ 6893 6892 void kfree_nolock(const void *object) 6894 6893 { 6895 - struct folio *folio; 6896 6894 struct slab *slab; 6897 6895 struct kmem_cache *s; 6898 6896 void *x = (void *)object; ··· 6899 6899 if (unlikely(ZERO_OR_NULL_PTR(object))) 6900 6900 return; 6901 6901 6902 - folio = virt_to_folio(object); 6903 - if (unlikely(!folio_test_slab(folio))) { 6902 + slab = virt_to_slab(object); 6903 + if (unlikely(!slab)) { 6904 6904 WARN_ONCE(1, "large_kmalloc is not supported by kfree_nolock()"); 6905 6905 return; 6906 6906 } 6907 6907 6908 - slab = folio_slab(folio); 6909 6908 s = slab->slab_cache; 6910 6909 6911 6910 memcg_slab_free_hook(s, slab, &x, 1); ··· 6968 6969 if (is_kfence_address(p)) { 6969 6970 ks = orig_size = kfence_ksize(p); 6970 6971 } else { 6971 - struct folio *folio; 6972 + struct page *page = virt_to_page(p); 6973 + struct slab *slab = page_slab(page); 6972 6974 6973 - folio = virt_to_folio(p); 6974 - if (unlikely(!folio_test_slab(folio))) { 6975 + if (!slab) { 6975 6976 /* Big kmalloc object */ 6976 - WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE); 6977 - WARN_ON(p != folio_address(folio)); 6978 - ks = folio_size(folio); 6977 + ks = page_size(page); 6978 + WARN_ON(ks <= KMALLOC_MAX_CACHE_SIZE); 6979 + WARN_ON(p != page_address(page)); 6979 6980 } else { 6980 - s = folio_slab(folio)->slab_cache; 6981 + s = slab->slab_cache; 6981 6982 orig_size = get_orig_size(s, (void *)p); 6982 6983 ks = s->object_size; 6983 6984 } ··· 7281 7282 { 7282 7283 int lookahead = 3; 7283 7284 void *object; 7284 - struct folio *folio; 7285 + struct page *page; 7286 + struct slab *slab; 7285 7287 size_t same; 7286 7288 7287 7289 object = p[--size]; 7288 - folio = virt_to_folio(object); 7290 + page = virt_to_page(object); 7291 + slab = page_slab(page); 7289 7292 if (!s) { 7290 7293 /* Handle kalloc'ed objects */ 7291 - if (unlikely(!folio_test_slab(folio))) { 7292 - free_large_kmalloc(folio, object); 7294 + if (!slab) { 7295 + free_large_kmalloc(page, object); 7293 7296 df->slab = NULL; 7294 7297 return size; 7295 7298 } 7296 7299 /* Derive kmem_cache from object */ 7297 - df->slab = folio_slab(folio); 7298 - df->s = df->slab->slab_cache; 7300 + df->slab = slab; 7301 + df->s = slab->slab_cache; 7299 7302 } else { 7300 - df->slab = folio_slab(folio); 7303 + df->slab = slab; 7301 7304 df->s = cache_from_obj(s, object); /* Support for memcg */ 7302 7305 } 7303 7306

+16 -8

mm/usercopy.c

··· 164 164 { 165 165 unsigned long addr = (unsigned long)ptr; 166 166 unsigned long offset; 167 - struct folio *folio; 167 + struct page *page; 168 + struct slab *slab; 168 169 169 170 if (is_kmap_addr(ptr)) { 170 171 offset = offset_in_page(ptr); ··· 190 189 if (!virt_addr_valid(ptr)) 191 190 return; 192 191 193 - folio = virt_to_folio(ptr); 194 - 195 - if (folio_test_slab(folio)) { 192 + page = virt_to_page(ptr); 193 + slab = page_slab(page); 194 + if (slab) { 196 195 /* Check slab allocator for flags and size. */ 197 - __check_heap_object(ptr, n, folio_slab(folio), to_user); 198 - } else if (folio_test_large(folio)) { 199 - offset = ptr - folio_address(folio); 200 - if (n > folio_size(folio) - offset) 196 + __check_heap_object(ptr, n, slab, to_user); 197 + } else if (PageCompound(page)) { 198 + page = compound_head(page); 199 + offset = ptr - page_address(page); 200 + if (n > page_size(page) - offset) 201 201 usercopy_abort("page alloc", NULL, to_user, offset, n); 202 202 } 203 + 204 + /* 205 + * We cannot check non-compound pages. They might be part of 206 + * a large allocation, in which case crossing a page boundary 207 + * is fine. 208 + */ 203 209 } 204 210 205 211 DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_HARDENED_USERCOPY_DEFAULT_ON,

Configure Feed

Configure Feed