Merge tag 'slab-for-5.20_or_6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab

+26 -14

include/trace/events/kmem.h

··· 13 13 14 14 TP_PROTO(unsigned long call_site, 15 15 const void *ptr, 16 + struct kmem_cache *s, 16 17 size_t bytes_req, 17 18 size_t bytes_alloc, 18 19 gfp_t gfp_flags), 19 20 20 - TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags), 21 + TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags), 21 22 22 23 TP_STRUCT__entry( 23 24 __field( unsigned long, call_site ) ··· 26 25 __field( size_t, bytes_req ) 27 26 __field( size_t, bytes_alloc ) 28 27 __field( unsigned long, gfp_flags ) 28 + __field( bool, accounted ) 29 29 ), 30 30 31 31 TP_fast_assign( ··· 35 33 __entry->bytes_req = bytes_req; 36 34 __entry->bytes_alloc = bytes_alloc; 37 35 __entry->gfp_flags = (__force unsigned long)gfp_flags; 36 + __entry->accounted = IS_ENABLED(CONFIG_MEMCG_KMEM) ? 37 + ((gfp_flags & __GFP_ACCOUNT) || 38 + (s && s->flags & SLAB_ACCOUNT)) : false; 38 39 ), 39 40 40 - TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s", 41 + TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s accounted=%s", 41 42 (void *)__entry->call_site, 42 43 __entry->ptr, 43 44 __entry->bytes_req, 44 45 __entry->bytes_alloc, 45 - show_gfp_flags(__entry->gfp_flags)) 46 + show_gfp_flags(__entry->gfp_flags), 47 + __entry->accounted ? "true" : "false") 46 48 ); 47 49 48 50 DEFINE_EVENT(kmem_alloc, kmalloc, 49 51 50 - TP_PROTO(unsigned long call_site, const void *ptr, 52 + TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s, 51 53 size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), 52 54 53 - TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags) 55 + TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags) 54 56 ); 55 57 56 58 DEFINE_EVENT(kmem_alloc, kmem_cache_alloc, 57 59 58 - TP_PROTO(unsigned long call_site, const void *ptr, 60 + TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s, 59 61 size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), 60 62 61 - TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags) 63 + TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags) 62 64 ); 63 65 64 66 DECLARE_EVENT_CLASS(kmem_alloc_node, 65 67 66 68 TP_PROTO(unsigned long call_site, 67 69 const void *ptr, 70 + struct kmem_cache *s, 68 71 size_t bytes_req, 69 72 size_t bytes_alloc, 70 73 gfp_t gfp_flags, 71 74 int node), 72 75 73 - TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node), 76 + TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node), 74 77 75 78 TP_STRUCT__entry( 76 79 __field( unsigned long, call_site ) ··· 84 77 __field( size_t, bytes_alloc ) 85 78 __field( unsigned long, gfp_flags ) 86 79 __field( int, node ) 80 + __field( bool, accounted ) 87 81 ), 88 82 89 83 TP_fast_assign( ··· 94 86 __entry->bytes_alloc = bytes_alloc; 95 87 __entry->gfp_flags = (__force unsigned long)gfp_flags; 96 88 __entry->node = node; 89 + __entry->accounted = IS_ENABLED(CONFIG_MEMCG_KMEM) ? 90 + ((gfp_flags & __GFP_ACCOUNT) || 91 + (s && s->flags & SLAB_ACCOUNT)) : false; 97 92 ), 98 93 99 - TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d", 94 + TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d accounted=%s", 100 95 (void *)__entry->call_site, 101 96 __entry->ptr, 102 97 __entry->bytes_req, 103 98 __entry->bytes_alloc, 104 99 show_gfp_flags(__entry->gfp_flags), 105 - __entry->node) 100 + __entry->node, 101 + __entry->accounted ? "true" : "false") 106 102 ); 107 103 108 104 DEFINE_EVENT(kmem_alloc_node, kmalloc_node, 109 105 110 106 TP_PROTO(unsigned long call_site, const void *ptr, 111 - size_t bytes_req, size_t bytes_alloc, 107 + struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc, 112 108 gfp_t gfp_flags, int node), 113 109 114 - TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) 110 + TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node) 115 111 ); 116 112 117 113 DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node, 118 114 119 115 TP_PROTO(unsigned long call_site, const void *ptr, 120 - size_t bytes_req, size_t bytes_alloc, 116 + struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc, 121 117 gfp_t gfp_flags, int node), 122 118 123 - TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) 119 + TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node) 124 120 ); 125 121 126 122 TRACE_EVENT(kfree,

+10 -10

mm/slab.c

··· 3230 3230 } 3231 3231 /* ___cache_alloc_node can fall back to other nodes */ 3232 3232 ptr = ____cache_alloc_node(cachep, flags, nodeid); 3233 - out: 3233 + out: 3234 3234 local_irq_restore(save_flags); 3235 3235 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); 3236 3236 init = slab_want_init_on_alloc(flags, cachep); ··· 3259 3259 if (!objp) 3260 3260 objp = ____cache_alloc_node(cache, flags, numa_mem_id()); 3261 3261 3262 - out: 3262 + out: 3263 3263 return objp; 3264 3264 } 3265 3265 #else ··· 3406 3406 { 3407 3407 bool init; 3408 3408 3409 + memcg_slab_free_hook(cachep, virt_to_slab(objp), &objp, 1); 3410 + 3409 3411 if (is_kfence_address(objp)) { 3410 3412 kmemleak_free_recursive(objp, cachep->flags); 3411 - memcg_slab_free_hook(cachep, &objp, 1); 3412 3413 __kfence_free(objp); 3413 3414 return; 3414 3415 } ··· 3442 3441 check_irq_off(); 3443 3442 kmemleak_free_recursive(objp, cachep->flags); 3444 3443 objp = cache_free_debugcheck(cachep, objp, caller); 3445 - memcg_slab_free_hook(cachep, &objp, 1); 3446 3444 3447 3445 /* 3448 3446 * Skip calling cache_free_alien() when the platform is not numa. ··· 3478 3478 { 3479 3479 void *ret = slab_alloc(cachep, lru, flags, cachep->object_size, _RET_IP_); 3480 3480 3481 - trace_kmem_cache_alloc(_RET_IP_, ret, 3481 + trace_kmem_cache_alloc(_RET_IP_, ret, cachep, 3482 3482 cachep->object_size, cachep->size, flags); 3483 3483 3484 3484 return ret; ··· 3553 3553 local_irq_enable(); 3554 3554 cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_); 3555 3555 slab_post_alloc_hook(s, objcg, flags, i, p, false); 3556 - __kmem_cache_free_bulk(s, i, p); 3556 + kmem_cache_free_bulk(s, i, p); 3557 3557 return 0; 3558 3558 } 3559 3559 EXPORT_SYMBOL(kmem_cache_alloc_bulk); ··· 3567 3567 ret = slab_alloc(cachep, NULL, flags, size, _RET_IP_); 3568 3568 3569 3569 ret = kasan_kmalloc(cachep, ret, size, flags); 3570 - trace_kmalloc(_RET_IP_, ret, 3570 + trace_kmalloc(_RET_IP_, ret, cachep, 3571 3571 size, cachep->size, flags); 3572 3572 return ret; 3573 3573 } ··· 3592 3592 { 3593 3593 void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_); 3594 3594 3595 - trace_kmem_cache_alloc_node(_RET_IP_, ret, 3595 + trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep, 3596 3596 cachep->object_size, cachep->size, 3597 3597 flags, nodeid); 3598 3598 ··· 3611 3611 ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_); 3612 3612 3613 3613 ret = kasan_kmalloc(cachep, ret, size, flags); 3614 - trace_kmalloc_node(_RET_IP_, ret, 3614 + trace_kmalloc_node(_RET_IP_, ret, cachep, 3615 3615 size, cachep->size, 3616 3616 flags, nodeid); 3617 3617 return ret; ··· 3694 3694 ret = slab_alloc(cachep, NULL, flags, size, caller); 3695 3695 3696 3696 ret = kasan_kmalloc(cachep, ret, size, flags); 3697 - trace_kmalloc(caller, ret, 3697 + trace_kmalloc(caller, ret, cachep, 3698 3698 size, cachep->size, flags); 3699 3699 3700 3700 return ret;

+8 -31

mm/slab.h

··· 380 380 ssize_t slabinfo_write(struct file *file, const char __user *buffer, 381 381 size_t count, loff_t *ppos); 382 382 383 - /* 384 - * Generic implementation of bulk operations 385 - * These are useful for situations in which the allocator cannot 386 - * perform optimizations. In that case segments of the object listed 387 - * may be allocated or freed using these operations. 388 - */ 389 - void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **); 390 - int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **); 391 - 392 383 static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) 393 384 { 394 385 return (s->flags & SLAB_RECLAIM_ACCOUNT) ? ··· 538 547 obj_cgroup_put(objcg); 539 548 } 540 549 541 - static inline void memcg_slab_free_hook(struct kmem_cache *s_orig, 550 + static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 542 551 void **p, int objects) 543 552 { 544 - struct kmem_cache *s; 545 553 struct obj_cgroup **objcgs; 546 - struct obj_cgroup *objcg; 547 - struct slab *slab; 548 - unsigned int off; 549 554 int i; 550 555 551 556 if (!memcg_kmem_enabled()) 552 557 return; 553 558 559 + objcgs = slab_objcgs(slab); 560 + if (!objcgs) 561 + return; 562 + 554 563 for (i = 0; i < objects; i++) { 555 - if (unlikely(!p[i])) 556 - continue; 557 - 558 - slab = virt_to_slab(p[i]); 559 - /* we could be given a kmalloc_large() object, skip those */ 560 - if (!slab) 561 - continue; 562 - 563 - objcgs = slab_objcgs(slab); 564 - if (!objcgs) 565 - continue; 566 - 567 - if (!s_orig) 568 - s = slab->slab_cache; 569 - else 570 - s = s_orig; 564 + struct obj_cgroup *objcg; 565 + unsigned int off; 571 566 572 567 off = obj_to_index(s, slab, p[i]); 573 568 objcg = objcgs[off]; ··· 605 628 { 606 629 } 607 630 608 - static inline void memcg_slab_free_hook(struct kmem_cache *s, 631 + static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 609 632 void **p, int objects) 610 633 { 611 634 }

+4 -32

mm/slab_common.c

··· 26 26 #include <linux/memcontrol.h> 27 27 #include <linux/stackdepot.h> 28 28 29 + #include "internal.h" 30 + #include "slab.h" 31 + 29 32 #define CREATE_TRACE_POINTS 30 33 #include <trace/events/kmem.h> 31 - 32 - #include "internal.h" 33 - 34 - #include "slab.h" 35 34 36 35 enum slab_state slab_state; 37 36 LIST_HEAD(slab_caches); ··· 103 104 return 0; 104 105 } 105 106 #endif 106 - 107 - void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p) 108 - { 109 - size_t i; 110 - 111 - for (i = 0; i < nr; i++) { 112 - if (s) 113 - kmem_cache_free(s, p[i]); 114 - else 115 - kfree(p[i]); 116 - } 117 - } 118 - 119 - int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, 120 - void **p) 121 - { 122 - size_t i; 123 - 124 - for (i = 0; i < nr; i++) { 125 - void *x = p[i] = kmem_cache_alloc(s, flags); 126 - if (!x) { 127 - __kmem_cache_free_bulk(s, i, p); 128 - return 0; 129 - } 130 - } 131 - return i; 132 - } 133 107 134 108 /* 135 109 * Figure out what the alignment of the objects will be given a set of ··· 931 959 void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 932 960 { 933 961 void *ret = kmalloc_order(size, flags, order); 934 - trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags); 962 + trace_kmalloc(_RET_IP_, ret, NULL, size, PAGE_SIZE << order, flags); 935 963 return ret; 936 964 } 937 965 EXPORT_SYMBOL(kmalloc_order_trace);

+25 -8

mm/slob.c

··· 507 507 *m = size; 508 508 ret = (void *)m + minalign; 509 509 510 - trace_kmalloc_node(caller, ret, 510 + trace_kmalloc_node(caller, ret, NULL, 511 511 size, size + minalign, gfp, node); 512 512 } else { 513 513 unsigned int order = get_order(size); ··· 516 516 gfp |= __GFP_COMP; 517 517 ret = slob_new_pages(gfp, order, node); 518 518 519 - trace_kmalloc_node(caller, ret, 519 + trace_kmalloc_node(caller, ret, NULL, 520 520 size, PAGE_SIZE << order, gfp, node); 521 521 } 522 522 ··· 616 616 617 617 if (c->size < PAGE_SIZE) { 618 618 b = slob_alloc(c->size, flags, c->align, node, 0); 619 - trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size, 619 + trace_kmem_cache_alloc_node(_RET_IP_, b, NULL, c->object_size, 620 620 SLOB_UNITS(c->size) * SLOB_UNIT, 621 621 flags, node); 622 622 } else { 623 623 b = slob_new_pages(flags, get_order(c->size), node); 624 - trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size, 624 + trace_kmem_cache_alloc_node(_RET_IP_, b, NULL, c->object_size, 625 625 PAGE_SIZE << get_order(c->size), 626 626 flags, node); 627 627 } ··· 692 692 } 693 693 EXPORT_SYMBOL(kmem_cache_free); 694 694 695 - void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) 695 + void kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p) 696 696 { 697 - __kmem_cache_free_bulk(s, size, p); 697 + size_t i; 698 + 699 + for (i = 0; i < nr; i++) { 700 + if (s) 701 + kmem_cache_free(s, p[i]); 702 + else 703 + kfree(p[i]); 704 + } 698 705 } 699 706 EXPORT_SYMBOL(kmem_cache_free_bulk); 700 707 701 - int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 708 + int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, 702 709 void **p) 703 710 { 704 - return __kmem_cache_alloc_bulk(s, flags, size, p); 711 + size_t i; 712 + 713 + for (i = 0; i < nr; i++) { 714 + void *x = p[i] = kmem_cache_alloc(s, flags); 715 + 716 + if (!x) { 717 + kmem_cache_free_bulk(s, i, p); 718 + return 0; 719 + } 720 + } 721 + return i; 705 722 } 706 723 EXPORT_SYMBOL(kmem_cache_alloc_bulk); 707 724

+37 -61

mm/slub.c

··· 3257 3257 { 3258 3258 void *ret = slab_alloc(s, lru, gfpflags, _RET_IP_, s->object_size); 3259 3259 3260 - trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, 3260 + trace_kmem_cache_alloc(_RET_IP_, ret, s, s->object_size, 3261 3261 s->size, gfpflags); 3262 3262 3263 3263 return ret; ··· 3280 3280 void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) 3281 3281 { 3282 3282 void *ret = slab_alloc(s, NULL, gfpflags, _RET_IP_, size); 3283 - trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); 3283 + trace_kmalloc(_RET_IP_, ret, s, size, s->size, gfpflags); 3284 3284 ret = kasan_kmalloc(s, ret, size, gfpflags); 3285 3285 return ret; 3286 3286 } ··· 3292 3292 { 3293 3293 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); 3294 3294 3295 - trace_kmem_cache_alloc_node(_RET_IP_, ret, 3295 + trace_kmem_cache_alloc_node(_RET_IP_, ret, s, 3296 3296 s->object_size, s->size, gfpflags, node); 3297 3297 3298 3298 return ret; ··· 3306 3306 { 3307 3307 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); 3308 3308 3309 - trace_kmalloc_node(_RET_IP_, ret, 3309 + trace_kmalloc_node(_RET_IP_, ret, s, 3310 3310 size, s->size, gfpflags, node); 3311 3311 3312 3312 ret = kasan_kmalloc(s, ret, size, gfpflags); ··· 3464 3464 struct kmem_cache_cpu *c; 3465 3465 unsigned long tid; 3466 3466 3467 - /* memcg_slab_free_hook() is already called for bulk free. */ 3468 - if (!tail) 3469 - memcg_slab_free_hook(s, &head, 1); 3470 3467 redo: 3471 3468 /* 3472 3469 * Determine the currently cpus per cpu slab. ··· 3523 3526 } 3524 3527 3525 3528 static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab, 3526 - void *head, void *tail, int cnt, 3529 + void *head, void *tail, void **p, int cnt, 3527 3530 unsigned long addr) 3528 3531 { 3532 + memcg_slab_free_hook(s, slab, p, cnt); 3529 3533 /* 3530 3534 * With KASAN enabled slab_free_freelist_hook modifies the freelist 3531 3535 * to remove objects, whose reuse must be delayed. ··· 3548 3550 if (!s) 3549 3551 return; 3550 3552 trace_kmem_cache_free(_RET_IP_, x, s->name); 3551 - slab_free(s, virt_to_slab(x), x, NULL, 1, _RET_IP_); 3553 + slab_free(s, virt_to_slab(x), x, NULL, &x, 1, _RET_IP_); 3552 3554 } 3553 3555 EXPORT_SYMBOL(kmem_cache_free); 3554 3556 ··· 3589 3591 int build_detached_freelist(struct kmem_cache *s, size_t size, 3590 3592 void **p, struct detached_freelist *df) 3591 3593 { 3592 - size_t first_skipped_index = 0; 3593 3594 int lookahead = 3; 3594 3595 void *object; 3595 3596 struct folio *folio; 3596 - struct slab *slab; 3597 + size_t same; 3597 3598 3598 - /* Always re-init detached_freelist */ 3599 - df->slab = NULL; 3600 - 3601 - do { 3602 - object = p[--size]; 3603 - /* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */ 3604 - } while (!object && size); 3605 - 3606 - if (!object) 3607 - return 0; 3608 - 3599 + object = p[--size]; 3609 3600 folio = virt_to_folio(object); 3610 3601 if (!s) { 3611 3602 /* Handle kalloc'ed objects */ 3612 3603 if (unlikely(!folio_test_slab(folio))) { 3613 3604 free_large_kmalloc(folio, object); 3614 - p[size] = NULL; /* mark object processed */ 3605 + df->slab = NULL; 3615 3606 return size; 3616 3607 } 3617 3608 /* Derive kmem_cache from object */ 3618 - slab = folio_slab(folio); 3619 - df->s = slab->slab_cache; 3609 + df->slab = folio_slab(folio); 3610 + df->s = df->slab->slab_cache; 3620 3611 } else { 3621 - slab = folio_slab(folio); 3612 + df->slab = folio_slab(folio); 3622 3613 df->s = cache_from_obj(s, object); /* Support for memcg */ 3623 3614 } 3624 3615 3625 - if (is_kfence_address(object)) { 3626 - slab_free_hook(df->s, object, false); 3627 - __kfence_free(object); 3628 - p[size] = NULL; /* mark object processed */ 3629 - return size; 3630 - } 3631 - 3632 3616 /* Start new detached freelist */ 3633 - df->slab = slab; 3634 - set_freepointer(df->s, object, NULL); 3635 3617 df->tail = object; 3636 3618 df->freelist = object; 3637 - p[size] = NULL; /* mark object processed */ 3638 3619 df->cnt = 1; 3639 3620 3621 + if (is_kfence_address(object)) 3622 + return size; 3623 + 3624 + set_freepointer(df->s, object, NULL); 3625 + 3626 + same = size; 3640 3627 while (size) { 3641 3628 object = p[--size]; 3642 - if (!object) 3643 - continue; /* Skip processed objects */ 3644 - 3645 3629 /* df->slab is always set at this point */ 3646 3630 if (df->slab == virt_to_slab(object)) { 3647 3631 /* Opportunity build freelist */ 3648 3632 set_freepointer(df->s, object, df->freelist); 3649 3633 df->freelist = object; 3650 3634 df->cnt++; 3651 - p[size] = NULL; /* mark object processed */ 3652 - 3635 + same--; 3636 + if (size != same) 3637 + swap(p[size], p[same]); 3653 3638 continue; 3654 3639 } 3655 3640 3656 3641 /* Limit look ahead search */ 3657 3642 if (!--lookahead) 3658 3643 break; 3659 - 3660 - if (!first_skipped_index) 3661 - first_skipped_index = size + 1; 3662 3644 } 3663 3645 3664 - return first_skipped_index; 3646 + return same; 3665 3647 } 3666 3648 3667 3649 /* Note that interrupts must be enabled when calling this function. */ 3668 3650 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) 3669 3651 { 3670 - if (WARN_ON(!size)) 3652 + if (!size) 3671 3653 return; 3672 3654 3673 - memcg_slab_free_hook(s, p, size); 3674 3655 do { 3675 3656 struct detached_freelist df; 3676 3657 ··· 3657 3680 if (!df.slab) 3658 3681 continue; 3659 3682 3660 - slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, _RET_IP_); 3683 + slab_free(df.s, df.slab, df.freelist, df.tail, &p[size], df.cnt, 3684 + _RET_IP_); 3661 3685 } while (likely(size)); 3662 3686 } 3663 3687 EXPORT_SYMBOL(kmem_cache_free_bulk); ··· 3738 3760 error: 3739 3761 slub_put_cpu_ptr(s->cpu_slab); 3740 3762 slab_post_alloc_hook(s, objcg, flags, i, p, false); 3741 - __kmem_cache_free_bulk(s, i, p); 3763 + kmem_cache_free_bulk(s, i, p); 3742 3764 return 0; 3743 3765 } 3744 3766 EXPORT_SYMBOL(kmem_cache_alloc_bulk); ··· 4419 4441 4420 4442 ret = slab_alloc(s, NULL, flags, _RET_IP_, size); 4421 4443 4422 - trace_kmalloc(_RET_IP_, ret, size, s->size, flags); 4444 + trace_kmalloc(_RET_IP_, ret, s, size, s->size, flags); 4423 4445 4424 4446 ret = kasan_kmalloc(s, ret, size, flags); 4425 4447 ··· 4453 4475 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 4454 4476 ret = kmalloc_large_node(size, flags, node); 4455 4477 4456 - trace_kmalloc_node(_RET_IP_, ret, 4478 + trace_kmalloc_node(_RET_IP_, ret, NULL, 4457 4479 size, PAGE_SIZE << get_order(size), 4458 4480 flags, node); 4459 4481 ··· 4467 4489 4468 4490 ret = slab_alloc_node(s, NULL, flags, node, _RET_IP_, size); 4469 4491 4470 - trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); 4492 + trace_kmalloc_node(_RET_IP_, ret, s, size, s->size, flags, node); 4471 4493 4472 4494 ret = kasan_kmalloc(s, ret, size, flags); 4473 4495 ··· 4559 4581 return; 4560 4582 } 4561 4583 slab = folio_slab(folio); 4562 - slab_free(slab->slab_cache, slab, object, NULL, 1, _RET_IP_); 4584 + slab_free(slab->slab_cache, slab, object, NULL, &object, 1, _RET_IP_); 4563 4585 } 4564 4586 EXPORT_SYMBOL(kfree); 4565 4587 ··· 4868 4890 4869 4891 s = find_mergeable(size, align, flags, name, ctor); 4870 4892 if (s) { 4893 + if (sysfs_slab_alias(s, name)) 4894 + return NULL; 4895 + 4871 4896 s->refcount++; 4872 4897 4873 4898 /* ··· 4879 4898 */ 4880 4899 s->object_size = max(s->object_size, size); 4881 4900 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *))); 4882 - 4883 - if (sysfs_slab_alias(s, name)) { 4884 - s->refcount--; 4885 - s = NULL; 4886 - } 4887 4901 } 4888 4902 4889 4903 return s; ··· 4924 4948 ret = slab_alloc(s, NULL, gfpflags, caller, size); 4925 4949 4926 4950 /* Honor the call site pointer we received. */ 4927 - trace_kmalloc(caller, ret, size, s->size, gfpflags); 4951 + trace_kmalloc(caller, ret, s, size, s->size, gfpflags); 4928 4952 4929 4953 return ret; 4930 4954 } ··· 4940 4964 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 4941 4965 ret = kmalloc_large_node(size, gfpflags, node); 4942 4966 4943 - trace_kmalloc_node(caller, ret, 4967 + trace_kmalloc_node(caller, ret, NULL, 4944 4968 size, PAGE_SIZE << get_order(size), 4945 4969 gfpflags, node); 4946 4970 ··· 4955 4979 ret = slab_alloc_node(s, NULL, gfpflags, node, caller, size); 4956 4980 4957 4981 /* Honor the call site pointer we received. */ 4958 - trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); 4982 + trace_kmalloc_node(caller, ret, s, size, s->size, gfpflags, node); 4959 4983 4960 4984 return ret; 4961 4985 }

+24 -2

tools/vm/slabinfo.c

··· 233 233 return l; 234 234 } 235 235 236 + static unsigned long read_debug_slab_obj(struct slabinfo *s, const char *name) 237 + { 238 + char x[128]; 239 + FILE *f; 240 + size_t l; 241 + 242 + snprintf(x, 128, "/sys/kernel/debug/slab/%s/%s", s->name, name); 243 + f = fopen(x, "r"); 244 + if (!f) { 245 + buffer[0] = 0; 246 + l = 0; 247 + } else { 248 + l = fread(buffer, 1, sizeof(buffer), f); 249 + buffer[l] = 0; 250 + fclose(f); 251 + } 252 + return l; 253 + } 236 254 237 255 /* 238 256 * Put a size string together ··· 427 409 { 428 410 printf("\n%s: Kernel object allocation\n", s->name); 429 411 printf("-----------------------------------------------------------------------\n"); 430 - if (read_slab_obj(s, "alloc_calls")) 412 + if (read_debug_slab_obj(s, "alloc_traces")) 413 + printf("%s", buffer); 414 + else if (read_slab_obj(s, "alloc_calls")) 431 415 printf("%s", buffer); 432 416 else 433 417 printf("No Data\n"); 434 418 435 419 printf("\n%s: Kernel object freeing\n", s->name); 436 420 printf("------------------------------------------------------------------------\n"); 437 - if (read_slab_obj(s, "free_calls")) 421 + if (read_debug_slab_obj(s, "free_traces")) 422 + printf("%s", buffer); 423 + else if (read_slab_obj(s, "free_calls")) 438 424 printf("%s", buffer); 439 425 else 440 426 printf("No Data\n");

Configure Feed

Configure Feed