tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm/slub: cleanup and repurpose some stat items

A number of stat items related to cpu slabs became unused, remove them.

Two of those were ALLOC_FASTPATH and FREE_FASTPATH. But instead of
removing those, use them instead of ALLOC_PCS and FREE_PCS, since
sheaves are the new (and only) fastpaths, Remove the recently added
_PCS variants instead.

Change where FREE_SLOWPATH is counted so that it only counts freeing of
objects by slab users that (for whatever reason) do not go to a percpu
sheaf, and not all (including internal) callers of __slab_free(). Thus
sheaf flushing (already counted by SHEAF_FLUSH) does not affect
FREE_SLOWPATH anymore. This matches how ALLOC_SLOWPATH doesn't count
sheaf refills (counted by SHEAF_REFILL).

Reviewed-by: Hao Li <hao.li@linux.dev>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

Vlastimil Babka 6f191218 fb016a5e

+26 -57
+26 -57
mm/slub.c
··· 335 335 }; 336 336 337 337 enum stat_item { 338 - ALLOC_PCS, /* Allocation from percpu sheaf */ 339 - ALLOC_FASTPATH, /* Allocation from cpu slab */ 340 - ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */ 341 - FREE_PCS, /* Free to percpu sheaf */ 338 + ALLOC_FASTPATH, /* Allocation from percpu sheaves */ 339 + ALLOC_SLOWPATH, /* Allocation from partial or new slab */ 342 340 FREE_RCU_SHEAF, /* Free to rcu_free sheaf */ 343 341 FREE_RCU_SHEAF_FAIL, /* Failed to free to a rcu_free sheaf */ 344 - FREE_FASTPATH, /* Free to cpu slab */ 345 - FREE_SLOWPATH, /* Freeing not to cpu slab */ 342 + FREE_FASTPATH, /* Free to percpu sheaves */ 343 + FREE_SLOWPATH, /* Free to a slab */ 346 344 FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */ 347 345 FREE_REMOVE_PARTIAL, /* Freeing removes last object */ 348 - ALLOC_FROM_PARTIAL, /* Cpu slab acquired from node partial list */ 349 - ALLOC_SLAB, /* Cpu slab acquired from page allocator */ 350 - ALLOC_REFILL, /* Refill cpu slab from slab freelist */ 351 - ALLOC_NODE_MISMATCH, /* Switching cpu slab */ 346 + ALLOC_SLAB, /* New slab acquired from page allocator */ 347 + ALLOC_NODE_MISMATCH, /* Requested node different from cpu sheaf */ 352 348 FREE_SLAB, /* Slab freed to the page allocator */ 353 - CPUSLAB_FLUSH, /* Abandoning of the cpu slab */ 354 - DEACTIVATE_FULL, /* Cpu slab was full when deactivated */ 355 - DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */ 356 - DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */ 357 - DEACTIVATE_BYPASS, /* Implicit deactivation */ 358 349 ORDER_FALLBACK, /* Number of times fallback was necessary */ 359 - CMPXCHG_DOUBLE_CPU_FAIL,/* Failures of this_cpu_cmpxchg_double */ 360 350 CMPXCHG_DOUBLE_FAIL, /* Failures of slab freelist update */ 361 - CPU_PARTIAL_ALLOC, /* Used cpu partial on alloc */ 362 - CPU_PARTIAL_FREE, /* Refill cpu partial on free */ 363 - CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */ 364 - CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */ 365 351 SHEAF_FLUSH, /* Objects flushed from a sheaf */ 366 352 SHEAF_REFILL, /* Objects refilled to a sheaf */ 367 353 SHEAF_ALLOC, /* Allocation of an empty sheaf */ ··· 4336 4350 * We assume the percpu sheaves contain only local objects although it's 4337 4351 * not completely guaranteed, so we verify later. 4338 4352 */ 4339 - if (unlikely(node_requested && node != numa_mem_id())) 4353 + if (unlikely(node_requested && node != numa_mem_id())) { 4354 + stat(s, ALLOC_NODE_MISMATCH); 4340 4355 return NULL; 4356 + } 4341 4357 4342 4358 if (!local_trylock(&s->cpu_sheaves->lock)) 4343 4359 return NULL; ··· 4362 4374 */ 4363 4375 if (page_to_nid(virt_to_page(object)) != node) { 4364 4376 local_unlock(&s->cpu_sheaves->lock); 4377 + stat(s, ALLOC_NODE_MISMATCH); 4365 4378 return NULL; 4366 4379 } 4367 4380 } ··· 4371 4382 4372 4383 local_unlock(&s->cpu_sheaves->lock); 4373 4384 4374 - stat(s, ALLOC_PCS); 4385 + stat(s, ALLOC_FASTPATH); 4375 4386 4376 4387 return object; 4377 4388 } ··· 4443 4454 4444 4455 local_unlock(&s->cpu_sheaves->lock); 4445 4456 4446 - stat_add(s, ALLOC_PCS, batch); 4457 + stat_add(s, ALLOC_FASTPATH, batch); 4447 4458 4448 4459 allocated += batch; 4449 4460 ··· 5106 5117 unsigned long flags; 5107 5118 bool on_node_partial; 5108 5119 5109 - stat(s, FREE_SLOWPATH); 5110 - 5111 5120 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 5112 5121 free_to_partial_list(s, slab, head, tail, cnt, addr); 5113 5122 return; ··· 5409 5422 5410 5423 local_unlock(&s->cpu_sheaves->lock); 5411 5424 5412 - stat(s, FREE_PCS); 5425 + stat(s, FREE_FASTPATH); 5413 5426 5414 5427 return true; 5415 5428 } ··· 5677 5690 5678 5691 local_unlock(&s->cpu_sheaves->lock); 5679 5692 5680 - stat_add(s, FREE_PCS, batch); 5693 + stat_add(s, FREE_FASTPATH, batch); 5681 5694 5682 5695 if (batch < size) { 5683 5696 p += batch; ··· 5699 5712 */ 5700 5713 fallback: 5701 5714 __kmem_cache_free_bulk(s, size, p); 5715 + stat_add(s, FREE_SLOWPATH, size); 5702 5716 5703 5717 flush_remote: 5704 5718 if (remote_nr) { 5705 5719 __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); 5720 + stat_add(s, FREE_SLOWPATH, remote_nr); 5706 5721 if (i < size) { 5707 5722 remote_nr = 0; 5708 5723 goto next_remote_batch; ··· 5758 5769 set_freepointer(s, x, NULL); 5759 5770 5760 5771 __slab_free(s, slab, x, x, 1, _THIS_IP_); 5772 + stat(s, FREE_SLOWPATH); 5761 5773 } 5762 5774 } 5763 5775 ··· 5800 5810 } 5801 5811 5802 5812 __slab_free(s, slab, object, object, 1, addr); 5813 + stat(s, FREE_SLOWPATH); 5803 5814 } 5804 5815 5805 5816 #ifdef CONFIG_MEMCG ··· 5823 5832 * With KASAN enabled slab_free_freelist_hook modifies the freelist 5824 5833 * to remove objects, whose reuse must be delayed. 5825 5834 */ 5826 - if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt))) 5835 + if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt))) { 5827 5836 __slab_free(s, slab, head, tail, cnt, addr); 5837 + stat_add(s, FREE_SLOWPATH, cnt); 5838 + } 5828 5839 } 5829 5840 5830 5841 #ifdef CONFIG_SLUB_RCU_DEBUG ··· 5851 5858 return; 5852 5859 5853 5860 /* resume freeing */ 5854 - if (slab_free_hook(s, object, slab_want_init_on_free(s), true)) 5861 + if (slab_free_hook(s, object, slab_want_init_on_free(s), true)) { 5855 5862 __slab_free(s, slab, object, object, 1, _THIS_IP_); 5863 + stat(s, FREE_SLOWPATH); 5864 + } 5856 5865 } 5857 5866 #endif /* CONFIG_SLUB_RCU_DEBUG */ 5858 5867 ··· 5862 5867 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr) 5863 5868 { 5864 5869 __slab_free(cache, virt_to_slab(x), x, x, 1, addr); 5870 + stat(cache, FREE_SLOWPATH); 5865 5871 } 5866 5872 #endif 5867 5873 ··· 6742 6746 i = refill_objects(s, p, flags, size, size); 6743 6747 if (i < size) 6744 6748 goto error; 6749 + stat_add(s, ALLOC_SLOWPATH, i); 6745 6750 } 6746 6751 6747 6752 return i; ··· 8746 8749 } \ 8747 8750 SLAB_ATTR(text); \ 8748 8751 8749 - STAT_ATTR(ALLOC_PCS, alloc_cpu_sheaf); 8750 8752 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); 8751 8753 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); 8752 - STAT_ATTR(FREE_PCS, free_cpu_sheaf); 8753 8754 STAT_ATTR(FREE_RCU_SHEAF, free_rcu_sheaf); 8754 8755 STAT_ATTR(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail); 8755 8756 STAT_ATTR(FREE_FASTPATH, free_fastpath); 8756 8757 STAT_ATTR(FREE_SLOWPATH, free_slowpath); 8757 8758 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); 8758 8759 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); 8759 - STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); 8760 8760 STAT_ATTR(ALLOC_SLAB, alloc_slab); 8761 - STAT_ATTR(ALLOC_REFILL, alloc_refill); 8762 8761 STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); 8763 8762 STAT_ATTR(FREE_SLAB, free_slab); 8764 - STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); 8765 - STAT_ATTR(DEACTIVATE_FULL, deactivate_full); 8766 - STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); 8767 - STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); 8768 - STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); 8769 8763 STAT_ATTR(ORDER_FALLBACK, order_fallback); 8770 - STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); 8771 8764 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); 8772 - STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); 8773 - STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); 8774 - STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); 8775 - STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); 8776 8765 STAT_ATTR(SHEAF_FLUSH, sheaf_flush); 8777 8766 STAT_ATTR(SHEAF_REFILL, sheaf_refill); 8778 8767 STAT_ATTR(SHEAF_ALLOC, sheaf_alloc); ··· 8834 8851 &remote_node_defrag_ratio_attr.attr, 8835 8852 #endif 8836 8853 #ifdef CONFIG_SLUB_STATS 8837 - &alloc_cpu_sheaf_attr.attr, 8838 8854 &alloc_fastpath_attr.attr, 8839 8855 &alloc_slowpath_attr.attr, 8840 - &free_cpu_sheaf_attr.attr, 8841 8856 &free_rcu_sheaf_attr.attr, 8842 8857 &free_rcu_sheaf_fail_attr.attr, 8843 8858 &free_fastpath_attr.attr, 8844 8859 &free_slowpath_attr.attr, 8845 8860 &free_add_partial_attr.attr, 8846 8861 &free_remove_partial_attr.attr, 8847 - &alloc_from_partial_attr.attr, 8848 8862 &alloc_slab_attr.attr, 8849 - &alloc_refill_attr.attr, 8850 8863 &alloc_node_mismatch_attr.attr, 8851 8864 &free_slab_attr.attr, 8852 - &cpuslab_flush_attr.attr, 8853 - &deactivate_full_attr.attr, 8854 - &deactivate_empty_attr.attr, 8855 - &deactivate_remote_frees_attr.attr, 8856 - &deactivate_bypass_attr.attr, 8857 8865 &order_fallback_attr.attr, 8858 8866 &cmpxchg_double_fail_attr.attr, 8859 - &cmpxchg_double_cpu_fail_attr.attr, 8860 - &cpu_partial_alloc_attr.attr, 8861 - &cpu_partial_free_attr.attr, 8862 - &cpu_partial_node_attr.attr, 8863 - &cpu_partial_drain_attr.attr, 8864 8867 &sheaf_flush_attr.attr, 8865 8868 &sheaf_refill_attr.attr, 8866 8869 &sheaf_alloc_attr.attr,