Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

net: shrink napi_skb_cache_{put,get}() and napi_skb_cache_get_bulk()

Following loop in napi_skb_cache_put() is unrolled by the compiler
even if CONFIG_KASAN is not enabled:

for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)
kasan_mempool_unpoison_object(nc->skb_cache[i],
kmem_cache_size(net_hotdata.skbuff_cache));

We have 32 times this sequence, for a total of 384 bytes.

48 8b 3d 00 00 00 00 net_hotdata.skbuff_cache,%rdi
e8 00 00 00 00 call kmem_cache_size

This is because kmem_cache_size() is not an inline and not const,
and kasan_unpoison_object_data() is an inline function.

Cache kmem_cache_size() result in a variable, so that
the compiler can remove dead code (and variable) when/if
CONFIG_KASAN is unset.

After this patch, napi_skb_cache_put() is inlined in its callers,
and we avoid one kmem_cache_size() call in napi_skb_cache_get()
and napi_skb_cache_get_bulk().

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Jacob Keller <jacob.e.keller@intel.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Reviewed-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Link: https://patch.msgid.link/20251016182911.1132792-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

authored by

Eric Dumazet and committed by
Jakub Kicinski
a5cd3a60 6ae022f8

+10 -5
+10 -5
net/core/skbuff.c
··· 274 274 } 275 275 EXPORT_SYMBOL(__netdev_alloc_frag_align); 276 276 277 + /* Cache kmem_cache_size(net_hotdata.skbuff_cache) to help the compiler 278 + * remove dead code (and skbuff_cache_size) when CONFIG_KASAN is unset. 279 + */ 280 + static u32 skbuff_cache_size __read_mostly; 281 + 277 282 static struct sk_buff *napi_skb_cache_get(void) 278 283 { 279 284 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); ··· 298 293 299 294 skb = nc->skb_cache[--nc->skb_count]; 300 295 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 301 - kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache)); 296 + kasan_mempool_unpoison_object(skb, skbuff_cache_size); 302 297 303 298 return skb; 304 299 } ··· 350 345 351 346 get: 352 347 for (u32 base = nc->skb_count - n, i = 0; i < n; i++) { 353 - u32 cache_size = kmem_cache_size(net_hotdata.skbuff_cache); 354 - 355 348 skbs[i] = nc->skb_cache[base + i]; 356 349 357 - kasan_mempool_unpoison_object(skbs[i], cache_size); 350 + kasan_mempool_unpoison_object(skbs[i], skbuff_cache_size); 358 351 memset(skbs[i], 0, offsetof(struct sk_buff, tail)); 359 352 } 360 353 ··· 1440 1437 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 1441 1438 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) 1442 1439 kasan_mempool_unpoison_object(nc->skb_cache[i], 1443 - kmem_cache_size(net_hotdata.skbuff_cache)); 1440 + skbuff_cache_size); 1444 1441 1445 1442 kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF, 1446 1443 nc->skb_cache + NAPI_SKB_CACHE_HALF); ··· 5128 5125 offsetof(struct sk_buff, cb), 5129 5126 sizeof_field(struct sk_buff, cb), 5130 5127 NULL); 5128 + skbuff_cache_size = kmem_cache_size(net_hotdata.skbuff_cache); 5129 + 5131 5130 net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 5132 5131 sizeof(struct sk_buff_fclones), 5133 5132 0,