Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'net-expand-napi_skb_cache-use'
Eric Dumazet says:
====================
net: expand napi_skb_cache use
This is a followup of commit e20dfbad8aab ("net: fix napi_consume_skb()
with alien skbs").
Now the per-cpu napi_skb_cache is populated from TX completion path,
we can make use of this cache, especially for cpus not used
from a driver NAPI poll (primary user of napi_cache).
With this series, I consistently reach 130 Mpps on my UDP tx stress test
and reduce SLUB spinlock contention to smaller values.
====================
Link: https://patch.msgid.link/20251116202717.1542829-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+31
-17
+31
-17
net/core/skbuff.c
+31
-17
net/core/skbuff.c
···
280
280
*/
281
281
static u32 skbuff_cache_size __read_mostly;
282
282
283
-
static struct sk_buff *napi_skb_cache_get(void)
283
+
static struct sk_buff *napi_skb_cache_get(bool alloc)
284
284
{
285
285
struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
286
286
struct sk_buff *skb;
287
287
288
288
local_lock_nested_bh(&napi_alloc_cache.bh_lock);
289
289
if (unlikely(!nc->skb_count)) {
290
-
nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
291
-
GFP_ATOMIC | __GFP_NOWARN,
292
-
NAPI_SKB_CACHE_BULK,
293
-
nc->skb_cache);
290
+
if (alloc)
291
+
nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
292
+
GFP_ATOMIC | __GFP_NOWARN,
293
+
NAPI_SKB_CACHE_BULK,
294
+
nc->skb_cache);
294
295
if (unlikely(!nc->skb_count)) {
295
296
local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
296
297
return NULL;
···
531
530
{
532
531
struct sk_buff *skb;
533
532
534
-
skb = napi_skb_cache_get();
533
+
skb = napi_skb_cache_get(true);
535
534
if (unlikely(!skb))
536
535
return NULL;
537
536
···
646
645
struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
647
646
int flags, int node)
648
647
{
648
+
struct sk_buff *skb = NULL;
649
649
struct kmem_cache *cache;
650
-
struct sk_buff *skb;
651
650
bool pfmemalloc;
652
651
u8 *data;
653
-
654
-
cache = (flags & SKB_ALLOC_FCLONE)
655
-
? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache;
656
652
657
653
if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
658
654
gfp_mask |= __GFP_MEMALLOC;
659
655
660
-
/* Get the HEAD */
661
-
if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI &&
662
-
likely(node == NUMA_NO_NODE || node == numa_mem_id()))
663
-
skb = napi_skb_cache_get();
664
-
else
656
+
if (flags & SKB_ALLOC_FCLONE) {
657
+
cache = net_hotdata.skbuff_fclone_cache;
658
+
goto fallback;
659
+
}
660
+
cache = net_hotdata.skbuff_cache;
661
+
if (unlikely(node != NUMA_NO_NODE && node != numa_mem_id()))
662
+
goto fallback;
663
+
664
+
if (flags & SKB_ALLOC_NAPI) {
665
+
skb = napi_skb_cache_get(true);
666
+
if (unlikely(!skb))
667
+
return NULL;
668
+
} else if (!in_hardirq() && !irqs_disabled()) {
669
+
local_bh_disable();
670
+
skb = napi_skb_cache_get(false);
671
+
local_bh_enable();
672
+
}
673
+
674
+
if (!skb) {
675
+
fallback:
665
676
skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node);
666
-
if (unlikely(!skb))
667
-
return NULL;
677
+
if (unlikely(!skb))
678
+
return NULL;
679
+
}
668
680
prefetchw(skb);
669
681
670
682
/* We do our best to align skb_shared_info on a separate cache