tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

net: use llist for sd->defer_list

Get rid of sd->defer_lock and adopt llist operations.

We optimize skb_attempt_defer_free() for the common case,
where the packet is queued. Otherwise sd->defer_count
is increasing, until skb_defer_free_flush() clears it.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Jason Xing <kerneljasonxing@gmail.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20250928084934.3266948-3-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

authored by

Eric Dumazet and committed by
Paolo Abeni 844c9db7 9c94ae6b

+17 -24
+4 -4
include/linux/netdevice.h
··· 3537 3537 struct numa_drop_counters drop_counters; 3538 3538 3539 3539 /* Another possibly contended cache line */ 3540 - spinlock_t defer_lock ____cacheline_aligned_in_smp; 3541 - atomic_t defer_count; 3542 - int defer_ipi_scheduled; 3543 - struct sk_buff *defer_list; 3540 + struct llist_head defer_list ____cacheline_aligned_in_smp; 3541 + atomic_long_t defer_count; 3542 + 3543 + int defer_ipi_scheduled ____cacheline_aligned_in_smp; 3544 3544 call_single_data_t defer_csd; 3545 3545 }; 3546 3546
+6 -12
net/core/dev.c
··· 6717 6717 6718 6718 static void skb_defer_free_flush(struct softnet_data *sd) 6719 6719 { 6720 + struct llist_node *free_list; 6720 6721 struct sk_buff *skb, *next; 6721 6722 6722 - /* Paired with WRITE_ONCE() in skb_attempt_defer_free() */ 6723 - if (!READ_ONCE(sd->defer_list)) 6723 + if (llist_empty(&sd->defer_list)) 6724 6724 return; 6725 + atomic_long_set(&sd->defer_count, 0); 6726 + free_list = llist_del_all(&sd->defer_list); 6725 6727 6726 - spin_lock(&sd->defer_lock); 6727 - skb = sd->defer_list; 6728 - sd->defer_list = NULL; 6729 - atomic_set(&sd->defer_count, 0); 6730 - spin_unlock(&sd->defer_lock); 6731 - 6732 - while (skb != NULL) { 6733 - next = skb->next; 6728 + llist_for_each_entry_safe(skb, next, free_list, ll_node) { 6734 6729 napi_consume_skb(skb, 1); 6735 - skb = next; 6736 6730 } 6737 6731 } 6738 6732 ··· 12989 12995 sd->cpu = i; 12990 12996 #endif 12991 12997 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd); 12992 - spin_lock_init(&sd->defer_lock); 12998 + init_llist_head(&sd->defer_list); 12993 12999 12994 13000 gro_init(&sd->backlog.gro); 12995 13001 sd->backlog.poll = process_backlog;
+7 -8
net/core/skbuff.c
··· 7185 7185 */ 7186 7186 void skb_attempt_defer_free(struct sk_buff *skb) 7187 7187 { 7188 + unsigned long defer_count; 7188 7189 int cpu = skb->alloc_cpu; 7189 7190 struct softnet_data *sd; 7190 7191 unsigned int defer_max; ··· 7203 7202 7204 7203 sd = &per_cpu(softnet_data, cpu); 7205 7204 defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max); 7206 - if (atomic_read(&sd->defer_count) >= defer_max) 7205 + defer_count = atomic_long_inc_return(&sd->defer_count); 7206 + 7207 + if (defer_count >= defer_max) 7207 7208 goto nodefer; 7208 7209 7209 - spin_lock_bh(&sd->defer_lock); 7210 - /* Send an IPI every time queue reaches half capacity. */ 7211 - kick = (atomic_inc_return(&sd->defer_count) - 1) == (defer_max >> 1); 7210 + llist_add(&skb->ll_node, &sd->defer_list); 7212 7211 7213 - skb->next = sd->defer_list; 7214 - /* Paired with READ_ONCE() in skb_defer_free_flush() */ 7215 - WRITE_ONCE(sd->defer_list, skb); 7216 - spin_unlock_bh(&sd->defer_lock); 7212 + /* Send an IPI every time queue reaches half capacity. */ 7213 + kick = (defer_count - 1) == (defer_max >> 1); 7217 7214 7218 7215 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU 7219 7216 * if we are unlucky enough (this seems very unlikely).