tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

net: optimize enqueue_to_backlog() for the fast path

Add likely() and unlikely() clauses for the common cases:

Device is running.
Queue is not full.
Queue is less than half capacity.

Add max_backlog parameter to skb_flow_limit() to avoid
a second READ_ONCE(net_hotdata.max_backlog).

skb_flow_limit() does not need the backlog_lock protection,
and can be called before we acquire the lock, for even better
resistance to attacks.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20251024090517.3289181-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

authored by

Eric Dumazet and committed by
Jakub Kicinski a086e986 34164142

+10 -8
+10 -8
net/core/dev.c
··· 5249 5249 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 5250 5250 #endif 5251 5251 5252 - static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) 5252 + static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen, 5253 + int max_backlog) 5253 5254 { 5254 5255 #ifdef CONFIG_NET_FLOW_LIMIT 5255 - struct sd_flow_limit *fl; 5256 - struct softnet_data *sd; 5257 5256 unsigned int old_flow, new_flow; 5257 + const struct softnet_data *sd; 5258 + struct sd_flow_limit *fl; 5258 5259 5259 - if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1)) 5260 + if (likely(qlen < (max_backlog >> 1))) 5260 5261 return false; 5261 5262 5262 5263 sd = this_cpu_ptr(&softnet_data); ··· 5302 5301 u32 tail; 5303 5302 5304 5303 reason = SKB_DROP_REASON_DEV_READY; 5305 - if (!netif_running(skb->dev)) 5304 + if (unlikely(!netif_running(skb->dev))) 5306 5305 goto bad_dev; 5307 5306 5308 - reason = SKB_DROP_REASON_CPU_BACKLOG; 5309 5307 sd = &per_cpu(softnet_data, cpu); 5310 5308 5311 5309 qlen = skb_queue_len_lockless(&sd->input_pkt_queue); 5312 5310 max_backlog = READ_ONCE(net_hotdata.max_backlog); 5313 - if (unlikely(qlen > max_backlog)) 5311 + if (unlikely(qlen > max_backlog) || 5312 + skb_flow_limit(skb, qlen, max_backlog)) 5314 5313 goto cpu_backlog_drop; 5315 5314 backlog_lock_irq_save(sd, &flags); 5316 5315 qlen = skb_queue_len(&sd->input_pkt_queue); 5317 - if (qlen <= max_backlog && !skb_flow_limit(skb, qlen)) { 5316 + if (likely(qlen <= max_backlog)) { 5318 5317 if (!qlen) { 5319 5318 /* Schedule NAPI for backlog device. We can use 5320 5319 * non atomic operation as we own the queue lock. ··· 5335 5334 backlog_unlock_irq_restore(sd, &flags); 5336 5335 5337 5336 cpu_backlog_drop: 5337 + reason = SKB_DROP_REASON_CPU_BACKLOG; 5338 5338 numa_drop_add(&sd->drop_counters, 1); 5339 5339 bad_dev: 5340 5340 dev_core_stats_rx_dropped_inc(skb->dev);