Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
net_sched: use qdisc_skb_cb(skb)->pkt_segs in bstats_update()
Avoid up to two cache line misses in qdisc dequeue() to fetch
skb_shinfo(skb)->gso_segs/gso_size while qdisc spinlock is held.
This gives a 5 % improvement in a TX intensive workload.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20251121083256.674562-6-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
authored by
Eric Dumazet
and committed by
Paolo Abeni
2773cb0b
f9e00e51
+16
-4
+10
-3
include/net/sch_generic.h
+10
-3
include/net/sch_generic.h
···
829
829
return qdisc_skb_cb(skb)->pkt_len;
830
830
}
831
831
832
+
static inline unsigned int qdisc_pkt_segs(const struct sk_buff *skb)
833
+
{
834
+
u32 pkt_segs = qdisc_skb_cb(skb)->pkt_segs;
835
+
836
+
DEBUG_NET_WARN_ON_ONCE(pkt_segs !=
837
+
(skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1));
838
+
return pkt_segs;
839
+
}
840
+
832
841
/* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */
833
842
enum net_xmit_qdisc_t {
834
843
__NET_XMIT_STOLEN = 0x00010000,
···
879
870
static inline void bstats_update(struct gnet_stats_basic_sync *bstats,
880
871
const struct sk_buff *skb)
881
872
{
882
-
_bstats_update(bstats,
883
-
qdisc_pkt_len(skb),
884
-
skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1);
873
+
_bstats_update(bstats, qdisc_pkt_len(skb), qdisc_pkt_segs(skb));
885
874
}
886
875
887
876
static inline void qdisc_bstats_cpu_update(struct Qdisc *sch,
+1
net/sched/sch_cake.c
+1
net/sched/sch_cake.c
···
1800
1800
skb_list_walk_safe(segs, segs, nskb) {
1801
1801
skb_mark_not_on_list(segs);
1802
1802
qdisc_skb_cb(segs)->pkt_len = segs->len;
1803
+
qdisc_skb_cb(segs)->pkt_segs = 1;
1803
1804
cobalt_set_enqueue_time(segs, now);
1804
1805
get_cobalt_cb(segs)->adjusted_len = cake_overhead(q,
1805
1806
segs);
+1
net/sched/sch_dualpi2.c
+1
net/sched/sch_dualpi2.c
···
475
475
* (3) Enqueue fragment & set ts in dualpi2_enqueue_skb
476
476
*/
477
477
qdisc_skb_cb(nskb)->pkt_len = nskb->len;
478
+
qdisc_skb_cb(nskb)->pkt_segs = 1;
478
479
dualpi2_skb_cb(nskb)->classified =
479
480
dualpi2_skb_cb(skb)->classified;
480
481
dualpi2_skb_cb(nskb)->ect = dualpi2_skb_cb(skb)->ect;
+1
net/sched/sch_netem.c
+1
net/sched/sch_netem.c
+1
-1
net/sched/sch_qfq.c
+1
-1
net/sched/sch_qfq.c
···
1250
1250
}
1251
1251
}
1252
1252
1253
-
gso_segs = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
1253
+
gso_segs = qdisc_pkt_segs(skb);
1254
1254
err = qdisc_enqueue(skb, cl->qdisc, to_free);
1255
1255
if (unlikely(err != NET_XMIT_SUCCESS)) {
1256
1256
pr_debug("qfq_enqueue: enqueue failed %d\n", err);
+1
net/sched/sch_taprio.c
+1
net/sched/sch_taprio.c
+1
net/sched/sch_tbf.c
+1
net/sched/sch_tbf.c