Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'net_sched-speedup-qdisc-dequeue'
Eric Dumazet says:
====================
net_sched: speedup qdisc dequeue
Avoid up to two cache line misses in qdisc dequeue() to fetch
skb_shinfo(skb)->gso_segs/gso_size while qdisc spinlock is held.
Idea is to cache gso_segs at enqueue time before spinlock is
acquired, in the first skb cache line, where we already
have qdisc_skb_cb(skb)->pkt_len.
This series gives a 8 % improvement in a TX intensive workload.
(120 Mpps -> 130 Mpps on a Turin host, IDPF with 32 TX queues)
v2: https://lore.kernel.org/netdev/20251111093204.1432437-1-edumazet@google.com/
v1: https://lore.kernel.org/netdev/20251110094505.3335073-1-edumazet@google.com/T/#m8f562ed148f807c02fd02c6cd243604d449615b9
====================
Link: https://patch.msgid.link/20251121083256.674562-1-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
+147
-80
+3
-2
include/net/pkt_sched.h
+3
-2
include/net/pkt_sched.h
···
114
114
115
115
void __qdisc_run(struct Qdisc *q);
116
116
117
-
static inline void qdisc_run(struct Qdisc *q)
117
+
static inline struct sk_buff *qdisc_run(struct Qdisc *q)
118
118
{
119
119
if (qdisc_run_begin(q)) {
120
120
__qdisc_run(q);
121
-
qdisc_run_end(q);
121
+
return qdisc_run_end(q);
122
122
}
123
+
return NULL;
123
124
}
124
125
125
126
extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
+75
-26
include/net/sch_generic.h
+75
-26
include/net/sch_generic.h
···
88
88
#define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */
89
89
#define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */
90
90
#define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */
91
+
#define TCQ_F_DEQUEUE_DROPS 0x400 /* ->dequeue() can drop packets in q->to_free */
92
+
91
93
u32 limit;
92
94
const struct Qdisc_ops *ops;
93
95
struct qdisc_size_table __rcu *stab;
···
105
103
int pad;
106
104
refcount_t refcnt;
107
105
108
-
/*
109
-
* For performance sake on SMP, we put highly modified fields at the end
110
-
*/
111
-
struct sk_buff_head gso_skb ____cacheline_aligned_in_smp;
112
-
struct qdisc_skb_head q;
113
-
struct gnet_stats_basic_sync bstats;
114
-
struct gnet_stats_queue qstats;
115
-
bool running; /* must be written under qdisc spinlock */
116
-
unsigned long state;
117
-
struct Qdisc *next_sched;
118
-
struct sk_buff_head skb_bad_txq;
106
+
/* Cache line potentially dirtied in dequeue() or __netif_reschedule(). */
107
+
__cacheline_group_begin(Qdisc_read_mostly) ____cacheline_aligned;
108
+
struct sk_buff_head gso_skb;
109
+
struct Qdisc *next_sched;
110
+
struct sk_buff_head skb_bad_txq;
111
+
__cacheline_group_end(Qdisc_read_mostly);
112
+
113
+
/* Fields dirtied in dequeue() fast path. */
114
+
__cacheline_group_begin(Qdisc_write) ____cacheline_aligned;
115
+
struct qdisc_skb_head q;
116
+
unsigned long state;
117
+
struct gnet_stats_basic_sync bstats;
118
+
bool running; /* must be written under qdisc spinlock */
119
+
120
+
/* Note : we only change qstats.backlog in fast path. */
121
+
struct gnet_stats_queue qstats;
122
+
123
+
struct sk_buff *to_free;
124
+
__cacheline_group_end(Qdisc_write);
125
+
119
126
120
127
atomic_long_t defer_count ____cacheline_aligned_in_smp;
121
128
struct llist_head defer_list;
···
222
211
return true;
223
212
}
224
213
225
-
static inline void qdisc_run_end(struct Qdisc *qdisc)
214
+
static inline struct sk_buff *qdisc_run_end(struct Qdisc *qdisc)
226
215
{
216
+
struct sk_buff *to_free = NULL;
217
+
227
218
if (qdisc->flags & TCQ_F_NOLOCK) {
228
219
spin_unlock(&qdisc->seqlock);
229
220
···
238
225
if (unlikely(test_bit(__QDISC_STATE_MISSED,
239
226
&qdisc->state)))
240
227
__netif_schedule(qdisc);
241
-
} else {
242
-
WRITE_ONCE(qdisc->running, false);
228
+
return NULL;
243
229
}
230
+
231
+
if (qdisc->flags & TCQ_F_DEQUEUE_DROPS) {
232
+
to_free = qdisc->to_free;
233
+
if (to_free)
234
+
qdisc->to_free = NULL;
235
+
}
236
+
WRITE_ONCE(qdisc->running, false);
237
+
return to_free;
244
238
}
245
239
246
240
static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)
···
449
429
};
450
430
451
431
struct qdisc_skb_cb {
452
-
struct {
453
-
unsigned int pkt_len;
454
-
u16 slave_dev_queue_mapping;
455
-
u16 tc_classid;
456
-
};
432
+
unsigned int pkt_len;
433
+
u16 pkt_segs;
434
+
u16 tc_classid;
457
435
#define QDISC_CB_PRIV_LEN 20
458
436
unsigned char data[QDISC_CB_PRIV_LEN];
437
+
438
+
u16 slave_dev_queue_mapping;
439
+
u8 post_ct:1;
440
+
u8 post_ct_snat:1;
441
+
u8 post_ct_dnat:1;
459
442
};
460
443
461
444
typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
···
849
826
return qdisc_skb_cb(skb)->pkt_len;
850
827
}
851
828
829
+
static inline unsigned int qdisc_pkt_segs(const struct sk_buff *skb)
830
+
{
831
+
u32 pkt_segs = qdisc_skb_cb(skb)->pkt_segs;
832
+
833
+
DEBUG_NET_WARN_ON_ONCE(pkt_segs !=
834
+
(skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1));
835
+
return pkt_segs;
836
+
}
837
+
852
838
/* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */
853
839
enum net_xmit_qdisc_t {
854
840
__NET_XMIT_STOLEN = 0x00010000,
···
899
867
static inline void bstats_update(struct gnet_stats_basic_sync *bstats,
900
868
const struct sk_buff *skb)
901
869
{
902
-
_bstats_update(bstats,
903
-
qdisc_pkt_len(skb),
904
-
skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1);
870
+
_bstats_update(bstats, qdisc_pkt_len(skb), qdisc_pkt_segs(skb));
905
871
}
906
872
907
873
static inline void qdisc_bstats_cpu_update(struct Qdisc *sch,
···
1094
1064
struct qdisc_skb_cb qdisc_cb;
1095
1065
u32 drop_reason;
1096
1066
1097
-
u16 zone; /* Only valid if post_ct = true */
1067
+
u16 zone; /* Only valid if qdisc_skb_cb(skb)->post_ct = true */
1098
1068
u16 mru;
1099
-
u8 post_ct:1;
1100
-
u8 post_ct_snat:1;
1101
-
u8 post_ct_dnat:1;
1102
1069
};
1103
1070
1104
1071
static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb)
···
1116
1089
enum skb_drop_reason reason)
1117
1090
{
1118
1091
tc_skb_cb(skb)->drop_reason = reason;
1092
+
}
1093
+
1094
+
static inline void tcf_kfree_skb_list(struct sk_buff *skb)
1095
+
{
1096
+
while (unlikely(skb)) {
1097
+
struct sk_buff *next = skb->next;
1098
+
1099
+
prefetch(next);
1100
+
kfree_skb_reason(skb, tcf_get_drop_reason(skb));
1101
+
skb = next;
1102
+
}
1103
+
}
1104
+
1105
+
static inline void qdisc_dequeue_drop(struct Qdisc *q, struct sk_buff *skb,
1106
+
enum skb_drop_reason reason)
1107
+
{
1108
+
DEBUG_NET_WARN_ON_ONCE(!(q->flags & TCQ_F_DEQUEUE_DROPS));
1109
+
DEBUG_NET_WARN_ON_ONCE(q->flags & TCQ_F_NOLOCK);
1110
+
1111
+
tcf_set_drop_reason(skb, reason);
1112
+
skb->next = q->to_free;
1113
+
q->to_free = skb;
1119
1114
}
1120
1115
1121
1116
/* Instead of calling kfree_skb() while root qdisc lock is held,
+35
-27
net/core/dev.c
+35
-27
net/core/dev.c
···
4069
4069
}
4070
4070
EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
4071
4071
4072
-
static void qdisc_pkt_len_init(struct sk_buff *skb)
4072
+
static void qdisc_pkt_len_segs_init(struct sk_buff *skb)
4073
4073
{
4074
-
const struct skb_shared_info *shinfo = skb_shinfo(skb);
4074
+
struct skb_shared_info *shinfo = skb_shinfo(skb);
4075
+
u16 gso_segs;
4075
4076
4076
4077
qdisc_skb_cb(skb)->pkt_len = skb->len;
4078
+
if (!shinfo->gso_size) {
4079
+
qdisc_skb_cb(skb)->pkt_segs = 1;
4080
+
return;
4081
+
}
4082
+
4083
+
qdisc_skb_cb(skb)->pkt_segs = gso_segs = shinfo->gso_segs;
4077
4084
4078
4085
/* To get more precise estimation of bytes sent on wire,
4079
4086
* we add to pkt_len the headers size of all segments
4080
4087
*/
4081
-
if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
4082
-
u16 gso_segs = shinfo->gso_segs;
4088
+
if (skb_transport_header_was_set(skb)) {
4083
4089
unsigned int hdr_len;
4084
4090
4085
4091
/* mac layer + network layer */
···
4118
4112
if (payload <= 0)
4119
4113
return;
4120
4114
gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size);
4115
+
shinfo->gso_segs = gso_segs;
4116
+
qdisc_skb_cb(skb)->pkt_segs = gso_segs;
4121
4117
}
4122
4118
qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
4123
4119
}
···
4141
4133
struct net_device *dev,
4142
4134
struct netdev_queue *txq)
4143
4135
{
4144
-
struct sk_buff *next, *to_free = NULL;
4136
+
struct sk_buff *next, *to_free = NULL, *to_free2 = NULL;
4145
4137
spinlock_t *root_lock = qdisc_lock(q);
4146
4138
struct llist_node *ll_list, *first_n;
4147
4139
unsigned long defer_count = 0;
···
4160
4152
if (unlikely(!nolock_qdisc_is_empty(q))) {
4161
4153
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4162
4154
__qdisc_run(q);
4163
-
qdisc_run_end(q);
4155
+
to_free2 = qdisc_run_end(q);
4164
4156
4165
-
goto no_lock_out;
4157
+
goto free_skbs;
4166
4158
}
4167
4159
4168
4160
qdisc_bstats_cpu_update(q, skb);
···
4170
4162
!nolock_qdisc_is_empty(q))
4171
4163
__qdisc_run(q);
4172
4164
4173
-
qdisc_run_end(q);
4174
-
return NET_XMIT_SUCCESS;
4165
+
to_free2 = qdisc_run_end(q);
4166
+
rc = NET_XMIT_SUCCESS;
4167
+
goto free_skbs;
4175
4168
}
4176
4169
4177
4170
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4178
-
qdisc_run(q);
4179
-
4180
-
no_lock_out:
4181
-
if (unlikely(to_free))
4182
-
kfree_skb_list_reason(to_free,
4183
-
tcf_get_drop_reason(to_free));
4184
-
return rc;
4171
+
to_free2 = qdisc_run(q);
4172
+
goto free_skbs;
4185
4173
}
4186
4174
4187
4175
/* Open code llist_add(&skb->ll_node, &q->defer_list) + queue limit.
···
4190
4186
do {
4191
4187
if (first_n && !defer_count) {
4192
4188
defer_count = atomic_long_inc_return(&q->defer_count);
4193
-
if (unlikely(defer_count > q->limit)) {
4189
+
if (unlikely(defer_count > READ_ONCE(q->limit))) {
4194
4190
kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_DROP);
4195
4191
return NET_XMIT_DROP;
4196
4192
}
···
4235
4231
qdisc_bstats_update(q, skb);
4236
4232
if (sch_direct_xmit(skb, q, dev, txq, root_lock, true))
4237
4233
__qdisc_run(q);
4238
-
qdisc_run_end(q);
4234
+
to_free2 = qdisc_run_end(q);
4239
4235
rc = NET_XMIT_SUCCESS;
4240
4236
} else {
4241
4237
int count = 0;
4242
4238
4243
4239
llist_for_each_entry_safe(skb, next, ll_list, ll_node) {
4244
4240
prefetch(next);
4241
+
prefetch(&next->priority);
4245
4242
skb_mark_not_on_list(skb);
4246
4243
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4247
4244
count++;
4248
4245
}
4249
-
qdisc_run(q);
4246
+
to_free2 = qdisc_run(q);
4250
4247
if (count != 1)
4251
4248
rc = NET_XMIT_SUCCESS;
4252
4249
}
4253
4250
unlock:
4254
4251
spin_unlock(root_lock);
4255
-
if (unlikely(to_free))
4256
-
kfree_skb_list_reason(to_free,
4257
-
tcf_get_drop_reason(to_free));
4252
+
4253
+
free_skbs:
4254
+
tcf_kfree_skb_list(to_free);
4255
+
tcf_kfree_skb_list(to_free2);
4258
4256
return rc;
4259
4257
}
4260
4258
···
4361
4355
return ret;
4362
4356
4363
4357
tc_skb_cb(skb)->mru = 0;
4364
-
tc_skb_cb(skb)->post_ct = false;
4358
+
qdisc_skb_cb(skb)->post_ct = false;
4365
4359
tcf_set_drop_reason(skb, *drop_reason);
4366
4360
4367
4361
mini_qdisc_bstats_cpu_update(miniq, skb);
···
4432
4426
*pt_prev = NULL;
4433
4427
}
4434
4428
4435
-
qdisc_skb_cb(skb)->pkt_len = skb->len;
4429
+
qdisc_pkt_len_segs_init(skb);
4436
4430
tcx_set_ingress(skb, true);
4437
4431
4438
4432
if (static_branch_unlikely(&tcx_needed_key)) {
···
4743
4737
4744
4738
skb_update_prio(skb);
4745
4739
4746
-
qdisc_pkt_len_init(skb);
4740
+
qdisc_pkt_len_segs_init(skb);
4747
4741
tcx_set_ingress(skb, false);
4748
4742
#ifdef CONFIG_NET_EGRESS
4749
4743
if (static_branch_unlikely(&egress_needed_key)) {
···
5749
5743
rcu_read_lock();
5750
5744
5751
5745
while (head) {
5752
-
struct Qdisc *q = head;
5753
5746
spinlock_t *root_lock = NULL;
5747
+
struct sk_buff *to_free;
5748
+
struct Qdisc *q = head;
5754
5749
5755
5750
head = head->next_sched;
5756
5751
···
5778
5771
}
5779
5772
5780
5773
clear_bit(__QDISC_STATE_SCHED, &q->state);
5781
-
qdisc_run(q);
5774
+
to_free = qdisc_run(q);
5782
5775
if (root_lock)
5783
5776
spin_unlock(root_lock);
5777
+
tcf_kfree_skb_list(to_free);
5784
5778
}
5785
5779
5786
5780
rcu_read_unlock();
+4
-4
net/sched/act_ct.c
+4
-4
net/sched/act_ct.c
···
948
948
return err & NF_VERDICT_MASK;
949
949
950
950
if (action & BIT(NF_NAT_MANIP_SRC))
951
-
tc_skb_cb(skb)->post_ct_snat = 1;
951
+
qdisc_skb_cb(skb)->post_ct_snat = 1;
952
952
if (action & BIT(NF_NAT_MANIP_DST))
953
-
tc_skb_cb(skb)->post_ct_dnat = 1;
953
+
qdisc_skb_cb(skb)->post_ct_dnat = 1;
954
954
955
955
return err;
956
956
#else
···
986
986
tcf_action_update_bstats(&c->common, skb);
987
987
988
988
if (clear) {
989
-
tc_skb_cb(skb)->post_ct = false;
989
+
qdisc_skb_cb(skb)->post_ct = false;
990
990
ct = nf_ct_get(skb, &ctinfo);
991
991
if (ct) {
992
992
nf_ct_put(ct);
···
1097
1097
out_push:
1098
1098
skb_push_rcsum(skb, nh_ofs);
1099
1099
1100
-
tc_skb_cb(skb)->post_ct = true;
1100
+
qdisc_skb_cb(skb)->post_ct = true;
1101
1101
tc_skb_cb(skb)->zone = p->zone;
1102
1102
out_clear:
1103
1103
if (defrag)
+3
-3
net/sched/cls_api.c
+3
-3
net/sched/cls_api.c
···
1872
1872
}
1873
1873
ext->chain = last_executed_chain;
1874
1874
ext->mru = cb->mru;
1875
-
ext->post_ct = cb->post_ct;
1876
-
ext->post_ct_snat = cb->post_ct_snat;
1877
-
ext->post_ct_dnat = cb->post_ct_dnat;
1875
+
ext->post_ct = qdisc_skb_cb(skb)->post_ct;
1876
+
ext->post_ct_snat = qdisc_skb_cb(skb)->post_ct_snat;
1877
+
ext->post_ct_dnat = qdisc_skb_cb(skb)->post_ct_dnat;
1878
1878
ext->zone = cb->zone;
1879
1879
}
1880
1880
}
+1
-1
net/sched/cls_flower.c
+1
-1
net/sched/cls_flower.c
···
326
326
struct tcf_result *res)
327
327
{
328
328
struct cls_fl_head *head = rcu_dereference_bh(tp->root);
329
-
bool post_ct = tc_skb_cb(skb)->post_ct;
329
+
bool post_ct = qdisc_skb_cb(skb)->post_ct;
330
330
u16 zone = tc_skb_cb(skb)->zone;
331
331
struct fl_flow_key skb_key;
332
332
struct fl_flow_mask *mask;
+8
-11
net/sched/sch_cake.c
+8
-11
net/sched/sch_cake.c
···
1398
1398
const struct skb_shared_info *shinfo = skb_shinfo(skb);
1399
1399
unsigned int hdr_len, last_len = 0;
1400
1400
u32 off = skb_network_offset(skb);
1401
+
u16 segs = qdisc_pkt_segs(skb);
1401
1402
u32 len = qdisc_pkt_len(skb);
1402
-
u16 segs = 1;
1403
1403
1404
1404
q->avg_netoff = cake_ewma(q->avg_netoff, off << 16, 8);
1405
1405
1406
-
if (!shinfo->gso_size)
1406
+
if (segs == 1)
1407
1407
return cake_calc_overhead(q, len, off);
1408
1408
1409
-
/* borrowed from qdisc_pkt_len_init() */
1409
+
/* borrowed from qdisc_pkt_len_segs_init() */
1410
1410
if (!skb->encapsulation)
1411
1411
hdr_len = skb_transport_offset(skb);
1412
1412
else
···
1429
1429
sizeof(_udphdr), &_udphdr))
1430
1430
hdr_len += sizeof(struct udphdr);
1431
1431
}
1432
-
1433
-
if (unlikely(shinfo->gso_type & SKB_GSO_DODGY))
1434
-
segs = DIV_ROUND_UP(skb->len - hdr_len,
1435
-
shinfo->gso_size);
1436
-
else
1437
-
segs = shinfo->gso_segs;
1438
1432
1439
1433
len = shinfo->gso_size + hdr_len;
1440
1434
last_len = skb->len - shinfo->gso_size * (segs - 1);
···
1782
1788
if (unlikely(len > b->max_skblen))
1783
1789
b->max_skblen = len;
1784
1790
1785
-
if (skb_is_gso(skb) && q->rate_flags & CAKE_FLAG_SPLIT_GSO) {
1791
+
if (qdisc_pkt_segs(skb) > 1 && q->rate_flags & CAKE_FLAG_SPLIT_GSO) {
1786
1792
struct sk_buff *segs, *nskb;
1787
1793
netdev_features_t features = netif_skb_features(skb);
1788
1794
unsigned int slen = 0, numsegs = 0;
···
1794
1800
skb_list_walk_safe(segs, segs, nskb) {
1795
1801
skb_mark_not_on_list(segs);
1796
1802
qdisc_skb_cb(segs)->pkt_len = segs->len;
1803
+
qdisc_skb_cb(segs)->pkt_segs = 1;
1797
1804
cobalt_set_enqueue_time(segs, now);
1798
1805
get_cobalt_cb(segs)->adjusted_len = cake_overhead(q,
1799
1806
segs);
···
2183
2188
b->tin_dropped++;
2184
2189
qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb));
2185
2190
qdisc_qstats_drop(sch);
2186
-
kfree_skb_reason(skb, reason);
2191
+
qdisc_dequeue_drop(sch, skb, reason);
2187
2192
if (q->rate_flags & CAKE_FLAG_INGRESS)
2188
2193
goto retry;
2189
2194
}
···
2724
2729
int i, j, err;
2725
2730
2726
2731
sch->limit = 10240;
2732
+
sch->flags |= TCQ_F_DEQUEUE_DROPS;
2733
+
2727
2734
q->tin_mode = CAKE_DIFFSERV_DIFFSERV3;
2728
2735
q->flow_mode = CAKE_FLOW_TRIPLE;
2729
2736
+3
-1
net/sched/sch_codel.c
+3
-1
net/sched/sch_codel.c
···
52
52
{
53
53
struct Qdisc *sch = ctx;
54
54
55
-
kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_CONGESTED);
55
+
qdisc_dequeue_drop(sch, skb, SKB_DROP_REASON_QDISC_CONGESTED);
56
56
qdisc_qstats_drop(sch);
57
57
}
58
58
···
181
181
sch->flags |= TCQ_F_CAN_BYPASS;
182
182
else
183
183
sch->flags &= ~TCQ_F_CAN_BYPASS;
184
+
185
+
sch->flags |= TCQ_F_DEQUEUE_DROPS;
184
186
185
187
return 0;
186
188
}
+1
net/sched/sch_dualpi2.c
+1
net/sched/sch_dualpi2.c
···
475
475
* (3) Enqueue fragment & set ts in dualpi2_enqueue_skb
476
476
*/
477
477
qdisc_skb_cb(nskb)->pkt_len = nskb->len;
478
+
qdisc_skb_cb(nskb)->pkt_segs = 1;
478
479
dualpi2_skb_cb(nskb)->classified =
479
480
dualpi2_skb_cb(skb)->classified;
480
481
dualpi2_skb_cb(nskb)->ect = dualpi2_skb_cb(skb)->ect;
+6
-3
net/sched/sch_fq.c
+6
-3
net/sched/sch_fq.c
···
480
480
struct sk_buff *skb)
481
481
{
482
482
if (skb == flow->head) {
483
-
flow->head = skb->next;
483
+
struct sk_buff *next = skb->next;
484
+
485
+
prefetch(next);
486
+
flow->head = next;
484
487
} else {
485
488
rb_erase(&skb->rbnode, &flow->t_root);
486
489
skb->dev = qdisc_dev(sch);
···
500
497
skb_mark_not_on_list(skb);
501
498
qdisc_qstats_backlog_dec(sch, skb);
502
499
sch->q.qlen--;
500
+
qdisc_bstats_update(sch, skb);
503
501
}
504
502
505
503
static void flow_queue_add(struct fq_flow *flow, struct sk_buff *skb)
···
715
711
goto begin;
716
712
}
717
713
prefetch(&skb->end);
714
+
fq_dequeue_skb(sch, f, skb);
718
715
if ((s64)(now - time_next_packet - q->ce_threshold) > 0) {
719
716
INET_ECN_set_ce(skb);
720
717
q->stat_ce_mark++;
···
723
718
if (--f->qlen == 0)
724
719
q->inactive_flows++;
725
720
q->band_pkt_count[fq_skb_cb(skb)->band]--;
726
-
fq_dequeue_skb(sch, f, skb);
727
721
} else {
728
722
head->first = f->next;
729
723
/* force a pass through old_flows to prevent starvation */
···
780
776
f->time_next_packet = now + len;
781
777
}
782
778
out:
783
-
qdisc_bstats_update(sch, skb);
784
779
return skb;
785
780
}
786
781
+4
-1
net/sched/sch_fq_codel.c
+4
-1
net/sched/sch_fq_codel.c
···
275
275
{
276
276
struct Qdisc *sch = ctx;
277
277
278
-
kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_CONGESTED);
278
+
qdisc_dequeue_drop(sch, skb, SKB_DROP_REASON_QDISC_CONGESTED);
279
279
qdisc_qstats_drop(sch);
280
280
}
281
281
···
519
519
sch->flags |= TCQ_F_CAN_BYPASS;
520
520
else
521
521
sch->flags &= ~TCQ_F_CAN_BYPASS;
522
+
523
+
sch->flags |= TCQ_F_DEQUEUE_DROPS;
524
+
522
525
return 0;
523
526
524
527
alloc_failure:
+1
net/sched/sch_netem.c
+1
net/sched/sch_netem.c
+1
-1
net/sched/sch_qfq.c
+1
-1
net/sched/sch_qfq.c
···
1250
1250
}
1251
1251
}
1252
1252
1253
-
gso_segs = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
1253
+
gso_segs = qdisc_pkt_segs(skb);
1254
1254
err = qdisc_enqueue(skb, cl->qdisc, to_free);
1255
1255
if (unlikely(err != NET_XMIT_SUCCESS)) {
1256
1256
pr_debug("qfq_enqueue: enqueue failed %d\n", err);
+1
net/sched/sch_taprio.c
+1
net/sched/sch_taprio.c
+1
net/sched/sch_tbf.c
+1
net/sched/sch_tbf.c