Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
net_sched: sch_sfq: handle bigger packets
SFQ has an assumption on dealing with packets smaller than 64KB.
Even before BIG TCP, TCA_STAB can provide arbitrary big values
in qdisc_pkt_len(skb)
It is time to switch (struct sfq_slot)->allot to a 32bit field.
sizeof(struct sfq_slot) is now 64 bytes, giving better cache locality.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://patch.msgid.link/20241008111603.653140-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
authored by
Eric Dumazet
and committed by
Jakub Kicinski
e4650d7a
0a316b16
+13
-26
+13
-26
net/sched/sch_sfq.c
+13
-26
net/sched/sch_sfq.c
···
77
77
#define SFQ_EMPTY_SLOT 0xffff
78
78
#define SFQ_DEFAULT_HASH_DIVISOR 1024
79
79
80
-
/* We use 16 bits to store allot, and want to handle packets up to 64K
81
-
* Scale allot by 8 (1<<3) so that no overflow occurs.
82
-
*/
83
-
#define SFQ_ALLOT_SHIFT 3
84
-
#define SFQ_ALLOT_SIZE(X) DIV_ROUND_UP(X, 1 << SFQ_ALLOT_SHIFT)
85
-
86
80
/* This type should contain at least SFQ_MAX_DEPTH + 1 + SFQ_MAX_FLOWS values */
87
81
typedef u16 sfq_index;
88
82
···
98
104
sfq_index next; /* next slot in sfq RR chain */
99
105
struct sfq_head dep; /* anchor in dep[] chains */
100
106
unsigned short hash; /* hash value (index in ht[]) */
101
-
short allot; /* credit for this slot */
107
+
int allot; /* credit for this slot */
102
108
103
109
unsigned int backlog;
104
110
struct red_vars vars;
···
114
120
siphash_key_t perturbation;
115
121
u8 cur_depth; /* depth of longest slot */
116
122
u8 flags;
117
-
unsigned short scaled_quantum; /* SFQ_ALLOT_SIZE(quantum) */
118
123
struct tcf_proto __rcu *filter_list;
119
124
struct tcf_block *block;
120
125
sfq_index *ht; /* Hash table ('divisor' slots) */
···
449
456
*/
450
457
q->tail = slot;
451
458
/* We could use a bigger initial quantum for new flows */
452
-
slot->allot = q->scaled_quantum;
459
+
slot->allot = q->quantum;
453
460
}
454
461
if (++sch->q.qlen <= q->limit)
455
462
return NET_XMIT_SUCCESS;
···
486
493
slot = &q->slots[a];
487
494
if (slot->allot <= 0) {
488
495
q->tail = slot;
489
-
slot->allot += q->scaled_quantum;
496
+
slot->allot += q->quantum;
490
497
goto next_slot;
491
498
}
492
499
skb = slot_dequeue_head(slot);
···
505
512
}
506
513
q->tail->next = next_a;
507
514
} else {
508
-
slot->allot -= SFQ_ALLOT_SIZE(qdisc_pkt_len(skb));
515
+
slot->allot -= qdisc_pkt_len(skb);
509
516
}
510
517
return skb;
511
518
}
···
588
595
q->tail->next = x;
589
596
}
590
597
q->tail = slot;
591
-
slot->allot = q->scaled_quantum;
598
+
slot->allot = q->quantum;
592
599
}
593
600
}
594
601
sch->q.qlen -= dropped;
···
621
628
rcu_read_unlock();
622
629
}
623
630
624
-
static int sfq_change(struct Qdisc *sch, struct nlattr *opt)
631
+
static int sfq_change(struct Qdisc *sch, struct nlattr *opt,
632
+
struct netlink_ext_ack *extack)
625
633
{
626
634
struct sfq_sched_data *q = qdisc_priv(sch);
627
635
struct tc_sfq_qopt *ctl = nla_data(opt);
···
640
646
(!is_power_of_2(ctl->divisor) || ctl->divisor > 65536))
641
647
return -EINVAL;
642
648
643
-
/* slot->allot is a short, make sure quantum is not too big. */
644
-
if (ctl->quantum) {
645
-
unsigned int scaled = SFQ_ALLOT_SIZE(ctl->quantum);
646
-
647
-
if (scaled <= 0 || scaled > SHRT_MAX)
648
-
return -EINVAL;
649
+
if ((int)ctl->quantum < 0) {
650
+
NL_SET_ERR_MSG_MOD(extack, "invalid quantum");
651
+
return -EINVAL;
649
652
}
650
-
651
653
if (ctl_v1 && !red_check_params(ctl_v1->qth_min, ctl_v1->qth_max,
652
654
ctl_v1->Wlog, ctl_v1->Scell_log, NULL))
653
655
return -EINVAL;
···
653
663
return -ENOMEM;
654
664
}
655
665
sch_tree_lock(sch);
656
-
if (ctl->quantum) {
666
+
if (ctl->quantum)
657
667
q->quantum = ctl->quantum;
658
-
q->scaled_quantum = SFQ_ALLOT_SIZE(q->quantum);
659
-
}
660
668
WRITE_ONCE(q->perturb_period, ctl->perturb_period * HZ);
661
669
if (ctl->flows)
662
670
q->maxflows = min_t(u32, ctl->flows, SFQ_MAX_FLOWS);
···
750
762
q->divisor = SFQ_DEFAULT_HASH_DIVISOR;
751
763
q->maxflows = SFQ_DEFAULT_FLOWS;
752
764
q->quantum = psched_mtu(qdisc_dev(sch));
753
-
q->scaled_quantum = SFQ_ALLOT_SIZE(q->quantum);
754
765
q->perturb_period = 0;
755
766
get_random_bytes(&q->perturbation, sizeof(q->perturbation));
756
767
757
768
if (opt) {
758
-
int err = sfq_change(sch, opt);
769
+
int err = sfq_change(sch, opt, extack);
759
770
if (err)
760
771
return err;
761
772
}
···
865
878
if (idx != SFQ_EMPTY_SLOT) {
866
879
const struct sfq_slot *slot = &q->slots[idx];
867
880
868
-
xstats.allot = slot->allot << SFQ_ALLOT_SHIFT;
881
+
xstats.allot = slot->allot;
869
882
qs.qlen = slot->qlen;
870
883
qs.backlog = slot->backlog;
871
884
}