Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Martin KaFai Lau says:

====================
pull-request: bpf-next 2025-05-02

We've added 14 non-merge commits during the last 10 day(s) which contain
a total of 13 files changed, 740 insertions(+), 121 deletions(-).

The main changes are:

1) Avoid skipping or repeating a sk when using a UDP bpf_iter,
from Jordan Rife.

2) Fixed a crash when a bpf qdisc is set in
the net.core.default_qdisc, from Amery Hung.

3) A few other fixes in the bpf qdisc, from Amery Hung.
- Always call qdisc_watchdog_init() in the .init prologue such that
the .reset/.destroy epilogue can always call qdisc_watchdog_cancel()
without issue.
- bpf_qdisc_init_prologue() was incorrectly returning an error
when the bpf qdisc is set as the default_qdisc and the mq is creating
the default_qdisc. It is now fixed.

* tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next:
selftests/bpf: Cleanup bpf qdisc selftests
selftests/bpf: Test attaching a bpf qdisc with incomplete operators
bpf: net_sched: Make some Qdisc_ops ops mandatory
selftests/bpf: Test setting and creating bpf qdisc as default qdisc
bpf: net_sched: Fix bpf qdisc init prologue when set as default qdisc
selftests/bpf: Add tests for bucket resume logic in UDP socket iterators
selftests/bpf: Return socket cookies from sock_iter_batch progs
bpf: udp: Avoid socket skips and repeats during iteration
bpf: udp: Use bpf_udp_iter_batch_item for bpf_udp_iter_state batch items
bpf: udp: Get rid of st_bucket_done
bpf: udp: Make sure iter->batch always contains a full bucket snapshot
bpf: udp: Make mem flags configurable through bpf_iter_udp_realloc_batch
bpf: net_sched: Fix using bpf qdisc as default qdisc
selftests/bpf: Fix compilation errors
====================

Link: https://patch.msgid.link/20250503010755.4030524-1-martin.lau@linux.dev
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

+741 -122
+3
include/linux/udp.h
··· 216 216 #define udp_portaddr_for_each_entry(__sk, list) \ 217 217 hlist_for_each_entry(__sk, list, __sk_common.skc_portaddr_node) 218 218 219 + #define udp_portaddr_for_each_entry_from(__sk) \ 220 + hlist_for_each_entry_from(__sk, __sk_common.skc_portaddr_node) 221 + 219 222 #define udp_portaddr_for_each_entry_rcu(__sk, list) \ 220 223 hlist_for_each_entry_rcu(__sk, list, __sk_common.skc_portaddr_node) 221 224
+117 -56
net/ipv4/udp.c
··· 93 93 #include <linux/inet.h> 94 94 #include <linux/netdevice.h> 95 95 #include <linux/slab.h> 96 + #include <linux/sock_diag.h> 96 97 #include <net/tcp_states.h> 97 98 #include <linux/skbuff.h> 98 99 #include <linux/proc_fs.h> ··· 3414 3413 int bucket __aligned(8); 3415 3414 }; 3416 3415 3416 + union bpf_udp_iter_batch_item { 3417 + struct sock *sk; 3418 + __u64 cookie; 3419 + }; 3420 + 3417 3421 struct bpf_udp_iter_state { 3418 3422 struct udp_iter_state state; 3419 3423 unsigned int cur_sk; 3420 3424 unsigned int end_sk; 3421 3425 unsigned int max_sk; 3422 - int offset; 3423 - struct sock **batch; 3424 - bool st_bucket_done; 3426 + union bpf_udp_iter_batch_item *batch; 3425 3427 }; 3426 3428 3427 3429 static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter, 3428 - unsigned int new_batch_sz); 3430 + unsigned int new_batch_sz, gfp_t flags); 3431 + static struct sock *bpf_iter_udp_resume(struct sock *first_sk, 3432 + union bpf_udp_iter_batch_item *cookies, 3433 + int n_cookies) 3434 + { 3435 + struct sock *sk = NULL; 3436 + int i; 3437 + 3438 + for (i = 0; i < n_cookies; i++) { 3439 + sk = first_sk; 3440 + udp_portaddr_for_each_entry_from(sk) 3441 + if (cookies[i].cookie == atomic64_read(&sk->sk_cookie)) 3442 + goto done; 3443 + } 3444 + done: 3445 + return sk; 3446 + } 3447 + 3429 3448 static struct sock *bpf_iter_udp_batch(struct seq_file *seq) 3430 3449 { 3431 3450 struct bpf_udp_iter_state *iter = seq->private; 3432 3451 struct udp_iter_state *state = &iter->state; 3452 + unsigned int find_cookie, end_cookie; 3433 3453 struct net *net = seq_file_net(seq); 3434 - int resume_bucket, resume_offset; 3435 3454 struct udp_table *udptable; 3436 3455 unsigned int batch_sks = 0; 3437 - bool resized = false; 3456 + int resume_bucket; 3457 + int resizes = 0; 3438 3458 struct sock *sk; 3459 + int err = 0; 3439 3460 3440 3461 resume_bucket = state->bucket; 3441 - resume_offset = iter->offset; 3442 3462 3443 3463 /* The current batch is done, so advance the bucket. */ 3444 - if (iter->st_bucket_done) 3464 + if (iter->cur_sk == iter->end_sk) 3445 3465 state->bucket++; 3446 3466 3447 3467 udptable = udp_get_table_seq(seq, net); ··· 3475 3453 * before releasing the bucket lock. This allows BPF programs that are 3476 3454 * called in seq_show to acquire the bucket lock if needed. 3477 3455 */ 3456 + find_cookie = iter->cur_sk; 3457 + end_cookie = iter->end_sk; 3478 3458 iter->cur_sk = 0; 3479 3459 iter->end_sk = 0; 3480 - iter->st_bucket_done = false; 3481 3460 batch_sks = 0; 3482 3461 3483 3462 for (; state->bucket <= udptable->mask; state->bucket++) { 3484 3463 struct udp_hslot *hslot2 = &udptable->hash2[state->bucket].hslot; 3485 3464 3486 3465 if (hlist_empty(&hslot2->head)) 3487 - continue; 3466 + goto next_bucket; 3488 3467 3489 - iter->offset = 0; 3490 3468 spin_lock_bh(&hslot2->lock); 3491 - udp_portaddr_for_each_entry(sk, &hslot2->head) { 3469 + sk = hlist_entry_safe(hslot2->head.first, struct sock, 3470 + __sk_common.skc_portaddr_node); 3471 + /* Resume from the first (in iteration order) unseen socket from 3472 + * the last batch that still exists in resume_bucket. Most of 3473 + * the time this will just be where the last iteration left off 3474 + * in resume_bucket unless that socket disappeared between 3475 + * reads. 3476 + */ 3477 + if (state->bucket == resume_bucket) 3478 + sk = bpf_iter_udp_resume(sk, &iter->batch[find_cookie], 3479 + end_cookie - find_cookie); 3480 + fill_batch: 3481 + udp_portaddr_for_each_entry_from(sk) { 3492 3482 if (seq_sk_match(seq, sk)) { 3493 - /* Resume from the last iterated socket at the 3494 - * offset in the bucket before iterator was stopped. 3495 - */ 3496 - if (state->bucket == resume_bucket && 3497 - iter->offset < resume_offset) { 3498 - ++iter->offset; 3499 - continue; 3500 - } 3501 3483 if (iter->end_sk < iter->max_sk) { 3502 3484 sock_hold(sk); 3503 - iter->batch[iter->end_sk++] = sk; 3485 + iter->batch[iter->end_sk++].sk = sk; 3504 3486 } 3505 3487 batch_sks++; 3506 3488 } 3507 3489 } 3490 + 3491 + /* Allocate a larger batch and try again. */ 3492 + if (unlikely(resizes <= 1 && iter->end_sk && 3493 + iter->end_sk != batch_sks)) { 3494 + resizes++; 3495 + 3496 + /* First, try with GFP_USER to maximize the chances of 3497 + * grabbing more memory. 3498 + */ 3499 + if (resizes == 1) { 3500 + spin_unlock_bh(&hslot2->lock); 3501 + err = bpf_iter_udp_realloc_batch(iter, 3502 + batch_sks * 3 / 2, 3503 + GFP_USER); 3504 + if (err) 3505 + return ERR_PTR(err); 3506 + /* Start over. */ 3507 + goto again; 3508 + } 3509 + 3510 + /* Next, hold onto the lock, so the bucket doesn't 3511 + * change while we get the rest of the sockets. 3512 + */ 3513 + err = bpf_iter_udp_realloc_batch(iter, batch_sks, 3514 + GFP_NOWAIT); 3515 + if (err) { 3516 + spin_unlock_bh(&hslot2->lock); 3517 + return ERR_PTR(err); 3518 + } 3519 + 3520 + /* Pick up where we left off. */ 3521 + sk = iter->batch[iter->end_sk - 1].sk; 3522 + sk = hlist_entry_safe(sk->__sk_common.skc_portaddr_node.next, 3523 + struct sock, 3524 + __sk_common.skc_portaddr_node); 3525 + batch_sks = iter->end_sk; 3526 + goto fill_batch; 3527 + } 3528 + 3508 3529 spin_unlock_bh(&hslot2->lock); 3509 3530 3510 3531 if (iter->end_sk) 3511 3532 break; 3533 + next_bucket: 3534 + resizes = 0; 3512 3535 } 3513 3536 3514 - /* All done: no batch made. */ 3515 - if (!iter->end_sk) 3516 - return NULL; 3517 - 3518 - if (iter->end_sk == batch_sks) { 3519 - /* Batching is done for the current bucket; return the first 3520 - * socket to be iterated from the batch. 3521 - */ 3522 - iter->st_bucket_done = true; 3523 - goto done; 3524 - } 3525 - if (!resized && !bpf_iter_udp_realloc_batch(iter, batch_sks * 3 / 2)) { 3526 - resized = true; 3527 - /* After allocating a larger batch, retry one more time to grab 3528 - * the whole bucket. 3529 - */ 3530 - goto again; 3531 - } 3532 - done: 3533 - return iter->batch[0]; 3537 + WARN_ON_ONCE(iter->end_sk != batch_sks); 3538 + return iter->end_sk ? iter->batch[0].sk : NULL; 3534 3539 } 3535 3540 3536 3541 static void *bpf_iter_udp_seq_next(struct seq_file *seq, void *v, loff_t *pos) ··· 3568 3519 /* Whenever seq_next() is called, the iter->cur_sk is 3569 3520 * done with seq_show(), so unref the iter->cur_sk. 3570 3521 */ 3571 - if (iter->cur_sk < iter->end_sk) { 3572 - sock_put(iter->batch[iter->cur_sk++]); 3573 - ++iter->offset; 3574 - } 3522 + if (iter->cur_sk < iter->end_sk) 3523 + sock_put(iter->batch[iter->cur_sk++].sk); 3575 3524 3576 3525 /* After updating iter->cur_sk, check if there are more sockets 3577 3526 * available in the current bucket batch. 3578 3527 */ 3579 3528 if (iter->cur_sk < iter->end_sk) 3580 - sk = iter->batch[iter->cur_sk]; 3529 + sk = iter->batch[iter->cur_sk].sk; 3581 3530 else 3582 3531 /* Prepare a new batch. */ 3583 3532 sk = bpf_iter_udp_batch(seq); ··· 3639 3592 3640 3593 static void bpf_iter_udp_put_batch(struct bpf_udp_iter_state *iter) 3641 3594 { 3642 - while (iter->cur_sk < iter->end_sk) 3643 - sock_put(iter->batch[iter->cur_sk++]); 3595 + union bpf_udp_iter_batch_item *item; 3596 + unsigned int cur_sk = iter->cur_sk; 3597 + __u64 cookie; 3598 + 3599 + /* Remember the cookies of the sockets we haven't seen yet, so we can 3600 + * pick up where we left off next time around. 3601 + */ 3602 + while (cur_sk < iter->end_sk) { 3603 + item = &iter->batch[cur_sk++]; 3604 + cookie = sock_gen_cookie(item->sk); 3605 + sock_put(item->sk); 3606 + item->cookie = cookie; 3607 + } 3644 3608 } 3645 3609 3646 3610 static void bpf_iter_udp_seq_stop(struct seq_file *seq, void *v) ··· 3667 3609 (void)udp_prog_seq_show(prog, &meta, v, 0, 0); 3668 3610 } 3669 3611 3670 - if (iter->cur_sk < iter->end_sk) { 3612 + if (iter->cur_sk < iter->end_sk) 3671 3613 bpf_iter_udp_put_batch(iter); 3672 - iter->st_bucket_done = false; 3673 - } 3674 3614 } 3675 3615 3676 3616 static const struct seq_operations bpf_iter_udp_seq_ops = { ··· 3919 3863 struct udp_sock *udp_sk, uid_t uid, int bucket) 3920 3864 3921 3865 static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter, 3922 - unsigned int new_batch_sz) 3866 + unsigned int new_batch_sz, gfp_t flags) 3923 3867 { 3924 - struct sock **new_batch; 3868 + union bpf_udp_iter_batch_item *new_batch; 3925 3869 3926 3870 new_batch = kvmalloc_array(new_batch_sz, sizeof(*new_batch), 3927 - GFP_USER | __GFP_NOWARN); 3871 + flags | __GFP_NOWARN); 3928 3872 if (!new_batch) 3929 3873 return -ENOMEM; 3930 3874 3931 - bpf_iter_udp_put_batch(iter); 3875 + if (flags != GFP_NOWAIT) 3876 + bpf_iter_udp_put_batch(iter); 3877 + 3878 + memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk); 3932 3879 kvfree(iter->batch); 3933 3880 iter->batch = new_batch; 3934 3881 iter->max_sk = new_batch_sz; ··· 3950 3891 if (ret) 3951 3892 return ret; 3952 3893 3953 - ret = bpf_iter_udp_realloc_batch(iter, INIT_BATCH_SZ); 3894 + ret = bpf_iter_udp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER); 3954 3895 if (ret) 3955 3896 bpf_iter_fini_seq_net(priv_data); 3897 + 3898 + iter->state.bucket = -1; 3956 3899 3957 3900 return ret; 3958 3901 }
+20 -6
net/sched/bpf_qdisc.c
··· 234 234 struct net_device *dev = qdisc_dev(sch); 235 235 struct Qdisc *p; 236 236 237 - if (sch->parent != TC_H_ROOT) { 238 - p = qdisc_lookup(dev, TC_H_MAJ(sch->parent)); 239 - if (!p) 240 - return -ENOENT; 237 + qdisc_watchdog_init(&q->watchdog, sch); 241 238 242 - if (!(p->flags & TCQ_F_MQROOT)) { 239 + if (sch->parent != TC_H_ROOT) { 240 + /* If qdisc_lookup() returns NULL, it means .init is called by 241 + * qdisc_create_dflt() in mq/mqprio_init and the parent qdisc 242 + * has not been added to qdisc_hash yet. 243 + */ 244 + p = qdisc_lookup(dev, TC_H_MAJ(sch->parent)); 245 + if (p && !(p->flags & TCQ_F_MQROOT)) { 243 246 NL_SET_ERR_MSG(extack, "BPF qdisc only supported on root or mq"); 244 247 return -EINVAL; 245 248 } 246 249 } 247 250 248 - qdisc_watchdog_init(&q->watchdog, sch); 249 251 return 0; 250 252 } 251 253 ··· 395 393 return unregister_qdisc(kdata); 396 394 } 397 395 396 + static int bpf_qdisc_validate(void *kdata) 397 + { 398 + struct Qdisc_ops *ops = (struct Qdisc_ops *)kdata; 399 + 400 + if (!ops->enqueue || !ops->dequeue || !ops->init || 401 + !ops->reset || !ops->destroy) 402 + return -EINVAL; 403 + 404 + return 0; 405 + } 406 + 398 407 static int Qdisc_ops__enqueue(struct sk_buff *skb__ref, struct Qdisc *sch, 399 408 struct sk_buff **to_free) 400 409 { ··· 443 430 .verifier_ops = &bpf_qdisc_verifier_ops, 444 431 .reg = bpf_qdisc_reg, 445 432 .unreg = bpf_qdisc_unreg, 433 + .validate = bpf_qdisc_validate, 446 434 .init_member = bpf_qdisc_init_member, 447 435 .init = bpf_qdisc_init, 448 436 .name = "Qdisc_ops",
+2 -2
net/sched/sch_api.c
··· 208 208 209 209 for (q = qdisc_base; q; q = q->next) { 210 210 if (!strcmp(name, q->id)) { 211 - if (!try_module_get(q->owner)) 211 + if (!bpf_try_module_get(q, q->owner)) 212 212 q = NULL; 213 213 break; 214 214 } ··· 238 238 239 239 if (ops) { 240 240 /* Set new default */ 241 - module_put(default_qdisc_ops->owner); 241 + bpf_module_put(default_qdisc_ops, default_qdisc_ops->owner); 242 242 default_qdisc_ops = ops; 243 243 } 244 244 write_unlock(&qdisc_mod_lock);
+2 -2
net/sched/sch_generic.c
··· 1002 1002 { 1003 1003 struct Qdisc *sch; 1004 1004 1005 - if (!try_module_get(ops->owner)) { 1005 + if (!bpf_try_module_get(ops, ops->owner)) { 1006 1006 NL_SET_ERR_MSG(extack, "Failed to increase module reference counter"); 1007 1007 return NULL; 1008 1008 } 1009 1009 1010 1010 sch = qdisc_alloc(dev_queue, ops, extack); 1011 1011 if (IS_ERR(sch)) { 1012 - module_put(ops->owner); 1012 + bpf_module_put(ops, ops->owner); 1013 1013 return NULL; 1014 1014 } 1015 1015 sch->parent = parentid;
+85 -34
tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c
··· 7 7 #include "network_helpers.h" 8 8 #include "bpf_qdisc_fifo.skel.h" 9 9 #include "bpf_qdisc_fq.skel.h" 10 + #include "bpf_qdisc_fail__incompl_ops.skel.h" 10 11 11 12 #define LO_IFINDEX 1 12 13 ··· 50 49 static void test_fifo(void) 51 50 { 52 51 struct bpf_qdisc_fifo *fifo_skel; 53 - struct bpf_link *link; 54 52 55 53 fifo_skel = bpf_qdisc_fifo__open_and_load(); 56 54 if (!ASSERT_OK_PTR(fifo_skel, "bpf_qdisc_fifo__open_and_load")) 57 55 return; 58 56 59 - link = bpf_map__attach_struct_ops(fifo_skel->maps.fifo); 60 - if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 61 - bpf_qdisc_fifo__destroy(fifo_skel); 62 - return; 63 - } 57 + if (!ASSERT_OK(bpf_qdisc_fifo__attach(fifo_skel), "bpf_qdisc_fifo__attach")) 58 + goto out; 64 59 65 60 do_test("bpf_fifo"); 66 - 67 - bpf_link__destroy(link); 61 + out: 68 62 bpf_qdisc_fifo__destroy(fifo_skel); 69 63 } 70 64 71 65 static void test_fq(void) 72 66 { 73 67 struct bpf_qdisc_fq *fq_skel; 74 - struct bpf_link *link; 75 68 76 69 fq_skel = bpf_qdisc_fq__open_and_load(); 77 70 if (!ASSERT_OK_PTR(fq_skel, "bpf_qdisc_fq__open_and_load")) 78 71 return; 79 72 80 - link = bpf_map__attach_struct_ops(fq_skel->maps.fq); 81 - if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 82 - bpf_qdisc_fq__destroy(fq_skel); 83 - return; 84 - } 73 + if (!ASSERT_OK(bpf_qdisc_fq__attach(fq_skel), "bpf_qdisc_fq__attach")) 74 + goto out; 85 75 86 76 do_test("bpf_fq"); 87 - 88 - bpf_link__destroy(link); 77 + out: 89 78 bpf_qdisc_fq__destroy(fq_skel); 90 79 } 91 80 ··· 87 96 .handle = 0x11 << 16, 88 97 .qdisc = "bpf_fifo"); 89 98 struct bpf_qdisc_fifo *fifo_skel; 90 - struct bpf_link *link; 91 99 int err; 92 100 93 101 fifo_skel = bpf_qdisc_fifo__open_and_load(); 94 102 if (!ASSERT_OK_PTR(fifo_skel, "bpf_qdisc_fifo__open_and_load")) 95 103 return; 96 104 97 - link = bpf_map__attach_struct_ops(fifo_skel->maps.fifo); 98 - if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 99 - bpf_qdisc_fifo__destroy(fifo_skel); 100 - return; 101 - } 105 + if (!ASSERT_OK(bpf_qdisc_fifo__attach(fifo_skel), "bpf_qdisc_fifo__attach")) 106 + goto out; 102 107 103 108 SYS(out, "ip link add veth0 type veth peer veth1"); 104 109 hook.ifindex = if_nametoindex("veth0"); ··· 107 120 108 121 SYS(out, "tc qdisc delete dev veth0 root mq"); 109 122 out: 110 - bpf_link__destroy(link); 111 123 bpf_qdisc_fifo__destroy(fifo_skel); 112 124 } 113 125 ··· 118 132 .handle = 0x11 << 16, 119 133 .qdisc = "bpf_fifo"); 120 134 struct bpf_qdisc_fifo *fifo_skel; 121 - struct bpf_link *link; 122 135 int err; 123 136 124 137 fifo_skel = bpf_qdisc_fifo__open_and_load(); 125 138 if (!ASSERT_OK_PTR(fifo_skel, "bpf_qdisc_fifo__open_and_load")) 126 139 return; 127 140 128 - link = bpf_map__attach_struct_ops(fifo_skel->maps.fifo); 129 - if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 130 - bpf_qdisc_fifo__destroy(fifo_skel); 131 - return; 132 - } 141 + if (!ASSERT_OK(bpf_qdisc_fifo__attach(fifo_skel), "bpf_qdisc_fifo__attach")) 142 + goto out; 133 143 134 144 SYS(out, "tc qdisc add dev lo root handle 1: htb"); 135 145 SYS(out_del_htb, "tc class add dev lo parent 1: classid 1:1 htb rate 75Kbit"); ··· 137 155 out_del_htb: 138 156 SYS(out, "tc qdisc delete dev lo root htb"); 139 157 out: 140 - bpf_link__destroy(link); 141 158 bpf_qdisc_fifo__destroy(fifo_skel); 142 159 } 143 160 144 - void test_bpf_qdisc(void) 161 + static void test_incompl_ops(void) 145 162 { 146 - struct netns_obj *netns; 163 + struct bpf_qdisc_fail__incompl_ops *skel; 164 + struct bpf_link *link; 165 + 166 + skel = bpf_qdisc_fail__incompl_ops__open_and_load(); 167 + if (!ASSERT_OK_PTR(skel, "bpf_qdisc_fifo__open_and_load")) 168 + return; 169 + 170 + link = bpf_map__attach_struct_ops(skel->maps.test); 171 + if (!ASSERT_ERR_PTR(link, "bpf_map__attach_struct_ops")) 172 + bpf_link__destroy(link); 173 + 174 + bpf_qdisc_fail__incompl_ops__destroy(skel); 175 + } 176 + 177 + static int get_default_qdisc(char *qdisc_name) 178 + { 179 + FILE *f; 180 + int num; 181 + 182 + f = fopen("/proc/sys/net/core/default_qdisc", "r"); 183 + if (!f) 184 + return -errno; 185 + 186 + num = fscanf(f, "%s", qdisc_name); 187 + fclose(f); 188 + 189 + return num == 1 ? 0 : -EFAULT; 190 + } 191 + 192 + static void test_default_qdisc_attach_to_mq(void) 193 + { 194 + char default_qdisc[IFNAMSIZ] = {}; 195 + struct bpf_qdisc_fifo *fifo_skel; 196 + struct netns_obj *netns = NULL; 197 + int err; 198 + 199 + fifo_skel = bpf_qdisc_fifo__open_and_load(); 200 + if (!ASSERT_OK_PTR(fifo_skel, "bpf_qdisc_fifo__open_and_load")) 201 + return; 202 + 203 + if (!ASSERT_OK(bpf_qdisc_fifo__attach(fifo_skel), "bpf_qdisc_fifo__attach")) 204 + goto out; 205 + 206 + err = get_default_qdisc(default_qdisc); 207 + if (!ASSERT_OK(err, "read sysctl net.core.default_qdisc")) 208 + goto out; 209 + 210 + err = write_sysctl("/proc/sys/net/core/default_qdisc", "bpf_fifo"); 211 + if (!ASSERT_OK(err, "write sysctl net.core.default_qdisc")) 212 + goto out; 147 213 148 214 netns = netns_new("bpf_qdisc_ns", true); 149 215 if (!ASSERT_OK_PTR(netns, "netns_new")) 150 - return; 216 + goto out; 151 217 218 + SYS(out, "ip link add veth0 type veth peer veth1"); 219 + SYS(out, "tc qdisc add dev veth0 root handle 1: mq"); 220 + 221 + ASSERT_EQ(fifo_skel->bss->init_called, true, "init_called"); 222 + 223 + SYS(out, "tc qdisc delete dev veth0 root mq"); 224 + out: 225 + netns_free(netns); 226 + if (default_qdisc[0]) 227 + write_sysctl("/proc/sys/net/core/default_qdisc", default_qdisc); 228 + 229 + bpf_qdisc_fifo__destroy(fifo_skel); 230 + } 231 + 232 + void test_ns_bpf_qdisc(void) 233 + { 152 234 if (test__start_subtest("fifo")) 153 235 test_fifo(); 154 236 if (test__start_subtest("fq")) ··· 221 175 test_qdisc_attach_to_mq(); 222 176 if (test__start_subtest("attach to non root")) 223 177 test_qdisc_attach_to_non_root(); 178 + if (test__start_subtest("incompl_ops")) 179 + test_incompl_ops(); 180 + } 224 181 225 - netns_free(netns); 182 + void serial_test_bpf_qdisc_default(void) 183 + { 184 + test_default_qdisc_attach_to_mq(); 226 185 }
+434 -13
tools/testing/selftests/bpf/prog_tests/sock_iter_batch.c
··· 7 7 8 8 #define TEST_NS "sock_iter_batch_netns" 9 9 10 + static const int init_batch_size = 16; 10 11 static const int nr_soreuse = 4; 12 + 13 + struct iter_out { 14 + int idx; 15 + __u64 cookie; 16 + } __packed; 17 + 18 + struct sock_count { 19 + __u64 cookie; 20 + int count; 21 + }; 22 + 23 + static int insert(__u64 cookie, struct sock_count counts[], int counts_len) 24 + { 25 + int insert = -1; 26 + int i = 0; 27 + 28 + for (; i < counts_len; i++) { 29 + if (!counts[i].cookie) { 30 + insert = i; 31 + } else if (counts[i].cookie == cookie) { 32 + insert = i; 33 + break; 34 + } 35 + } 36 + if (insert < 0) 37 + return insert; 38 + 39 + counts[insert].cookie = cookie; 40 + counts[insert].count++; 41 + 42 + return counts[insert].count; 43 + } 44 + 45 + static int read_n(int iter_fd, int n, struct sock_count counts[], 46 + int counts_len) 47 + { 48 + struct iter_out out; 49 + int nread = 1; 50 + int i = 0; 51 + 52 + for (; nread > 0 && (n < 0 || i < n); i++) { 53 + nread = read(iter_fd, &out, sizeof(out)); 54 + if (!nread || !ASSERT_EQ(nread, sizeof(out), "nread")) 55 + break; 56 + ASSERT_GE(insert(out.cookie, counts, counts_len), 0, "insert"); 57 + } 58 + 59 + ASSERT_TRUE(n < 0 || i == n, "n < 0 || i == n"); 60 + 61 + return i; 62 + } 63 + 64 + static __u64 socket_cookie(int fd) 65 + { 66 + __u64 cookie; 67 + socklen_t cookie_len = sizeof(cookie); 68 + 69 + if (!ASSERT_OK(getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, 70 + &cookie_len), "getsockopt(SO_COOKIE)")) 71 + return 0; 72 + return cookie; 73 + } 74 + 75 + static bool was_seen(int fd, struct sock_count counts[], int counts_len) 76 + { 77 + __u64 cookie = socket_cookie(fd); 78 + int i = 0; 79 + 80 + for (; cookie && i < counts_len; i++) 81 + if (cookie == counts[i].cookie) 82 + return true; 83 + 84 + return false; 85 + } 86 + 87 + static int get_seen_socket(int *fds, struct sock_count counts[], int n) 88 + { 89 + int i = 0; 90 + 91 + for (; i < n; i++) 92 + if (was_seen(fds[i], counts, n)) 93 + return i; 94 + return -1; 95 + } 96 + 97 + static int get_nth_socket(int *fds, int fds_len, struct bpf_link *link, int n) 98 + { 99 + int i, nread, iter_fd; 100 + int nth_sock_idx = -1; 101 + struct iter_out out; 102 + 103 + iter_fd = bpf_iter_create(bpf_link__fd(link)); 104 + if (!ASSERT_OK_FD(iter_fd, "bpf_iter_create")) 105 + return -1; 106 + 107 + for (; n >= 0; n--) { 108 + nread = read(iter_fd, &out, sizeof(out)); 109 + if (!nread || !ASSERT_GE(nread, 1, "nread")) 110 + goto done; 111 + } 112 + 113 + for (i = 0; i < fds_len && nth_sock_idx < 0; i++) 114 + if (fds[i] >= 0 && socket_cookie(fds[i]) == out.cookie) 115 + nth_sock_idx = i; 116 + done: 117 + close(iter_fd); 118 + return nth_sock_idx; 119 + } 120 + 121 + static int get_seen_count(int fd, struct sock_count counts[], int n) 122 + { 123 + __u64 cookie = socket_cookie(fd); 124 + int count = 0; 125 + int i = 0; 126 + 127 + for (; cookie && !count && i < n; i++) 128 + if (cookie == counts[i].cookie) 129 + count = counts[i].count; 130 + 131 + return count; 132 + } 133 + 134 + static void check_n_were_seen_once(int *fds, int fds_len, int n, 135 + struct sock_count counts[], int counts_len) 136 + { 137 + int seen_once = 0; 138 + int seen_cnt; 139 + int i = 0; 140 + 141 + for (; i < fds_len; i++) { 142 + /* Skip any sockets that were closed or that weren't seen 143 + * exactly once. 144 + */ 145 + if (fds[i] < 0) 146 + continue; 147 + seen_cnt = get_seen_count(fds[i], counts, counts_len); 148 + if (seen_cnt && ASSERT_EQ(seen_cnt, 1, "seen_cnt")) 149 + seen_once++; 150 + } 151 + 152 + ASSERT_EQ(seen_once, n, "seen_once"); 153 + } 154 + 155 + static void remove_seen(int family, int sock_type, const char *addr, __u16 port, 156 + int *socks, int socks_len, struct sock_count *counts, 157 + int counts_len, struct bpf_link *link, int iter_fd) 158 + { 159 + int close_idx; 160 + 161 + /* Iterate through the first socks_len - 1 sockets. */ 162 + read_n(iter_fd, socks_len - 1, counts, counts_len); 163 + 164 + /* Make sure we saw socks_len - 1 sockets exactly once. */ 165 + check_n_were_seen_once(socks, socks_len, socks_len - 1, counts, 166 + counts_len); 167 + 168 + /* Close a socket we've already seen to remove it from the bucket. */ 169 + close_idx = get_seen_socket(socks, counts, counts_len); 170 + if (!ASSERT_GE(close_idx, 0, "close_idx")) 171 + return; 172 + close(socks[close_idx]); 173 + socks[close_idx] = -1; 174 + 175 + /* Iterate through the rest of the sockets. */ 176 + read_n(iter_fd, -1, counts, counts_len); 177 + 178 + /* Make sure the last socket wasn't skipped and that there were no 179 + * repeats. 180 + */ 181 + check_n_were_seen_once(socks, socks_len, socks_len - 1, counts, 182 + counts_len); 183 + } 184 + 185 + static void remove_unseen(int family, int sock_type, const char *addr, 186 + __u16 port, int *socks, int socks_len, 187 + struct sock_count *counts, int counts_len, 188 + struct bpf_link *link, int iter_fd) 189 + { 190 + int close_idx; 191 + 192 + /* Iterate through the first socket. */ 193 + read_n(iter_fd, 1, counts, counts_len); 194 + 195 + /* Make sure we saw a socket from fds. */ 196 + check_n_were_seen_once(socks, socks_len, 1, counts, counts_len); 197 + 198 + /* Close what would be the next socket in the bucket to exercise the 199 + * condition where we need to skip past the first cookie we remembered. 200 + */ 201 + close_idx = get_nth_socket(socks, socks_len, link, 1); 202 + if (!ASSERT_GE(close_idx, 0, "close_idx")) 203 + return; 204 + close(socks[close_idx]); 205 + socks[close_idx] = -1; 206 + 207 + /* Iterate through the rest of the sockets. */ 208 + read_n(iter_fd, -1, counts, counts_len); 209 + 210 + /* Make sure the remaining sockets were seen exactly once and that we 211 + * didn't repeat the socket that was already seen. 212 + */ 213 + check_n_were_seen_once(socks, socks_len, socks_len - 1, counts, 214 + counts_len); 215 + } 216 + 217 + static void remove_all(int family, int sock_type, const char *addr, 218 + __u16 port, int *socks, int socks_len, 219 + struct sock_count *counts, int counts_len, 220 + struct bpf_link *link, int iter_fd) 221 + { 222 + int close_idx, i; 223 + 224 + /* Iterate through the first socket. */ 225 + read_n(iter_fd, 1, counts, counts_len); 226 + 227 + /* Make sure we saw a socket from fds. */ 228 + check_n_were_seen_once(socks, socks_len, 1, counts, counts_len); 229 + 230 + /* Close all remaining sockets to exhaust the list of saved cookies and 231 + * exit without putting any sockets into the batch on the next read. 232 + */ 233 + for (i = 0; i < socks_len - 1; i++) { 234 + close_idx = get_nth_socket(socks, socks_len, link, 1); 235 + if (!ASSERT_GE(close_idx, 0, "close_idx")) 236 + return; 237 + close(socks[close_idx]); 238 + socks[close_idx] = -1; 239 + } 240 + 241 + /* Make sure there are no more sockets returned */ 242 + ASSERT_EQ(read_n(iter_fd, -1, counts, counts_len), 0, "read_n"); 243 + } 244 + 245 + static void add_some(int family, int sock_type, const char *addr, __u16 port, 246 + int *socks, int socks_len, struct sock_count *counts, 247 + int counts_len, struct bpf_link *link, int iter_fd) 248 + { 249 + int *new_socks = NULL; 250 + 251 + /* Iterate through the first socks_len - 1 sockets. */ 252 + read_n(iter_fd, socks_len - 1, counts, counts_len); 253 + 254 + /* Make sure we saw socks_len - 1 sockets exactly once. */ 255 + check_n_were_seen_once(socks, socks_len, socks_len - 1, counts, 256 + counts_len); 257 + 258 + /* Double the number of sockets in the bucket. */ 259 + new_socks = start_reuseport_server(family, sock_type, addr, port, 0, 260 + socks_len); 261 + if (!ASSERT_OK_PTR(new_socks, "start_reuseport_server")) 262 + goto done; 263 + 264 + /* Iterate through the rest of the sockets. */ 265 + read_n(iter_fd, -1, counts, counts_len); 266 + 267 + /* Make sure each of the original sockets was seen exactly once. */ 268 + check_n_were_seen_once(socks, socks_len, socks_len, counts, 269 + counts_len); 270 + done: 271 + free_fds(new_socks, socks_len); 272 + } 273 + 274 + static void force_realloc(int family, int sock_type, const char *addr, 275 + __u16 port, int *socks, int socks_len, 276 + struct sock_count *counts, int counts_len, 277 + struct bpf_link *link, int iter_fd) 278 + { 279 + int *new_socks = NULL; 280 + 281 + /* Iterate through the first socket just to initialize the batch. */ 282 + read_n(iter_fd, 1, counts, counts_len); 283 + 284 + /* Double the number of sockets in the bucket to force a realloc on the 285 + * next read. 286 + */ 287 + new_socks = start_reuseport_server(family, sock_type, addr, port, 0, 288 + socks_len); 289 + if (!ASSERT_OK_PTR(new_socks, "start_reuseport_server")) 290 + goto done; 291 + 292 + /* Iterate through the rest of the sockets. */ 293 + read_n(iter_fd, -1, counts, counts_len); 294 + 295 + /* Make sure each socket from the first set was seen exactly once. */ 296 + check_n_were_seen_once(socks, socks_len, socks_len, counts, 297 + counts_len); 298 + done: 299 + free_fds(new_socks, socks_len); 300 + } 301 + 302 + struct test_case { 303 + void (*test)(int family, int sock_type, const char *addr, __u16 port, 304 + int *socks, int socks_len, struct sock_count *counts, 305 + int counts_len, struct bpf_link *link, int iter_fd); 306 + const char *description; 307 + int init_socks; 308 + int max_socks; 309 + int sock_type; 310 + int family; 311 + }; 312 + 313 + static struct test_case resume_tests[] = { 314 + { 315 + .description = "udp: resume after removing a seen socket", 316 + .init_socks = nr_soreuse, 317 + .max_socks = nr_soreuse, 318 + .sock_type = SOCK_DGRAM, 319 + .family = AF_INET6, 320 + .test = remove_seen, 321 + }, 322 + { 323 + .description = "udp: resume after removing one unseen socket", 324 + .init_socks = nr_soreuse, 325 + .max_socks = nr_soreuse, 326 + .sock_type = SOCK_DGRAM, 327 + .family = AF_INET6, 328 + .test = remove_unseen, 329 + }, 330 + { 331 + .description = "udp: resume after removing all unseen sockets", 332 + .init_socks = nr_soreuse, 333 + .max_socks = nr_soreuse, 334 + .sock_type = SOCK_DGRAM, 335 + .family = AF_INET6, 336 + .test = remove_all, 337 + }, 338 + { 339 + .description = "udp: resume after adding a few sockets", 340 + .init_socks = nr_soreuse, 341 + .max_socks = nr_soreuse, 342 + .sock_type = SOCK_DGRAM, 343 + /* Use AF_INET so that new sockets are added to the head of the 344 + * bucket's list. 345 + */ 346 + .family = AF_INET, 347 + .test = add_some, 348 + }, 349 + { 350 + .description = "udp: force a realloc to occur", 351 + .init_socks = init_batch_size, 352 + .max_socks = init_batch_size * 2, 353 + .sock_type = SOCK_DGRAM, 354 + /* Use AF_INET6 so that new sockets are added to the tail of the 355 + * bucket's list, needing to be added to the next batch to force 356 + * a realloc. 357 + */ 358 + .family = AF_INET6, 359 + .test = force_realloc, 360 + }, 361 + }; 362 + 363 + static void do_resume_test(struct test_case *tc) 364 + { 365 + struct sock_iter_batch *skel = NULL; 366 + static const __u16 port = 10001; 367 + struct bpf_link *link = NULL; 368 + struct sock_count *counts; 369 + int err, iter_fd = -1; 370 + const char *addr; 371 + int *fds = NULL; 372 + int local_port; 373 + 374 + counts = calloc(tc->max_socks, sizeof(*counts)); 375 + if (!ASSERT_OK_PTR(counts, "counts")) 376 + goto done; 377 + skel = sock_iter_batch__open(); 378 + if (!ASSERT_OK_PTR(skel, "sock_iter_batch__open")) 379 + goto done; 380 + 381 + /* Prepare a bucket of sockets in the kernel hashtable */ 382 + addr = tc->family == AF_INET6 ? "::1" : "127.0.0.1"; 383 + fds = start_reuseport_server(tc->family, tc->sock_type, addr, port, 0, 384 + tc->init_socks); 385 + if (!ASSERT_OK_PTR(fds, "start_reuseport_server")) 386 + goto done; 387 + local_port = get_socket_local_port(*fds); 388 + if (!ASSERT_GE(local_port, 0, "get_socket_local_port")) 389 + goto done; 390 + skel->rodata->ports[0] = ntohs(local_port); 391 + skel->rodata->sf = tc->family; 392 + 393 + err = sock_iter_batch__load(skel); 394 + if (!ASSERT_OK(err, "sock_iter_batch__load")) 395 + goto done; 396 + 397 + link = bpf_program__attach_iter(tc->sock_type == SOCK_STREAM ? 398 + skel->progs.iter_tcp_soreuse : 399 + skel->progs.iter_udp_soreuse, 400 + NULL); 401 + if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter")) 402 + goto done; 403 + 404 + iter_fd = bpf_iter_create(bpf_link__fd(link)); 405 + if (!ASSERT_OK_FD(iter_fd, "bpf_iter_create")) 406 + goto done; 407 + 408 + tc->test(tc->family, tc->sock_type, addr, port, fds, tc->init_socks, 409 + counts, tc->max_socks, link, iter_fd); 410 + done: 411 + free(counts); 412 + free_fds(fds, tc->init_socks); 413 + if (iter_fd >= 0) 414 + close(iter_fd); 415 + bpf_link__destroy(link); 416 + sock_iter_batch__destroy(skel); 417 + } 418 + 419 + static void do_resume_tests(void) 420 + { 421 + int i; 422 + 423 + for (i = 0; i < ARRAY_SIZE(resume_tests); i++) { 424 + if (test__start_subtest(resume_tests[i].description)) { 425 + do_resume_test(&resume_tests[i]); 426 + } 427 + } 428 + } 11 429 12 430 static void do_test(int sock_type, bool onebyone) 13 431 { 14 432 int err, i, nread, to_read, total_read, iter_fd = -1; 15 - int first_idx, second_idx, indices[nr_soreuse]; 433 + struct iter_out outputs[nr_soreuse]; 16 434 struct bpf_link *link = NULL; 17 435 struct sock_iter_batch *skel; 436 + int first_idx, second_idx; 18 437 int *fds[2] = {}; 19 438 20 439 skel = sock_iter_batch__open(); ··· 453 34 goto done; 454 35 skel->rodata->ports[i] = ntohs(local_port); 455 36 } 37 + skel->rodata->sf = AF_INET6; 456 38 457 39 err = sock_iter_batch__load(skel); 458 40 if (!ASSERT_OK(err, "sock_iter_batch__load")) ··· 475 55 * from a bucket and leave one socket out from 476 56 * that bucket on purpose. 477 57 */ 478 - to_read = (nr_soreuse - 1) * sizeof(*indices); 58 + to_read = (nr_soreuse - 1) * sizeof(*outputs); 479 59 total_read = 0; 480 60 first_idx = -1; 481 61 do { 482 - nread = read(iter_fd, indices, onebyone ? sizeof(*indices) : to_read); 483 - if (nread <= 0 || nread % sizeof(*indices)) 62 + nread = read(iter_fd, outputs, onebyone ? sizeof(*outputs) : to_read); 63 + if (nread <= 0 || nread % sizeof(*outputs)) 484 64 break; 485 65 total_read += nread; 486 66 487 67 if (first_idx == -1) 488 - first_idx = indices[0]; 489 - for (i = 0; i < nread / sizeof(*indices); i++) 490 - ASSERT_EQ(indices[i], first_idx, "first_idx"); 68 + first_idx = outputs[0].idx; 69 + for (i = 0; i < nread / sizeof(*outputs); i++) 70 + ASSERT_EQ(outputs[i].idx, first_idx, "first_idx"); 491 71 } while (total_read < to_read); 492 - ASSERT_EQ(nread, onebyone ? sizeof(*indices) : to_read, "nread"); 72 + ASSERT_EQ(nread, onebyone ? sizeof(*outputs) : to_read, "nread"); 493 73 ASSERT_EQ(total_read, to_read, "total_read"); 494 74 495 75 free_fds(fds[first_idx], nr_soreuse); 496 76 fds[first_idx] = NULL; 497 77 498 78 /* Read the "whole" second bucket */ 499 - to_read = nr_soreuse * sizeof(*indices); 79 + to_read = nr_soreuse * sizeof(*outputs); 500 80 total_read = 0; 501 81 second_idx = !first_idx; 502 82 do { 503 - nread = read(iter_fd, indices, onebyone ? sizeof(*indices) : to_read); 504 - if (nread <= 0 || nread % sizeof(*indices)) 83 + nread = read(iter_fd, outputs, onebyone ? sizeof(*outputs) : to_read); 84 + if (nread <= 0 || nread % sizeof(*outputs)) 505 85 break; 506 86 total_read += nread; 507 87 508 - for (i = 0; i < nread / sizeof(*indices); i++) 509 - ASSERT_EQ(indices[i], second_idx, "second_idx"); 88 + for (i = 0; i < nread / sizeof(*outputs); i++) 89 + ASSERT_EQ(outputs[i].idx, second_idx, "second_idx"); 510 90 } while (total_read <= to_read); 511 91 ASSERT_EQ(nread, 0, "nread"); 512 92 /* Both so_reuseport ports should be in different buckets, so ··· 548 128 do_test(SOCK_DGRAM, true); 549 129 do_test(SOCK_DGRAM, false); 550 130 } 131 + do_resume_tests(); 551 132 close_netns(nstoken); 552 133 553 134 done:
+1 -5
tools/testing/selftests/bpf/progs/bpf_qdisc_common.h
··· 12 12 13 13 #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) 14 14 15 - u32 bpf_skb_get_hash(struct sk_buff *p) __ksym; 16 - void bpf_kfree_skb(struct sk_buff *p) __ksym; 17 - void bpf_qdisc_skb_drop(struct sk_buff *p, struct bpf_sk_buff_ptr *to_free) __ksym; 18 - void bpf_qdisc_watchdog_schedule(struct Qdisc *sch, u64 expire, u64 delta_ns) __ksym; 19 - void bpf_qdisc_bstats_update(struct Qdisc *sch, const struct sk_buff *skb) __ksym; 15 + struct bpf_sk_buff_ptr; 20 16 21 17 static struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 22 18 {
+41
tools/testing/selftests/bpf/progs/bpf_qdisc_fail__incompl_ops.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + #include <vmlinux.h> 4 + #include "bpf_experimental.h" 5 + #include "bpf_qdisc_common.h" 6 + 7 + char _license[] SEC("license") = "GPL"; 8 + 9 + SEC("struct_ops") 10 + int BPF_PROG(bpf_qdisc_test_enqueue, struct sk_buff *skb, struct Qdisc *sch, 11 + struct bpf_sk_buff_ptr *to_free) 12 + { 13 + bpf_qdisc_skb_drop(skb, to_free); 14 + return NET_XMIT_DROP; 15 + } 16 + 17 + SEC("struct_ops") 18 + struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) 19 + { 20 + return NULL; 21 + } 22 + 23 + SEC("struct_ops") 24 + void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) 25 + { 26 + } 27 + 28 + SEC("struct_ops") 29 + void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) 30 + { 31 + } 32 + 33 + SEC(".struct_ops") 34 + struct Qdisc_ops test = { 35 + .enqueue = (void *)bpf_qdisc_test_enqueue, 36 + .dequeue = (void *)bpf_qdisc_test_dequeue, 37 + .reset = (void *)bpf_qdisc_test_reset, 38 + .destroy = (void *)bpf_qdisc_test_destroy, 39 + .id = "bpf_qdisc_test", 40 + }; 41 +
+9
tools/testing/selftests/bpf/progs/bpf_qdisc_fifo.c
··· 14 14 private(A) struct bpf_spin_lock q_fifo_lock; 15 15 private(A) struct bpf_list_head q_fifo __contains(skb_node, node); 16 16 17 + bool init_called; 18 + 17 19 SEC("struct_ops/bpf_fifo_enqueue") 18 20 int BPF_PROG(bpf_fifo_enqueue, struct sk_buff *skb, struct Qdisc *sch, 19 21 struct bpf_sk_buff_ptr *to_free) ··· 79 77 struct netlink_ext_ack *extack) 80 78 { 81 79 sch->limit = 1000; 80 + init_called = true; 82 81 return 0; 83 82 } 84 83 ··· 109 106 sch->q.qlen = 0; 110 107 } 111 108 109 + SEC("struct_ops") 110 + void BPF_PROG(bpf_fifo_destroy, struct Qdisc *sch) 111 + { 112 + } 113 + 112 114 SEC(".struct_ops") 113 115 struct Qdisc_ops fifo = { 114 116 .enqueue = (void *)bpf_fifo_enqueue, 115 117 .dequeue = (void *)bpf_fifo_dequeue, 116 118 .init = (void *)bpf_fifo_init, 117 119 .reset = (void *)bpf_fifo_reset, 120 + .destroy = (void *)bpf_fifo_destroy, 118 121 .id = "bpf_fifo", 119 122 }; 120 123
+6
tools/testing/selftests/bpf/progs/bpf_qdisc_fq.c
··· 740 740 return 0; 741 741 } 742 742 743 + SEC("struct_ops") 744 + void BPF_PROG(bpf_fq_destroy, struct Qdisc *sch) 745 + { 746 + } 747 + 743 748 SEC(".struct_ops") 744 749 struct Qdisc_ops fq = { 745 750 .enqueue = (void *)bpf_fq_enqueue, 746 751 .dequeue = (void *)bpf_fq_dequeue, 747 752 .reset = (void *)bpf_fq_reset, 748 753 .init = (void *)bpf_fq_init, 754 + .destroy = (void *)bpf_fq_destroy, 749 755 .id = "bpf_fq", 750 756 };
+1
tools/testing/selftests/bpf/progs/bpf_tracing_net.h
··· 128 128 #define sk_refcnt __sk_common.skc_refcnt 129 129 #define sk_state __sk_common.skc_state 130 130 #define sk_net __sk_common.skc_net 131 + #define sk_rcv_saddr __sk_common.skc_rcv_saddr 131 132 #define sk_v6_daddr __sk_common.skc_v6_daddr 132 133 #define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr 133 134 #define sk_flags __sk_common.skc_flags
+20 -4
tools/testing/selftests/bpf/progs/sock_iter_batch.c
··· 17 17 a->s6_addr32[2] | (a->s6_addr32[3] ^ bpf_htonl(1))) == 0; 18 18 } 19 19 20 + static bool ipv4_addr_loopback(__be32 a) 21 + { 22 + return a == bpf_ntohl(0x7f000001); 23 + } 24 + 25 + volatile const unsigned int sf; 20 26 volatile const __u16 ports[2]; 21 27 unsigned int bucket[2]; 22 28 ··· 32 26 struct sock *sk = (struct sock *)ctx->sk_common; 33 27 struct inet_hashinfo *hinfo; 34 28 unsigned int hash; 29 + __u64 sock_cookie; 35 30 struct net *net; 36 31 int idx; 37 32 38 33 if (!sk) 39 34 return 0; 40 35 36 + sock_cookie = bpf_get_socket_cookie(sk); 41 37 sk = bpf_core_cast(sk, struct sock); 42 - if (sk->sk_family != AF_INET6 || 38 + if (sk->sk_family != sf || 43 39 sk->sk_state != TCP_LISTEN || 44 - !ipv6_addr_loopback(&sk->sk_v6_rcv_saddr)) 40 + sk->sk_family == AF_INET6 ? 41 + !ipv6_addr_loopback(&sk->sk_v6_rcv_saddr) : 42 + !ipv4_addr_loopback(sk->sk_rcv_saddr)) 45 43 return 0; 46 44 47 45 if (sk->sk_num == ports[0]) ··· 62 52 hinfo = net->ipv4.tcp_death_row.hashinfo; 63 53 bucket[idx] = hash & hinfo->lhash2_mask; 64 54 bpf_seq_write(ctx->meta->seq, &idx, sizeof(idx)); 55 + bpf_seq_write(ctx->meta->seq, &sock_cookie, sizeof(sock_cookie)); 65 56 66 57 return 0; 67 58 } ··· 74 63 { 75 64 struct sock *sk = (struct sock *)ctx->udp_sk; 76 65 struct udp_table *udptable; 66 + __u64 sock_cookie; 77 67 int idx; 78 68 79 69 if (!sk) 80 70 return 0; 81 71 72 + sock_cookie = bpf_get_socket_cookie(sk); 82 73 sk = bpf_core_cast(sk, struct sock); 83 - if (sk->sk_family != AF_INET6 || 84 - !ipv6_addr_loopback(&sk->sk_v6_rcv_saddr)) 74 + if (sk->sk_family != sf || 75 + sk->sk_family == AF_INET6 ? 76 + !ipv6_addr_loopback(&sk->sk_v6_rcv_saddr) : 77 + !ipv4_addr_loopback(sk->sk_rcv_saddr)) 85 78 return 0; 86 79 87 80 if (sk->sk_num == ports[0]) ··· 99 84 udptable = sk->sk_net.net->ipv4.udp_table; 100 85 bucket[idx] = udp_sk(sk)->udp_portaddr_hash & udptable->mask; 101 86 bpf_seq_write(ctx->meta->seq, &idx, sizeof(idx)); 87 + bpf_seq_write(ctx->meta->seq, &sock_cookie, sizeof(sock_cookie)); 102 88 103 89 return 0; 104 90 }