Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'net-bpf_net_context-cleanups'
Sebastian Andrzej Siewior says:
====================
net: bpf_net_context cleanups.
a small series with bpf_net_context cleanups/ improvements.
Jakub asked for #1 and #2 and while looking around I made #3.
====================
Link: https://patch.msgid.link/20240628103020.1766241-1-bigeasy@linutronix.de
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
+71
-68
+4
-6
include/linux/bpf.h
+4
-6
include/linux/bpf.h
···
2494
2494
struct bpf_dtab_netdev;
2495
2495
struct bpf_cpu_map_entry;
2496
2496
2497
-
void __dev_flush(void);
2497
+
void __dev_flush(struct list_head *flush_list);
2498
2498
int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2499
2499
struct net_device *dev_rx);
2500
2500
int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
···
2507
2507
struct bpf_prog *xdp_prog, struct bpf_map *map,
2508
2508
bool exclude_ingress);
2509
2509
2510
-
void __cpu_map_flush(void);
2510
+
void __cpu_map_flush(struct list_head *flush_list);
2511
2511
int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
2512
2512
struct net_device *dev_rx);
2513
2513
int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
···
2644
2644
void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
2645
2645
void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
2646
2646
2647
-
bool dev_check_flush(void);
2648
-
bool cpu_map_check_flush(void);
2649
2647
#else /* !CONFIG_BPF_SYSCALL */
2650
2648
static inline struct bpf_prog *bpf_prog_get(u32 ufd)
2651
2649
{
···
2736
2738
return ERR_PTR(-EOPNOTSUPP);
2737
2739
}
2738
2740
2739
-
static inline void __dev_flush(void)
2741
+
static inline void __dev_flush(struct list_head *flush_list)
2740
2742
{
2741
2743
}
2742
2744
···
2782
2784
return 0;
2783
2785
}
2784
2786
2785
-
static inline void __cpu_map_flush(void)
2787
+
static inline void __cpu_map_flush(struct list_head *flush_list)
2786
2788
{
2787
2789
}
2788
2790
+27
include/linux/filter.h
+27
include/linux/filter.h
···
829
829
return &bpf_net_ctx->xskmap_map_flush_list;
830
830
}
831
831
832
+
static inline void bpf_net_ctx_get_all_used_flush_lists(struct list_head **lh_map,
833
+
struct list_head **lh_dev,
834
+
struct list_head **lh_xsk)
835
+
{
836
+
struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get();
837
+
u32 kern_flags = bpf_net_ctx->ri.kern_flags;
838
+
struct list_head *lh;
839
+
840
+
*lh_map = *lh_dev = *lh_xsk = NULL;
841
+
842
+
if (!IS_ENABLED(CONFIG_BPF_SYSCALL))
843
+
return;
844
+
845
+
lh = &bpf_net_ctx->dev_map_flush_list;
846
+
if (kern_flags & BPF_RI_F_DEV_MAP_INIT && !list_empty(lh))
847
+
*lh_dev = lh;
848
+
849
+
lh = &bpf_net_ctx->cpu_map_flush_list;
850
+
if (kern_flags & BPF_RI_F_CPU_MAP_INIT && !list_empty(lh))
851
+
*lh_map = lh;
852
+
853
+
lh = &bpf_net_ctx->xskmap_map_flush_list;
854
+
if (IS_ENABLED(CONFIG_XDP_SOCKETS) &&
855
+
kern_flags & BPF_RI_F_XSK_MAP_INIT && !list_empty(lh))
856
+
*lh_xsk = lh;
857
+
}
858
+
832
859
/* Compute the linear packet data range [data, data_end) which
833
860
* will be accessed by various program types (cls_bpf, act_bpf,
834
861
* lwt, ...). Subsystems allowing direct data access must (!)
+2
-12
include/net/xdp_sock.h
+2
-12
include/net/xdp_sock.h
···
121
121
122
122
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
123
123
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
124
-
void __xsk_map_flush(void);
124
+
void __xsk_map_flush(struct list_head *flush_list);
125
125
126
126
/**
127
127
* xsk_tx_metadata_to_compl - Save enough relevant metadata information
···
206
206
return -EOPNOTSUPP;
207
207
}
208
208
209
-
static inline void __xsk_map_flush(void)
209
+
static inline void __xsk_map_flush(struct list_head *flush_list)
210
210
{
211
211
}
212
212
···
228
228
}
229
229
230
230
#endif /* CONFIG_XDP_SOCKETS */
231
-
232
-
#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
233
-
bool xsk_map_check_flush(void);
234
-
#else
235
-
static inline bool xsk_map_check_flush(void)
236
-
{
237
-
return false;
238
-
}
239
-
#endif
240
-
241
231
#endif /* _LINUX_XDP_SOCK_H */
+5
-14
kernel/bpf/cpumap.c
+5
-14
kernel/bpf/cpumap.c
···
707
707
*/
708
708
static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
709
709
{
710
-
struct list_head *flush_list = bpf_net_ctx_get_cpu_map_flush_list();
711
710
struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
712
711
713
712
if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
···
723
724
*/
724
725
bq->q[bq->count++] = xdpf;
725
726
726
-
if (!bq->flush_node.prev)
727
+
if (!bq->flush_node.prev) {
728
+
struct list_head *flush_list = bpf_net_ctx_get_cpu_map_flush_list();
729
+
727
730
list_add(&bq->flush_node, flush_list);
731
+
}
728
732
}
729
733
730
734
int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
···
759
757
return ret;
760
758
}
761
759
762
-
void __cpu_map_flush(void)
760
+
void __cpu_map_flush(struct list_head *flush_list)
763
761
{
764
-
struct list_head *flush_list = bpf_net_ctx_get_cpu_map_flush_list();
765
762
struct xdp_bulk_queue *bq, *tmp;
766
763
767
764
list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
···
770
769
wake_up_process(bq->obj->kthread);
771
770
}
772
771
}
773
-
774
-
#ifdef CONFIG_DEBUG_NET
775
-
bool cpu_map_check_flush(void)
776
-
{
777
-
if (list_empty(bpf_net_ctx_get_cpu_map_flush_list()))
778
-
return false;
779
-
__cpu_map_flush();
780
-
return true;
781
-
}
782
-
#endif
+3
-13
kernel/bpf/devmap.c
+3
-13
kernel/bpf/devmap.c
···
412
412
* driver before returning from its napi->poll() routine. See the comment above
413
413
* xdp_do_flush() in filter.c.
414
414
*/
415
-
void __dev_flush(void)
415
+
void __dev_flush(struct list_head *flush_list)
416
416
{
417
-
struct list_head *flush_list = bpf_net_ctx_get_dev_flush_list();
418
417
struct xdp_dev_bulk_queue *bq, *tmp;
419
418
420
419
list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
···
423
424
__list_del_clearprev(&bq->flush_node);
424
425
}
425
426
}
426
-
427
-
#ifdef CONFIG_DEBUG_NET
428
-
bool dev_check_flush(void)
429
-
{
430
-
if (list_empty(bpf_net_ctx_get_dev_flush_list()))
431
-
return false;
432
-
__dev_flush();
433
-
return true;
434
-
}
435
-
#endif
436
427
437
428
/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
438
429
* by local_bh_disable() (from XDP calls inside NAPI). The
···
448
459
static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
449
460
struct net_device *dev_rx, struct bpf_prog *xdp_prog)
450
461
{
451
-
struct list_head *flush_list = bpf_net_ctx_get_dev_flush_list();
452
462
struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq);
453
463
454
464
if (unlikely(bq->count == DEV_MAP_BULK_SIZE))
···
461
473
* are only ever modified together.
462
474
*/
463
475
if (!bq->dev_rx) {
476
+
struct list_head *flush_list = bpf_net_ctx_get_dev_flush_list();
477
+
464
478
bq->dev_rx = dev_rx;
465
479
bq->xdp_prog = xdp_prog;
466
480
list_add(&bq->flush_node, flush_list);
-1
kernel/fork.c
-1
kernel/fork.c
+25
-8
net/core/filter.c
+25
-8
net/core/filter.c
···
4277
4277
*/
4278
4278
void xdp_do_flush(void)
4279
4279
{
4280
-
__dev_flush();
4281
-
__cpu_map_flush();
4282
-
__xsk_map_flush();
4280
+
struct list_head *lh_map, *lh_dev, *lh_xsk;
4281
+
4282
+
bpf_net_ctx_get_all_used_flush_lists(&lh_map, &lh_dev, &lh_xsk);
4283
+
if (lh_dev)
4284
+
__dev_flush(lh_dev);
4285
+
if (lh_map)
4286
+
__cpu_map_flush(lh_map);
4287
+
if (lh_xsk)
4288
+
__xsk_map_flush(lh_xsk);
4283
4289
}
4284
4290
EXPORT_SYMBOL_GPL(xdp_do_flush);
4285
4291
4286
4292
#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
4287
4293
void xdp_do_check_flushed(struct napi_struct *napi)
4288
4294
{
4289
-
bool ret;
4295
+
struct list_head *lh_map, *lh_dev, *lh_xsk;
4296
+
bool missed = false;
4290
4297
4291
-
ret = dev_check_flush();
4292
-
ret |= cpu_map_check_flush();
4293
-
ret |= xsk_map_check_flush();
4298
+
bpf_net_ctx_get_all_used_flush_lists(&lh_map, &lh_dev, &lh_xsk);
4299
+
if (lh_dev) {
4300
+
__dev_flush(lh_dev);
4301
+
missed = true;
4302
+
}
4303
+
if (lh_map) {
4304
+
__cpu_map_flush(lh_map);
4305
+
missed = true;
4306
+
}
4307
+
if (lh_xsk) {
4308
+
__xsk_map_flush(lh_xsk);
4309
+
missed = true;
4310
+
}
4294
4311
4295
-
WARN_ONCE(ret, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
4312
+
WARN_ONCE(missed, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
4296
4313
napi->poll);
4297
4314
}
4298
4315
#endif
+5
-14
net/xdp/xsk.c
+5
-14
net/xdp/xsk.c
···
370
370
371
371
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
372
372
{
373
-
struct list_head *flush_list = bpf_net_ctx_get_xskmap_flush_list();
374
373
int err;
375
374
376
375
err = xsk_rcv(xs, xdp);
377
376
if (err)
378
377
return err;
379
378
380
-
if (!xs->flush_node.prev)
379
+
if (!xs->flush_node.prev) {
380
+
struct list_head *flush_list = bpf_net_ctx_get_xskmap_flush_list();
381
+
381
382
list_add(&xs->flush_node, flush_list);
383
+
}
382
384
383
385
return 0;
384
386
}
385
387
386
-
void __xsk_map_flush(void)
388
+
void __xsk_map_flush(struct list_head *flush_list)
387
389
{
388
-
struct list_head *flush_list = bpf_net_ctx_get_xskmap_flush_list();
389
390
struct xdp_sock *xs, *tmp;
390
391
391
392
list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
···
394
393
__list_del_clearprev(&xs->flush_node);
395
394
}
396
395
}
397
-
398
-
#ifdef CONFIG_DEBUG_NET
399
-
bool xsk_map_check_flush(void)
400
-
{
401
-
if (list_empty(bpf_net_ctx_get_xskmap_flush_list()))
402
-
return false;
403
-
__xsk_map_flush();
404
-
return true;
405
-
}
406
-
#endif
407
396
408
397
void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
409
398
{