Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'for-7.0/io_uring-zcrx-large-buffers-20260206' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux
Pull io_uring large rx buffer support from Jens Axboe:
"Now that the networking updates are upstream, here's the support for
large buffers for zcrx.
Using larger (bigger than 4K) rx buffers can increase the effiency of
zcrx. For example, it's been shown that using 32K buffers can decrease
CPU usage by ~30% compared to 4K buffers"
* tag 'for-7.0/io_uring-zcrx-large-buffers-20260206' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux:
io_uring/zcrx: implement large rx buffer support
+34
-6
+1
-1
include/uapi/linux/io_uring.h
+1
-1
include/uapi/linux/io_uring.h
+33
-5
io_uring/zcrx.c
+33
-5
io_uring/zcrx.c
···
55
55
return area->mem.pages[net_iov_idx(niov) << niov_pages_shift];
56
56
}
57
57
58
+
static int io_area_max_shift(struct io_zcrx_mem *mem)
59
+
{
60
+
struct sg_table *sgt = mem->sgt;
61
+
struct scatterlist *sg;
62
+
unsigned shift = -1U;
63
+
unsigned i;
64
+
65
+
for_each_sgtable_dma_sg(sgt, sg, i)
66
+
shift = min(shift, __ffs(sg->length));
67
+
return shift;
68
+
}
69
+
58
70
static int io_populate_area_dma(struct io_zcrx_ifq *ifq,
59
71
struct io_zcrx_area *area)
60
72
{
···
429
417
}
430
418
431
419
static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
432
-
struct io_uring_zcrx_area_reg *area_reg)
420
+
struct io_uring_zcrx_area_reg *area_reg,
421
+
struct io_uring_zcrx_ifq_reg *reg)
433
422
{
423
+
int buf_size_shift = PAGE_SHIFT;
434
424
struct io_zcrx_area *area;
435
425
unsigned nr_iovs;
436
426
int i, ret;
427
+
428
+
if (reg->rx_buf_len) {
429
+
if (!is_power_of_2(reg->rx_buf_len) ||
430
+
reg->rx_buf_len < PAGE_SIZE)
431
+
return -EINVAL;
432
+
buf_size_shift = ilog2(reg->rx_buf_len);
433
+
}
437
434
438
435
ret = -ENOMEM;
439
436
area = kzalloc(sizeof(*area), GFP_KERNEL);
···
454
433
if (ret)
455
434
goto err;
456
435
457
-
ifq->niov_shift = PAGE_SHIFT;
436
+
if (buf_size_shift > io_area_max_shift(&area->mem)) {
437
+
ret = -ERANGE;
438
+
goto err;
439
+
}
440
+
441
+
ifq->niov_shift = buf_size_shift;
458
442
nr_iovs = area->mem.size >> ifq->niov_shift;
459
443
area->nia.num_niovs = nr_iovs;
460
444
···
769
743
return -EINVAL;
770
744
if (copy_from_user(®, arg, sizeof(reg)))
771
745
return -EFAULT;
772
-
if (!mem_is_zero(®.__resv, sizeof(reg.__resv)) ||
773
-
reg.__resv2 || reg.zcrx_id)
746
+
if (!mem_is_zero(®.__resv, sizeof(reg.__resv)) || reg.zcrx_id)
774
747
return -EINVAL;
775
748
if (reg.flags & ZCRX_REG_IMPORT)
776
749
return import_zcrx(ctx, arg, ®);
···
826
801
}
827
802
get_device(ifq->dev);
828
803
829
-
ret = io_zcrx_create_area(ifq, &area);
804
+
ret = io_zcrx_create_area(ifq, &area, ®);
830
805
if (ret)
831
806
goto netdev_put_unlock;
832
807
808
+
mp_param.rx_page_size = 1U << ifq->niov_shift;
833
809
mp_param.mp_ops = &io_uring_pp_zc_ops;
834
810
mp_param.mp_priv = ifq;
835
811
ret = __net_mp_open_rxq(ifq->netdev, reg.if_rxq, &mp_param, NULL);
···
847
821
if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL))
848
822
goto err;
849
823
}
824
+
825
+
reg.rx_buf_len = 1U << ifq->niov_shift;
850
826
851
827
if (copy_to_user(arg, ®, sizeof(reg)) ||
852
828
copy_to_user(u64_to_user_ptr(reg.region_ptr), &rd, sizeof(rd)) ||