io_uring/zcrx: implement device-less mode for zcrx

Allow creating a zcrx instance without attaching it to a net device.
All data will be copied through the fallback path. The user is also
expected to use ZCRX_CTRL_FLUSH_RQ to handle overflows as it normally
should even with a netdev, but it becomes even more relevant as there
will likely be no one to automatically pick up buffers.

Apart from that, it follows the zcrx uapi for the I/O path, and is
useful for testing, experimentation, and potentially for the copy
receive path in the future if improved.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/674f8ad679c5a0bc79d538352b3042cf0999596e.1774261953.git.asml.silence@gmail.com
[axboe: fix spelling error in uapi header and commit message]
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by

Jens Axboe 1 month ago 825f2764 06fc3b6d

+36 -16

3 changed files

expand all

include

uapi

linux

io_uring

zcrx.h

io_uring

zcrx.c

zcrx.h

+8 -1

include/uapi/linux/io_uring/zcrx.h

··· 49 49 }; 50 50 51 51 enum zcrx_reg_flags { 52 - ZCRX_REG_IMPORT = 1, 52 + ZCRX_REG_IMPORT = 1, 53 + 54 + /* 55 + * Register a zcrx instance without a net device. All data will be 56 + * copied. The refill queue entries might not be automatically 57 + * consumed and need to be flushed, see ZCRX_CTRL_FLUSH_RQ. 58 + */ 59 + ZCRX_REG_NODEV = 2, 53 60 }; 54 61 55 62 enum zcrx_features {

+27 -14

io_uring/zcrx.c

··· 127 127 int dmabuf_fd = area_reg->dmabuf_fd; 128 128 int i, ret; 129 129 130 + if (!ifq->dev) 131 + return -EINVAL; 130 132 if (off) 131 133 return -EINVAL; 132 - if (WARN_ON_ONCE(!ifq->dev)) 133 - return -EFAULT; 134 134 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 135 135 return -EINVAL; 136 136 ··· 211 211 if (ret) 212 212 goto out_err; 213 213 214 - ret = dma_map_sgtable(ifq->dev, &mem->page_sg_table, 215 - DMA_FROM_DEVICE, IO_DMA_ATTR); 216 - if (ret < 0) 217 - goto out_err; 218 - mapped = true; 214 + if (ifq->dev) { 215 + ret = dma_map_sgtable(ifq->dev, &mem->page_sg_table, 216 + DMA_FROM_DEVICE, IO_DMA_ATTR); 217 + if (ret < 0) 218 + goto out_err; 219 + mapped = true; 220 + } 219 221 220 222 mem->account_pages = io_count_account_pages(pages, nr_pages); 221 223 ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages); ··· 452 450 ret = io_import_area(ifq, &area->mem, area_reg); 453 451 if (ret) 454 452 goto err; 455 - area->is_mapped = true; 453 + if (ifq->dev) 454 + area->is_mapped = true; 456 455 457 456 if (buf_size_shift > io_area_max_shift(&area->mem)) { 458 457 ret = -ERANGE; ··· 489 486 niov->type = NET_IOV_IOURING; 490 487 } 491 488 492 - ret = io_populate_area_dma(ifq, area); 493 - if (ret) 494 - goto err; 489 + if (ifq->dev) { 490 + ret = io_populate_area_dma(ifq, area); 491 + if (ret) 492 + goto err; 493 + } 495 494 496 495 area->free_count = nr_iovs; 497 496 /* we're only supporting one area per ifq for now */ ··· 831 826 return -EFAULT; 832 827 if (reg.if_rxq == -1 || !reg.rq_entries) 833 828 return -EINVAL; 829 + if ((reg.if_rxq || reg.if_idx) && (reg.flags & ZCRX_REG_NODEV)) 830 + return -EINVAL; 834 831 if (reg.rq_entries > IO_RQ_MAX_ENTRIES) { 835 832 if (!(ctx->flags & IORING_SETUP_CLAMP)) 836 833 return -EINVAL; ··· 868 861 if (ret) 869 862 goto err; 870 863 871 - ret = zcrx_register_netdev(ifq, &reg, &area); 872 - if (ret) 873 - goto err; 864 + if (!(reg.flags & ZCRX_REG_NODEV)) { 865 + ret = zcrx_register_netdev(ifq, &reg, &area); 866 + if (ret) 867 + goto err; 868 + } else { 869 + ret = io_zcrx_create_area(ifq, &area, &reg); 870 + if (ret) 871 + goto err; 872 + } 874 873 875 874 reg.zcrx_id = id; 876 875

+1 -1

io_uring/zcrx.h

··· 8 8 #include <net/page_pool/types.h> 9 9 #include <net/net_trackers.h> 10 10 11 - #define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT) 11 + #define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT | ZCRX_REG_NODEV) 12 12 #define ZCRX_FEATURES (ZCRX_FEATURE_RX_PAGE_SIZE) 13 13 14 14 struct io_zcrx_mem {

Configure Feed

Configure Feed