io_uring/zcrx: prepare fallback for larger pages

io_zcrx_copy_chunk() processes one page at a time, which won't be
sufficient when the net_iov size grows. Introduce a structure keeping
the target niov page and other parameters, it's more convenient and can
be reused later. And add a helper function that can efficient copy
buffers of an arbitrary length. For 64bit archs the loop inside should
be compiled out.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/e84bc705a4e1edeb9aefff470d96558d8232388f.1751466461.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by

Jens Axboe 11 months ago e67645bb 1b4dc1ff

+56 -27

1 changed file

expand all

io_uring

zcrx.c

+56 -27

io_uring/zcrx.c

··· 929 929 return niov; 930 930 } 931 931 932 + struct io_copy_cache { 933 + struct page *page; 934 + unsigned long offset; 935 + size_t size; 936 + }; 937 + 938 + static ssize_t io_copy_page(struct io_copy_cache *cc, struct page *src_page, 939 + unsigned int src_offset, size_t len) 940 + { 941 + size_t copied = 0; 942 + 943 + len = min(len, cc->size); 944 + 945 + while (len) { 946 + void *src_addr, *dst_addr; 947 + struct page *dst_page = cc->page; 948 + unsigned dst_offset = cc->offset; 949 + size_t n = len; 950 + 951 + if (folio_test_partial_kmap(page_folio(dst_page)) || 952 + folio_test_partial_kmap(page_folio(src_page))) { 953 + dst_page = nth_page(dst_page, dst_offset / PAGE_SIZE); 954 + dst_offset = offset_in_page(dst_offset); 955 + src_page = nth_page(src_page, src_offset / PAGE_SIZE); 956 + src_offset = offset_in_page(src_offset); 957 + n = min(PAGE_SIZE - src_offset, PAGE_SIZE - dst_offset); 958 + n = min(n, len); 959 + } 960 + 961 + dst_addr = kmap_local_page(dst_page) + dst_offset; 962 + src_addr = kmap_local_page(src_page) + src_offset; 963 + 964 + memcpy(dst_addr, src_addr, n); 965 + 966 + kunmap_local(src_addr); 967 + kunmap_local(dst_addr); 968 + 969 + cc->size -= n; 970 + cc->offset += n; 971 + len -= n; 972 + copied += n; 973 + } 974 + return copied; 975 + } 976 + 932 977 static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 933 978 struct page *src_page, unsigned int src_offset, 934 979 size_t len) ··· 986 941 return -EFAULT; 987 942 988 943 while (len) { 989 - size_t copy_size = min_t(size_t, PAGE_SIZE, len); 990 - const int dst_off = 0; 944 + struct io_copy_cache cc; 991 945 struct net_iov *niov; 992 - struct page *dst_page; 993 - void *dst_addr, *src_addr; 946 + size_t n; 994 947 995 948 niov = io_zcrx_alloc_fallback(area); 996 949 if (!niov) { ··· 996 953 break; 997 954 } 998 955 999 - dst_page = io_zcrx_iov_page(niov); 1000 - dst_addr = kmap_local_page(dst_page); 1001 - src_addr = kmap_local_page(src_page); 956 + cc.page = io_zcrx_iov_page(niov); 957 + cc.offset = 0; 958 + cc.size = PAGE_SIZE; 1002 959 1003 - memcpy(dst_addr, src_addr + src_offset, copy_size); 960 + n = io_copy_page(&cc, src_page, src_offset, len); 1004 961 1005 - kunmap_local(src_addr); 1006 - kunmap_local(dst_addr); 1007 - 1008 - if (!io_zcrx_queue_cqe(req, niov, ifq, dst_off, copy_size)) { 962 + if (!io_zcrx_queue_cqe(req, niov, ifq, 0, n)) { 1009 963 io_zcrx_return_niov(niov); 1010 964 ret = -ENOSPC; 1011 965 break; 1012 966 } 1013 967 1014 968 io_zcrx_get_niov_uref(niov); 1015 - src_offset += copy_size; 1016 - len -= copy_size; 1017 - copied += copy_size; 969 + src_offset += n; 970 + len -= n; 971 + copied += n; 1018 972 } 1019 973 1020 974 return copied ? copied : ret; ··· 1021 981 const skb_frag_t *frag, int off, int len) 1022 982 { 1023 983 struct page *page = skb_frag_page(frag); 1024 - u32 p_off, p_len, t, copied = 0; 1025 - int ret = 0; 1026 984 1027 - off += skb_frag_off(frag); 1028 - 1029 - skb_frag_foreach_page(frag, off, len, 1030 - page, p_off, p_len, t) { 1031 - ret = io_zcrx_copy_chunk(req, ifq, page, p_off, p_len); 1032 - if (ret < 0) 1033 - return copied ? copied : ret; 1034 - copied += ret; 1035 - } 1036 - return copied; 985 + return io_zcrx_copy_chunk(req, ifq, page, off + skb_frag_off(frag), len); 1037 986 } 1038 987 1039 988 static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,

Configure Feed

Configure Feed