io_uring/zcrx: allocate sgtable for umem areas

Currently, dma addresses for umem areas are stored directly in niovs.
It's memory efficient but inconvenient. I need a better format 1) to
share code with dmabuf areas, and 2) for disentangling page, folio and
niov sizes. dmabuf already provides sg_table, create one for user memory
as well.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: David Wei <dw@davidwei.uk>
Link: https://lore.kernel.org/r/f3c15081827c1bf5427d3a2e693bc526476b87ee.1751466461.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by

Jens Axboe 11 months ago b84621d9 54e89a93

+29 -52

2 changed files

expand all

io_uring

zcrx.c

zcrx.h

+28 -52

io_uring/zcrx.c

··· 161 161 struct io_uring_zcrx_area_reg *area_reg) 162 162 { 163 163 struct page **pages; 164 - int nr_pages; 164 + int nr_pages, ret; 165 165 166 166 if (area_reg->dmabuf_fd) 167 167 return -EINVAL; ··· 171 171 &nr_pages); 172 172 if (IS_ERR(pages)) 173 173 return PTR_ERR(pages); 174 + 175 + ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages, 176 + 0, nr_pages << PAGE_SHIFT, 177 + GFP_KERNEL_ACCOUNT); 178 + if (ret) 179 + return ret; 174 180 175 181 mem->pages = pages; 176 182 mem->nr_folios = nr_pages; ··· 192 186 } 193 187 if (mem->pages) { 194 188 unpin_user_pages(mem->pages, mem->nr_folios); 189 + sg_free_table(&mem->page_sg_table); 195 190 kvfree(mem->pages); 196 191 } 197 192 } ··· 214 207 return io_import_umem(ifq, mem, area_reg); 215 208 } 216 209 217 - static void io_zcrx_unmap_umem(struct io_zcrx_ifq *ifq, 218 - struct io_zcrx_area *area, int nr_mapped) 210 + static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, 211 + struct io_zcrx_area *area) 219 212 { 220 213 int i; 221 214 222 - for (i = 0; i < nr_mapped; i++) { 223 - netmem_ref netmem = net_iov_to_netmem(&area->nia.niovs[i]); 224 - dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem); 225 - 226 - dma_unmap_page_attrs(ifq->dev, dma, PAGE_SIZE, 227 - DMA_FROM_DEVICE, IO_DMA_ATTR); 228 - } 229 - } 230 - 231 - static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, 232 - struct io_zcrx_area *area, int nr_mapped) 233 - { 234 - int i; 235 - 236 - if (area->mem.is_dmabuf) 237 - io_release_dmabuf(&area->mem); 238 - else 239 - io_zcrx_unmap_umem(ifq, area, nr_mapped); 215 + guard(mutex)(&ifq->dma_lock); 216 + if (!area->is_mapped) 217 + return; 218 + area->is_mapped = false; 240 219 241 220 for (i = 0; i < area->nia.num_niovs; i++) 242 221 net_mp_niov_set_dma_addr(&area->nia.niovs[i], 0); 222 + 223 + if (area->mem.is_dmabuf) { 224 + io_release_dmabuf(&area->mem); 225 + } else { 226 + dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table, 227 + DMA_FROM_DEVICE, IO_DMA_ATTR); 228 + } 243 229 } 244 230 245 - static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) 231 + static unsigned io_zcrx_map_area_umem(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) 246 232 { 247 - guard(mutex)(&ifq->dma_lock); 233 + int ret; 248 234 249 - if (area->is_mapped) 250 - __io_zcrx_unmap_area(ifq, area, area->nia.num_niovs); 251 - area->is_mapped = false; 252 - } 253 - 254 - static int io_zcrx_map_area_umem(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) 255 - { 256 - int i; 257 - 258 - for (i = 0; i < area->nia.num_niovs; i++) { 259 - struct net_iov *niov = &area->nia.niovs[i]; 260 - dma_addr_t dma; 261 - 262 - dma = dma_map_page_attrs(ifq->dev, area->mem.pages[i], 0, 263 - PAGE_SIZE, DMA_FROM_DEVICE, IO_DMA_ATTR); 264 - if (dma_mapping_error(ifq->dev, dma)) 265 - break; 266 - if (net_mp_niov_set_dma_addr(niov, dma)) { 267 - dma_unmap_page_attrs(ifq->dev, dma, PAGE_SIZE, 268 - DMA_FROM_DEVICE, IO_DMA_ATTR); 269 - break; 270 - } 271 - } 272 - 273 - if (i != area->nia.num_niovs) { 274 - __io_zcrx_unmap_area(ifq, area, i); 275 - return -EINVAL; 276 - } 277 - return 0; 235 + ret = dma_map_sgtable(ifq->dev, &area->mem.page_sg_table, 236 + DMA_FROM_DEVICE, IO_DMA_ATTR); 237 + if (ret < 0) 238 + return ret; 239 + return io_populate_area_dma(ifq, area, &area->mem.page_sg_table, 0); 278 240 } 279 241 280 242 static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)

io_uring/zcrx.h

··· 14 14 15 15 struct page **pages; 16 16 unsigned long nr_folios; 17 + struct sg_table page_sg_table; 17 18 18 19 struct dma_buf_attachment *attach; 19 20 struct dma_buf *dmabuf;

Configure Feed

Configure Feed