Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/zcrx: split out memory holders from area
In the data path users of struct io_zcrx_area don't need to know what
kind of memory it's backed by. Only keep there generic bits in there and
and split out memory type dependent fields into a new structure. It also
logically separates the step that actually imports the memory, e.g.
pinning user pages, from the generic area initialisation.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/b60fc09c76921bf69e77eb17e07eb4decedb3bf4.1746097431.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Pavel Begunkov
and committed by
Jens Axboe
782dfa32
6c9589aa
+56
-26
+47
-24
io_uring/zcrx.c
+47
-24
io_uring/zcrx.c
···
26
26
#include "zcrx.h"
27
27
#include "rsrc.h"
28
28
29
+
#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
30
+
29
31
static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)
30
32
{
31
33
return pp->mp_priv;
···
44
42
{
45
43
struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
46
44
47
-
return area->pages[net_iov_idx(niov)];
45
+
return area->mem.pages[net_iov_idx(niov)];
48
46
}
49
47
50
-
#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
48
+
static void io_release_area_mem(struct io_zcrx_mem *mem)
49
+
{
50
+
if (mem->pages) {
51
+
unpin_user_pages(mem->pages, mem->nr_folios);
52
+
kvfree(mem->pages);
53
+
}
54
+
}
55
+
56
+
static int io_import_area(struct io_zcrx_ifq *ifq,
57
+
struct io_zcrx_mem *mem,
58
+
struct io_uring_zcrx_area_reg *area_reg)
59
+
{
60
+
struct page **pages;
61
+
int nr_pages;
62
+
int ret;
63
+
64
+
ret = io_validate_user_buf_range(area_reg->addr, area_reg->len);
65
+
if (ret)
66
+
return ret;
67
+
if (!area_reg->addr)
68
+
return -EFAULT;
69
+
if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK)
70
+
return -EINVAL;
71
+
72
+
pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len,
73
+
&nr_pages);
74
+
if (IS_ERR(pages))
75
+
return PTR_ERR(pages);
76
+
77
+
mem->pages = pages;
78
+
mem->nr_folios = nr_pages;
79
+
mem->size = area_reg->len;
80
+
return 0;
81
+
}
51
82
52
83
static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
53
84
struct io_zcrx_area *area, int nr_mapped)
···
119
84
struct net_iov *niov = &area->nia.niovs[i];
120
85
dma_addr_t dma;
121
86
122
-
dma = dma_map_page_attrs(ifq->dev, area->pages[i], 0, PAGE_SIZE,
123
-
DMA_FROM_DEVICE, IO_DMA_ATTR);
87
+
dma = dma_map_page_attrs(ifq->dev, area->mem.pages[i], 0,
88
+
PAGE_SIZE, DMA_FROM_DEVICE, IO_DMA_ATTR);
124
89
if (dma_mapping_error(ifq->dev, dma))
125
90
break;
126
91
if (net_mp_niov_set_dma_addr(niov, dma)) {
···
227
192
static void io_zcrx_free_area(struct io_zcrx_area *area)
228
193
{
229
194
io_zcrx_unmap_area(area->ifq, area);
195
+
io_release_area_mem(&area->mem);
230
196
231
197
kvfree(area->freelist);
232
198
kvfree(area->nia.niovs);
233
199
kvfree(area->user_refs);
234
-
if (area->pages) {
235
-
unpin_user_pages(area->pages, area->nr_folios);
236
-
kvfree(area->pages);
237
-
}
238
200
kfree(area);
239
201
}
240
202
···
240
208
struct io_uring_zcrx_area_reg *area_reg)
241
209
{
242
210
struct io_zcrx_area *area;
243
-
int i, ret, nr_pages, nr_iovs;
211
+
unsigned nr_iovs;
212
+
int i, ret;
244
213
245
214
if (area_reg->flags || area_reg->rq_area_token)
246
215
return -EINVAL;
247
216
if (area_reg->__resv1 || area_reg->__resv2[0] || area_reg->__resv2[1])
248
217
return -EINVAL;
249
-
if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK)
250
-
return -EINVAL;
251
-
252
-
ret = io_validate_user_buf_range(area_reg->addr, area_reg->len);
253
-
if (ret)
254
-
return ret;
255
-
if (!area_reg->addr)
256
-
return -EFAULT;
257
218
258
219
ret = -ENOMEM;
259
220
area = kzalloc(sizeof(*area), GFP_KERNEL);
260
221
if (!area)
261
222
goto err;
262
223
263
-
area->pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len,
264
-
&nr_pages);
265
-
if (IS_ERR(area->pages)) {
266
-
ret = PTR_ERR(area->pages);
267
-
area->pages = NULL;
224
+
ret = io_import_area(ifq, &area->mem, area_reg);
225
+
if (ret)
268
226
goto err;
269
-
}
270
-
area->nr_folios = nr_iovs = nr_pages;
227
+
228
+
nr_iovs = area->mem.size >> PAGE_SHIFT;
271
229
area->nia.num_niovs = nr_iovs;
272
230
231
+
ret = -ENOMEM;
273
232
area->nia.niovs = kvmalloc_array(nr_iovs, sizeof(area->nia.niovs[0]),
274
233
GFP_KERNEL | __GFP_ZERO);
275
234
if (!area->nia.niovs)
+9
-2
io_uring/zcrx.h
+9
-2
io_uring/zcrx.h
···
7
7
#include <net/page_pool/types.h>
8
8
#include <net/net_trackers.h>
9
9
10
+
struct io_zcrx_mem {
11
+
unsigned long size;
12
+
13
+
struct page **pages;
14
+
unsigned long nr_folios;
15
+
};
16
+
10
17
struct io_zcrx_area {
11
18
struct net_iov_area nia;
12
19
struct io_zcrx_ifq *ifq;
···
21
14
22
15
bool is_mapped;
23
16
u16 area_id;
24
-
struct page **pages;
25
-
unsigned long nr_folios;
26
17
27
18
/* freelist */
28
19
spinlock_t freelist_lock ____cacheline_aligned_in_smp;
29
20
u32 free_count;
30
21
u32 *freelist;
22
+
23
+
struct io_zcrx_mem mem;
31
24
};
32
25
33
26
struct io_zcrx_ifq {