Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/kbuf: vmap pinned buffer ring
This avoids needing to care about HIGHMEM, and it makes the buffer
indexing easier as both ring provided buffer methods are now virtually
mapped in a contigious fashion.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+15
-24
+15
-24
io_uring/kbuf.c
+15
-24
io_uring/kbuf.c
···
7
7
#include <linux/slab.h>
8
8
#include <linux/namei.h>
9
9
#include <linux/poll.h>
10
+
#include <linux/vmalloc.h>
10
11
#include <linux/io_uring.h>
11
12
12
13
#include <uapi/linux/io_uring.h>
···
147
146
req->flags |= REQ_F_BL_EMPTY;
148
147
149
148
head &= bl->mask;
150
-
/* mmaped buffers are always contig */
151
-
if (bl->is_mmap || head < IO_BUFFER_LIST_BUF_PER_PAGE) {
152
-
buf = &br->bufs[head];
153
-
} else {
154
-
int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1);
155
-
int index = head / IO_BUFFER_LIST_BUF_PER_PAGE;
156
-
buf = page_address(bl->buf_pages[index]);
157
-
buf += off;
158
-
}
149
+
buf = &br->bufs[head];
159
150
if (*len == 0 || *len > buf->len)
160
151
*len = buf->len;
161
152
req->flags |= REQ_F_BUFFER_RING;
···
234
241
for (j = 0; j < bl->buf_nr_pages; j++)
235
242
unpin_user_page(bl->buf_pages[j]);
236
243
kvfree(bl->buf_pages);
244
+
vunmap(bl->buf_ring);
237
245
bl->buf_pages = NULL;
238
246
bl->buf_nr_pages = 0;
239
247
}
···
492
498
static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg,
493
499
struct io_buffer_list *bl)
494
500
{
495
-
struct io_uring_buf_ring *br;
501
+
struct io_uring_buf_ring *br = NULL;
502
+
int nr_pages, ret, i;
496
503
struct page **pages;
497
-
int i, nr_pages;
498
504
499
505
pages = io_pin_pages(reg->ring_addr,
500
506
flex_array_size(br, bufs, reg->ring_entries),
···
502
508
if (IS_ERR(pages))
503
509
return PTR_ERR(pages);
504
510
505
-
/*
506
-
* Apparently some 32-bit boxes (ARM) will return highmem pages,
507
-
* which then need to be mapped. We could support that, but it'd
508
-
* complicate the code and slowdown the common cases quite a bit.
509
-
* So just error out, returning -EINVAL just like we did on kernels
510
-
* that didn't support mapped buffer rings.
511
-
*/
512
-
for (i = 0; i < nr_pages; i++)
513
-
if (PageHighMem(pages[i]))
514
-
goto error_unpin;
511
+
br = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
512
+
if (!br) {
513
+
ret = -ENOMEM;
514
+
goto error_unpin;
515
+
}
515
516
516
-
br = page_address(pages[0]);
517
517
#ifdef SHM_COLOUR
518
518
/*
519
519
* On platforms that have specific aliasing requirements, SHM_COLOUR
···
518
530
* should use IOU_PBUF_RING_MMAP instead, and liburing will handle
519
531
* this transparently.
520
532
*/
521
-
if ((reg->ring_addr | (unsigned long) br) & (SHM_COLOUR - 1))
533
+
if ((reg->ring_addr | (unsigned long) br) & (SHM_COLOUR - 1)) {
534
+
ret = -EINVAL;
522
535
goto error_unpin;
536
+
}
523
537
#endif
524
538
bl->buf_pages = pages;
525
539
bl->buf_nr_pages = nr_pages;
···
533
543
for (i = 0; i < nr_pages; i++)
534
544
unpin_user_page(pages[i]);
535
545
kvfree(pages);
536
-
return -EINVAL;
546
+
vunmap(br);
547
+
return ret;
537
548
}
538
549
539
550
/*