io_uring/kbuf: remove legacy kbuf caching

-3

include/linux/io_uring_types.h

··· 360 360 361 361 spinlock_t completion_lock; 362 362 363 - struct list_head io_buffers_comp; 364 363 struct list_head cq_overflow_list; 365 364 366 365 struct hlist_head waitid_list; ··· 377 378 378 379 unsigned int file_alloc_start; 379 380 unsigned int file_alloc_end; 380 - 381 - struct list_head io_buffers_cache; 382 381 383 382 /* Keep this last, we don't need it for the fast path */ 384 383 struct wait_queue_head poll_wq;

-2

io_uring/io_uring.c

··· 323 323 init_waitqueue_head(&ctx->sqo_sq_wait); 324 324 INIT_LIST_HEAD(&ctx->sqd_list); 325 325 INIT_LIST_HEAD(&ctx->cq_overflow_list); 326 - INIT_LIST_HEAD(&ctx->io_buffers_cache); 327 326 ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX, 328 327 sizeof(struct async_poll), 0); 329 328 ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX, ··· 347 348 spin_lock_init(&ctx->completion_lock); 348 349 raw_spin_lock_init(&ctx->timeout_lock); 349 350 INIT_WQ_LIST(&ctx->iopoll_list); 350 - INIT_LIST_HEAD(&ctx->io_buffers_comp); 351 351 INIT_LIST_HEAD(&ctx->defer_list); 352 352 INIT_LIST_HEAD(&ctx->timeout_list); 353 353 INIT_LIST_HEAD(&ctx->ltimeout_list);

+7 -50

io_uring/kbuf.c

··· 70 70 71 71 void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags) 72 72 { 73 - spin_lock(&req->ctx->completion_lock); 74 73 __io_put_kbuf_list(req, len); 75 - spin_unlock(&req->ctx->completion_lock); 76 74 } 77 75 78 76 static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len, ··· 343 345 struct io_buffer *nxt; 344 346 345 347 nxt = list_first_entry(&bl->buf_list, struct io_buffer, list); 346 - list_move(&nxt->list, &ctx->io_buffers_cache); 348 + list_del(&nxt->list); 349 + kfree(nxt); 350 + 347 351 if (++i == nbufs) 348 352 return i; 349 353 cond_resched(); ··· 363 363 void io_destroy_buffers(struct io_ring_ctx *ctx) 364 364 { 365 365 struct io_buffer_list *bl; 366 - struct list_head *item, *tmp; 367 - struct io_buffer *buf; 368 366 369 367 while (1) { 370 368 unsigned long index = 0; ··· 375 377 if (!bl) 376 378 break; 377 379 io_put_bl(ctx, bl); 378 - } 379 - 380 - /* 381 - * Move deferred locked entries to cache before pruning 382 - */ 383 - spin_lock(&ctx->completion_lock); 384 - if (!list_empty(&ctx->io_buffers_comp)) 385 - list_splice_init(&ctx->io_buffers_comp, &ctx->io_buffers_cache); 386 - spin_unlock(&ctx->completion_lock); 387 - 388 - list_for_each_safe(item, tmp, &ctx->io_buffers_cache) { 389 - buf = list_entry(item, struct io_buffer, list); 390 - kfree(buf); 391 380 } 392 381 } 393 382 ··· 464 479 return 0; 465 480 } 466 481 467 - static int io_refill_buffer_cache(struct io_ring_ctx *ctx) 468 - { 469 - struct io_buffer *buf; 470 - 471 - /* 472 - * Completions that don't happen inline (eg not under uring_lock) will 473 - * add to ->io_buffers_comp. If we don't have any free buffers, check 474 - * the completion list and splice those entries first. 475 - */ 476 - if (!list_empty_careful(&ctx->io_buffers_comp)) { 477 - spin_lock(&ctx->completion_lock); 478 - if (!list_empty(&ctx->io_buffers_comp)) { 479 - list_splice_init(&ctx->io_buffers_comp, 480 - &ctx->io_buffers_cache); 481 - spin_unlock(&ctx->completion_lock); 482 - return 0; 483 - } 484 - spin_unlock(&ctx->completion_lock); 485 - } 486 - 487 - buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT); 488 - if (!buf) 489 - return -ENOMEM; 490 - list_add_tail(&buf->list, &ctx->io_buffers_cache); 491 - return 0; 492 - } 493 - 494 482 static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, 495 483 struct io_buffer_list *bl) 496 484 { ··· 472 514 int i, bid = pbuf->bid; 473 515 474 516 for (i = 0; i < pbuf->nbufs; i++) { 475 - if (list_empty(&ctx->io_buffers_cache) && 476 - io_refill_buffer_cache(ctx)) 517 + buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT); 518 + if (!buf) 477 519 break; 478 - buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer, 479 - list); 480 - list_move_tail(&buf->list, &bl->buf_list); 520 + 521 + list_add_tail(&buf->list, &bl->buf_list); 481 522 buf->addr = addr; 482 523 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); 483 524 buf->bid = bid;

+2 -3

io_uring/kbuf.h

··· 166 166 __io_put_kbuf_ring(req, len, 1); 167 167 } else { 168 168 req->buf_index = req->kbuf->bgid; 169 - list_add(&req->kbuf->list, &req->ctx->io_buffers_comp); 170 169 req->flags &= ~REQ_F_BUFFER_SELECTED; 170 + kfree(req->kbuf); 171 + req->kbuf = NULL; 171 172 } 172 173 } 173 174 ··· 177 176 if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))) 178 177 return; 179 178 180 - spin_lock(&req->ctx->completion_lock); 181 179 /* len == 0 is fine here, non-ring will always drop all of it */ 182 180 __io_put_kbuf_list(req, 0); 183 - spin_unlock(&req->ctx->completion_lock); 184 181 } 185 182 186 183 static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,

Configure Feed

Configure Feed