Merge tag 'io_uring-6.9-20240405' of git://git.kernel.dk/linux

-1

include/linux/io_uring_types.h

··· 294 294 295 295 struct io_submit_state submit_state; 296 296 297 - struct io_buffer_list *io_bl; 298 297 struct xarray io_bl_xa; 299 298 300 299 struct io_hash_table cancel_table_locked;

+19 -12

io_uring/io_uring.c

··· 147 147 static void io_queue_sqe(struct io_kiocb *req); 148 148 149 149 struct kmem_cache *req_cachep; 150 + static struct workqueue_struct *iou_wq __ro_after_init; 150 151 151 152 static int __read_mostly sysctl_io_uring_disabled; 152 153 static int __read_mostly sysctl_io_uring_group = -1; ··· 351 350 err: 352 351 kfree(ctx->cancel_table.hbs); 353 352 kfree(ctx->cancel_table_locked.hbs); 354 - kfree(ctx->io_bl); 355 353 xa_destroy(&ctx->io_bl_xa); 356 354 kfree(ctx); 357 355 return NULL; ··· 1982 1982 err = -EBADFD; 1983 1983 if (!io_file_can_poll(req)) 1984 1984 goto fail; 1985 - err = -ECANCELED; 1986 - if (io_arm_poll_handler(req, issue_flags) != IO_APOLL_OK) 1987 - goto fail; 1988 - return; 1985 + if (req->file->f_flags & O_NONBLOCK || 1986 + req->file->f_mode & FMODE_NOWAIT) { 1987 + err = -ECANCELED; 1988 + if (io_arm_poll_handler(req, issue_flags) != IO_APOLL_OK) 1989 + goto fail; 1990 + return; 1991 + } else { 1992 + req->flags &= ~REQ_F_APOLL_MULTISHOT; 1993 + } 1989 1994 } 1990 1995 1991 1996 if (req->flags & REQ_F_FORCE_ASYNC) { ··· 2931 2926 io_napi_free(ctx); 2932 2927 kfree(ctx->cancel_table.hbs); 2933 2928 kfree(ctx->cancel_table_locked.hbs); 2934 - kfree(ctx->io_bl); 2935 2929 xa_destroy(&ctx->io_bl_xa); 2936 2930 kfree(ctx); 2937 2931 } ··· 3165 3161 * noise and overhead, there's no discernable change in runtime 3166 3162 * over using system_wq. 3167 3163 */ 3168 - queue_work(system_unbound_wq, &ctx->exit_work); 3164 + queue_work(iou_wq, &ctx->exit_work); 3169 3165 } 3170 3166 3171 3167 static int io_uring_release(struct inode *inode, struct file *file) ··· 3447 3443 ptr = ctx->sq_sqes; 3448 3444 break; 3449 3445 case IORING_OFF_PBUF_RING: { 3446 + struct io_buffer_list *bl; 3450 3447 unsigned int bgid; 3451 3448 3452 3449 bgid = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_PBUF_SHIFT; 3453 - rcu_read_lock(); 3454 - ptr = io_pbuf_get_address(ctx, bgid); 3455 - rcu_read_unlock(); 3456 - if (!ptr) 3457 - return ERR_PTR(-EINVAL); 3450 + bl = io_pbuf_get_bl(ctx, bgid); 3451 + if (IS_ERR(bl)) 3452 + return bl; 3453 + ptr = bl->buf_ring; 3454 + io_put_bl(ctx, bl); 3458 3455 break; 3459 3456 } 3460 3457 default: ··· 4189 4184 sizeof_field(struct io_kiocb, cmd.data), NULL); 4190 4185 io_buf_cachep = KMEM_CACHE(io_buffer, 4191 4186 SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); 4187 + 4188 + iou_wq = alloc_workqueue("iou_exit", WQ_UNBOUND, 64); 4192 4189 4193 4190 #ifdef CONFIG_SYSCTL 4194 4191 register_sysctl_init("kernel", kernel_io_uring_disabled_table);

+40 -76

io_uring/kbuf.c

··· 17 17 18 18 #define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf)) 19 19 20 - #define BGID_ARRAY 64 21 - 22 20 /* BIDs are addressed by a 16-bit field in a CQE */ 23 21 #define MAX_BIDS_PER_BGID (1 << 16) 24 22 ··· 38 40 int inuse; 39 41 }; 40 42 41 - static struct io_buffer_list *__io_buffer_get_list(struct io_ring_ctx *ctx, 42 - struct io_buffer_list *bl, 43 - unsigned int bgid) 43 + static inline struct io_buffer_list *__io_buffer_get_list(struct io_ring_ctx *ctx, 44 + unsigned int bgid) 44 45 { 45 - if (bl && bgid < BGID_ARRAY) 46 - return &bl[bgid]; 47 - 48 46 return xa_load(&ctx->io_bl_xa, bgid); 49 47 } 50 48 ··· 49 55 { 50 56 lockdep_assert_held(&ctx->uring_lock); 51 57 52 - return __io_buffer_get_list(ctx, ctx->io_bl, bgid); 58 + return __io_buffer_get_list(ctx, bgid); 53 59 } 54 60 55 61 static int io_buffer_add_list(struct io_ring_ctx *ctx, ··· 61 67 * always under the ->uring_lock, but the RCU lookup from mmap does. 62 68 */ 63 69 bl->bgid = bgid; 64 - smp_store_release(&bl->is_ready, 1); 65 - 66 - if (bgid < BGID_ARRAY) 67 - return 0; 68 - 70 + atomic_set(&bl->refs, 1); 69 71 return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL)); 70 72 } 71 73 ··· 198 208 return ret; 199 209 } 200 210 201 - static __cold int io_init_bl_list(struct io_ring_ctx *ctx) 202 - { 203 - struct io_buffer_list *bl; 204 - int i; 205 - 206 - bl = kcalloc(BGID_ARRAY, sizeof(struct io_buffer_list), GFP_KERNEL); 207 - if (!bl) 208 - return -ENOMEM; 209 - 210 - for (i = 0; i < BGID_ARRAY; i++) { 211 - INIT_LIST_HEAD(&bl[i].buf_list); 212 - bl[i].bgid = i; 213 - } 214 - 215 - smp_store_release(&ctx->io_bl, bl); 216 - return 0; 217 - } 218 - 219 211 /* 220 212 * Mark the given mapped range as free for reuse 221 213 */ ··· 266 294 return i; 267 295 } 268 296 297 + void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) 298 + { 299 + if (atomic_dec_and_test(&bl->refs)) { 300 + __io_remove_buffers(ctx, bl, -1U); 301 + kfree_rcu(bl, rcu); 302 + } 303 + } 304 + 269 305 void io_destroy_buffers(struct io_ring_ctx *ctx) 270 306 { 271 307 struct io_buffer_list *bl; 272 308 struct list_head *item, *tmp; 273 309 struct io_buffer *buf; 274 310 unsigned long index; 275 - int i; 276 - 277 - for (i = 0; i < BGID_ARRAY; i++) { 278 - if (!ctx->io_bl) 279 - break; 280 - __io_remove_buffers(ctx, &ctx->io_bl[i], -1U); 281 - } 282 311 283 312 xa_for_each(&ctx->io_bl_xa, index, bl) { 284 313 xa_erase(&ctx->io_bl_xa, bl->bgid); 285 - __io_remove_buffers(ctx, bl, -1U); 286 - kfree_rcu(bl, rcu); 314 + io_put_bl(ctx, bl); 287 315 } 288 316 289 317 /* ··· 461 489 462 490 io_ring_submit_lock(ctx, issue_flags); 463 491 464 - if (unlikely(p->bgid < BGID_ARRAY && !ctx->io_bl)) { 465 - ret = io_init_bl_list(ctx); 466 - if (ret) 467 - goto err; 468 - } 469 - 470 492 bl = io_buffer_get_list(ctx, p->bgid); 471 493 if (unlikely(!bl)) { 472 494 bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT); ··· 473 507 if (ret) { 474 508 /* 475 509 * Doesn't need rcu free as it was never visible, but 476 - * let's keep it consistent throughout. Also can't 477 - * be a lower indexed array group, as adding one 478 - * where lookup failed cannot happen. 510 + * let's keep it consistent throughout. 479 511 */ 480 - if (p->bgid >= BGID_ARRAY) 481 - kfree_rcu(bl, rcu); 482 - else 483 - WARN_ON_ONCE(1); 512 + kfree_rcu(bl, rcu); 484 513 goto err; 485 514 } 486 515 } ··· 640 679 if (reg.ring_entries >= 65536) 641 680 return -EINVAL; 642 681 643 - if (unlikely(reg.bgid < BGID_ARRAY && !ctx->io_bl)) { 644 - int ret = io_init_bl_list(ctx); 645 - if (ret) 646 - return ret; 647 - } 648 - 649 682 bl = io_buffer_get_list(ctx, reg.bgid); 650 683 if (bl) { 651 684 /* if mapped buffer ring OR classic exists, don't allow */ ··· 688 733 if (!bl->is_buf_ring) 689 734 return -EINVAL; 690 735 691 - __io_remove_buffers(ctx, bl, -1U); 692 - if (bl->bgid >= BGID_ARRAY) { 693 - xa_erase(&ctx->io_bl_xa, bl->bgid); 694 - kfree_rcu(bl, rcu); 695 - } 736 + xa_erase(&ctx->io_bl_xa, bl->bgid); 737 + io_put_bl(ctx, bl); 696 738 return 0; 697 739 } 698 740 ··· 719 767 return 0; 720 768 } 721 769 722 - void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid) 770 + struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx, 771 + unsigned long bgid) 723 772 { 724 773 struct io_buffer_list *bl; 774 + bool ret; 725 775 726 - bl = __io_buffer_get_list(ctx, smp_load_acquire(&ctx->io_bl), bgid); 727 - 728 - if (!bl || !bl->is_mmap) 729 - return NULL; 730 776 /* 731 - * Ensure the list is fully setup. Only strictly needed for RCU lookup 732 - * via mmap, and in that case only for the array indexed groups. For 733 - * the xarray lookups, it's either visible and ready, or not at all. 777 + * We have to be a bit careful here - we're inside mmap and cannot grab 778 + * the uring_lock. This means the buffer_list could be simultaneously 779 + * going away, if someone is trying to be sneaky. Look it up under rcu 780 + * so we know it's not going away, and attempt to grab a reference to 781 + * it. If the ref is already zero, then fail the mapping. If successful, 782 + * the caller will call io_put_bl() to drop the the reference at at the 783 + * end. This may then safely free the buffer_list (and drop the pages) 784 + * at that point, vm_insert_pages() would've already grabbed the 785 + * necessary vma references. 734 786 */ 735 - if (!smp_load_acquire(&bl->is_ready)) 736 - return NULL; 787 + rcu_read_lock(); 788 + bl = xa_load(&ctx->io_bl_xa, bgid); 789 + /* must be a mmap'able buffer ring and have pages */ 790 + ret = false; 791 + if (bl && bl->is_mmap) 792 + ret = atomic_inc_not_zero(&bl->refs); 793 + rcu_read_unlock(); 737 794 738 - return bl->buf_ring; 795 + if (ret) 796 + return bl; 797 + 798 + return ERR_PTR(-EINVAL); 739 799 } 740 800 741 801 /*

+5 -3

io_uring/kbuf.h

··· 25 25 __u16 head; 26 26 __u16 mask; 27 27 28 + atomic_t refs; 29 + 28 30 /* ring mapped provided buffers */ 29 31 __u8 is_buf_ring; 30 32 /* ring mapped provided buffers, but mmap'ed by application */ 31 33 __u8 is_mmap; 32 - /* bl is visible from an RCU point of view for lookup */ 33 - __u8 is_ready; 34 34 }; 35 35 36 36 struct io_buffer { ··· 61 61 62 62 bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags); 63 63 64 - void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid); 64 + void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl); 65 + struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx, 66 + unsigned long bgid); 65 67 66 68 static inline bool io_kbuf_recycle_ring(struct io_kiocb *req) 67 69 {

+8 -1

io_uring/rw.c

··· 937 937 ret = __io_read(req, issue_flags); 938 938 939 939 /* 940 + * If the file doesn't support proper NOWAIT, then disable multishot 941 + * and stay in single shot mode. 942 + */ 943 + if (!io_file_supports_nowait(req)) 944 + req->flags &= ~REQ_F_APOLL_MULTISHOT; 945 + 946 + /* 940 947 * If we get -EAGAIN, recycle our buffer and just let normal poll 941 948 * handling arm it. 942 949 */ ··· 962 955 /* 963 956 * Any successful return value will keep the multishot read armed. 964 957 */ 965 - if (ret > 0) { 958 + if (ret > 0 && req->flags & REQ_F_APOLL_MULTISHOT) { 966 959 /* 967 960 * Put our buffer and post a CQE. If we fail to post a CQE, then 968 961 * jump to the termination path. This request is then done.

Configure Feed

Configure Feed