ublk: use copy_{to,from}_iter() for user copy

ublk_copy_user_pages()/ublk_copy_io_pages() currently uses
iov_iter_get_pages2() to extract the pages from the iov_iter and
memcpy()s between the bvec_iter and the iov_iter's pages one at a time.
Switch to using copy_to_iter()/copy_from_iter() instead. This avoids the
user page reference count increments and decrements and needing to split
the memcpy() at user page boundaries. It also simplifies the code
considerably.
Ming reports a 40% throughput improvement when issuing I/O to the
selftests null ublk server with zero-copy disabled.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Caleb Sander Mateos and committed by

Jens Axboe 7 months ago 2299ceec 15638d52

+14 -48

1 changed file

expand all

drivers

block

ublk_drv.c

+14 -48

drivers/block/ublk_drv.c

··· 913 913 .report_zones = ublk_report_zones, 914 914 }; 915 915 916 - #define UBLK_MAX_PIN_PAGES 32 917 - 918 916 struct ublk_io_iter { 919 - struct page *pages[UBLK_MAX_PIN_PAGES]; 920 917 struct bio *bio; 921 918 struct bvec_iter iter; 922 919 }; 923 920 924 - /* return how many pages are copied */ 925 - static void ublk_copy_io_pages(struct ublk_io_iter *data, 926 - size_t total, size_t pg_off, int dir) 921 + /* return how many bytes are copied */ 922 + static size_t ublk_copy_io_pages(struct ublk_io_iter *data, 923 + struct iov_iter *uiter, int dir) 927 924 { 928 - unsigned done = 0; 929 - unsigned pg_idx = 0; 925 + size_t done = 0; 930 926 931 - while (done < total) { 927 + for (;;) { 932 928 struct bio_vec bv = bio_iter_iovec(data->bio, data->iter); 933 - unsigned int bytes = min3(bv.bv_len, (unsigned)total - done, 934 - (unsigned)(PAGE_SIZE - pg_off)); 935 929 void *bv_buf = bvec_kmap_local(&bv); 936 - void *pg_buf = kmap_local_page(data->pages[pg_idx]); 930 + size_t copied; 937 931 938 932 if (dir == ITER_DEST) 939 - memcpy(pg_buf + pg_off, bv_buf, bytes); 933 + copied = copy_to_iter(bv_buf, bv.bv_len, uiter); 940 934 else 941 - memcpy(bv_buf, pg_buf + pg_off, bytes); 935 + copied = copy_from_iter(bv_buf, bv.bv_len, uiter); 942 936 943 - kunmap_local(pg_buf); 944 937 kunmap_local(bv_buf); 945 938 946 - /* advance page array */ 947 - pg_off += bytes; 948 - if (pg_off == PAGE_SIZE) { 949 - pg_idx += 1; 950 - pg_off = 0; 951 - } 952 - 953 - done += bytes; 939 + done += copied; 940 + if (copied < bv.bv_len) 941 + break; 954 942 955 943 /* advance bio */ 956 - bio_advance_iter_single(data->bio, &data->iter, bytes); 944 + bio_advance_iter_single(data->bio, &data->iter, copied); 957 945 if (!data->iter.bi_size) { 958 946 data->bio = data->bio->bi_next; 959 947 if (data->bio == NULL) ··· 949 961 data->iter = data->bio->bi_iter; 950 962 } 951 963 } 964 + return done; 952 965 } 953 966 954 967 static bool ublk_advance_io_iter(const struct request *req, ··· 977 988 unsigned offset, struct iov_iter *uiter, int dir) 978 989 { 979 990 struct ublk_io_iter iter; 980 - size_t done = 0; 981 991 982 992 if (!ublk_advance_io_iter(req, &iter, offset)) 983 993 return 0; 984 994 985 - while (iov_iter_count(uiter) && iter.bio) { 986 - unsigned nr_pages; 987 - ssize_t len; 988 - size_t off; 989 - int i; 990 - 991 - len = iov_iter_get_pages2(uiter, iter.pages, 992 - iov_iter_count(uiter), 993 - UBLK_MAX_PIN_PAGES, &off); 994 - if (len <= 0) 995 - return done; 996 - 997 - ublk_copy_io_pages(&iter, len, off, dir); 998 - nr_pages = DIV_ROUND_UP(len + off, PAGE_SIZE); 999 - for (i = 0; i < nr_pages; i++) { 1000 - if (dir == ITER_DEST) 1001 - set_page_dirty(iter.pages[i]); 1002 - put_page(iter.pages[i]); 1003 - } 1004 - done += len; 1005 - } 1006 - 1007 - return done; 995 + return ublk_copy_io_pages(&iter, uiter, dir); 1008 996 } 1009 997 1010 998 static inline bool ublk_need_map_req(const struct request *req)

Configure Feed

Configure Feed