Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
ublk: optimize UBLK_IO_REGISTER_IO_BUF on daemon task
ublk_register_io_buf() performs an expensive atomic refcount increment,
as well as a lot of pointer chasing to look up the struct request.
Create a separate ublk_daemon_register_io_buf() for the daemon task to
call. Initialize ublk_io's reference count to a large number, introduce
a field task_registered_buffers to count the buffers registered on the
daemon task, and atomically subtract the large number minus
task_registered_buffers in ublk_commit_and_fetch().
Also obtain the struct request directly from ublk_io's req field instead
of looking it up on the tagset.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250620151008.3976463-12-csander@purestorage.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Caleb Sander Mateos
and committed by
Jens Axboe
8a8fe42d
7ab74108
+61
-9
+61
-9
drivers/block/ublk_drv.c
+61
-9
drivers/block/ublk_drv.c
···
148
148
/* atomic RW with ubq->cancel_lock */
149
149
#define UBLK_IO_FLAG_CANCELED 0x80000000
150
150
151
+
/*
152
+
* Initialize refcount to a large number to include any registered buffers.
153
+
* UBLK_IO_COMMIT_AND_FETCH_REQ will release these references minus those for
154
+
* any buffers registered on the io daemon task.
155
+
*/
156
+
#define UBLK_REFCOUNT_INIT (REFCOUNT_MAX / 2)
157
+
151
158
struct ublk_io {
152
159
/* userspace buffer address from io cmd */
153
160
__u64 addr;
···
173
166
/*
174
167
* The number of uses of this I/O by the ublk server
175
168
* if user copy or zero copy are enabled:
176
-
* - 1 from dispatch to the server until UBLK_IO_COMMIT_AND_FETCH_REQ
169
+
* - UBLK_REFCOUNT_INIT from dispatch to the server
170
+
* until UBLK_IO_COMMIT_AND_FETCH_REQ
177
171
* - 1 for each inflight ublk_ch_{read,write}_iter() call
178
-
* - 1 for each io_uring registered buffer
172
+
* - 1 for each io_uring registered buffer not registered on task
179
173
* The I/O can only be completed once all references are dropped.
180
174
* User copy and buffer registration operations are only permitted
181
175
* if the reference count is nonzero.
182
176
*/
183
177
refcount_t ref;
178
+
/* Count of buffers registered on task and not yet unregistered */
179
+
unsigned task_registered_buffers;
184
180
185
181
/* auto-registered buffer, valid if UBLK_IO_FLAG_AUTO_BUF_REG is set */
186
182
u16 buf_index;
···
696
686
struct ublk_io *io)
697
687
{
698
688
if (ublk_need_req_ref(ubq))
699
-
refcount_set(&io->ref, 1);
689
+
refcount_set(&io->ref, UBLK_REFCOUNT_INIT);
700
690
}
701
691
702
692
static inline bool ublk_get_req_ref(const struct ublk_queue *ubq,
···
717
707
} else {
718
708
__ublk_complete_rq(req);
719
709
}
710
+
}
711
+
712
+
static inline void ublk_sub_req_ref(struct ublk_io *io, struct request *req)
713
+
{
714
+
unsigned sub_refs = UBLK_REFCOUNT_INIT - io->task_registered_buffers;
715
+
716
+
io->task_registered_buffers = 0;
717
+
if (refcount_sub_and_test(sub_refs, &io->ref))
718
+
__ublk_complete_rq(req);
720
719
}
721
720
722
721
static inline bool ublk_need_get_data(const struct ublk_queue *ubq)
···
1216
1197
struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag);
1217
1198
1218
1199
iod->op_flags |= UBLK_IO_F_NEED_REG_BUF;
1219
-
refcount_set(&io->ref, 1);
1220
1200
}
1221
1201
1222
1202
static bool ublk_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
···
1234
1216
blk_mq_end_request(req, BLK_STS_IOERR);
1235
1217
return false;
1236
1218
}
1237
-
/* one extra reference is dropped by ublk_io_release */
1238
-
refcount_set(&io->ref, 2);
1239
1219
1220
+
io->task_registered_buffers = 1;
1240
1221
io->buf_ctx_handle = io_uring_cmd_ctx_handle(io->cmd);
1241
1222
/* store buffer index in request payload */
1242
1223
io->buf_index = pdu->buf.index;
···
1247
1230
struct request *req, struct ublk_io *io,
1248
1231
unsigned int issue_flags)
1249
1232
{
1233
+
ublk_init_req_ref(ubq, io);
1250
1234
if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req))
1251
1235
return ublk_auto_buf_reg(ubq, req, io, issue_flags);
1252
1236
1253
-
ublk_init_req_ref(ubq, io);
1254
1237
return true;
1255
1238
}
1256
1239
···
1523
1506
}
1524
1507
1525
1508
WARN_ON_ONCE(refcount_read(&io->ref));
1509
+
WARN_ON_ONCE(io->task_registered_buffers);
1526
1510
}
1527
1511
}
1528
1512
···
2059
2041
return 0;
2060
2042
}
2061
2043
2044
+
static int
2045
+
ublk_daemon_register_io_buf(struct io_uring_cmd *cmd,
2046
+
const struct ublk_queue *ubq, struct ublk_io *io,
2047
+
unsigned index, unsigned issue_flags)
2048
+
{
2049
+
unsigned new_registered_buffers;
2050
+
struct request *req = io->req;
2051
+
int ret;
2052
+
2053
+
/*
2054
+
* Ensure there are still references for ublk_sub_req_ref() to release.
2055
+
* If not, fall back on the thread-safe buffer registration.
2056
+
*/
2057
+
new_registered_buffers = io->task_registered_buffers + 1;
2058
+
if (unlikely(new_registered_buffers >= UBLK_REFCOUNT_INIT))
2059
+
return ublk_register_io_buf(cmd, ubq, io, index, issue_flags);
2060
+
2061
+
if (!ublk_support_zero_copy(ubq) || !ublk_rq_has_data(req))
2062
+
return -EINVAL;
2063
+
2064
+
ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index,
2065
+
issue_flags);
2066
+
if (ret)
2067
+
return ret;
2068
+
2069
+
io->task_registered_buffers = new_registered_buffers;
2070
+
return 0;
2071
+
}
2072
+
2062
2073
static int ublk_unregister_io_buf(struct io_uring_cmd *cmd,
2063
2074
const struct ublk_device *ub,
2064
2075
unsigned int index, unsigned int issue_flags)
···
2211
2164
if (unlikely(blk_should_fake_timeout(req->q)))
2212
2165
return 0;
2213
2166
2214
-
ublk_put_req_ref(ubq, io, req);
2167
+
if (ublk_need_req_ref(ubq))
2168
+
ublk_sub_req_ref(io, req);
2169
+
else
2170
+
__ublk_complete_rq(req);
2215
2171
return 0;
2216
2172
}
2217
2173
···
2312
2262
2313
2263
switch (_IOC_NR(cmd_op)) {
2314
2264
case UBLK_IO_REGISTER_IO_BUF:
2315
-
return ublk_register_io_buf(cmd, ubq, io, ub_cmd->addr, issue_flags);
2265
+
return ublk_daemon_register_io_buf(cmd, ubq, io, ub_cmd->addr,
2266
+
issue_flags);
2316
2267
case UBLK_IO_COMMIT_AND_FETCH_REQ:
2317
2268
ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags);
2318
2269
if (ret)
···
2552
2501
if (io->task)
2553
2502
put_task_struct(io->task);
2554
2503
WARN_ON_ONCE(refcount_read(&io->ref));
2504
+
WARN_ON_ONCE(io->task_registered_buffers);
2555
2505
}
2556
2506
2557
2507
if (ubq->io_cmd_buf)