ublk: inline __ublk_ch_uring_cmd()

ublk_ch_uring_cmd_local() is a thin wrapper around __ublk_ch_uring_cmd()
that copies the ublksrv_io_cmd from user-mapped memory to the stack
using READ_ONCE(). This ublksrv_io_cmd is passed by pointer to
__ublk_ch_uring_cmd() and __ublk_ch_uring_cmd() is a large function
unlikely to be inlined, so __ublk_ch_uring_cmd() will have to load the
ublksrv_io_cmd fields back from the stack. Inline __ublk_ch_uring_cmd()
into ublk_ch_uring_cmd_local() and load the ublksrv_io_cmd fields into
local variables with READ_ONCE(). This allows the compiler to delay
loading the fields until they are needed and choose whether to store
them in registers or on the stack.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250808153251.282107-1-csander@purestorage.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Caleb Sander Mateos and committed by

Jens Axboe 8 months ago 225dc96f 4dbe13c7

+23 -39

1 changed file

expand all

drivers

block

ublk_drv.c

+23 -39

drivers/block/ublk_drv.c

··· 2265 2265 return ublk_start_io(ubq, req, io); 2266 2266 } 2267 2267 2268 - static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, 2269 - unsigned int issue_flags, 2270 - const struct ublksrv_io_cmd *ub_cmd) 2268 + static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, 2269 + unsigned int issue_flags) 2271 2270 { 2271 + /* May point to userspace-mapped memory */ 2272 + const struct ublksrv_io_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe); 2272 2273 u16 buf_idx = UBLK_INVALID_BUF_IDX; 2273 2274 struct ublk_device *ub = cmd->file->private_data; 2274 2275 struct ublk_queue *ubq; 2275 2276 struct ublk_io *io; 2276 2277 u32 cmd_op = cmd->cmd_op; 2277 - unsigned tag = ub_cmd->tag; 2278 + u16 q_id = READ_ONCE(ub_src->q_id); 2279 + u16 tag = READ_ONCE(ub_src->tag); 2280 + s32 result = READ_ONCE(ub_src->result); 2281 + u64 addr = READ_ONCE(ub_src->addr); /* unioned with zone_append_lba */ 2278 2282 struct request *req; 2279 2283 int ret; 2280 2284 bool compl; 2281 2285 2286 + WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED); 2287 + 2282 2288 pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n", 2283 - __func__, cmd->cmd_op, ub_cmd->q_id, tag, 2284 - ub_cmd->result); 2289 + __func__, cmd->cmd_op, q_id, tag, result); 2285 2290 2286 2291 ret = ublk_check_cmd_op(cmd_op); 2287 2292 if (ret) ··· 2297 2292 * so no need to validate the q_id, tag, or task 2298 2293 */ 2299 2294 if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF) 2300 - return ublk_unregister_io_buf(cmd, ub, ub_cmd->addr, 2301 - issue_flags); 2295 + return ublk_unregister_io_buf(cmd, ub, addr, issue_flags); 2302 2296 2303 2297 ret = -EINVAL; 2304 - if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) 2298 + if (q_id >= ub->dev_info.nr_hw_queues) 2305 2299 goto out; 2306 2300 2307 - ubq = ublk_get_queue(ub, ub_cmd->q_id); 2301 + ubq = ublk_get_queue(ub, q_id); 2308 2302 2309 2303 if (tag >= ubq->q_depth) 2310 2304 goto out; ··· 2311 2307 io = &ubq->ios[tag]; 2312 2308 /* UBLK_IO_FETCH_REQ can be handled on any task, which sets io->task */ 2313 2309 if (unlikely(_IOC_NR(cmd_op) == UBLK_IO_FETCH_REQ)) { 2314 - ret = ublk_check_fetch_buf(ubq, ub_cmd->addr); 2310 + ret = ublk_check_fetch_buf(ubq, addr); 2315 2311 if (ret) 2316 2312 goto out; 2317 - ret = ublk_fetch(cmd, ubq, io, ub_cmd->addr); 2313 + ret = ublk_fetch(cmd, ubq, io, addr); 2318 2314 if (ret) 2319 2315 goto out; 2320 2316 ··· 2328 2324 * so can be handled on any task 2329 2325 */ 2330 2326 if (_IOC_NR(cmd_op) == UBLK_IO_REGISTER_IO_BUF) 2331 - return ublk_register_io_buf(cmd, ubq, io, ub_cmd->addr, 2327 + return ublk_register_io_buf(cmd, ubq, io, addr, 2332 2328 issue_flags); 2333 2329 2334 2330 goto out; ··· 2350 2346 2351 2347 switch (_IOC_NR(cmd_op)) { 2352 2348 case UBLK_IO_REGISTER_IO_BUF: 2353 - return ublk_daemon_register_io_buf(cmd, ubq, io, ub_cmd->addr, 2349 + return ublk_daemon_register_io_buf(cmd, ubq, io, addr, 2354 2350 issue_flags); 2355 2351 case UBLK_IO_COMMIT_AND_FETCH_REQ: 2356 - ret = ublk_check_commit_and_fetch(ubq, io, ub_cmd->addr); 2352 + ret = ublk_check_commit_and_fetch(ubq, io, addr); 2357 2353 if (ret) 2358 2354 goto out; 2359 - io->res = ub_cmd->result; 2355 + io->res = result; 2360 2356 req = ublk_fill_io_cmd(io, cmd); 2361 - ret = ublk_config_io_buf(ubq, io, cmd, ub_cmd->addr, &buf_idx); 2357 + ret = ublk_config_io_buf(ubq, io, cmd, addr, &buf_idx); 2362 2358 compl = ublk_need_complete_req(ubq, io); 2363 2359 2364 2360 /* can't touch 'ublk_io' any more */ 2365 2361 if (buf_idx != UBLK_INVALID_BUF_IDX) 2366 2362 io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); 2367 2363 if (req_op(req) == REQ_OP_ZONE_APPEND) 2368 - req->__sector = ub_cmd->zone_append_lba; 2364 + req->__sector = addr; 2369 2365 if (compl) 2370 2366 __ublk_complete_rq(req); 2371 2367 ··· 2379 2375 * request 2380 2376 */ 2381 2377 req = ublk_fill_io_cmd(io, cmd); 2382 - ret = ublk_config_io_buf(ubq, io, cmd, ub_cmd->addr, NULL); 2378 + ret = ublk_config_io_buf(ubq, io, cmd, addr, NULL); 2383 2379 WARN_ON_ONCE(ret); 2384 2380 if (likely(ublk_get_data(ubq, io, req))) { 2385 2381 __ublk_prep_compl_io_cmd(io, req); ··· 2428 2424 fail_put: 2429 2425 ublk_put_req_ref(io, req); 2430 2426 return NULL; 2431 - } 2432 - 2433 - static inline int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, 2434 - unsigned int issue_flags) 2435 - { 2436 - /* 2437 - * Not necessary for async retry, but let's keep it simple and always 2438 - * copy the values to avoid any potential reuse. 2439 - */ 2440 - const struct ublksrv_io_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe); 2441 - const struct ublksrv_io_cmd ub_cmd = { 2442 - .q_id = READ_ONCE(ub_src->q_id), 2443 - .tag = READ_ONCE(ub_src->tag), 2444 - .result = READ_ONCE(ub_src->result), 2445 - .addr = READ_ONCE(ub_src->addr) 2446 - }; 2447 - 2448 - WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED); 2449 - 2450 - return __ublk_ch_uring_cmd(cmd, issue_flags, &ub_cmd); 2451 2427 } 2452 2428 2453 2429 static void ublk_ch_uring_cmd_cb(struct io_uring_cmd *cmd,

Configure Feed

Configure Feed