+47 -62

block/blk-mq.c

reviewed

··· 1152 1152 } 1153 1153 EXPORT_SYMBOL(blk_mq_start_request); 1154 1154 1155 1155 - /** 1156 1156 - * blk_end_sync_rq - executes a completion event on a request 1157 1157 - * @rq: request to complete 1158 1158 - * @error: end I/O status of the request 1159 1159 - */ 1160 1160 - static void blk_end_sync_rq(struct request *rq, blk_status_t error) 1161 1161 - { 1162 1162 - struct completion *waiting = rq->end_io_data; 1163 1163 - 1164 1164 - rq->end_io_data = (void *)(uintptr_t)error; 1165 1165 - 1166 1166 - /* 1167 1167 - * complete last, if this is a stack request the process (and thus 1168 1168 - * the rq pointer) could be invalid right after this complete() 1169 1169 - */ 1170 1170 - complete(waiting); 1171 1171 - } 1172 1172 - 1173 1155 /* 1174 1156 * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple 1175 1157 * queues. This is important for md arrays to benefit from merging ··· 1186 1204 plug->rq_count++; 1187 1205 } 1188 1206 1189 1189 - static void __blk_execute_rq_nowait(struct request *rq, bool at_head, 1190 1190 - rq_end_io_fn *done, bool use_plug) 1191 1191 - { 1192 1192 - WARN_ON(irqs_disabled()); 1193 1193 - WARN_ON(!blk_rq_is_passthrough(rq)); 1194 1194 - 1195 1195 - rq->end_io = done; 1196 1196 - 1197 1197 - blk_account_io_start(rq); 1198 1198 - 1199 1199 - if (use_plug && current->plug) { 1200 1200 - blk_add_rq_to_plug(current->plug, rq); 1201 1201 - return; 1202 1202 - } 1203 1203 - /* 1204 1204 - * don't check dying flag for MQ because the request won't 1205 1205 - * be reused after dying flag is set 1206 1206 - */ 1207 1207 - blk_mq_sched_insert_request(rq, at_head, true, false); 1208 1208 - } 1209 1209 - 1210 1210 - 1211 1207 /** 1212 1208 * blk_execute_rq_nowait - insert a request to I/O scheduler for execution 1213 1209 * @rq: request to insert 1214 1210 * @at_head: insert request at head or tail of queue 1215 1215 - * @done: I/O completion handler 1216 1211 * 1217 1212 * Description: 1218 1213 * Insert a fully prepared request at the back of the I/O scheduler queue ··· 1198 1239 * Note: 1199 1240 * This function will invoke @done directly if the queue is dead. 1200 1241 */ 1201 1201 - void blk_execute_rq_nowait(struct request *rq, bool at_head, rq_end_io_fn *done) 1242 1242 + void blk_execute_rq_nowait(struct request *rq, bool at_head) 1202 1243 { 1203 1203 - __blk_execute_rq_nowait(rq, at_head, done, true); 1244 1244 + WARN_ON(irqs_disabled()); 1245 1245 + WARN_ON(!blk_rq_is_passthrough(rq)); 1204 1246 1247 1247 + blk_account_io_start(rq); 1248 1248 + if (current->plug) 1249 1249 + blk_add_rq_to_plug(current->plug, rq); 1250 1250 + else 1251 1251 + blk_mq_sched_insert_request(rq, at_head, true, false); 1205 1252 } 1206 1253 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); 1254 1254 + 1255 1255 + struct blk_rq_wait { 1256 1256 + struct completion done; 1257 1257 + blk_status_t ret; 1258 1258 + }; 1259 1259 + 1260 1260 + static void blk_end_sync_rq(struct request *rq, blk_status_t ret) 1261 1261 + { 1262 1262 + struct blk_rq_wait *wait = rq->end_io_data; 1263 1263 + 1264 1264 + wait->ret = ret; 1265 1265 + complete(&wait->done); 1266 1266 + } 1207 1267 1208 1268 static bool blk_rq_is_poll(struct request *rq) 1209 1269 { ··· 1255 1277 */ 1256 1278 blk_status_t blk_execute_rq(struct request *rq, bool at_head) 1257 1279 { 1258 1258 - DECLARE_COMPLETION_ONSTACK(wait); 1259 1259 - unsigned long hang_check; 1280 1280 + struct blk_rq_wait wait = { 1281 1281 + .done = COMPLETION_INITIALIZER_ONSTACK(wait.done), 1282 1282 + }; 1260 1283 1261 1261 - /* 1262 1262 - * iopoll requires request to be submitted to driver, so can't 1263 1263 - * use plug 1264 1264 - */ 1284 1284 + WARN_ON(irqs_disabled()); 1285 1285 + WARN_ON(!blk_rq_is_passthrough(rq)); 1286 1286 + 1265 1287 rq->end_io_data = &wait; 1266 1266 - __blk_execute_rq_nowait(rq, at_head, blk_end_sync_rq, 1267 1267 - !blk_rq_is_poll(rq)); 1288 1288 + rq->end_io = blk_end_sync_rq; 1268 1289 1269 1269 - /* Prevent hang_check timer from firing at us during very long I/O */ 1270 1270 - hang_check = sysctl_hung_task_timeout_secs; 1290 1290 + blk_account_io_start(rq); 1291 1291 + blk_mq_sched_insert_request(rq, at_head, true, false); 1271 1292 1272 1272 - if (blk_rq_is_poll(rq)) 1273 1273 - blk_rq_poll_completion(rq, &wait); 1274 1274 - else if (hang_check) 1275 1275 - while (!wait_for_completion_io_timeout(&wait, 1276 1276 - hang_check * (HZ/2))) 1277 1277 - ; 1278 1278 - else 1279 1279 - wait_for_completion_io(&wait); 1293 1293 + if (blk_rq_is_poll(rq)) { 1294 1294 + blk_rq_poll_completion(rq, &wait.done); 1295 1295 + } else { 1296 1296 + /* 1297 1297 + * Prevent hang_check timer from firing at us during very long 1298 1298 + * I/O 1299 1299 + */ 1300 1300 + unsigned long hang_check = sysctl_hung_task_timeout_secs; 1280 1301 1281 1281 - return (blk_status_t)(uintptr_t)rq->end_io_data; 1302 1302 + if (hang_check) 1303 1303 + while (!wait_for_completion_io_timeout(&wait.done, 1304 1304 + hang_check * (HZ/2))) 1305 1305 + ; 1306 1306 + else 1307 1307 + wait_for_completion_io(&wait.done); 1308 1308 + } 1309 1309 + 1310 1310 + return wait.ret; 1282 1311 } 1283 1312 EXPORT_SYMBOL(blk_execute_rq); 1284 1313

+2 -2

drivers/block/sx8.c

reviewed

··· 540 540 spin_unlock_irq(&host->lock); 541 541 542 542 DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag); 543 543 - blk_execute_rq_nowait(rq, true, NULL); 543 543 + blk_execute_rq_nowait(rq, true); 544 544 545 545 return 0; 546 546 ··· 579 579 crq->msg_bucket = (u32) rc; 580 580 581 581 DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag); 582 582 - blk_execute_rq_nowait(rq, true, NULL); 582 582 + blk_execute_rq_nowait(rq, true); 583 583 584 584 return 0; 585 585 }

+2 -1

drivers/nvme/host/core.c

reviewed

··· 1206 1206 nvme_init_request(rq, &ctrl->ka_cmd); 1207 1207 1208 1208 rq->timeout = ctrl->kato * HZ; 1209 1209 + rq->end_io = nvme_keep_alive_end_io; 1209 1210 rq->end_io_data = ctrl; 1210 1211 rq->rq_flags |= RQF_QUIET; 1211 1211 - blk_execute_rq_nowait(rq, false, nvme_keep_alive_end_io); 1212 1212 + blk_execute_rq_nowait(rq, false); 1212 1213 } 1213 1214 1214 1215 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)

+2 -1

drivers/nvme/host/ioctl.c

reviewed

··· 453 453 blk_flags); 454 454 if (IS_ERR(req)) 455 455 return PTR_ERR(req); 456 456 + req->end_io = nvme_uring_cmd_end_io; 456 457 req->end_io_data = ioucmd; 457 458 458 459 /* to free bio on completion, as req->bio will be null at that time */ ··· 462 461 pdu->meta_buffer = nvme_to_user_ptr(d.metadata); 463 462 pdu->meta_len = d.metadata_len; 464 463 465 465 - blk_execute_rq_nowait(req, 0, nvme_uring_cmd_end_io); 464 464 + blk_execute_rq_nowait(req, false); 466 465 return -EIOCBQUEUED; 467 466 } 468 467

+7 -3

drivers/nvme/host/pci.c

reviewed

··· 1438 1438 } 1439 1439 nvme_init_request(abort_req, &cmd); 1440 1440 1441 1441 + abort_req->end_io = abort_endio; 1441 1442 abort_req->end_io_data = NULL; 1442 1443 abort_req->rq_flags |= RQF_QUIET; 1443 1443 - blk_execute_rq_nowait(abort_req, false, abort_endio); 1444 1444 + blk_execute_rq_nowait(abort_req, false); 1444 1445 1445 1446 /* 1446 1447 * The aborted req will be completed on receiving the abort req. ··· 2486 2485 return PTR_ERR(req); 2487 2486 nvme_init_request(req, &cmd); 2488 2487 2488 2488 + if (opcode == nvme_admin_delete_cq) 2489 2489 + req->end_io = nvme_del_cq_end; 2490 2490 + else 2491 2491 + req->end_io = nvme_del_queue_end; 2489 2492 req->end_io_data = nvmeq; 2490 2493 2491 2494 init_completion(&nvmeq->delete_done); 2492 2495 req->rq_flags |= RQF_QUIET; 2493 2493 - blk_execute_rq_nowait(req, false, opcode == nvme_admin_delete_cq ? 2494 2494 - nvme_del_cq_end : nvme_del_queue_end); 2496 2496 + blk_execute_rq_nowait(req, false); 2495 2497 return 0; 2496 2498 } 2497 2499

+2 -1

drivers/nvme/target/passthru.c

reviewed

··· 285 285 req->p.rq = rq; 286 286 queue_work(nvmet_wq, &req->p.work); 287 287 } else { 288 288 + rq->end_io = nvmet_passthru_req_done; 288 289 rq->end_io_data = req; 289 289 - blk_execute_rq_nowait(rq, false, nvmet_passthru_req_done); 290 290 + blk_execute_rq_nowait(rq, false); 290 291 } 291 292 292 293 if (ns)

+3 -2

drivers/scsi/scsi_error.c

reviewed

··· 2039 2039 scmd->cmnd[4] = SCSI_REMOVAL_PREVENT; 2040 2040 scmd->cmnd[5] = 0; 2041 2041 scmd->cmd_len = COMMAND_SIZE(scmd->cmnd[0]); 2042 2042 + scmd->allowed = 5; 2042 2043 2043 2044 req->rq_flags |= RQF_QUIET; 2044 2045 req->timeout = 10 * HZ; 2045 2045 - scmd->allowed = 5; 2046 2046 + req->end_io = eh_lock_door_done; 2046 2047 2047 2047 - blk_execute_rq_nowait(req, true, eh_lock_door_done); 2048 2048 + blk_execute_rq_nowait(req, true); 2048 2049 } 2049 2050 2050 2051 /**

+2 -1

drivers/scsi/sg.c

reviewed

··· 831 831 832 832 srp->rq->timeout = timeout; 833 833 kref_get(&sfp->f_ref); /* sg_rq_end_io() does kref_put(). */ 834 834 - blk_execute_rq_nowait(srp->rq, at_head, sg_rq_end_io); 834 834 + srp->rq->end_io = sg_rq_end_io; 835 835 + blk_execute_rq_nowait(srp->rq, at_head); 835 836 return 0; 836 837 } 837 838

+2 -1

drivers/scsi/st.c

reviewed

··· 579 579 memcpy(scmd->cmnd, cmd, scmd->cmd_len); 580 580 req->timeout = timeout; 581 581 scmd->allowed = retries; 582 582 + req->end_io = st_scsi_execute_end; 582 583 req->end_io_data = SRpnt; 583 584 584 584 - blk_execute_rq_nowait(req, true, st_scsi_execute_end); 585 585 + blk_execute_rq_nowait(req, true); 585 586 return 0; 586 587 } 587 588

+4 -2

drivers/scsi/ufs/ufshpb.c

reviewed

··· 671 671 672 672 req->timeout = 0; 673 673 req->end_io_data = umap_req; 674 674 + req->end_io = ufshpb_umap_req_compl_fn; 674 675 675 676 ufshpb_set_unmap_cmd(scmd->cmnd, rgn); 676 677 scmd->cmd_len = HPB_WRITE_BUFFER_CMD_LENGTH; 677 678 678 678 - blk_execute_rq_nowait(req, true, ufshpb_umap_req_compl_fn); 679 679 + blk_execute_rq_nowait(req, true); 679 680 680 681 hpb->stats.umap_req_cnt++; 681 682 } ··· 708 707 blk_rq_append_bio(req, map_req->bio); 709 708 710 709 req->end_io_data = map_req; 710 710 + req->end_io = ufshpb_map_req_compl_fn; 711 711 712 712 if (unlikely(last)) 713 713 mem_size = hpb->last_srgn_entries * HPB_ENTRY_SIZE; ··· 718 716 map_req->rb.srgn_idx, mem_size); 719 717 scmd->cmd_len = HPB_READ_BUFFER_CMD_LENGTH; 720 718 721 721 - blk_execute_rq_nowait(req, true, ufshpb_map_req_compl_fn); 719 719 + blk_execute_rq_nowait(req, true); 722 720 723 721 hpb->stats.map_req_cnt++; 724 722 return 0;

+1 -2

drivers/target/target_core_pscsi.c

reviewed

··· 972 972 973 973 cmd->priv = scmd->cmnd; 974 974 975 975 - blk_execute_rq_nowait(req, cmd->sam_task_attr == TCM_HEAD_TAG, 976 976 - pscsi_req_done); 975 975 + blk_execute_rq_nowait(req, cmd->sam_task_attr == TCM_HEAD_TAG); 977 976 978 977 return 0; 979 978

+1 -2

include/linux/blk-mq.h

reviewed

··· 969 969 int blk_rq_map_kern(struct request_queue *, struct request *, void *, 970 970 unsigned int, gfp_t); 971 971 int blk_rq_append_bio(struct request *rq, struct bio *bio); 972 972 - void blk_execute_rq_nowait(struct request *rq, bool at_head, 973 973 - rq_end_io_fn *end_io); 972 972 + void blk_execute_rq_nowait(struct request *rq, bool at_head); 974 973 blk_status_t blk_execute_rq(struct request *rq, bool at_head); 975 974 976 975 struct req_iterator {