Merge patch series "Optimize the hot path in the UFS driver"

+15

drivers/scsi/hosts.c

··· 231 231 goto fail; 232 232 } 233 233 234 + if (shost->nr_reserved_cmds && !sht->queue_reserved_command) { 235 + shost_printk(KERN_ERR, shost, 236 + "nr_reserved_cmds set but no method to queue\n"); 237 + goto fail; 238 + } 239 + 234 240 /* Use min_t(int, ...) in case shost->can_queue exceeds SHRT_MAX */ 235 241 shost->cmd_per_lun = min_t(int, shost->cmd_per_lun, 236 242 shost->can_queue); ··· 312 306 error = scsi_sysfs_add_host(shost); 313 307 if (error) 314 308 goto out_del_dev; 309 + 310 + if (shost->nr_reserved_cmds) { 311 + shost->pseudo_sdev = scsi_get_pseudo_sdev(shost); 312 + if (!shost->pseudo_sdev) { 313 + error = -ENOMEM; 314 + goto out_del_dev; 315 + } 316 + } 315 317 316 318 scsi_proc_host_add(shost); 317 319 scsi_autopm_put_host(shost); ··· 450 436 shost->hostt = sht; 451 437 shost->this_id = sht->this_id; 452 438 shost->can_queue = sht->can_queue; 439 + shost->nr_reserved_cmds = sht->nr_reserved_cmds; 453 440 shost->sg_tablesize = sht->sg_tablesize; 454 441 shost->sg_prot_tablesize = sht->sg_prot_tablesize; 455 442 shost->cmd_per_lun = sht->cmd_per_lun;

+10 -2

drivers/scsi/scsi.c

··· 216 216 */ 217 217 int scsi_change_queue_depth(struct scsi_device *sdev, int depth) 218 218 { 219 + if (!sdev->budget_map.map) 220 + return -EINVAL; 221 + 219 222 depth = min_t(int, depth, scsi_device_max_queue_depth(sdev)); 220 223 221 224 if (depth > 0) { ··· 258 255 */ 259 256 int scsi_track_queue_full(struct scsi_device *sdev, int depth) 260 257 { 258 + if (!sdev->budget_map.map) 259 + return 0; 261 260 262 261 /* 263 262 * Don't let QUEUE_FULLs on the same ··· 831 826 spin_lock_irqsave(shost->host_lock, flags); 832 827 while (list->next != &shost->__devices) { 833 828 next = list_entry(list->next, struct scsi_device, siblings); 834 - /* skip devices that we can't get a reference to */ 835 - if (!scsi_device_get(next)) 829 + /* 830 + * Skip pseudo devices and also devices we can't get a 831 + * reference to. 832 + */ 833 + if (!scsi_device_is_pseudo_dev(next) && !scsi_device_get(next)) 836 834 break; 837 835 next = NULL; 838 836 list = list->next;

+104 -10

drivers/scsi/scsi_debug.c

··· 6752 6752 return false; 6753 6753 } 6754 6754 6755 + struct sdebug_abort_cmd { 6756 + u32 unique_tag; 6757 + }; 6758 + 6759 + enum sdebug_internal_cmd_type { 6760 + SCSI_DEBUG_ABORT_CMD, 6761 + }; 6762 + 6763 + struct sdebug_internal_cmd { 6764 + enum sdebug_internal_cmd_type type; 6765 + 6766 + union { 6767 + struct sdebug_abort_cmd abort_cmd; 6768 + }; 6769 + }; 6770 + 6771 + union sdebug_priv { 6772 + struct sdebug_scsi_cmd cmd; 6773 + struct sdebug_internal_cmd internal_cmd; 6774 + }; 6775 + 6755 6776 /* 6756 - * Called from scsi_debug_abort() only, which is for timed-out cmd. 6777 + * Abort SCSI command @cmnd. Only called from scsi_debug_abort(). Although 6778 + * it would be possible to call scsi_debug_stop_cmnd() directly, an internal 6779 + * command is allocated and submitted to trigger the reserved command 6780 + * infrastructure. 6757 6781 */ 6758 6782 static bool scsi_debug_abort_cmnd(struct scsi_cmnd *cmnd) 6759 6783 { 6760 - struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmnd); 6761 - unsigned long flags; 6762 - bool res; 6784 + struct Scsi_Host *shost = cmnd->device->host; 6785 + struct request *rq = scsi_cmd_to_rq(cmnd); 6786 + u32 unique_tag = blk_mq_unique_tag(rq); 6787 + struct sdebug_internal_cmd *internal_cmd; 6788 + struct scsi_cmnd *abort_cmd; 6789 + struct request *abort_rq; 6790 + blk_status_t res; 6763 6791 6764 - spin_lock_irqsave(&sdsc->lock, flags); 6765 - res = scsi_debug_stop_cmnd(cmnd); 6766 - spin_unlock_irqrestore(&sdsc->lock, flags); 6767 - 6768 - return res; 6792 + abort_cmd = scsi_get_internal_cmd(shost->pseudo_sdev, DMA_NONE, 6793 + BLK_MQ_REQ_RESERVED); 6794 + if (!abort_cmd) 6795 + return false; 6796 + internal_cmd = scsi_cmd_priv(abort_cmd); 6797 + *internal_cmd = (struct sdebug_internal_cmd) { 6798 + .type = SCSI_DEBUG_ABORT_CMD, 6799 + .abort_cmd = { 6800 + .unique_tag = unique_tag, 6801 + }, 6802 + }; 6803 + abort_rq = scsi_cmd_to_rq(abort_cmd); 6804 + abort_rq->timeout = secs_to_jiffies(3); 6805 + res = blk_execute_rq(abort_rq, true); 6806 + scsi_put_internal_cmd(abort_cmd); 6807 + return res == BLK_STS_OK; 6769 6808 } 6770 6809 6771 6810 /* ··· 9259 9220 return ret; 9260 9221 } 9261 9222 9223 + /* Process @scp, a request to abort a SCSI command by tag. */ 9224 + static void scsi_debug_abort_cmd(struct Scsi_Host *shost, struct scsi_cmnd *scp) 9225 + { 9226 + struct sdebug_internal_cmd *internal_cmd = scsi_cmd_priv(scp); 9227 + struct sdebug_abort_cmd *abort_cmd = &internal_cmd->abort_cmd; 9228 + const u32 unique_tag = abort_cmd->unique_tag; 9229 + struct scsi_cmnd *to_be_aborted_scmd = 9230 + scsi_host_find_tag(shost, unique_tag); 9231 + struct sdebug_scsi_cmd *to_be_aborted_sdsc = 9232 + scsi_cmd_priv(to_be_aborted_scmd); 9233 + bool res = false; 9234 + 9235 + if (!to_be_aborted_scmd) { 9236 + pr_err("%s: command with tag %#x not found\n", __func__, 9237 + unique_tag); 9238 + return; 9239 + } 9240 + 9241 + scoped_guard(spinlock_irqsave, &to_be_aborted_sdsc->lock) 9242 + res = scsi_debug_stop_cmnd(to_be_aborted_scmd); 9243 + 9244 + if (res) 9245 + pr_info("%s: aborted command with tag %#x\n", 9246 + __func__, unique_tag); 9247 + else 9248 + pr_err("%s: failed to abort command with tag %#x\n", 9249 + __func__, unique_tag); 9250 + 9251 + set_host_byte(scp, res ? DID_OK : DID_ERROR); 9252 + } 9253 + 9254 + static int scsi_debug_process_reserved_command(struct Scsi_Host *shost, 9255 + struct scsi_cmnd *scp) 9256 + { 9257 + struct sdebug_internal_cmd *internal_cmd = scsi_cmd_priv(scp); 9258 + 9259 + switch (internal_cmd->type) { 9260 + case SCSI_DEBUG_ABORT_CMD: 9261 + scsi_debug_abort_cmd(shost, scp); 9262 + break; 9263 + default: 9264 + WARN_ON_ONCE(true); 9265 + set_host_byte(scp, DID_ERROR); 9266 + break; 9267 + } 9268 + 9269 + scsi_done(scp); 9270 + return 0; 9271 + } 9272 + 9262 9273 static int scsi_debug_queuecommand(struct Scsi_Host *shost, 9263 9274 struct scsi_cmnd *scp) 9264 9275 { ··· 9509 9420 struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmd); 9510 9421 struct sdebug_defer *sd_dp = &sdsc->sd_dp; 9511 9422 9423 + if (blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd))) 9424 + return 0; 9425 + 9512 9426 spin_lock_init(&sdsc->lock); 9513 9427 hrtimer_setup(&sd_dp->hrt, sdebug_q_cmd_hrt_complete, CLOCK_MONOTONIC, 9514 9428 HRTIMER_MODE_REL_PINNED); ··· 9531 9439 .sdev_destroy = scsi_debug_sdev_destroy, 9532 9440 .ioctl = scsi_debug_ioctl, 9533 9441 .queuecommand = scsi_debug_queuecommand, 9442 + .queue_reserved_command = scsi_debug_process_reserved_command, 9534 9443 .change_queue_depth = sdebug_change_qdepth, 9535 9444 .map_queues = sdebug_map_queues, 9536 9445 .mq_poll = sdebug_blk_mq_poll, ··· 9541 9448 .eh_bus_reset_handler = scsi_debug_bus_reset, 9542 9449 .eh_host_reset_handler = scsi_debug_host_reset, 9543 9450 .can_queue = SDEBUG_CANQUEUE, 9451 + .nr_reserved_cmds = 1, 9544 9452 .this_id = 7, 9545 9453 .sg_tablesize = SG_MAX_SEGMENTS, 9546 9454 .cmd_per_lun = DEF_CMD_PER_LUN, ··· 9550 9456 .module = THIS_MODULE, 9551 9457 .skip_settle_delay = 1, 9552 9458 .track_queue_depth = 1, 9553 - .cmd_size = sizeof(struct sdebug_scsi_cmd), 9459 + .cmd_size = sizeof(union sdebug_priv), 9554 9460 .init_cmd_priv = sdebug_init_cmd_priv, 9555 9461 .target_alloc = sdebug_target_alloc, 9556 9462 .target_destroy = sdebug_target_destroy,

+3

drivers/scsi/scsi_error.c

··· 749 749 const struct scsi_host_template *sht = sdev->host->hostt; 750 750 struct scsi_device *tmp_sdev; 751 751 752 + if (!sdev->budget_map.map) 753 + return; 754 + 752 755 if (!sht->track_queue_depth || 753 756 sdev->queue_depth >= sdev->max_queue_depth) 754 757 return;

+85 -19

drivers/scsi/scsi_lib.c

··· 396 396 if (starget->can_queue > 0) 397 397 atomic_dec(&starget->target_busy); 398 398 399 - sbitmap_put(&sdev->budget_map, cmd->budget_token); 399 + if (sdev->budget_map.map) 400 + sbitmap_put(&sdev->budget_map, cmd->budget_token); 400 401 cmd->budget_token = -1; 401 402 } 402 403 ··· 1361 1360 { 1362 1361 int token; 1363 1362 1363 + if (!sdev->budget_map.map) 1364 + return INT_MAX; 1365 + 1364 1366 token = sbitmap_get(&sdev->budget_map); 1365 1367 if (token < 0) 1366 1368 return -1; ··· 1533 1529 { 1534 1530 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 1535 1531 enum scsi_disposition disposition; 1532 + 1533 + if (blk_mq_is_reserved_rq(rq)) { 1534 + /* Only pass-through requests are supported in this code path. */ 1535 + WARN_ON_ONCE(!blk_rq_is_passthrough(scsi_cmd_to_rq(cmd))); 1536 + scsi_mq_uninit_cmd(cmd); 1537 + __blk_mq_end_request(rq, scsi_result_to_blk_status(cmd->result)); 1538 + return; 1539 + } 1536 1540 1537 1541 INIT_LIST_HEAD(&cmd->eh_entry); 1538 1542 ··· 1761 1749 { 1762 1750 struct scsi_device *sdev = q->queuedata; 1763 1751 1764 - sbitmap_put(&sdev->budget_map, budget_token); 1752 + if (sdev->budget_map.map) 1753 + sbitmap_put(&sdev->budget_map, budget_token); 1765 1754 } 1766 1755 1767 1756 /* ··· 1831 1818 WARN_ON_ONCE(cmd->budget_token < 0); 1832 1819 1833 1820 /* 1834 - * If the device is not in running state we will reject some or all 1835 - * commands. 1821 + * Bypass the SCSI device, SCSI target and SCSI host checks for 1822 + * reserved commands. 1836 1823 */ 1837 - if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { 1838 - ret = scsi_device_state_check(sdev, req); 1839 - if (ret != BLK_STS_OK) 1840 - goto out_put_budget; 1841 - } 1824 + if (!blk_mq_is_reserved_rq(req)) { 1825 + /* 1826 + * If the device is not in running state we will reject some or 1827 + * all commands. 1828 + */ 1829 + if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { 1830 + ret = scsi_device_state_check(sdev, req); 1831 + if (ret != BLK_STS_OK) 1832 + goto out_put_budget; 1833 + } 1842 1834 1843 - ret = BLK_STS_RESOURCE; 1844 - if (!scsi_target_queue_ready(shost, sdev)) 1845 - goto out_put_budget; 1846 - if (unlikely(scsi_host_in_recovery(shost))) { 1847 - if (cmd->flags & SCMD_FAIL_IF_RECOVERING) 1848 - ret = BLK_STS_OFFLINE; 1849 - goto out_dec_target_busy; 1835 + ret = BLK_STS_RESOURCE; 1836 + if (!scsi_target_queue_ready(shost, sdev)) 1837 + goto out_put_budget; 1838 + if (unlikely(scsi_host_in_recovery(shost))) { 1839 + if (cmd->flags & SCMD_FAIL_IF_RECOVERING) 1840 + ret = BLK_STS_OFFLINE; 1841 + goto out_dec_target_busy; 1842 + } 1843 + if (!scsi_host_queue_ready(q, shost, sdev, cmd)) 1844 + goto out_dec_target_busy; 1850 1845 } 1851 - if (!scsi_host_queue_ready(q, shost, sdev, cmd)) 1852 - goto out_dec_target_busy; 1853 1846 1854 1847 /* 1855 1848 * Only clear the driver-private command data if the LLD does not supply ··· 1884 1865 cmd->submitter = SUBMITTED_BY_BLOCK_LAYER; 1885 1866 1886 1867 blk_mq_start_request(req); 1868 + if (blk_mq_is_reserved_rq(req)) { 1869 + reason = shost->hostt->queue_reserved_command(shost, cmd); 1870 + if (reason) { 1871 + ret = BLK_STS_RESOURCE; 1872 + goto out_put_budget; 1873 + } 1874 + return BLK_STS_OK; 1875 + } 1887 1876 reason = scsi_dispatch_cmd(cmd); 1888 1877 if (reason) { 1889 1878 scsi_set_blocked(cmd, reason); ··· 2110 2083 tag_set->ops = &scsi_mq_ops_no_commit; 2111 2084 tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1; 2112 2085 tag_set->nr_maps = shost->nr_maps ? : 1; 2113 - tag_set->queue_depth = shost->can_queue; 2086 + tag_set->queue_depth = shost->can_queue + shost->nr_reserved_cmds; 2087 + tag_set->reserved_tags = shost->nr_reserved_cmds; 2114 2088 tag_set->cmd_size = cmd_size; 2115 2089 tag_set->numa_node = dev_to_node(shost->dma_dev); 2116 2090 if (shost->hostt->tag_alloc_policy_rr) ··· 2133 2105 blk_mq_free_tag_set(&shost->tag_set); 2134 2106 complete(&shost->tagset_freed); 2135 2107 } 2108 + 2109 + /** 2110 + * scsi_get_internal_cmd() - Allocate an internal SCSI command. 2111 + * @sdev: SCSI device from which to allocate the command 2112 + * @data_direction: Data direction for the allocated command 2113 + * @flags: request allocation flags, e.g. BLK_MQ_REQ_RESERVED or 2114 + * BLK_MQ_REQ_NOWAIT. 2115 + * 2116 + * Allocates a SCSI command for internal LLDD use. 2117 + */ 2118 + struct scsi_cmnd *scsi_get_internal_cmd(struct scsi_device *sdev, 2119 + enum dma_data_direction data_direction, 2120 + blk_mq_req_flags_t flags) 2121 + { 2122 + enum req_op op = data_direction == DMA_TO_DEVICE ? REQ_OP_DRV_OUT : 2123 + REQ_OP_DRV_IN; 2124 + struct scsi_cmnd *scmd; 2125 + struct request *rq; 2126 + 2127 + rq = scsi_alloc_request(sdev->request_queue, op, flags); 2128 + if (IS_ERR(rq)) 2129 + return NULL; 2130 + scmd = blk_mq_rq_to_pdu(rq); 2131 + scmd->device = sdev; 2132 + 2133 + return scmd; 2134 + } 2135 + EXPORT_SYMBOL_GPL(scsi_get_internal_cmd); 2136 + 2137 + /** 2138 + * scsi_put_internal_cmd() - Free an internal SCSI command. 2139 + * @scmd: SCSI command to be freed 2140 + */ 2141 + void scsi_put_internal_cmd(struct scsi_cmnd *scmd) 2142 + { 2143 + blk_mq_free_request(blk_mq_rq_from_pdu(scmd)); 2144 + } 2145 + EXPORT_SYMBOL_GPL(scsi_put_internal_cmd); 2136 2146 2137 2147 /** 2138 2148 * scsi_device_from_queue - return sdev associated with a request_queue

+1

drivers/scsi/scsi_priv.h

··· 135 135 extern int scsi_scan_host_selected(struct Scsi_Host *, unsigned int, 136 136 unsigned int, u64, enum scsi_scan_mode); 137 137 extern void scsi_forget_host(struct Scsi_Host *); 138 + struct scsi_device *scsi_get_pseudo_sdev(struct Scsi_Host *); 138 139 139 140 /* scsi_sysctl.c */ 140 141 #ifdef CONFIG_SYSCTL

+70 -4

drivers/scsi/scsi_scan.c

··· 347 347 kref_get(&sdev->host->tagset_refcnt); 348 348 sdev->request_queue = q; 349 349 350 + scsi_sysfs_device_initialize(sdev); 351 + 352 + if (scsi_device_is_pseudo_dev(sdev)) 353 + return sdev; 354 + 350 355 depth = sdev->host->cmd_per_lun ?: 1; 351 356 352 357 /* ··· 367 362 } 368 363 369 364 scsi_change_queue_depth(sdev, depth); 370 - 371 - scsi_sysfs_device_initialize(sdev); 372 365 373 366 if (shost->hostt->sdev_init) { 374 367 ret = shost->hostt->sdev_init(sdev); ··· 1071 1068 1072 1069 transport_configure_device(&sdev->sdev_gendev); 1073 1070 1071 + sdev->sdev_bflags = *bflags; 1072 + 1073 + if (scsi_device_is_pseudo_dev(sdev)) 1074 + return SCSI_SCAN_LUN_PRESENT; 1075 + 1074 1076 /* 1075 1077 * No need to freeze the queue as it isn't reachable to anyone else yet. 1076 1078 */ ··· 1121 1113 1122 1114 sdev->max_queue_depth = sdev->queue_depth; 1123 1115 WARN_ON_ONCE(sdev->max_queue_depth > sdev->budget_map.depth); 1124 - sdev->sdev_bflags = *bflags; 1125 1116 1126 1117 /* 1127 1118 * Ok, the device is now all set up, we can ··· 1218 1211 sdev = scsi_alloc_sdev(starget, lun, hostdata); 1219 1212 if (!sdev) 1220 1213 goto out; 1214 + 1215 + if (scsi_device_is_pseudo_dev(sdev)) { 1216 + if (bflagsp) 1217 + *bflagsp = BLIST_NOLUN; 1218 + return SCSI_SCAN_LUN_PRESENT; 1219 + } 1221 1220 1222 1221 result = kmalloc(result_len, GFP_KERNEL); 1223 1222 if (!result) ··· 2096 2083 restart: 2097 2084 spin_lock_irqsave(shost->host_lock, flags); 2098 2085 list_for_each_entry(sdev, &shost->__devices, siblings) { 2099 - if (sdev->sdev_state == SDEV_DEL) 2086 + if (scsi_device_is_pseudo_dev(sdev) || 2087 + sdev->sdev_state == SDEV_DEL) 2100 2088 continue; 2101 2089 spin_unlock_irqrestore(shost->host_lock, flags); 2102 2090 __scsi_remove_device(sdev); 2103 2091 goto restart; 2104 2092 } 2105 2093 spin_unlock_irqrestore(shost->host_lock, flags); 2094 + 2095 + /* 2096 + * Remove the pseudo device last since it may be needed during removal 2097 + * of other SCSI devices. 2098 + */ 2099 + if (shost->pseudo_sdev) 2100 + __scsi_remove_device(shost->pseudo_sdev); 2106 2101 } 2107 2102 2103 + /** 2104 + * scsi_get_pseudo_sdev() - Attach a pseudo SCSI device to a SCSI host 2105 + * @shost: Host that needs a pseudo SCSI device 2106 + * 2107 + * Lock status: None assumed. 2108 + * 2109 + * Returns: The scsi_device or NULL 2110 + * 2111 + * Notes: 2112 + * Attach a single scsi_device to the Scsi_Host. The primary aim for this 2113 + * device is to serve as a container from which SCSI commands can be 2114 + * allocated. Each SCSI command will carry a command tag allocated by the 2115 + * block layer. These SCSI commands can be used by the LLDD to send 2116 + * internal or passthrough commands without having to manage tag allocation 2117 + * inside the LLDD. 2118 + */ 2119 + struct scsi_device *scsi_get_pseudo_sdev(struct Scsi_Host *shost) 2120 + { 2121 + struct scsi_device *sdev = NULL; 2122 + struct scsi_target *starget; 2123 + 2124 + guard(mutex)(&shost->scan_mutex); 2125 + 2126 + if (!scsi_host_scan_allowed(shost)) 2127 + goto out; 2128 + 2129 + starget = scsi_alloc_target(&shost->shost_gendev, 0, shost->max_id); 2130 + if (!starget) 2131 + goto out; 2132 + 2133 + sdev = scsi_alloc_sdev(starget, U64_MAX, NULL); 2134 + if (!sdev) { 2135 + scsi_target_reap(starget); 2136 + goto put_target; 2137 + } 2138 + 2139 + sdev->borken = 0; 2140 + 2141 + put_target: 2142 + /* See also the get_device(dev) call in scsi_alloc_target(). */ 2143 + put_device(&starget->dev); 2144 + 2145 + out: 2146 + return sdev; 2147 + }

+4 -1

drivers/scsi/scsi_sysfs.c

··· 1348 1348 int error; 1349 1349 struct scsi_target *starget = sdev->sdev_target; 1350 1350 1351 + if (WARN_ON_ONCE(scsi_device_is_pseudo_dev(sdev))) 1352 + return -EINVAL; 1353 + 1351 1354 error = scsi_target_add(starget); 1352 1355 if (error) 1353 1356 return error; ··· 1458 1455 kref_put(&sdev->host->tagset_refcnt, scsi_mq_free_tags); 1459 1456 cancel_work_sync(&sdev->requeue_work); 1460 1457 1461 - if (sdev->host->hostt->sdev_destroy) 1458 + if (!scsi_device_is_pseudo_dev(sdev) && sdev->host->hostt->sdev_destroy) 1462 1459 sdev->host->hostt->sdev_destroy(sdev); 1463 1460 transport_destroy_device(dev); 1464 1461

+20 -36

drivers/ufs/core/ufs-mcq.c

··· 134 134 EXPORT_SYMBOL_GPL(ufshcd_mcq_queue_cfg_addr); 135 135 136 136 /** 137 - * ufshcd_mcq_decide_queue_depth - decide the queue depth 137 + * ufshcd_get_hba_mac - Maximum number of commands supported by the host 138 + * controller. 138 139 * @hba: per adapter instance 139 140 * 140 - * Return: queue-depth on success, non-zero on error 141 + * Return: queue depth on success; negative upon error. 141 142 * 142 - * MAC - Max. Active Command of the Host Controller (HC) 143 - * HC wouldn't send more than this commands to the device. 144 - * Calculates and adjusts the queue depth based on the depth 145 - * supported by the HC and ufs device. 143 + * MAC = Maximum number of Active Commands supported by the Host Controller. 146 144 */ 147 - int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba) 145 + int ufshcd_get_hba_mac(struct ufs_hba *hba) 148 146 { 149 147 int mac; 150 148 ··· 160 162 mac = hba->vops->get_hba_mac(hba); 161 163 } 162 164 if (mac < 0) 163 - goto err; 164 - 165 - WARN_ON_ONCE(!hba->dev_info.bqueuedepth); 166 - /* 167 - * max. value of bqueuedepth = 256, mac is host dependent. 168 - * It is mandatory for UFS device to define bQueueDepth if 169 - * shared queuing architecture is enabled. 170 - */ 171 - return min_t(int, mac, hba->dev_info.bqueuedepth); 172 - 173 - err: 174 - dev_err(hba->dev, "Failed to get mac, err=%d\n", mac); 165 + dev_err(hba->dev, "Failed to get mac, err=%d\n", mac); 175 166 return mac; 176 167 } 177 168 ··· 294 307 struct ufs_hw_queue *hwq) 295 308 { 296 309 struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); 297 - int tag = ufshcd_mcq_get_tag(hba, cqe); 298 310 299 311 if (cqe->command_desc_base_addr) { 312 + int tag = ufshcd_mcq_get_tag(hba, cqe); 313 + 300 314 ufshcd_compl_one_cqe(hba, tag, cqe); 301 315 /* After processed the cqe, mark it empty (invalid) entry */ 302 316 cqe->command_desc_base_addr = 0; ··· 479 491 mutex_init(&hwq->sq_mutex); 480 492 } 481 493 482 - /* The very first HW queue serves device commands */ 483 - hba->dev_cmd_queue = &hba->uhq[0]; 484 - 485 494 host->host_tagset = 1; 486 495 return 0; 487 496 } ··· 531 546 */ 532 547 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) 533 548 { 534 - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 535 - struct scsi_cmnd *cmd = lrbp->cmd; 549 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 550 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 551 + struct request *rq = scsi_cmd_to_rq(cmd); 536 552 struct ufs_hw_queue *hwq; 537 553 void __iomem *reg, *opr_sqd_base; 538 554 u32 nexus, id, val; ··· 542 556 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 543 557 return -ETIMEDOUT; 544 558 545 - if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) { 546 - if (!cmd) 547 - return -EINVAL; 548 - hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 549 - if (!hwq) 550 - return 0; 551 - } else { 552 - hwq = hba->dev_cmd_queue; 553 - } 559 + if (!cmd) 560 + return -EINVAL; 561 + 562 + hwq = ufshcd_mcq_req_to_hwq(hba, rq); 563 + if (!hwq) 564 + return 0; 554 565 555 566 id = hwq->id; 556 567 ··· 613 630 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, 614 631 struct ufs_hw_queue *hwq, int task_tag) 615 632 { 616 - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 633 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 634 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 617 635 struct utp_transfer_req_desc *utrd; 618 636 __le64 cmd_desc_base_addr; 619 637 bool ret = false; ··· 665 681 struct Scsi_Host *host = cmd->device->host; 666 682 struct ufs_hba *hba = shost_priv(host); 667 683 int tag = scsi_cmd_to_rq(cmd)->tag; 668 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 684 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 669 685 struct ufs_hw_queue *hwq; 670 686 int err; 671 687

+11 -7

drivers/ufs/core/ufshcd-crypto.h

··· 38 38 } 39 39 40 40 static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, 41 - struct ufshcd_lrb *lrbp) 41 + struct scsi_cmnd *cmd) 42 42 { 43 - struct scsi_cmnd *cmd = lrbp->cmd; 44 43 const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx; 44 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 45 45 46 46 if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt) 47 47 return hba->vops->fill_crypto_prdt(hba, crypt_ctx, ··· 51 51 } 52 52 53 53 static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, 54 - struct ufshcd_lrb *lrbp) 54 + struct scsi_cmnd *cmd) 55 55 { 56 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 57 + 56 58 if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT)) 57 59 return; 58 60 59 - if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx)) 61 + if (!(scsi_cmd_to_rq(cmd)->crypt_ctx)) 60 62 return; 61 63 62 64 /* Zeroize the PRDT because it can contain cryptographic keys. */ 63 65 memzero_explicit(lrbp->ucd_prdt_ptr, 64 - ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd)); 66 + ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd)); 65 67 } 66 68 67 69 bool ufshcd_crypto_enable(struct ufs_hba *hba); ··· 84 82 struct request_desc_header *h) { } 85 83 86 84 static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, 87 - struct ufshcd_lrb *lrbp) 85 + struct scsi_cmnd *cmd) 88 86 { 89 87 return 0; 90 88 } 91 89 92 90 static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, 93 - struct ufshcd_lrb *lrbp) { } 91 + struct scsi_cmnd *cmd) 92 + { 93 + } 94 94 95 95 static inline bool ufshcd_crypto_enable(struct ufs_hba *hba) 96 96 {

+17 -3

drivers/ufs/core/ufshcd-priv.h

··· 67 67 struct cq_entry *cqe); 68 68 int ufshcd_mcq_init(struct ufs_hba *hba); 69 69 void ufshcd_mcq_disable(struct ufs_hba *hba); 70 - int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba); 70 + int ufshcd_get_hba_mac(struct ufs_hba *hba); 71 71 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba); 72 72 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, 73 73 struct request *req); ··· 77 77 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag); 78 78 int ufshcd_mcq_abort(struct scsi_cmnd *cmd); 79 79 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag); 80 - void ufshcd_release_scsi_cmd(struct ufs_hba *hba, 81 - struct ufshcd_lrb *lrbp); 80 + void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd); 82 81 83 82 #define SD_ASCII_STD true 84 83 #define SD_RAW false ··· 360 361 return false; 361 362 } 362 363 return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported); 364 + } 365 + 366 + /* 367 + * Convert a block layer tag into a SCSI command pointer. This function is 368 + * called once per I/O completion path and is also called from error paths. 369 + */ 370 + static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag) 371 + { 372 + struct blk_mq_tags *tags = hba->host->tag_set.shared_tags; 373 + struct request *rq = blk_mq_tag_to_rq(tags, tag); 374 + 375 + if (WARN_ON_ONCE(!rq)) 376 + return NULL; 377 + 378 + return blk_mq_rq_to_pdu(rq); 363 379 } 364 380 365 381 static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)

+423 -379

drivers/ufs/core/ufshcd.c

··· 28 28 #include <scsi/scsi_dbg.h> 29 29 #include <scsi/scsi_driver.h> 30 30 #include <scsi/scsi_eh.h> 31 + #include <scsi/scsi_tcq.h> 31 32 #include "ufshcd-priv.h" 32 33 #include <ufs/ufs_quirks.h> 33 34 #include <ufs/unipro.h> ··· 404 403 ufshcd_wb_toggle_buf_flush(hba, true); 405 404 } 406 405 407 - static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, 406 + static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, 407 + struct ufshcd_lrb *lrb, 408 408 enum ufs_trace_str_t str_t) 409 409 { 410 - struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr; 410 + struct utp_upiu_req *rq = lrb->ucd_req_ptr; 411 411 struct utp_upiu_header *header; 412 412 413 413 if (!trace_ufshcd_upiu_enabled()) ··· 417 415 if (str_t == UFS_CMD_SEND) 418 416 header = &rq->header; 419 417 else 420 - header = &hba->lrb[tag].ucd_rsp_ptr->header; 418 + header = &lrb->ucd_rsp_ptr->header; 421 419 422 420 trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, 423 421 UFS_TSF_CDB); ··· 474 472 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); 475 473 } 476 474 477 - static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, 475 + static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, 478 476 enum ufs_trace_str_t str_t) 479 477 { 480 478 u64 lba = 0; ··· 482 480 u32 doorbell = 0; 483 481 u32 intr; 484 482 u32 hwq_id = 0; 485 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 486 - struct scsi_cmnd *cmd = lrbp->cmd; 487 483 struct request *rq = scsi_cmd_to_rq(cmd); 484 + unsigned int tag = rq->tag; 485 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 488 486 int transfer_len = -1; 489 487 490 - if (!cmd) 491 - return; 492 - 493 488 /* trace UPIU also */ 494 - ufshcd_add_cmd_upiu_trace(hba, tag, str_t); 489 + ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t); 495 490 if (!trace_ufshcd_command_enabled()) 496 491 return; 497 492 ··· 502 503 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); 503 504 lba = scsi_get_lba(cmd); 504 505 if (opcode == WRITE_10) 505 - group_id = lrbp->cmd->cmnd[6]; 506 + group_id = cmd->cmnd[6]; 506 507 } else if (opcode == UNMAP) { 507 508 /* 508 509 * The number of Bytes to be unmapped beginning with the lba. ··· 595 596 ufshcd_vops_dbg_register_dump(hba); 596 597 } 597 598 598 - static 599 - void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt) 599 + static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, 600 + bool pr_prdt) 600 601 { 601 - const struct ufshcd_lrb *lrbp; 602 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 603 + const int tag = scsi_cmd_to_rq(cmd)->tag; 602 604 int prdt_length; 603 - 604 - lrbp = &hba->lrb[tag]; 605 605 606 606 if (hba->monitor.enabled) { 607 607 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, ··· 644 646 struct Scsi_Host *shost = sdev->host; 645 647 struct ufs_hba *hba = shost_priv(shost); 646 648 647 - ufshcd_print_tr(hba, req->tag, *(bool *)priv); 649 + if (!blk_mq_is_reserved_rq(req)) 650 + ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); 648 651 649 652 return true; 650 653 } ··· 2294 2295 return -EINVAL; 2295 2296 } 2296 2297 2298 + /* Must only be called for SCSI commands. */ 2297 2299 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, 2298 - struct ufshcd_lrb *lrbp) 2300 + struct scsi_cmnd *cmd) 2299 2301 { 2300 2302 const struct ufs_hba_monitor *m = &hba->monitor; 2303 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2301 2304 2302 - return (m->enabled && lrbp && lrbp->cmd && 2303 - (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) && 2304 - ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp)); 2305 + return m->enabled && 2306 + (!m->chunk_size || m->chunk_size == cmd->sdb.length) && 2307 + ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp); 2305 2308 } 2306 2309 2307 - static void ufshcd_start_monitor(struct ufs_hba *hba, 2308 - const struct ufshcd_lrb *lrbp) 2310 + static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2309 2311 { 2310 - int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); 2312 + int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2311 2313 unsigned long flags; 2312 2314 2313 2315 spin_lock_irqsave(hba->host->host_lock, flags); ··· 2317 2317 spin_unlock_irqrestore(hba->host->host_lock, flags); 2318 2318 } 2319 2319 2320 - static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp) 2320 + static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2321 2321 { 2322 - int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); 2322 + struct request *req = scsi_cmd_to_rq(cmd); 2323 + const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2324 + int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2323 2325 unsigned long flags; 2324 2326 2325 2327 spin_lock_irqsave(hba->host->host_lock, flags); 2326 2328 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { 2327 - const struct request *req = scsi_cmd_to_rq(lrbp->cmd); 2328 2329 struct ufs_hba_monitor *m = &hba->monitor; 2329 2330 ktime_t now, inc, lat; 2330 2331 ··· 2350 2349 spin_unlock_irqrestore(hba->host->host_lock, flags); 2351 2350 } 2352 2351 2352 + /* Returns %true for SCSI commands and %false for device management commands. */ 2353 + static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) 2354 + { 2355 + return !blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd)); 2356 + } 2357 + 2353 2358 /** 2354 2359 * ufshcd_send_command - Send SCSI or device management commands 2355 2360 * @hba: per adapter instance 2356 - * @task_tag: Task tag of the command 2361 + * @cmd: SCSI command or device management command pointer 2357 2362 * @hwq: pointer to hardware queue instance 2358 2363 */ 2359 - static inline 2360 - void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, 2361 - struct ufs_hw_queue *hwq) 2364 + static inline void ufshcd_send_command(struct ufs_hba *hba, 2365 + struct scsi_cmnd *cmd, 2366 + struct ufs_hw_queue *hwq) 2362 2367 { 2363 - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 2368 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2369 + const int tag = scsi_cmd_to_rq(cmd)->tag; 2364 2370 unsigned long flags; 2365 2371 2366 2372 if (hba->monitor.enabled) { ··· 2376 2368 lrbp->compl_time_stamp = ktime_set(0, 0); 2377 2369 lrbp->compl_time_stamp_local_clock = 0; 2378 2370 } 2379 - ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); 2380 - if (lrbp->cmd) 2371 + if (ufshcd_is_scsi_cmd(cmd)) { 2372 + ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); 2381 2373 ufshcd_clk_scaling_start_busy(hba); 2382 - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) 2383 - ufshcd_start_monitor(hba, lrbp); 2374 + if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 2375 + ufshcd_start_monitor(hba, cmd); 2376 + } 2384 2377 2385 2378 if (hba->mcq_enabled) { 2386 2379 int utrd_size = sizeof(struct utp_transfer_req_desc); ··· 2396 2387 } else { 2397 2388 spin_lock_irqsave(&hba->outstanding_lock, flags); 2398 2389 if (hba->vops && hba->vops->setup_xfer_req) 2399 - hba->vops->setup_xfer_req(hba, lrbp->task_tag, 2400 - !!lrbp->cmd); 2401 - __set_bit(lrbp->task_tag, &hba->outstanding_reqs); 2402 - ufshcd_writel(hba, 1 << lrbp->task_tag, 2403 - REG_UTP_TRANSFER_REQ_DOOR_BELL); 2390 + hba->vops->setup_xfer_req(hba, tag, 2391 + ufshcd_is_scsi_cmd(cmd)); 2392 + __set_bit(tag, &hba->outstanding_reqs); 2393 + ufshcd_writel(hba, 1 << tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); 2404 2394 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 2405 2395 } 2406 2396 } 2407 2397 2408 2398 /** 2409 2399 * ufshcd_copy_sense_data - Copy sense data in case of check condition 2410 - * @lrbp: pointer to local reference block 2400 + * @cmd: SCSI command 2411 2401 */ 2412 - static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) 2402 + static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd) 2413 2403 { 2414 - u8 *const sense_buffer = lrbp->cmd->sense_buffer; 2404 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2405 + u8 *const sense_buffer = cmd->sense_buffer; 2415 2406 u16 resp_len; 2416 2407 int len; 2417 2408 ··· 2484 2475 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; 2485 2476 hba->nutmrs = 2486 2477 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; 2487 - hba->reserved_slot = hba->nutrs - 1; 2488 2478 2489 2479 hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; 2490 2480 ··· 2715 2707 /** 2716 2708 * ufshcd_map_sg - Map scatter-gather list to prdt 2717 2709 * @hba: per adapter instance 2718 - * @lrbp: pointer to local reference block 2710 + * @cmd: SCSI command 2719 2711 * 2720 2712 * Return: 0 in case of success, non-zero value in case of failure. 2721 2713 */ 2722 - static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2714 + static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2723 2715 { 2724 - struct scsi_cmnd *cmd = lrbp->cmd; 2716 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2725 2717 int sg_segments = scsi_dma_map(cmd); 2726 2718 2727 2719 if (sg_segments < 0) ··· 2729 2721 2730 2722 ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); 2731 2723 2732 - return ufshcd_crypto_fill_prdt(hba, lrbp); 2724 + return ufshcd_crypto_fill_prdt(hba, cmd); 2733 2725 } 2734 2726 2735 2727 /** ··· 2788 2780 /** 2789 2781 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 2790 2782 * for scsi commands 2791 - * @lrbp: local reference block pointer 2783 + * @cmd: SCSI command 2792 2784 * @upiu_flags: flags 2793 2785 */ 2794 - static 2795 - void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) 2786 + static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, 2787 + u8 upiu_flags) 2796 2788 { 2797 - struct scsi_cmnd *cmd = lrbp->cmd; 2789 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2790 + const int tag = scsi_cmd_to_rq(cmd)->tag; 2798 2791 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2799 2792 unsigned short cdb_len; 2800 2793 ··· 2803 2794 .transaction_code = UPIU_TRANSACTION_COMMAND, 2804 2795 .flags = upiu_flags, 2805 2796 .lun = lrbp->lun, 2806 - .task_tag = lrbp->task_tag, 2797 + .task_tag = tag, 2807 2798 .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, 2808 2799 }; 2809 2800 2810 - WARN_ON_ONCE(ucd_req_ptr->header.task_tag != lrbp->task_tag); 2801 + WARN_ON_ONCE(ucd_req_ptr->header.task_tag != tag); 2811 2802 2812 2803 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); 2813 2804 ··· 2820 2811 /** 2821 2812 * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request 2822 2813 * @hba: UFS hba 2823 - * @lrbp: local reference block pointer 2814 + * @cmd: SCSI command pointer 2824 2815 * @upiu_flags: flags 2825 2816 */ 2826 2817 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, 2827 - struct ufshcd_lrb *lrbp, u8 upiu_flags) 2818 + struct scsi_cmnd *cmd, u8 upiu_flags) 2828 2819 { 2820 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2829 2821 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2822 + const int tag = scsi_cmd_to_rq(cmd)->tag; 2830 2823 struct ufs_query *query = &hba->dev_cmd.query; 2831 2824 u16 len = be16_to_cpu(query->request.upiu_req.length); 2832 2825 ··· 2837 2826 .transaction_code = UPIU_TRANSACTION_QUERY_REQ, 2838 2827 .flags = upiu_flags, 2839 2828 .lun = lrbp->lun, 2840 - .task_tag = lrbp->task_tag, 2829 + .task_tag = tag, 2841 2830 .query_function = query->request.query_func, 2842 2831 /* Data segment length only need for WRITE_DESC */ 2843 2832 .data_segment_length = ··· 2856 2845 memcpy(ucd_req_ptr + 1, query->descriptor, len); 2857 2846 } 2858 2847 2859 - static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) 2848 + static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd) 2860 2849 { 2850 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2861 2851 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2852 + const int tag = scsi_cmd_to_rq(cmd)->tag; 2862 2853 2863 2854 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); 2864 2855 2865 2856 ucd_req_ptr->header = (struct utp_upiu_header){ 2866 2857 .transaction_code = UPIU_TRANSACTION_NOP_OUT, 2867 - .task_tag = lrbp->task_tag, 2858 + .task_tag = tag, 2868 2859 }; 2869 2860 } 2870 2861 ··· 2874 2861 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) 2875 2862 * for Device Management Purposes 2876 2863 * @hba: per adapter instance 2877 - * @lrbp: pointer to local reference block 2864 + * @cmd: SCSI command pointer 2878 2865 * 2879 2866 * Return: 0 upon success; < 0 upon failure. 2880 2867 */ 2881 2868 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, 2882 - struct ufshcd_lrb *lrbp) 2869 + struct scsi_cmnd *cmd) 2883 2870 { 2871 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2884 2872 u8 upiu_flags; 2885 2873 int ret = 0; 2886 2874 2887 2875 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 2888 2876 2889 2877 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) 2890 - ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags); 2878 + ufshcd_prepare_utp_query_req_upiu(hba, cmd, upiu_flags); 2891 2879 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) 2892 - ufshcd_prepare_utp_nop_upiu(lrbp); 2880 + ufshcd_prepare_utp_nop_upiu(cmd); 2893 2881 else 2894 2882 ret = -EINVAL; 2895 2883 ··· 2903 2889 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) 2904 2890 * for SCSI Purposes 2905 2891 * @hba: per adapter instance 2906 - * @lrbp: pointer to local reference block 2892 + * @cmd: SCSI command 2907 2893 */ 2908 - static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2894 + static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2909 2895 { 2910 - struct request *rq = scsi_cmd_to_rq(lrbp->cmd); 2896 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2897 + struct request *rq = scsi_cmd_to_rq(cmd); 2911 2898 unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); 2912 2899 u8 upiu_flags; 2913 2900 2914 - ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0); 2901 + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, 2902 + cmd->sc_data_direction, 0); 2915 2903 if (ioprio_class == IOPRIO_CLASS_RT) 2916 2904 upiu_flags |= UPIU_CMD_FLAGS_CP; 2917 - ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); 2905 + ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags); 2918 2906 } 2919 2907 2920 - static void __ufshcd_setup_cmd(struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd, u8 lun, int tag) 2908 + static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2921 2909 { 2910 + const int i = scsi_cmd_to_rq(cmd)->tag; 2911 + struct utp_transfer_cmd_desc *cmd_descp = 2912 + (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); 2913 + struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; 2914 + dma_addr_t cmd_desc_element_addr = 2915 + hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); 2916 + u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); 2917 + u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); 2918 + struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd); 2919 + 2920 + lrb->utr_descriptor_ptr = utrdlp + i; 2921 + lrb->utrd_dma_addr = 2922 + hba->utrdl_dma_addr + i * sizeof(struct utp_transfer_req_desc); 2923 + lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; 2924 + lrb->ucd_req_dma_addr = cmd_desc_element_addr; 2925 + lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; 2926 + lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; 2927 + lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; 2928 + lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; 2929 + } 2930 + 2931 + static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2932 + u8 lun, int tag) 2933 + { 2934 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2935 + 2936 + ufshcd_init_lrb(hba, cmd); 2937 + 2922 2938 memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); 2923 2939 2924 - lrbp->cmd = cmd; 2925 - lrbp->task_tag = tag; 2926 2940 lrbp->lun = lun; 2927 - ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp); 2941 + ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? 2942 + scsi_cmd_to_rq(cmd) : NULL, lrbp); 2928 2943 } 2929 2944 2930 - static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 2931 - struct scsi_cmnd *cmd, u8 lun, int tag) 2945 + static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2946 + u8 lun, int tag) 2932 2947 { 2933 - __ufshcd_setup_cmd(lrbp, cmd, lun, tag); 2948 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2949 + 2950 + __ufshcd_setup_cmd(hba, cmd, lun, tag); 2934 2951 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); 2935 2952 lrbp->req_abort_skip = false; 2936 2953 2937 - ufshcd_comp_scsi_upiu(hba, lrbp); 2954 + ufshcd_comp_scsi_upiu(hba, cmd); 2938 2955 } 2939 2956 2940 2957 /** ··· 3016 2971 } 3017 2972 } 3018 2973 3019 - static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) 2974 + /* 2975 + * The only purpose of this function is to make the SCSI core skip the memset() 2976 + * call for the private command data. 2977 + */ 2978 + static int ufshcd_init_cmd_priv(struct Scsi_Host *host, struct scsi_cmnd *cmd) 3020 2979 { 3021 - struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr + 3022 - i * ufshcd_get_ucd_size(hba); 3023 - struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; 3024 - dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr + 3025 - i * ufshcd_get_ucd_size(hba); 3026 - u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); 3027 - u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); 3028 - 3029 - lrb->utr_descriptor_ptr = utrdlp + i; 3030 - lrb->utrd_dma_addr = hba->utrdl_dma_addr + 3031 - i * sizeof(struct utp_transfer_req_desc); 3032 - lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; 3033 - lrb->ucd_req_dma_addr = cmd_desc_element_addr; 3034 - lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; 3035 - lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; 3036 - lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; 3037 - lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; 2980 + return 0; 3038 2981 } 3039 2982 3040 2983 /** ··· 3036 3003 { 3037 3004 struct ufs_hba *hba = shost_priv(host); 3038 3005 int tag = scsi_cmd_to_rq(cmd)->tag; 3039 - struct ufshcd_lrb *lrbp; 3040 3006 int err = 0; 3041 3007 struct ufs_hw_queue *hwq = NULL; 3042 3008 ··· 3086 3054 3087 3055 ufshcd_hold(hba); 3088 3056 3089 - lrbp = &hba->lrb[tag]; 3057 + ufshcd_setup_scsi_cmd(hba, cmd, 3058 + ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); 3090 3059 3091 - ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); 3092 - 3093 - err = ufshcd_map_sg(hba, lrbp); 3060 + err = ufshcd_map_sg(hba, cmd); 3094 3061 if (err) { 3095 3062 ufshcd_release(hba); 3096 3063 goto out; ··· 3098 3067 if (hba->mcq_enabled) 3099 3068 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 3100 3069 3101 - ufshcd_send_command(hba, tag, hwq); 3070 + ufshcd_send_command(hba, cmd, hwq); 3102 3071 3103 3072 out: 3104 3073 if (ufs_trigger_eh(hba)) { ··· 3112 3081 return err; 3113 3082 } 3114 3083 3115 - static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 3116 - enum dev_cmd_type cmd_type, u8 lun, int tag) 3084 + static int ufshcd_queue_reserved_command(struct Scsi_Host *host, 3085 + struct scsi_cmnd *cmd) 3117 3086 { 3118 - __ufshcd_setup_cmd(lrbp, NULL, lun, tag); 3087 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3088 + struct request *rq = scsi_cmd_to_rq(cmd); 3089 + struct ufs_hba *hba = shost_priv(host); 3090 + struct ufs_hw_queue *hwq = 3091 + hba->mcq_enabled ? ufshcd_mcq_req_to_hwq(hba, rq) : NULL; 3092 + 3093 + ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 3094 + ufshcd_send_command(hba, cmd, hwq); 3095 + return 0; 3096 + } 3097 + 3098 + static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3099 + enum dev_cmd_type cmd_type, u8 lun, int tag) 3100 + { 3101 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3102 + 3103 + __ufshcd_setup_cmd(hba, cmd, lun, tag); 3119 3104 lrbp->intr_cmd = true; /* No interrupt aggregation */ 3120 3105 hba->dev_cmd.type = cmd_type; 3121 3106 } ··· 3139 3092 /* 3140 3093 * Return: 0 upon success; < 0 upon failure. 3141 3094 */ 3142 - static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, 3143 - struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) 3095 + static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3096 + enum dev_cmd_type cmd_type, int tag) 3144 3097 { 3145 - ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); 3098 + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 3146 3099 3147 - return ufshcd_compose_devman_upiu(hba, lrbp); 3100 + return ufshcd_compose_devman_upiu(hba, cmd); 3148 3101 } 3149 3102 3150 3103 /* ··· 3255 3208 return err; 3256 3209 } 3257 3210 3258 - /* 3259 - * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3260 - * < 0 if another error occurred. 3261 - */ 3262 - static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, 3263 - struct ufshcd_lrb *lrbp, int max_timeout) 3264 - { 3265 - unsigned long time_left = msecs_to_jiffies(max_timeout); 3266 - unsigned long flags; 3267 - bool pending; 3268 - int err; 3269 - 3270 - retry: 3271 - time_left = wait_for_completion_timeout(&hba->dev_cmd.complete, 3272 - time_left); 3273 - 3274 - if (likely(time_left)) { 3275 - err = ufshcd_get_tr_ocs(lrbp, NULL); 3276 - if (!err) 3277 - err = ufshcd_dev_cmd_completion(hba, lrbp); 3278 - } else { 3279 - err = -ETIMEDOUT; 3280 - dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n", 3281 - __func__, lrbp->task_tag); 3282 - 3283 - /* MCQ mode */ 3284 - if (hba->mcq_enabled) { 3285 - /* successfully cleared the command, retry if needed */ 3286 - if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) 3287 - err = -EAGAIN; 3288 - return err; 3289 - } 3290 - 3291 - /* SDB mode */ 3292 - if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) { 3293 - /* successfully cleared the command, retry if needed */ 3294 - err = -EAGAIN; 3295 - /* 3296 - * Since clearing the command succeeded we also need to 3297 - * clear the task tag bit from the outstanding_reqs 3298 - * variable. 3299 - */ 3300 - spin_lock_irqsave(&hba->outstanding_lock, flags); 3301 - pending = test_bit(lrbp->task_tag, 3302 - &hba->outstanding_reqs); 3303 - if (pending) 3304 - __clear_bit(lrbp->task_tag, 3305 - &hba->outstanding_reqs); 3306 - spin_unlock_irqrestore(&hba->outstanding_lock, flags); 3307 - 3308 - if (!pending) { 3309 - /* 3310 - * The completion handler ran while we tried to 3311 - * clear the command. 3312 - */ 3313 - time_left = 1; 3314 - goto retry; 3315 - } 3316 - } else { 3317 - dev_err(hba->dev, "%s: failed to clear tag %d\n", 3318 - __func__, lrbp->task_tag); 3319 - 3320 - spin_lock_irqsave(&hba->outstanding_lock, flags); 3321 - pending = test_bit(lrbp->task_tag, 3322 - &hba->outstanding_reqs); 3323 - spin_unlock_irqrestore(&hba->outstanding_lock, flags); 3324 - 3325 - if (!pending) { 3326 - /* 3327 - * The completion handler ran while we tried to 3328 - * clear the command. 3329 - */ 3330 - time_left = 1; 3331 - goto retry; 3332 - } 3333 - } 3334 - } 3335 - 3336 - return err; 3337 - } 3338 - 3339 3211 static void ufshcd_dev_man_lock(struct ufs_hba *hba) 3340 3212 { 3341 3213 ufshcd_hold(hba); ··· 3269 3303 ufshcd_release(hba); 3270 3304 } 3271 3305 3306 + static struct scsi_cmnd *ufshcd_get_dev_mgmt_cmd(struct ufs_hba *hba) 3307 + { 3308 + /* 3309 + * The caller must hold this lock to guarantee that the NOWAIT 3310 + * allocation will succeed. 3311 + */ 3312 + lockdep_assert_held(&hba->dev_cmd.lock); 3313 + 3314 + return scsi_get_internal_cmd( 3315 + hba->host->pseudo_sdev, DMA_TO_DEVICE, 3316 + BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); 3317 + } 3318 + 3319 + static void ufshcd_put_dev_mgmt_cmd(struct scsi_cmnd *cmd) 3320 + { 3321 + scsi_put_internal_cmd(cmd); 3322 + } 3323 + 3272 3324 /* 3273 3325 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3274 3326 * < 0 if another error occurred. 3275 3327 */ 3276 - static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 3277 - const u32 tag, int timeout) 3328 + static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3329 + const u32 tag, int timeout) 3278 3330 { 3279 - int err; 3331 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3332 + struct request *rq = scsi_cmd_to_rq(cmd); 3333 + blk_status_t sts; 3280 3334 3281 - ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 3282 - ufshcd_send_command(hba, tag, hba->dev_cmd_queue); 3283 - err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); 3284 - 3285 - ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, 3286 - (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); 3287 - 3288 - return err; 3335 + rq->timeout = timeout; 3336 + sts = blk_execute_rq(rq, true); 3337 + if (sts != BLK_STS_OK) 3338 + return blk_status_to_errno(sts); 3339 + return lrbp->utr_descriptor_ptr->header.ocs; 3289 3340 } 3290 3341 3291 3342 /** ··· 3320 3337 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, 3321 3338 enum dev_cmd_type cmd_type, int timeout) 3322 3339 { 3323 - const u32 tag = hba->reserved_slot; 3324 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 3340 + struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); 3341 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3342 + u32 tag; 3325 3343 int err; 3326 3344 3327 - /* Protects use of hba->reserved_slot. */ 3345 + /* Protects use of hba->dev_cmd. */ 3328 3346 lockdep_assert_held(&hba->dev_cmd.lock); 3329 3347 3330 - err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); 3331 - if (unlikely(err)) 3332 - return err; 3348 + if (WARN_ON_ONCE(!cmd)) 3349 + return -ENOMEM; 3333 3350 3334 - return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout); 3351 + tag = scsi_cmd_to_rq(cmd)->tag; 3352 + 3353 + err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); 3354 + if (unlikely(err)) 3355 + goto out; 3356 + 3357 + err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); 3358 + if (err == 0) 3359 + err = ufshcd_dev_cmd_completion(hba, lrbp); 3360 + 3361 + out: 3362 + ufshcd_put_dev_mgmt_cmd(cmd); 3363 + 3364 + return err; 3335 3365 } 3336 3366 3337 3367 /** ··· 3974 3978 } 3975 3979 3976 3980 skip_utmrdl: 3977 - /* Allocate memory for local reference block */ 3978 - hba->lrb = devm_kcalloc(hba->dev, 3979 - hba->nutrs, sizeof(struct ufshcd_lrb), 3980 - GFP_KERNEL); 3981 - if (!hba->lrb) { 3982 - dev_err(hba->dev, "LRB Memory allocation failed\n"); 3983 - goto out; 3984 - } 3985 3981 return 0; 3986 3982 out: 3987 3983 return -ENOMEM; ··· 4035 4047 utrdlp[i].response_upiu_length = 4036 4048 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); 4037 4049 } 4038 - 4039 - ufshcd_init_lrb(hba, &hba->lrb[i], i); 4040 4050 } 4041 4051 } 4042 4052 ··· 5386 5400 5387 5401 /** 5388 5402 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 5389 - * @lrbp: pointer to local reference block of completed command 5403 + * @cmd: SCSI command 5390 5404 * @scsi_status: SCSI command status 5391 5405 * 5392 5406 * Return: value base on SCSI command status. 5393 5407 */ 5394 - static inline int 5395 - ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) 5408 + static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status) 5396 5409 { 5397 5410 int result = 0; 5398 5411 5399 5412 switch (scsi_status) { 5400 5413 case SAM_STAT_CHECK_CONDITION: 5401 - ufshcd_copy_sense_data(lrbp); 5414 + ufshcd_copy_sense_data(cmd); 5402 5415 fallthrough; 5403 5416 case SAM_STAT_GOOD: 5404 5417 result |= DID_OK << 16 | scsi_status; ··· 5405 5420 case SAM_STAT_TASK_SET_FULL: 5406 5421 case SAM_STAT_BUSY: 5407 5422 case SAM_STAT_TASK_ABORTED: 5408 - ufshcd_copy_sense_data(lrbp); 5423 + ufshcd_copy_sense_data(cmd); 5409 5424 result |= scsi_status; 5410 5425 break; 5411 5426 default: ··· 5419 5434 /** 5420 5435 * ufshcd_transfer_rsp_status - Get overall status of the response 5421 5436 * @hba: per adapter instance 5422 - * @lrbp: pointer to local reference block of completed command 5437 + * @cmd: SCSI command 5423 5438 * @cqe: pointer to the completion queue entry 5424 5439 * 5425 5440 * Return: result of the command to notify SCSI midlayer. 5426 5441 */ 5427 - static inline int 5428 - ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 5429 - struct cq_entry *cqe) 5442 + static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, 5443 + struct scsi_cmnd *cmd, 5444 + struct cq_entry *cqe) 5430 5445 { 5446 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5447 + const int tag = scsi_cmd_to_rq(cmd)->tag; 5431 5448 int result = 0; 5432 5449 int scsi_status; 5433 5450 enum utp_ocs ocs; ··· 5443 5456 * not set either flag. 5444 5457 */ 5445 5458 if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) 5446 - scsi_set_resid(lrbp->cmd, resid); 5459 + scsi_set_resid(cmd, resid); 5447 5460 5448 5461 /* overall command status of utrd */ 5449 5462 ocs = ufshcd_get_tr_ocs(lrbp, cqe); ··· 5464 5477 * to notify the SCSI midlayer of the command status 5465 5478 */ 5466 5479 scsi_status = lrbp->ucd_rsp_ptr->header.status; 5467 - result = ufshcd_scsi_cmd_status(lrbp, scsi_status); 5480 + result = ufshcd_scsi_cmd_status(cmd, scsi_status); 5468 5481 5469 5482 /* 5470 5483 * Currently we are only supporting BKOPs exception ··· 5501 5514 case OCS_ABORTED: 5502 5515 case OCS_INVALID_COMMAND_STATUS: 5503 5516 result |= DID_REQUEUE << 16; 5504 - dev_warn(hba->dev, 5505 - "OCS %s from controller for tag %d\n", 5506 - (ocs == OCS_ABORTED ? "aborted" : "invalid"), 5507 - lrbp->task_tag); 5517 + dev_warn(hba->dev, "OCS %s from controller for tag %d\n", 5518 + ocs == OCS_ABORTED ? "aborted" : "invalid", tag); 5508 5519 break; 5509 5520 case OCS_INVALID_CMD_TABLE_ATTR: 5510 5521 case OCS_INVALID_PRDT_ATTR: ··· 5515 5530 case OCS_GENERAL_CRYPTO_ERROR: 5516 5531 default: 5517 5532 result |= DID_ERROR << 16; 5518 - dev_err(hba->dev, 5519 - "OCS error from controller = %x for tag %d\n", 5520 - ocs, lrbp->task_tag); 5533 + dev_err(hba->dev, "OCS error from controller = %x for tag %d\n", 5534 + ocs, tag); 5521 5535 ufshcd_print_evt_hist(hba); 5522 5536 ufshcd_print_host_state(hba); 5523 5537 break; ··· 5526 5542 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { 5527 5543 if (cqe) 5528 5544 ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); 5529 - ufshcd_print_tr(hba, lrbp->task_tag, true); 5545 + ufshcd_print_tr(hba, cmd, true); 5530 5546 } 5531 5547 return result; 5532 5548 } ··· 5593 5609 } 5594 5610 5595 5611 /* Release the resources allocated for processing a SCSI command. */ 5596 - void ufshcd_release_scsi_cmd(struct ufs_hba *hba, 5597 - struct ufshcd_lrb *lrbp) 5612 + void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) 5598 5613 { 5599 - struct scsi_cmnd *cmd = lrbp->cmd; 5600 - 5601 5614 scsi_dma_unmap(cmd); 5602 - ufshcd_crypto_clear_prdt(hba, lrbp); 5615 + ufshcd_crypto_clear_prdt(hba, cmd); 5603 5616 ufshcd_release(hba); 5604 5617 ufshcd_clk_scaling_update_busy(hba); 5605 5618 } ··· 5610 5629 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, 5611 5630 struct cq_entry *cqe) 5612 5631 { 5613 - struct ufshcd_lrb *lrbp; 5614 - struct scsi_cmnd *cmd; 5632 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 5633 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5615 5634 enum utp_ocs ocs; 5616 5635 5617 - lrbp = &hba->lrb[task_tag]; 5636 + if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", 5637 + le64_to_cpu(cqe->command_desc_base_addr))) 5638 + return; 5639 + 5618 5640 if (hba->monitor.enabled) { 5619 5641 lrbp->compl_time_stamp = ktime_get(); 5620 5642 lrbp->compl_time_stamp_local_clock = local_clock(); 5621 5643 } 5622 - cmd = lrbp->cmd; 5623 - if (cmd) { 5624 - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) 5625 - ufshcd_update_monitor(hba, lrbp); 5626 - ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP); 5627 - cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); 5628 - ufshcd_release_scsi_cmd(hba, lrbp); 5629 - /* Do not touch lrbp after scsi done */ 5630 - scsi_done(cmd); 5644 + if (ufshcd_is_scsi_cmd(cmd)) { 5645 + if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 5646 + ufshcd_update_monitor(hba, cmd); 5647 + ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); 5648 + cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); 5649 + ufshcd_release_scsi_cmd(hba, cmd); 5631 5650 } else { 5632 5651 if (cqe) { 5633 5652 ocs = cqe->overall_status & MASK_OCS; 5634 5653 lrbp->utr_descriptor_ptr->header.ocs = ocs; 5654 + } else { 5655 + ocs = lrbp->utr_descriptor_ptr->header.ocs; 5635 5656 } 5636 - complete(&hba->dev_cmd.complete); 5657 + ufshcd_add_query_upiu_trace( 5658 + hba, 5659 + ocs == OCS_SUCCESS ? UFS_QUERY_COMP : UFS_QUERY_ERR, 5660 + (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); 5661 + cmd->result = 0; 5637 5662 } 5663 + /* Do not touch lrbp after scsi_done() has been called. */ 5664 + scsi_done(cmd); 5638 5665 } 5639 5666 5640 5667 /** ··· 5670 5681 int tag; 5671 5682 5672 5683 for_each_set_bit(tag, completed_reqs, hba->nutrs) { 5673 - struct scsi_cmnd *cmd = hba->lrb[tag].cmd; 5684 + struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag); 5674 5685 5675 5686 if (!cmd) 5676 5687 continue; ··· 5715 5726 return completed_reqs != 0; 5716 5727 } 5717 5728 5729 + static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) 5730 + { 5731 + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 5732 + struct scsi_device *sdev = rq->q->queuedata; 5733 + struct Scsi_Host *shost = sdev->host; 5734 + struct ufs_hba *hba = shost_priv(shost); 5735 + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5736 + 5737 + if (blk_mq_is_reserved_rq(rq) || !hwq) 5738 + return true; 5739 + 5740 + ufshcd_mcq_compl_all_cqes_lock(hba, hwq); 5741 + 5742 + /* 5743 + * For those cmds of which the cqes are not present in the cq, complete 5744 + * them explicitly. 5745 + */ 5746 + scoped_guard(spinlock_irqsave, &hwq->cq_lock) { 5747 + if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { 5748 + set_host_byte(cmd, DID_REQUEUE); 5749 + ufshcd_release_scsi_cmd(hba, cmd); 5750 + scsi_done(cmd); 5751 + } 5752 + } 5753 + 5754 + return true; 5755 + } 5756 + 5757 + static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) 5758 + { 5759 + struct scsi_device *sdev = rq->q->queuedata; 5760 + struct Scsi_Host *shost = sdev->host; 5761 + struct ufs_hba *hba = shost_priv(shost); 5762 + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5763 + 5764 + if (!blk_mq_is_reserved_rq(rq) && hwq) 5765 + ufshcd_mcq_poll_cqe_lock(hba, hwq); 5766 + 5767 + return true; 5768 + } 5769 + 5718 5770 /** 5719 5771 * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is 5720 5772 * invoked from the error handler context or ufshcd_host_reset_and_restore() ··· 5770 5740 static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, 5771 5741 bool force_compl) 5772 5742 { 5773 - struct ufs_hw_queue *hwq; 5774 - struct ufshcd_lrb *lrbp; 5775 - struct scsi_cmnd *cmd; 5776 - unsigned long flags; 5777 - int tag; 5778 - 5779 - for (tag = 0; tag < hba->nutrs; tag++) { 5780 - lrbp = &hba->lrb[tag]; 5781 - cmd = lrbp->cmd; 5782 - if (!ufshcd_cmd_inflight(cmd) || 5783 - test_bit(SCMD_STATE_COMPLETE, &cmd->state)) 5784 - continue; 5785 - 5786 - hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 5787 - if (!hwq) 5788 - continue; 5789 - 5790 - if (force_compl) { 5791 - ufshcd_mcq_compl_all_cqes_lock(hba, hwq); 5792 - /* 5793 - * For those cmds of which the cqes are not present 5794 - * in the cq, complete them explicitly. 5795 - */ 5796 - spin_lock_irqsave(&hwq->cq_lock, flags); 5797 - if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { 5798 - set_host_byte(cmd, DID_REQUEUE); 5799 - ufshcd_release_scsi_cmd(hba, lrbp); 5800 - scsi_done(cmd); 5801 - } 5802 - spin_unlock_irqrestore(&hwq->cq_lock, flags); 5803 - } else { 5804 - ufshcd_mcq_poll_cqe_lock(hba, hwq); 5805 - } 5806 - } 5743 + blk_mq_tagset_busy_iter(&hba->host->tag_set, 5744 + force_compl ? ufshcd_mcq_force_compl_one : 5745 + ufshcd_mcq_compl_one, 5746 + NULL); 5807 5747 } 5808 5748 5809 5749 /** ··· 6618 6618 struct Scsi_Host *shost = sdev->host; 6619 6619 struct ufs_hba *hba = shost_priv(shost); 6620 6620 6621 + if (blk_mq_is_reserved_rq(rq)) 6622 + return true; 6623 + 6621 6624 *ret = ufshcd_try_to_abort_task(hba, tag); 6622 6625 dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, 6623 - hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, 6626 + ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, 6624 6627 *ret ? "failed" : "succeeded"); 6625 6628 6626 6629 return *ret == 0; ··· 7352 7349 enum dev_cmd_type cmd_type, 7353 7350 enum query_opcode desc_op) 7354 7351 { 7355 - const u32 tag = hba->reserved_slot; 7356 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7352 + struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); 7353 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7354 + u32 tag; 7357 7355 int err = 0; 7358 7356 u8 upiu_flags; 7359 7357 7360 - /* Protects use of hba->reserved_slot. */ 7358 + /* Protects use of hba->dev_cmd. */ 7361 7359 lockdep_assert_held(&hba->dev_cmd.lock); 7362 7360 7363 - ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); 7361 + if (WARN_ON_ONCE(!cmd)) 7362 + return -ENOMEM; 7363 + 7364 + tag = scsi_cmd_to_rq(cmd)->tag; 7365 + 7366 + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 7364 7367 7365 7368 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 7366 7369 ··· 7386 7377 7387 7378 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7388 7379 7389 - /* 7390 - * ignore the returning value here - ufshcd_check_query_response is 7391 - * bound to fail since dev_cmd.query and dev_cmd.type were left empty. 7392 - * read the response directly ignoring all errors. 7393 - */ 7394 - ufshcd_issue_dev_cmd(hba, lrbp, tag, dev_cmd_timeout); 7380 + err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); 7381 + if (err) 7382 + goto put_dev_mgmt_cmd; 7395 7383 7396 7384 /* just copy the upiu response as it is */ 7397 7385 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); ··· 7408 7402 err = -EINVAL; 7409 7403 } 7410 7404 } 7405 + 7406 + put_dev_mgmt_cmd: 7407 + ufshcd_put_dev_mgmt_cmd(cmd); 7411 7408 7412 7409 return err; 7413 7410 } ··· 7505 7496 struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, 7506 7497 enum dma_data_direction dir) 7507 7498 { 7508 - const u32 tag = hba->reserved_slot; 7509 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7499 + struct scsi_cmnd *cmd; 7500 + struct ufshcd_lrb *lrbp; 7501 + u32 tag; 7510 7502 int err = 0; 7511 7503 int result; 7512 7504 u8 upiu_flags; ··· 7515 7505 u16 ehs_len; 7516 7506 int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0; 7517 7507 7518 - /* Protects use of hba->reserved_slot. */ 7519 7508 ufshcd_dev_man_lock(hba); 7520 7509 7521 - ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag); 7510 + cmd = ufshcd_get_dev_mgmt_cmd(hba); 7511 + 7512 + if (WARN_ON_ONCE(!cmd)) { 7513 + err = -ENOMEM; 7514 + goto unlock; 7515 + } 7516 + 7517 + lrbp = scsi_cmd_priv(cmd); 7518 + tag = scsi_cmd_to_rq(cmd)->tag; 7519 + 7520 + ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, 7521 + tag); 7522 7522 7523 7523 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); 7524 7524 ··· 7545 7525 7546 7526 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7547 7527 7548 - err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT); 7528 + err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); 7529 + if (err) 7530 + goto put_dev_mgmt_cmd; 7549 7531 7532 + err = ufshcd_dev_cmd_completion(hba, lrbp); 7550 7533 if (!err) { 7551 7534 /* Just copy the upiu response as it is */ 7552 7535 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); ··· 7573 7550 } 7574 7551 } 7575 7552 7553 + put_dev_mgmt_cmd: 7554 + ufshcd_put_dev_mgmt_cmd(cmd); 7555 + 7556 + unlock: 7576 7557 ufshcd_dev_man_unlock(hba); 7577 7558 7578 7559 return err ? : result; 7560 + } 7561 + 7562 + static bool ufshcd_clear_lu_cmds(struct request *req, void *priv) 7563 + { 7564 + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 7565 + struct scsi_device *sdev = cmd->device; 7566 + struct Scsi_Host *shost = sdev->host; 7567 + struct ufs_hba *hba = shost_priv(shost); 7568 + const u64 lun = *(u64 *)priv; 7569 + const u32 tag = req->tag; 7570 + 7571 + if (blk_mq_is_reserved_rq(req) || sdev->lun != lun) 7572 + return true; 7573 + 7574 + if (ufshcd_clear_cmd(hba, tag) < 0) { 7575 + dev_err(hba->dev, "%s: failed to clear request %d\n", __func__, 7576 + tag); 7577 + return true; 7578 + } 7579 + 7580 + if (hba->mcq_enabled) { 7581 + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, req); 7582 + 7583 + if (hwq) 7584 + ufshcd_mcq_poll_cqe_lock(hba, hwq); 7585 + return true; 7586 + } 7587 + 7588 + ufshcd_compl_one_cqe(hba, tag, NULL); 7589 + return true; 7579 7590 } 7580 7591 7581 7592 /** ··· 7620 7563 */ 7621 7564 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) 7622 7565 { 7623 - unsigned long flags, pending_reqs = 0, not_cleared = 0; 7624 7566 struct Scsi_Host *host; 7625 7567 struct ufs_hba *hba; 7626 - struct ufs_hw_queue *hwq; 7627 - struct ufshcd_lrb *lrbp; 7628 - u32 pos, not_cleared_mask = 0; 7629 7568 int err; 7630 7569 u8 resp = 0xF, lun; 7631 7570 ··· 7630 7577 7631 7578 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 7632 7579 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); 7633 - if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7634 - if (!err) 7635 - err = resp; 7636 - goto out; 7580 + if (err) { 7581 + } else if (resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7582 + err = resp; 7583 + } else { 7584 + /* clear the commands that were pending for corresponding LUN */ 7585 + blk_mq_tagset_busy_iter(&hba->host->tag_set, 7586 + ufshcd_clear_lu_cmds, 7587 + &cmd->device->lun); 7637 7588 } 7638 7589 7639 - if (hba->mcq_enabled) { 7640 - for (pos = 0; pos < hba->nutrs; pos++) { 7641 - lrbp = &hba->lrb[pos]; 7642 - if (ufshcd_cmd_inflight(lrbp->cmd) && 7643 - lrbp->lun == lun) { 7644 - ufshcd_clear_cmd(hba, pos); 7645 - hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd)); 7646 - ufshcd_mcq_poll_cqe_lock(hba, hwq); 7647 - } 7648 - } 7649 - err = 0; 7650 - goto out; 7651 - } 7652 - 7653 - /* clear the commands that were pending for corresponding LUN */ 7654 - spin_lock_irqsave(&hba->outstanding_lock, flags); 7655 - for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) 7656 - if (hba->lrb[pos].lun == lun) 7657 - __set_bit(pos, &pending_reqs); 7658 - hba->outstanding_reqs &= ~pending_reqs; 7659 - spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7660 - 7661 - for_each_set_bit(pos, &pending_reqs, hba->nutrs) { 7662 - if (ufshcd_clear_cmd(hba, pos) < 0) { 7663 - spin_lock_irqsave(&hba->outstanding_lock, flags); 7664 - not_cleared = 1U << pos & 7665 - ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7666 - hba->outstanding_reqs |= not_cleared; 7667 - not_cleared_mask |= not_cleared; 7668 - spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7669 - 7670 - dev_err(hba->dev, "%s: failed to clear request %d\n", 7671 - __func__, pos); 7672 - } 7673 - } 7674 - __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask); 7675 - 7676 - out: 7677 7590 hba->req_abort_count = 0; 7678 7591 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); 7679 7592 if (!err) { ··· 7653 7634 7654 7635 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) 7655 7636 { 7656 - struct ufshcd_lrb *lrbp; 7657 7637 int tag; 7658 7638 7659 7639 for_each_set_bit(tag, &bitmap, hba->nutrs) { 7660 - lrbp = &hba->lrb[tag]; 7640 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7641 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7642 + 7661 7643 lrbp->req_abort_skip = true; 7662 7644 } 7663 7645 } ··· 7666 7646 /** 7667 7647 * ufshcd_try_to_abort_task - abort a specific task 7668 7648 * @hba: Pointer to adapter instance 7669 - * @tag: Task tag/index to be aborted 7649 + * @tag: Tag of the task to be aborted 7670 7650 * 7671 7651 * Abort the pending command in device by sending UFS_ABORT_TASK task management 7672 7652 * command, and in host controller by clearing the door-bell register. There can ··· 7678 7658 */ 7679 7659 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) 7680 7660 { 7681 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7661 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7662 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7682 7663 int err; 7683 7664 int poll_cnt; 7684 7665 u8 resp = 0xF; 7685 7666 7686 7667 for (poll_cnt = 100; poll_cnt; poll_cnt--) { 7687 - err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 7688 - UFS_QUERY_TASK, &resp); 7668 + err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_QUERY_TASK, 7669 + &resp); 7689 7670 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { 7690 7671 /* cmd pending in the device */ 7691 7672 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", ··· 7701 7680 hba->dev, 7702 7681 "%s: cmd with tag %d not pending in the device.\n", 7703 7682 __func__, tag); 7704 - if (!ufshcd_cmd_inflight(lrbp->cmd)) { 7683 + if (!ufshcd_cmd_inflight(cmd)) { 7705 7684 dev_info(hba->dev, 7706 7685 "%s: cmd with tag=%d completed.\n", 7707 7686 __func__, tag); ··· 7719 7698 if (!poll_cnt) 7720 7699 return -EBUSY; 7721 7700 7722 - err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 7723 - UFS_ABORT_TASK, &resp); 7701 + err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_ABORT_TASK, &resp); 7724 7702 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7725 7703 if (!err) { 7726 7704 err = resp; /* service response error */ ··· 7747 7727 { 7748 7728 struct Scsi_Host *host = cmd->device->host; 7749 7729 struct ufs_hba *hba = shost_priv(host); 7750 - int tag = scsi_cmd_to_rq(cmd)->tag; 7751 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7730 + struct request *rq = scsi_cmd_to_rq(cmd); 7731 + int tag = rq->tag; 7732 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7752 7733 unsigned long flags; 7753 7734 int err = FAILED; 7754 7735 bool outstanding; ··· 7778 7757 * to reduce repeated printouts. For other aborted requests only print 7779 7758 * basic details. 7780 7759 */ 7781 - scsi_print_command(cmd); 7760 + if (ufshcd_is_scsi_cmd(cmd)) 7761 + scsi_print_command(cmd); 7782 7762 if (!hba->req_abort_count) { 7783 7763 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); 7784 7764 ufshcd_print_evt_hist(hba); 7785 7765 ufshcd_print_host_state(hba); 7786 7766 ufshcd_print_pwr_info(hba); 7787 - ufshcd_print_tr(hba, tag, true); 7767 + ufshcd_print_tr(hba, cmd, true); 7788 7768 } else { 7789 - ufshcd_print_tr(hba, tag, false); 7769 + ufshcd_print_tr(hba, cmd, false); 7790 7770 } 7791 7771 hba->req_abort_count++; 7792 7772 ··· 7831 7809 goto release; 7832 7810 } 7833 7811 7834 - err = ufshcd_try_to_abort_task(hba, tag); 7812 + if (blk_mq_is_reserved_rq(rq)) 7813 + err = ufshcd_clear_cmd(hba, tag); 7814 + else 7815 + err = ufshcd_try_to_abort_task(hba, tag); 7835 7816 if (err) { 7836 7817 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7837 7818 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); ··· 7851 7826 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7852 7827 7853 7828 if (outstanding) 7854 - ufshcd_release_scsi_cmd(hba, lrbp); 7829 + ufshcd_release_scsi_cmd(hba, cmd); 7855 7830 7856 7831 err = SUCCESS; 7857 7832 ··· 8488 8463 8489 8464 static int ufs_get_device_desc(struct ufs_hba *hba) 8490 8465 { 8466 + struct ufs_dev_info *dev_info = &hba->dev_info; 8467 + struct Scsi_Host *shost = hba->host; 8491 8468 int err; 8492 8469 u8 model_index; 8493 8470 u8 *desc_buf; 8494 - struct ufs_dev_info *dev_info = &hba->dev_info; 8495 8471 8496 8472 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8497 8473 if (!desc_buf) { ··· 8519 8493 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | 8520 8494 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; 8521 8495 dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; 8496 + 8497 + /* 8498 + * According to the UFS standard, the UFS device queue depth 8499 + * (bQueueDepth) must be in the range 1..255 if the shared queueing 8500 + * architecture is supported. bQueueDepth is zero if the shared queueing 8501 + * architecture is not supported. 8502 + */ 8503 + if (dev_info->bqueuedepth) 8504 + shost->cmd_per_lun = min(hba->nutrs, dev_info->bqueuedepth) - 8505 + UFSHCD_NUM_RESERVED; 8506 + else 8507 + shost->cmd_per_lun = shost->can_queue; 8522 8508 8523 8509 dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; 8524 8510 ··· 8931 8893 utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; 8932 8894 dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, 8933 8895 hba->utrdl_dma_addr); 8934 - 8935 - devm_kfree(hba->dev, hba->lrb); 8936 8896 } 8937 8897 8938 8898 static int ufshcd_alloc_mcq(struct ufs_hba *hba) ··· 8938 8902 int ret; 8939 8903 int old_nutrs = hba->nutrs; 8940 8904 8941 - ret = ufshcd_mcq_decide_queue_depth(hba); 8905 + ret = ufshcd_get_hba_mac(hba); 8942 8906 if (ret < 0) 8943 8907 return ret; 8944 8908 ··· 8964 8928 goto err; 8965 8929 8966 8930 hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 8967 - hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED; 8968 8931 8969 8932 return 0; 8970 8933 err: ··· 9201 9166 .name = UFSHCD, 9202 9167 .proc_name = UFSHCD, 9203 9168 .map_queues = ufshcd_map_queues, 9169 + .cmd_size = sizeof(struct ufshcd_lrb), 9170 + .init_cmd_priv = ufshcd_init_cmd_priv, 9204 9171 .queuecommand = ufshcd_queuecommand, 9172 + .queue_reserved_command = ufshcd_queue_reserved_command, 9173 + .nr_reserved_cmds = UFSHCD_NUM_RESERVED, 9205 9174 .mq_poll = ufshcd_poll, 9206 9175 .sdev_init = ufshcd_sdev_init, 9207 9176 .sdev_configure = ufshcd_sdev_configure, ··· 10628 10589 { 10629 10590 int err; 10630 10591 10592 + WARN_ON_ONCE(!hba->host->can_queue); 10593 + WARN_ON_ONCE(!hba->host->cmd_per_lun); 10594 + 10631 10595 if (is_mcq_supported(hba)) { 10632 10596 ufshcd_mcq_enable(hba); 10633 10597 err = ufshcd_alloc_mcq(hba); ··· 10757 10715 */ 10758 10716 hba->vcc_off_delay_us = 2000; 10759 10717 10760 - init_completion(&hba->dev_cmd.complete); 10761 - 10762 10718 err = ufshcd_hba_init(hba); 10763 10719 if (err) 10764 10720 goto out_error; ··· 10789 10749 ufshcd_host_memory_configure(hba); 10790 10750 10791 10751 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 10792 - host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED; 10752 + /* 10753 + * Set the queue depth for WLUNs. ufs_get_device_desc() will increase 10754 + * host->cmd_per_lun to a larger value. 10755 + */ 10756 + host->cmd_per_lun = 1; 10793 10757 host->max_id = UFSHCD_MAX_ID; 10794 10758 host->max_lun = UFS_MAX_LUNS; 10795 10759 host->max_channel = UFSHCD_MAX_CHANNEL; ··· 10885 10841 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); 10886 10842 } 10887 10843 10844 + err = ufshcd_add_scsi_host(hba); 10845 + if (err) 10846 + goto out_disable; 10847 + 10888 10848 /* Hold auto suspend until async scan completes */ 10889 10849 pm_runtime_get_sync(dev); 10890 10850 ··· 10936 10888 10937 10889 initialized: 10938 10890 ufshcd_process_probe_result(hba, probe_start, err); 10939 - if (err) 10940 - goto out_disable; 10941 - 10942 - err = ufshcd_add_scsi_host(hba); 10943 10891 if (err) 10944 10892 goto out_disable; 10945 10893

+20

include/scsi/scsi_device.h

··· 564 564 const struct scsi_exec_args *args); 565 565 void scsi_failures_reset_retries(struct scsi_failures *failures); 566 566 567 + struct scsi_cmnd *scsi_get_internal_cmd(struct scsi_device *sdev, 568 + enum dma_data_direction data_direction, 569 + blk_mq_req_flags_t flags); 570 + void scsi_put_internal_cmd(struct scsi_cmnd *scmd); 567 571 extern void sdev_disable_disk_events(struct scsi_device *sdev); 568 572 extern void sdev_enable_disk_events(struct scsi_device *sdev); 569 573 extern int scsi_vpd_lun_id(struct scsi_device *, char *, size_t); ··· 598 594 599 595 #define scmd_id(scmd) sdev_id((scmd)->device) 600 596 #define scmd_channel(scmd) sdev_channel((scmd)->device) 597 + 598 + /** 599 + * scsi_device_is_pseudo_dev() - Whether a device is a pseudo SCSI device. 600 + * @sdev: SCSI device to examine 601 + * 602 + * A pseudo SCSI device can be used to allocate SCSI commands but does not show 603 + * up in sysfs. Additionally, the logical unit information in *@sdev is made up. 604 + * 605 + * This function tests the LUN number instead of comparing @sdev with 606 + * @sdev->host->pseudo_sdev because this function may be called before 607 + * @sdev->host->pseudo_sdev has been initialized. 608 + */ 609 + static inline bool scsi_device_is_pseudo_dev(struct scsi_device *sdev) 610 + { 611 + return sdev->lun == U64_MAX; 612 + } 601 613 602 614 /* 603 615 * checks for positions of the SCSI state machine

+32 -1

include/scsi/scsi_host.h

··· 87 87 int (* queuecommand)(struct Scsi_Host *, struct scsi_cmnd *); 88 88 89 89 /* 90 + * Queue a reserved command (BLK_MQ_REQ_RESERVED). The .queuecommand() 91 + * documentation also applies to the .queue_reserved_command() callback. 92 + */ 93 + int (*queue_reserved_command)(struct Scsi_Host *, struct scsi_cmnd *); 94 + 95 + /* 90 96 * The commit_rqs function is used to trigger a hardware 91 97 * doorbell after some requests have been queued with 92 98 * queuecommand, when an error is encountered before sending ··· 381 375 /* 382 376 * This determines if we will use a non-interrupt driven 383 377 * or an interrupt driven scheme. It is set to the maximum number 384 - * of simultaneous commands a single hw queue in HBA will accept. 378 + * of simultaneous commands a single hw queue in HBA will accept 379 + * excluding internal commands. 385 380 */ 386 381 int can_queue; 382 + 383 + /* 384 + * This determines how many commands the HBA will set aside 385 + * for internal commands. This number will be added to 386 + * @can_queue to calculate the maximum number of simultaneous 387 + * commands sent to the host. 388 + */ 389 + int nr_reserved_cmds; 387 390 388 391 /* 389 392 * In many instances, especially where disconnect / reconnect are ··· 626 611 unsigned short max_cmd_len; 627 612 628 613 int this_id; 614 + 615 + /* 616 + * Number of commands this host can handle at the same time. 617 + * This excludes reserved commands as specified by nr_reserved_cmds. 618 + */ 629 619 int can_queue; 620 + /* 621 + * Number of reserved commands to allocate, if any. 622 + */ 623 + unsigned int nr_reserved_cmds; 624 + 630 625 short cmd_per_lun; 631 626 short unsigned int sg_tablesize; 632 627 short unsigned int sg_prot_tablesize; ··· 726 701 727 702 /* ldm bits */ 728 703 struct device shost_gendev, shost_dev; 704 + 705 + /* 706 + * A SCSI device structure used for sending internal commands to the 707 + * HBA. There is no corresponding logical unit inside the SCSI device. 708 + */ 709 + struct scsi_device *pseudo_sdev; 729 710 730 711 /* 731 712 * Points to the transport data (if any) which is allocated

-12

include/ufs/ufshcd.h

··· 161 161 * @ucd_prdt_dma_addr: PRDT dma address for debug 162 162 * @ucd_rsp_dma_addr: UPIU response dma address for debug 163 163 * @ucd_req_dma_addr: UPIU request dma address for debug 164 - * @cmd: pointer to SCSI command 165 164 * @scsi_status: SCSI status of the command 166 165 * @command_type: SCSI, UFS, Query. 167 166 * @task_tag: Task tag of the command ··· 185 186 dma_addr_t ucd_rsp_dma_addr; 186 187 dma_addr_t ucd_prdt_dma_addr; 187 188 188 - struct scsi_cmnd *cmd; 189 189 int scsi_status; 190 190 191 191 int command_type; 192 - int task_tag; 193 192 u8 lun; /* UPIU LUN id field is only 8-bit wide */ 194 193 bool intr_cmd; 195 194 bool req_abort_skip; ··· 236 239 * struct ufs_dev_cmd - all assosiated fields with device management commands 237 240 * @type: device management command type - Query, NOP OUT 238 241 * @lock: lock to allow one command at a time 239 - * @complete: internal commands completion 240 242 * @query: Device management query information 241 243 */ 242 244 struct ufs_dev_cmd { 243 245 enum dev_cmd_type type; 244 246 struct mutex lock; 245 - struct completion complete; 246 247 struct ufs_query query; 247 248 }; 248 249 ··· 828 833 * @spm_lvl: desired UFS power management level during system PM. 829 834 * @pm_op_in_progress: whether or not a PM operation is in progress. 830 835 * @ahit: value of Auto-Hibernate Idle Timer register. 831 - * @lrb: local reference block 832 836 * @outstanding_tasks: Bits representing outstanding task requests 833 837 * @outstanding_lock: Protects @outstanding_reqs. 834 838 * @outstanding_reqs: Bits representing outstanding transfer requests ··· 836 842 * @nutrs: Transfer Request Queue depth supported by controller 837 843 * @nortt - Max outstanding RTTs supported by controller 838 844 * @nutmrs: Task Management Queue depth supported by controller 839 - * @reserved_slot: Used to submit device commands. Protected by @dev_cmd.lock. 840 845 * @ufs_version: UFS Version to which controller complies 841 846 * @vops: pointer to variant specific operations 842 847 * @vps: pointer to variant specific parameters ··· 926 933 * @res: array of resource info of MCQ registers 927 934 * @mcq_base: Multi circular queue registers base address 928 935 * @uhq: array of supported hardware queues 929 - * @dev_cmd_queue: Queue for issuing device management commands 930 936 * @mcq_opr: MCQ operation and runtime registers 931 937 * @ufs_rtc_update_work: A work for UFS RTC periodic update 932 938 * @pm_qos_req: PM QoS request handle ··· 968 976 /* Auto-Hibernate Idle Timer register value */ 969 977 u32 ahit; 970 978 971 - struct ufshcd_lrb *lrb; 972 - 973 979 unsigned long outstanding_tasks; 974 980 spinlock_t outstanding_lock; 975 981 unsigned long outstanding_reqs; ··· 977 987 int nortt; 978 988 u32 mcq_capabilities; 979 989 int nutmrs; 980 - u32 reserved_slot; 981 990 u32 ufs_version; 982 991 const struct ufs_hba_variant_ops *vops; 983 992 struct ufs_hba_variant_params *vps; ··· 1094 1105 bool mcq_esi_enabled; 1095 1106 void __iomem *mcq_base; 1096 1107 struct ufs_hw_queue *uhq; 1097 - struct ufs_hw_queue *dev_cmd_queue; 1098 1108 struct ufshcd_mcq_opr_info_t mcq_opr[OPR_MAX]; 1099 1109 1100 1110 struct delayed_work ufs_rtc_update_work;

Configure Feed

Configure Feed