scsi: ufs: core: Optimize the hot path · tjh.dev/kernel@22089c2

+5 -4

drivers/ufs/core/ufs-mcq.c

··· 534 534 */ 535 535 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) 536 536 { 537 - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 538 - struct scsi_cmnd *cmd = lrbp->cmd; 537 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 538 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 539 539 struct ufs_hw_queue *hwq; 540 540 void __iomem *reg, *opr_sqd_base; 541 541 u32 nexus, id, val; ··· 618 618 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, 619 619 struct ufs_hw_queue *hwq, int task_tag) 620 620 { 621 - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 621 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 622 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 622 623 struct utp_transfer_req_desc *utrd; 623 624 __le64 cmd_desc_base_addr; 624 625 bool ret = false; ··· 670 669 struct Scsi_Host *host = cmd->device->host; 671 670 struct ufs_hba *hba = shost_priv(host); 672 671 int tag = scsi_cmd_to_rq(cmd)->tag; 673 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 672 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 674 673 struct ufs_hw_queue *hwq; 675 674 int err; 676 675

+11 -7

drivers/ufs/core/ufshcd-crypto.h

··· 38 38 } 39 39 40 40 static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, 41 - struct ufshcd_lrb *lrbp) 41 + struct scsi_cmnd *cmd) 42 42 { 43 - struct scsi_cmnd *cmd = lrbp->cmd; 44 43 const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx; 44 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 45 45 46 46 if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt) 47 47 return hba->vops->fill_crypto_prdt(hba, crypt_ctx, ··· 51 51 } 52 52 53 53 static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, 54 - struct ufshcd_lrb *lrbp) 54 + struct scsi_cmnd *cmd) 55 55 { 56 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 57 + 56 58 if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT)) 57 59 return; 58 60 59 - if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx)) 61 + if (!(scsi_cmd_to_rq(cmd)->crypt_ctx)) 60 62 return; 61 63 62 64 /* Zeroize the PRDT because it can contain cryptographic keys. */ 63 65 memzero_explicit(lrbp->ucd_prdt_ptr, 64 - ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd)); 66 + ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd)); 65 67 } 66 68 67 69 bool ufshcd_crypto_enable(struct ufs_hba *hba); ··· 84 82 struct request_desc_header *h) { } 85 83 86 84 static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba, 87 - struct ufshcd_lrb *lrbp) 85 + struct scsi_cmnd *cmd) 88 86 { 89 87 return 0; 90 88 } 91 89 92 90 static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba, 93 - struct ufshcd_lrb *lrbp) { } 91 + struct scsi_cmnd *cmd) 92 + { 93 + } 94 94 95 95 static inline bool ufshcd_crypto_enable(struct ufs_hba *hba) 96 96 {

+39 -2

drivers/ufs/core/ufshcd-priv.h

··· 77 77 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag); 78 78 int ufshcd_mcq_abort(struct scsi_cmnd *cmd); 79 79 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag); 80 - void ufshcd_release_scsi_cmd(struct ufs_hba *hba, 81 - struct ufshcd_lrb *lrbp); 80 + void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd); 82 81 83 82 #define SD_ASCII_STD true 84 83 #define SD_RAW false ··· 360 361 return false; 361 362 } 362 363 return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported); 364 + } 365 + 366 + /* 367 + * Convert a block layer tag into a SCSI command pointer. This function is 368 + * called once per I/O completion path and is also called from error paths. 369 + */ 370 + static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag) 371 + { 372 + struct blk_mq_tags *tags = hba->host->tag_set.shared_tags; 373 + struct request *rq; 374 + 375 + /* 376 + * Handle reserved tags differently because the UFS driver does not 377 + * call blk_mq_alloc_request() for allocating reserved requests. 378 + * Allocating reserved tags with blk_mq_alloc_request() would require 379 + * the following: 380 + * - Allocate an additional request queue from &hba->host->tag_set for 381 + * allocating reserved requests from. 382 + * - For that request queue, allocate a SCSI device. 383 + * - Calling blk_mq_alloc_request(hba->dev_mgmt_queue, REQ_OP_DRV_OUT, 384 + * BLK_MQ_REQ_RESERVED) for allocating a reserved request and 385 + * blk_mq_free_request() for freeing reserved requests. 386 + * - Set the .device pointer for these reserved requests. 387 + * - Submit reserved requests with blk_execute_rq(). 388 + * - Modify ufshcd_queuecommand() such that it handles reserved requests 389 + * in another way than SCSI requests. 390 + * - Modify ufshcd_compl_one_cqe() such that it calls scsi_done() for 391 + * device management commands. 392 + * - Modify all callback functions called by blk_mq_tagset_busy_iter() 393 + * calls in the UFS driver and skip device management commands. 394 + */ 395 + rq = tag < UFSHCD_NUM_RESERVED ? tags->static_rqs[tag] : 396 + blk_mq_tag_to_rq(tags, tag); 397 + 398 + if (WARN_ON_ONCE(!rq)) 399 + return NULL; 400 + 401 + return blk_mq_rq_to_pdu(rq); 363 402 } 364 403 365 404 static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)

+124 -111

drivers/ufs/core/ufshcd.c

··· 28 28 #include <scsi/scsi_dbg.h> 29 29 #include <scsi/scsi_driver.h> 30 30 #include <scsi/scsi_eh.h> 31 + #include <scsi/scsi_tcq.h> 31 32 #include "ufshcd-priv.h" 32 33 #include <ufs/ufs_quirks.h> 33 34 #include <ufs/unipro.h> ··· 484 483 u32 hwq_id = 0; 485 484 struct request *rq = scsi_cmd_to_rq(cmd); 486 485 unsigned int tag = rq->tag; 487 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 486 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 488 487 int transfer_len = -1; 489 488 490 489 /* trace UPIU also */ ··· 595 594 ufshcd_vops_dbg_register_dump(hba); 596 595 } 597 596 598 - static 599 - void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt) 597 + static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, 598 + bool pr_prdt) 600 599 { 601 - const struct ufshcd_lrb *lrbp; 600 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 601 + const int tag = lrbp->task_tag; 602 602 int prdt_length; 603 - 604 - lrbp = &hba->lrb[tag]; 605 603 606 604 if (hba->monitor.enabled) { 607 605 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, ··· 644 644 struct Scsi_Host *shost = sdev->host; 645 645 struct ufs_hba *hba = shost_priv(shost); 646 646 647 - ufshcd_print_tr(hba, req->tag, *(bool *)priv); 647 + ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); 648 648 649 649 return true; 650 650 } ··· 2298 2298 struct scsi_cmnd *cmd) 2299 2299 { 2300 2300 const struct ufs_hba_monitor *m = &hba->monitor; 2301 - struct request *rq = scsi_cmd_to_rq(cmd); 2302 - struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag]; 2301 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2303 2302 2304 2303 return m->enabled && 2305 2304 (!m->chunk_size || m->chunk_size == cmd->sdb.length) && ··· 2319 2320 static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2320 2321 { 2321 2322 struct request *req = scsi_cmd_to_rq(cmd); 2322 - struct ufshcd_lrb *lrbp = &hba->lrb[req->tag]; 2323 + const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2323 2324 int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2324 2325 unsigned long flags; 2325 2326 ··· 2349 2350 spin_unlock_irqrestore(hba->host->host_lock, flags); 2350 2351 } 2351 2352 2353 + /* 2354 + * Returns %true for SCSI commands and %false for device management commands. 2355 + * Must not be called for SCSI commands that have not yet been started. 2356 + */ 2357 + static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) 2358 + { 2359 + return blk_mq_request_started(scsi_cmd_to_rq(cmd)); 2360 + } 2361 + 2352 2362 /** 2353 2363 * ufshcd_send_command - Send SCSI or device management commands 2354 2364 * @hba: per adapter instance 2355 - * @lrbp: Local reference block of SCSI command 2365 + * @cmd: SCSI command or device management command pointer 2356 2366 * @hwq: pointer to hardware queue instance 2357 2367 */ 2358 2368 static inline void ufshcd_send_command(struct ufs_hba *hba, 2359 - struct ufshcd_lrb *lrbp, 2369 + struct scsi_cmnd *cmd, 2360 2370 struct ufs_hw_queue *hwq) 2361 2371 { 2362 - struct scsi_cmnd *cmd = lrbp->cmd; 2372 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2363 2373 unsigned long flags; 2364 2374 2365 2375 if (hba->monitor.enabled) { ··· 2377 2369 lrbp->compl_time_stamp = ktime_set(0, 0); 2378 2370 lrbp->compl_time_stamp_local_clock = 0; 2379 2371 } 2380 - if (cmd) { 2372 + if (ufshcd_is_scsi_cmd(cmd)) { 2381 2373 ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); 2382 2374 ufshcd_clk_scaling_start_busy(hba); 2383 2375 if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) ··· 2397 2389 } else { 2398 2390 spin_lock_irqsave(&hba->outstanding_lock, flags); 2399 2391 if (hba->vops && hba->vops->setup_xfer_req) 2400 - hba->vops->setup_xfer_req(hba, lrbp->task_tag, !!cmd); 2392 + hba->vops->setup_xfer_req(hba, lrbp->task_tag, 2393 + ufshcd_is_scsi_cmd(cmd)); 2401 2394 __set_bit(lrbp->task_tag, &hba->outstanding_reqs); 2402 2395 ufshcd_writel(hba, 1 << lrbp->task_tag, 2403 2396 REG_UTP_TRANSFER_REQ_DOOR_BELL); ··· 2408 2399 2409 2400 /** 2410 2401 * ufshcd_copy_sense_data - Copy sense data in case of check condition 2411 - * @lrbp: pointer to local reference block 2402 + * @cmd: SCSI command 2412 2403 */ 2413 - static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) 2404 + static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd) 2414 2405 { 2415 - u8 *const sense_buffer = lrbp->cmd->sense_buffer; 2406 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2407 + u8 *const sense_buffer = cmd->sense_buffer; 2416 2408 u16 resp_len; 2417 2409 int len; 2418 2410 ··· 2718 2708 /** 2719 2709 * ufshcd_map_sg - Map scatter-gather list to prdt 2720 2710 * @hba: per adapter instance 2721 - * @lrbp: pointer to local reference block 2711 + * @cmd: SCSI command 2722 2712 * 2723 2713 * Return: 0 in case of success, non-zero value in case of failure. 2724 2714 */ 2725 - static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2715 + static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2726 2716 { 2727 - struct scsi_cmnd *cmd = lrbp->cmd; 2717 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2728 2718 int sg_segments = scsi_dma_map(cmd); 2729 2719 2730 2720 if (sg_segments < 0) ··· 2732 2722 2733 2723 ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); 2734 2724 2735 - return ufshcd_crypto_fill_prdt(hba, lrbp); 2725 + return ufshcd_crypto_fill_prdt(hba, cmd); 2736 2726 } 2737 2727 2738 2728 /** ··· 2791 2781 /** 2792 2782 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 2793 2783 * for scsi commands 2794 - * @lrbp: local reference block pointer 2784 + * @cmd: SCSI command 2795 2785 * @upiu_flags: flags 2796 2786 */ 2797 - static 2798 - void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) 2787 + static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, 2788 + u8 upiu_flags) 2799 2789 { 2800 - struct scsi_cmnd *cmd = lrbp->cmd; 2790 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2801 2791 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2802 2792 unsigned short cdb_len; 2803 2793 ··· 2900 2890 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) 2901 2891 * for SCSI Purposes 2902 2892 * @hba: per adapter instance 2903 - * @lrbp: pointer to local reference block 2893 + * @cmd: SCSI command 2904 2894 */ 2905 - static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2895 + static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2906 2896 { 2907 - struct request *rq = scsi_cmd_to_rq(lrbp->cmd); 2897 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2898 + struct request *rq = scsi_cmd_to_rq(cmd); 2908 2899 unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); 2909 2900 u8 upiu_flags; 2910 2901 2911 - ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0); 2902 + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, 2903 + cmd->sc_data_direction, 0); 2912 2904 if (ioprio_class == IOPRIO_CLASS_RT) 2913 2905 upiu_flags |= UPIU_CMD_FLAGS_CP; 2914 - ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); 2906 + ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags); 2915 2907 } 2916 2908 2917 - static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) 2909 + static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2918 2910 { 2911 + const int i = scsi_cmd_to_rq(cmd)->tag; 2919 2912 struct utp_transfer_cmd_desc *cmd_descp = 2920 2913 (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); 2921 2914 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; ··· 2926 2913 hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); 2927 2914 u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); 2928 2915 u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); 2916 + struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd); 2929 2917 2930 2918 lrb->utr_descriptor_ptr = utrdlp + i; 2931 2919 lrb->utrd_dma_addr = ··· 2939 2925 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; 2940 2926 } 2941 2927 2942 - static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 2943 - struct scsi_cmnd *cmd, u8 lun, int tag) 2928 + static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2929 + u8 lun, int tag) 2944 2930 { 2945 - ufshcd_init_lrb(hba, lrbp, tag); 2931 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2932 + 2933 + ufshcd_init_lrb(hba, cmd); 2946 2934 2947 2935 memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); 2948 2936 2949 - lrbp->cmd = cmd; 2950 2937 lrbp->task_tag = tag; 2951 2938 lrbp->lun = lun; 2952 - ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp); 2939 + ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? 2940 + scsi_cmd_to_rq(cmd) : NULL, lrbp); 2953 2941 } 2954 2942 2955 - static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 2956 - struct scsi_cmnd *cmd, u8 lun, int tag) 2943 + static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2944 + u8 lun, int tag) 2957 2945 { 2958 - __ufshcd_setup_cmd(hba, lrbp, cmd, lun, tag); 2946 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2947 + 2948 + __ufshcd_setup_cmd(hba, cmd, lun, tag); 2959 2949 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); 2960 2950 lrbp->req_abort_skip = false; 2961 2951 2962 - ufshcd_comp_scsi_upiu(hba, lrbp); 2952 + ufshcd_comp_scsi_upiu(hba, cmd); 2963 2953 } 2964 2954 2965 2955 /** ··· 3034 3016 { 3035 3017 struct ufs_hba *hba = shost_priv(host); 3036 3018 int tag = scsi_cmd_to_rq(cmd)->tag; 3037 - struct ufshcd_lrb *lrbp; 3038 3019 int err = 0; 3039 3020 struct ufs_hw_queue *hwq = NULL; 3040 3021 ··· 3084 3067 3085 3068 ufshcd_hold(hba); 3086 3069 3087 - lrbp = &hba->lrb[tag]; 3070 + ufshcd_setup_scsi_cmd(hba, cmd, 3071 + ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); 3088 3072 3089 - ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); 3090 - 3091 - err = ufshcd_map_sg(hba, lrbp); 3073 + err = ufshcd_map_sg(hba, cmd); 3092 3074 if (err) { 3093 3075 ufshcd_release(hba); 3094 3076 goto out; ··· 3096 3080 if (hba->mcq_enabled) 3097 3081 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 3098 3082 3099 - ufshcd_send_command(hba, lrbp, hwq); 3083 + ufshcd_send_command(hba, cmd, hwq); 3100 3084 3101 3085 out: 3102 3086 if (ufs_trigger_eh(hba)) { ··· 3110 3094 return err; 3111 3095 } 3112 3096 3113 - static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 3114 - enum dev_cmd_type cmd_type, u8 lun, int tag) 3097 + static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3098 + enum dev_cmd_type cmd_type, u8 lun, int tag) 3115 3099 { 3116 - __ufshcd_setup_cmd(hba, lrbp, NULL, lun, tag); 3100 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3101 + 3102 + __ufshcd_setup_cmd(hba, cmd, lun, tag); 3117 3103 lrbp->intr_cmd = true; /* No interrupt aggregation */ 3118 3104 hba->dev_cmd.type = cmd_type; 3119 3105 } ··· 3123 3105 /* 3124 3106 * Return: 0 upon success; < 0 upon failure. 3125 3107 */ 3126 - static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, 3127 - struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) 3108 + static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3109 + enum dev_cmd_type cmd_type, int tag) 3128 3110 { 3129 - ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); 3111 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3112 + 3113 + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 3130 3114 3131 3115 return ufshcd_compose_devman_upiu(hba, lrbp); 3132 3116 } ··· 3340 3320 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3341 3321 * < 0 if another error occurred. 3342 3322 */ 3343 - static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 3344 - const u32 tag, int timeout) 3323 + static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3324 + const u32 tag, int timeout) 3345 3325 { 3326 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3346 3327 int err; 3347 3328 3348 3329 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 3349 - ufshcd_send_command(hba, lrbp, hba->dev_cmd_queue); 3330 + ufshcd_send_command(hba, cmd, hba->dev_cmd_queue); 3350 3331 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); 3351 3332 3352 3333 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, ··· 3372 3351 enum dev_cmd_type cmd_type, int timeout) 3373 3352 { 3374 3353 const u32 tag = hba->reserved_slot; 3375 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 3354 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 3376 3355 int err; 3377 3356 3378 3357 /* Protects use of hba->reserved_slot. */ 3379 3358 lockdep_assert_held(&hba->dev_cmd.lock); 3380 3359 3381 - err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); 3360 + err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); 3382 3361 if (unlikely(err)) 3383 3362 return err; 3384 3363 3385 - return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout); 3364 + return ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); 3386 3365 } 3387 3366 3388 3367 /** ··· 4012 3991 } 4013 3992 4014 3993 skip_utmrdl: 4015 - /* Allocate memory for local reference block */ 4016 - hba->lrb = devm_kcalloc(hba->dev, 4017 - hba->nutrs, sizeof(struct ufshcd_lrb), 4018 - GFP_KERNEL); 4019 - if (!hba->lrb) { 4020 - dev_err(hba->dev, "LRB Memory allocation failed\n"); 4021 - goto out; 4022 - } 4023 3994 return 0; 4024 3995 out: 4025 3996 return -ENOMEM; ··· 5424 5411 5425 5412 /** 5426 5413 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 5427 - * @lrbp: pointer to local reference block of completed command 5414 + * @cmd: SCSI command 5428 5415 * @scsi_status: SCSI command status 5429 5416 * 5430 5417 * Return: value base on SCSI command status. 5431 5418 */ 5432 - static inline int 5433 - ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) 5419 + static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status) 5434 5420 { 5435 5421 int result = 0; 5436 5422 5437 5423 switch (scsi_status) { 5438 5424 case SAM_STAT_CHECK_CONDITION: 5439 - ufshcd_copy_sense_data(lrbp); 5425 + ufshcd_copy_sense_data(cmd); 5440 5426 fallthrough; 5441 5427 case SAM_STAT_GOOD: 5442 5428 result |= DID_OK << 16 | scsi_status; ··· 5443 5431 case SAM_STAT_TASK_SET_FULL: 5444 5432 case SAM_STAT_BUSY: 5445 5433 case SAM_STAT_TASK_ABORTED: 5446 - ufshcd_copy_sense_data(lrbp); 5434 + ufshcd_copy_sense_data(cmd); 5447 5435 result |= scsi_status; 5448 5436 break; 5449 5437 default: ··· 5457 5445 /** 5458 5446 * ufshcd_transfer_rsp_status - Get overall status of the response 5459 5447 * @hba: per adapter instance 5460 - * @lrbp: pointer to local reference block of completed command 5448 + * @cmd: SCSI command 5461 5449 * @cqe: pointer to the completion queue entry 5462 5450 * 5463 5451 * Return: result of the command to notify SCSI midlayer. 5464 5452 */ 5465 - static inline int 5466 - ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 5467 - struct cq_entry *cqe) 5453 + static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, 5454 + struct scsi_cmnd *cmd, 5455 + struct cq_entry *cqe) 5468 5456 { 5457 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5469 5458 int result = 0; 5470 5459 int scsi_status; 5471 5460 enum utp_ocs ocs; ··· 5480 5467 * not set either flag. 5481 5468 */ 5482 5469 if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) 5483 - scsi_set_resid(lrbp->cmd, resid); 5470 + scsi_set_resid(cmd, resid); 5484 5471 5485 5472 /* overall command status of utrd */ 5486 5473 ocs = ufshcd_get_tr_ocs(lrbp, cqe); ··· 5501 5488 * to notify the SCSI midlayer of the command status 5502 5489 */ 5503 5490 scsi_status = lrbp->ucd_rsp_ptr->header.status; 5504 - result = ufshcd_scsi_cmd_status(lrbp, scsi_status); 5491 + result = ufshcd_scsi_cmd_status(cmd, scsi_status); 5505 5492 5506 5493 /* 5507 5494 * Currently we are only supporting BKOPs exception ··· 5566 5553 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { 5567 5554 if (cqe) 5568 5555 ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); 5569 - ufshcd_print_tr(hba, lrbp->task_tag, true); 5556 + ufshcd_print_tr(hba, cmd, true); 5570 5557 } 5571 5558 return result; 5572 5559 } ··· 5633 5620 } 5634 5621 5635 5622 /* Release the resources allocated for processing a SCSI command. */ 5636 - void ufshcd_release_scsi_cmd(struct ufs_hba *hba, 5637 - struct ufshcd_lrb *lrbp) 5623 + void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) 5638 5624 { 5639 - struct scsi_cmnd *cmd = lrbp->cmd; 5640 - 5641 5625 scsi_dma_unmap(cmd); 5642 - ufshcd_crypto_clear_prdt(hba, lrbp); 5626 + ufshcd_crypto_clear_prdt(hba, cmd); 5643 5627 ufshcd_release(hba); 5644 5628 ufshcd_clk_scaling_update_busy(hba); 5645 5629 } ··· 5650 5640 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, 5651 5641 struct cq_entry *cqe) 5652 5642 { 5653 - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 5654 - struct scsi_cmnd *cmd = lrbp->cmd; 5643 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 5644 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5655 5645 enum utp_ocs ocs; 5656 5646 5657 5647 if (hba->monitor.enabled) { 5658 5648 lrbp->compl_time_stamp = ktime_get(); 5659 5649 lrbp->compl_time_stamp_local_clock = local_clock(); 5660 5650 } 5661 - if (cmd) { 5651 + if (ufshcd_is_scsi_cmd(cmd)) { 5662 5652 if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 5663 5653 ufshcd_update_monitor(hba, cmd); 5664 5654 ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); 5665 - cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); 5666 - ufshcd_release_scsi_cmd(hba, lrbp); 5655 + cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); 5656 + ufshcd_release_scsi_cmd(hba, cmd); 5667 5657 /* Do not touch lrbp after scsi done */ 5668 5658 scsi_done(cmd); 5669 5659 } else { ··· 5700 5690 int tag; 5701 5691 5702 5692 for_each_set_bit(tag, completed_reqs, hba->nutrs) { 5703 - struct scsi_cmnd *cmd = hba->lrb[tag].cmd; 5693 + struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag); 5704 5694 5705 5695 if (!cmd) 5706 5696 continue; ··· 5751 5741 struct scsi_device *sdev = rq->q->queuedata; 5752 5742 struct Scsi_Host *shost = sdev->host; 5753 5743 struct ufs_hba *hba = shost_priv(shost); 5754 - struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag]; 5755 5744 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5756 5745 5757 5746 if (!hwq) ··· 5765 5756 scoped_guard(spinlock_irqsave, &hwq->cq_lock) { 5766 5757 if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { 5767 5758 set_host_byte(cmd, DID_REQUEUE); 5768 - ufshcd_release_scsi_cmd(hba, lrbp); 5759 + ufshcd_release_scsi_cmd(hba, cmd); 5769 5760 scsi_done(cmd); 5770 5761 } 5771 5762 } ··· 6650 6641 6651 6642 *ret = ufshcd_try_to_abort_task(hba, tag); 6652 6643 dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, 6653 - hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, 6644 + ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, 6654 6645 *ret ? "failed" : "succeeded"); 6655 6646 6656 6647 return *ret == 0; ··· 7380 7371 enum query_opcode desc_op) 7381 7372 { 7382 7373 const u32 tag = hba->reserved_slot; 7383 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7374 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7375 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7384 7376 int err = 0; 7385 7377 u8 upiu_flags; 7386 7378 7387 7379 /* Protects use of hba->reserved_slot. */ 7388 7380 lockdep_assert_held(&hba->dev_cmd.lock); 7389 7381 7390 - ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); 7382 + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 7391 7383 7392 7384 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 7393 7385 ··· 7413 7403 * bound to fail since dev_cmd.query and dev_cmd.type were left empty. 7414 7404 * read the response directly ignoring all errors. 7415 7405 */ 7416 - ufshcd_issue_dev_cmd(hba, lrbp, tag, dev_cmd_timeout); 7406 + ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); 7417 7407 7418 7408 /* just copy the upiu response as it is */ 7419 7409 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); ··· 7528 7518 enum dma_data_direction dir) 7529 7519 { 7530 7520 const u32 tag = hba->reserved_slot; 7531 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7521 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7522 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7532 7523 int err = 0; 7533 7524 int result; 7534 7525 u8 upiu_flags; ··· 7540 7529 /* Protects use of hba->reserved_slot. */ 7541 7530 ufshcd_dev_man_lock(hba); 7542 7531 7543 - ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag); 7532 + ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, 7533 + tag); 7544 7534 7545 7535 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); 7546 7536 ··· 7558 7546 7559 7547 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7560 7548 7561 - err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT); 7549 + err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); 7562 7550 7563 7551 if (!err) { 7564 7552 /* Just copy the upiu response as it is */ ··· 7659 7647 7660 7648 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) 7661 7649 { 7662 - struct ufshcd_lrb *lrbp; 7663 7650 int tag; 7664 7651 7665 7652 for_each_set_bit(tag, &bitmap, hba->nutrs) { 7666 - lrbp = &hba->lrb[tag]; 7653 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7654 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7655 + 7667 7656 lrbp->req_abort_skip = true; 7668 7657 } 7669 7658 } ··· 7672 7659 /** 7673 7660 * ufshcd_try_to_abort_task - abort a specific task 7674 7661 * @hba: Pointer to adapter instance 7675 - * @tag: Task tag/index to be aborted 7662 + * @tag: Tag of the task to be aborted 7676 7663 * 7677 7664 * Abort the pending command in device by sending UFS_ABORT_TASK task management 7678 7665 * command, and in host controller by clearing the door-bell register. There can ··· 7684 7671 */ 7685 7672 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) 7686 7673 { 7687 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7674 + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7675 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7688 7676 int err; 7689 7677 int poll_cnt; 7690 7678 u8 resp = 0xF; ··· 7707 7693 hba->dev, 7708 7694 "%s: cmd with tag %d not pending in the device.\n", 7709 7695 __func__, tag); 7710 - if (!ufshcd_cmd_inflight(lrbp->cmd)) { 7696 + if (!ufshcd_cmd_inflight(cmd)) { 7711 7697 dev_info(hba->dev, 7712 7698 "%s: cmd with tag=%d completed.\n", 7713 7699 __func__, tag); ··· 7755 7741 struct Scsi_Host *host = cmd->device->host; 7756 7742 struct ufs_hba *hba = shost_priv(host); 7757 7743 int tag = scsi_cmd_to_rq(cmd)->tag; 7758 - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7744 + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7759 7745 unsigned long flags; 7760 7746 int err = FAILED; 7761 7747 bool outstanding; ··· 7790 7776 ufshcd_print_evt_hist(hba); 7791 7777 ufshcd_print_host_state(hba); 7792 7778 ufshcd_print_pwr_info(hba); 7793 - ufshcd_print_tr(hba, tag, true); 7779 + ufshcd_print_tr(hba, cmd, true); 7794 7780 } else { 7795 - ufshcd_print_tr(hba, tag, false); 7781 + ufshcd_print_tr(hba, cmd, false); 7796 7782 } 7797 7783 hba->req_abort_count++; 7798 7784 ··· 7836 7822 goto release; 7837 7823 } 7838 7824 7839 - err = ufshcd_try_to_abort_task(hba, tag); 7825 + err = ufshcd_try_to_abort_task(hba, lrbp->task_tag); 7840 7826 if (err) { 7841 7827 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7842 7828 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); ··· 7853 7839 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7854 7840 7855 7841 if (outstanding) 7856 - ufshcd_release_scsi_cmd(hba, lrbp); 7842 + ufshcd_release_scsi_cmd(hba, cmd); 7857 7843 7858 7844 err = SUCCESS; 7859 7845 ··· 8933 8919 utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; 8934 8920 dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, 8935 8921 hba->utrdl_dma_addr); 8936 - 8937 - devm_kfree(hba->dev, hba->lrb); 8938 8922 } 8939 8923 8940 8924 static int ufshcd_alloc_mcq(struct ufs_hba *hba) ··· 9203 9191 .name = UFSHCD, 9204 9192 .proc_name = UFSHCD, 9205 9193 .map_queues = ufshcd_map_queues, 9194 + .cmd_size = sizeof(struct ufshcd_lrb), 9206 9195 .init_cmd_priv = ufshcd_init_cmd_priv, 9207 9196 .queuecommand = ufshcd_queuecommand, 9208 9197 .nr_reserved_cmds = UFSHCD_NUM_RESERVED,

-5

include/ufs/ufshcd.h

··· 161 161 * @ucd_prdt_dma_addr: PRDT dma address for debug 162 162 * @ucd_rsp_dma_addr: UPIU response dma address for debug 163 163 * @ucd_req_dma_addr: UPIU request dma address for debug 164 - * @cmd: pointer to SCSI command 165 164 * @scsi_status: SCSI status of the command 166 165 * @command_type: SCSI, UFS, Query. 167 166 * @task_tag: Task tag of the command ··· 185 186 dma_addr_t ucd_rsp_dma_addr; 186 187 dma_addr_t ucd_prdt_dma_addr; 187 188 188 - struct scsi_cmnd *cmd; 189 189 int scsi_status; 190 190 191 191 int command_type; ··· 831 833 * @spm_lvl: desired UFS power management level during system PM. 832 834 * @pm_op_in_progress: whether or not a PM operation is in progress. 833 835 * @ahit: value of Auto-Hibernate Idle Timer register. 834 - * @lrb: local reference block 835 836 * @outstanding_tasks: Bits representing outstanding task requests 836 837 * @outstanding_lock: Protects @outstanding_reqs. 837 838 * @outstanding_reqs: Bits representing outstanding transfer requests ··· 972 975 973 976 /* Auto-Hibernate Idle Timer register value */ 974 977 u32 ahit; 975 - 976 - struct ufshcd_lrb *lrb; 977 978 978 979 unsigned long outstanding_tasks; 979 980 spinlock_t outstanding_lock;

Configure Feed

Configure Feed