tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

scsi: ufs: Implement polling support

The time spent in io_schedule() and also the interrupt latency are
significant when submitting direct I/O to a UFS device. Hence this patch
that implements polling support. User space software can enable polling by
passing the RWF_HIPRI flag to the preadv2() system call or the
IORING_SETUP_IOPOLL flag to the io_uring interface.

Although the block layer supports to partition the tag space for
interrupt-based completions (HCTX_TYPE_DEFAULT) purposes and polling
(HCTX_TYPE_POLL), the choice has been made to use the same hardware queue
for both hctx types because partitioning the tag space would negatively
affect performance.

On my test setup this patch increases IOPS from 2736 to 22000 (8x) for the
following test:

for hipri in 0 1; do
fio --ioengine=io_uring --iodepth=1 --rw=randread \
--runtime=60 --time_based=1 --direct=1 --name=qd1 \
--filename=/dev/block/sda --ioscheduler=none --gtod_reduce=1 \
--norandommap --hipri=$hipri
done

Link: https://lore.kernel.org/r/20211203231950.193369-18-bvanassche@acm.org
Tested-by: Bean Huo <beanhuo@micron.com>
Reviewed-by: Bean Huo <beanhuo@micron.com>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

authored by

Bart Van Assche and committed by
Martin K. Petersen eaab9b57 8d077ede

+67 -18
+67 -18
drivers/scsi/ufs/ufshcd.c
··· 2662 2662 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN); 2663 2663 } 2664 2664 2665 + /* 2666 + * Associate the UFS controller queue with the default and poll HCTX types. 2667 + * Initialize the mq_map[] arrays. 2668 + */ 2669 + static int ufshcd_map_queues(struct Scsi_Host *shost) 2670 + { 2671 + int i, ret; 2672 + 2673 + for (i = 0; i < shost->nr_maps; i++) { 2674 + struct blk_mq_queue_map *map = &shost->tag_set.map[i]; 2675 + 2676 + switch (i) { 2677 + case HCTX_TYPE_DEFAULT: 2678 + case HCTX_TYPE_POLL: 2679 + map->nr_queues = 1; 2680 + break; 2681 + case HCTX_TYPE_READ: 2682 + map->nr_queues = 0; 2683 + break; 2684 + default: 2685 + WARN_ON_ONCE(true); 2686 + } 2687 + map->queue_offset = 0; 2688 + ret = blk_mq_map_queues(map); 2689 + WARN_ON_ONCE(ret); 2690 + } 2691 + 2692 + return 0; 2693 + } 2694 + 2665 2695 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) 2666 2696 { 2667 2697 struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr; ··· 2727 2697 struct ufshcd_lrb *lrbp; 2728 2698 int err = 0; 2729 2699 2730 - WARN_ONCE(tag < 0, "Invalid tag %d\n", tag); 2700 + WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag); 2731 2701 2732 2702 /* 2733 2703 * Allows the UFS error handler to wait for prior ufshcd_queuecommand() ··· 5318 5288 } 5319 5289 } 5320 5290 5291 + /* 5292 + * Returns > 0 if one or more commands have been completed or 0 if no 5293 + * requests have been completed. 5294 + */ 5295 + static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) 5296 + { 5297 + struct ufs_hba *hba = shost_priv(shost); 5298 + unsigned long completed_reqs, flags; 5299 + u32 tr_doorbell; 5300 + 5301 + spin_lock_irqsave(&hba->outstanding_lock, flags); 5302 + tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 5303 + completed_reqs = ~tr_doorbell & hba->outstanding_reqs; 5304 + WARN_ONCE(completed_reqs & ~hba->outstanding_reqs, 5305 + "completed: %#lx; outstanding: %#lx\n", completed_reqs, 5306 + hba->outstanding_reqs); 5307 + hba->outstanding_reqs &= ~completed_reqs; 5308 + spin_unlock_irqrestore(&hba->outstanding_lock, flags); 5309 + 5310 + if (completed_reqs) 5311 + __ufshcd_transfer_req_compl(hba, completed_reqs); 5312 + 5313 + return completed_reqs; 5314 + } 5315 + 5321 5316 /** 5322 5317 * ufshcd_transfer_req_compl - handle SCSI and query command completion 5323 5318 * @hba: per adapter instance ··· 5353 5298 */ 5354 5299 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) 5355 5300 { 5356 - unsigned long completed_reqs, flags; 5357 - u32 tr_doorbell; 5358 - 5359 5301 /* Resetting interrupt aggregation counters first and reading the 5360 5302 * DOOR_BELL afterward allows us to handle all the completed requests. 5361 5303 * In order to prevent other interrupts starvation the DB is read once ··· 5367 5315 if (ufs_fail_completion()) 5368 5316 return IRQ_HANDLED; 5369 5317 5370 - spin_lock_irqsave(&hba->outstanding_lock, flags); 5371 - tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 5372 - completed_reqs = ~tr_doorbell & hba->outstanding_reqs; 5373 - WARN_ONCE(completed_reqs & ~hba->outstanding_reqs, 5374 - "completed: %#lx; outstanding: %#lx\n", completed_reqs, 5375 - hba->outstanding_reqs); 5376 - hba->outstanding_reqs &= ~completed_reqs; 5377 - spin_unlock_irqrestore(&hba->outstanding_lock, flags); 5318 + /* 5319 + * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we 5320 + * do not want polling to trigger spurious interrupt complaints. 5321 + */ 5322 + ufshcd_poll(hba->host, 0); 5378 5323 5379 - if (completed_reqs) { 5380 - __ufshcd_transfer_req_compl(hba, completed_reqs); 5381 - return IRQ_HANDLED; 5382 - } else { 5383 - return IRQ_NONE; 5384 - } 5324 + return IRQ_HANDLED; 5385 5325 } 5386 5326 5387 5327 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask) ··· 6625 6581 spin_lock_irqsave(host->host_lock, flags); 6626 6582 6627 6583 task_tag = req->tag; 6584 + WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n", 6585 + task_tag); 6628 6586 hba->tmf_rqs[req->tag] = req; 6629 6587 treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag); 6630 6588 ··· 8190 8144 .module = THIS_MODULE, 8191 8145 .name = UFSHCD, 8192 8146 .proc_name = UFSHCD, 8147 + .map_queues = ufshcd_map_queues, 8193 8148 .queuecommand = ufshcd_queuecommand, 8149 + .mq_poll = ufshcd_poll, 8194 8150 .slave_alloc = ufshcd_slave_alloc, 8195 8151 .slave_configure = ufshcd_slave_configure, 8196 8152 .slave_destroy = ufshcd_slave_destroy, ··· 9480 9432 err = -ENOMEM; 9481 9433 goto out_error; 9482 9434 } 9435 + host->nr_maps = HCTX_TYPE_POLL + 1; 9483 9436 hba = shost_priv(host); 9484 9437 hba->host = host; 9485 9438 hba->dev = dev;