tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

blk-mq: Replace tags->lock with SRCU for tag iterators

Replace the spinlock in blk_mq_find_and_get_req() with an SRCU read lock
around the tag iterators.

This is done by:

- Holding the SRCU read lock in blk_mq_queue_tag_busy_iter(),
blk_mq_tagset_busy_iter(), and blk_mq_hctx_has_requests().

- Removing the now-redundant tags->lock from blk_mq_find_and_get_req().

This change fixes lockup issue in scsi_host_busy() in case of shost->host_blocked.

Also avoids big tags->lock when reading disk sysfs attribute `inflight`.

Reviewed-by: Hannes Reinecke <hare@suse.de>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Yu Kuai <yukuai3@huawei.com>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Ming Lei and committed by
Jens Axboe 995412e2 135b8521

+12 -24
+8 -4
block/blk-mq-tag.c
··· 256 256 unsigned int bitnr) 257 257 { 258 258 struct request *rq; 259 - unsigned long flags; 260 259 261 - spin_lock_irqsave(&tags->lock, flags); 262 260 rq = tags->rqs[bitnr]; 263 261 if (!rq || rq->tag != bitnr || !req_ref_inc_not_zero(rq)) 264 262 rq = NULL; 265 - spin_unlock_irqrestore(&tags->lock, flags); 266 263 return rq; 267 264 } 268 265 ··· 437 440 busy_tag_iter_fn *fn, void *priv) 438 441 { 439 442 unsigned int flags = tagset->flags; 440 - int i, nr_tags; 443 + int i, nr_tags, srcu_idx; 444 + 445 + srcu_idx = srcu_read_lock(&tagset->tags_srcu); 441 446 442 447 nr_tags = blk_mq_is_shared_tags(flags) ? 1 : tagset->nr_hw_queues; 443 448 ··· 448 449 __blk_mq_all_tag_iter(tagset->tags[i], fn, priv, 449 450 BT_TAG_ITER_STARTED); 450 451 } 452 + srcu_read_unlock(&tagset->tags_srcu, srcu_idx); 451 453 } 452 454 EXPORT_SYMBOL(blk_mq_tagset_busy_iter); 453 455 ··· 499 499 void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, 500 500 void *priv) 501 501 { 502 + int srcu_idx; 503 + 502 504 /* 503 505 * __blk_mq_update_nr_hw_queues() updates nr_hw_queues and hctx_table 504 506 * while the queue is frozen. So we can use q_usage_counter to avoid ··· 509 507 if (!percpu_ref_tryget(&q->q_usage_counter)) 510 508 return; 511 509 510 + srcu_idx = srcu_read_lock(&q->tag_set->tags_srcu); 512 511 if (blk_mq_is_shared_tags(q->tag_set->flags)) { 513 512 struct blk_mq_tags *tags = q->tag_set->shared_tags; 514 513 struct sbitmap_queue *bresv = &tags->breserved_tags; ··· 539 536 bt_for_each(hctx, q, btags, fn, priv, false); 540 537 } 541 538 } 539 + srcu_read_unlock(&q->tag_set->tags_srcu, srcu_idx); 542 540 blk_queue_exit(q); 543 541 } 544 542
+4 -20
block/blk-mq.c
··· 3415 3415 struct blk_mq_tags *tags) 3416 3416 { 3417 3417 struct page *page; 3418 - unsigned long flags; 3419 3418 3420 3419 /* 3421 3420 * There is no need to clear mapping if driver tags is not initialized ··· 3438 3439 } 3439 3440 } 3440 3441 } 3441 - 3442 - /* 3443 - * Wait until all pending iteration is done. 3444 - * 3445 - * Request reference is cleared and it is guaranteed to be observed 3446 - * after the ->lock is released. 3447 - */ 3448 - spin_lock_irqsave(&drv_tags->lock, flags); 3449 - spin_unlock_irqrestore(&drv_tags->lock, flags); 3450 3442 } 3451 3443 3452 3444 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, ··· 3660 3670 struct rq_iter_data data = { 3661 3671 .hctx = hctx, 3662 3672 }; 3673 + int srcu_idx; 3663 3674 3675 + srcu_idx = srcu_read_lock(&hctx->queue->tag_set->tags_srcu); 3664 3676 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data); 3677 + srcu_read_unlock(&hctx->queue->tag_set->tags_srcu, srcu_idx); 3678 + 3665 3679 return data.has_rq; 3666 3680 } 3667 3681 ··· 3885 3891 unsigned int queue_depth, struct request *flush_rq) 3886 3892 { 3887 3893 int i; 3888 - unsigned long flags; 3889 3894 3890 3895 /* The hw queue may not be mapped yet */ 3891 3896 if (!tags) ··· 3894 3901 3895 3902 for (i = 0; i < queue_depth; i++) 3896 3903 cmpxchg(&tags->rqs[i], flush_rq, NULL); 3897 - 3898 - /* 3899 - * Wait until all pending iteration is done. 3900 - * 3901 - * Request reference is cleared and it is guaranteed to be observed 3902 - * after the ->lock is released. 3903 - */ 3904 - spin_lock_irqsave(&tags->lock, flags); 3905 - spin_unlock_irqrestore(&tags->lock, flags); 3906 3904 } 3907 3905 3908 3906 static void blk_free_flush_queue_callback(struct rcu_head *head)