block: don't update BLK_FEAT_POLL in __blk_mq_update_nr_hw_queues

When __blk_mq_update_nr_hw_queues changes the number of tag sets, it
might have to disable poll queues. Currently it does so by adjusting
the BLK_FEAT_POLL, which is a bit against the intent of features that
describe hardware / driver capabilities, but more importantly causes
nasty lock order problems with the broadly held freeze when updating the
number of hardware queues and the limits lock. Fix this by leaving
BLK_FEAT_POLL alone, and instead check for the number of poll queues in
the bio submission and poll handlers. While this adds extra work to the
fast path, the variables are in cache lines used by these operations
anyway, so it should be cheap enough.

Fixes: 8023e144f9d6 ("block: move the poll flag to queue_limits")
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Damien Le Moal <dlemoal@kernel.org>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
Reviewed-by: Nilay Shroff <nilay@linux.ibm.com>
Link: https://lore.kernel.org/r/20250110054726.1499538-5-hch@lst.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Christoph Hellwig and committed by

Jens Axboe 1 year ago d432c817 958148a6

+22 -26

4 changed files

expand all

block

blk-core.c

blk-mq.c

blk-mq.h

blk-sysfs.c

+3 -4

block/blk-core.c

··· 951 951 */ 952 952 if (!percpu_ref_tryget(&q->q_usage_counter)) 953 953 return 0; 954 - if (!(q->limits.features & BLK_FEAT_POLL)) { 955 - ret = 0; 956 - } else if (queue_is_mq(q)) { 954 + if (queue_is_mq(q)) { 957 955 ret = blk_mq_poll(q, cookie, iob, flags); 958 956 } else { 959 957 struct gendisk *disk = q->disk; 960 958 961 - if (disk && disk->fops->poll_bio) 959 + if ((q->limits.features & BLK_FEAT_POLL) && disk && 960 + disk->fops->poll_bio) 962 961 ret = disk->fops->poll_bio(bio, iob, flags); 963 962 } 964 963 blk_queue_exit(q);

+5 -21

block/blk-mq.c

··· 3105 3105 goto queue_exit; 3106 3106 } 3107 3107 3108 - if ((bio->bi_opf & REQ_POLLED) && 3109 - !(q->limits.features & BLK_FEAT_POLL)) { 3108 + if ((bio->bi_opf & REQ_POLLED) && !blk_mq_can_poll(q)) { 3110 3109 bio->bi_status = BLK_STS_NOTSUPP; 3111 3110 bio_endio(bio); 3112 3111 goto queue_exit; ··· 4327 4328 blk_mq_sysfs_deinit(q); 4328 4329 } 4329 4330 4330 - static bool blk_mq_can_poll(struct blk_mq_tag_set *set) 4331 - { 4332 - return set->nr_maps > HCTX_TYPE_POLL && 4333 - set->map[HCTX_TYPE_POLL].nr_queues; 4334 - } 4335 - 4336 4331 struct request_queue *blk_mq_alloc_queue(struct blk_mq_tag_set *set, 4337 4332 struct queue_limits *lim, void *queuedata) 4338 4333 { ··· 4337 4344 if (!lim) 4338 4345 lim = &default_lim; 4339 4346 lim->features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT; 4340 - if (blk_mq_can_poll(set)) 4347 + if (set->nr_maps > HCTX_TYPE_POLL) 4341 4348 lim->features |= BLK_FEAT_POLL; 4342 4349 4343 4350 q = blk_alloc_queue(lim, set->numa_node); ··· 5025 5032 fallback: 5026 5033 blk_mq_update_queue_map(set); 5027 5034 list_for_each_entry(q, &set->tag_list, tag_set_list) { 5028 - struct queue_limits lim; 5029 - 5030 5035 blk_mq_realloc_hw_ctxs(set, q); 5031 5036 5032 5037 if (q->nr_hw_queues != set->nr_hw_queues) { ··· 5038 5047 set->nr_hw_queues = prev_nr_hw_queues; 5039 5048 goto fallback; 5040 5049 } 5041 - lim = queue_limits_start_update(q); 5042 - if (blk_mq_can_poll(set)) 5043 - lim.features |= BLK_FEAT_POLL; 5044 - else 5045 - lim.features &= ~BLK_FEAT_POLL; 5046 - if (queue_limits_commit_update(q, &lim) < 0) 5047 - pr_warn("updating the poll flag failed\n"); 5048 5050 blk_mq_map_swqueue(q); 5049 5051 } 5050 5052 ··· 5097 5113 int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, 5098 5114 struct io_comp_batch *iob, unsigned int flags) 5099 5115 { 5100 - struct blk_mq_hw_ctx *hctx = xa_load(&q->hctx_table, cookie); 5101 - 5102 - return blk_hctx_poll(q, hctx, iob, flags); 5116 + if (!blk_mq_can_poll(q)) 5117 + return 0; 5118 + return blk_hctx_poll(q, xa_load(&q->hctx_table, cookie), iob, flags); 5103 5119 } 5104 5120 5105 5121 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,

block/blk-mq.h

··· 448 448 #define blk_mq_run_dispatch_ops(q, dispatch_ops) \ 449 449 __blk_mq_run_dispatch_ops(q, true, dispatch_ops) \ 450 450 451 + static inline bool blk_mq_can_poll(struct request_queue *q) 452 + { 453 + return (q->limits.features & BLK_FEAT_POLL) && 454 + q->tag_set->map[HCTX_TYPE_POLL].nr_queues; 455 + } 456 + 451 457 #endif

+8 -1

block/blk-sysfs.c

··· 245 245 !!(disk->queue->limits.features & _feature)); \ 246 246 } 247 247 248 - QUEUE_SYSFS_FEATURE_SHOW(poll, BLK_FEAT_POLL); 249 248 QUEUE_SYSFS_FEATURE_SHOW(fua, BLK_FEAT_FUA); 250 249 QUEUE_SYSFS_FEATURE_SHOW(dax, BLK_FEAT_DAX); 250 + 251 + static ssize_t queue_poll_show(struct gendisk *disk, char *page) 252 + { 253 + if (queue_is_mq(disk->queue)) 254 + return sysfs_emit(page, "%u\n", blk_mq_can_poll(disk->queue)); 255 + return sysfs_emit(page, "%u\n", 256 + !!(disk->queue->limits.features & BLK_FEAT_POLL)); 257 + } 251 258 252 259 static ssize_t queue_zoned_show(struct gendisk *disk, char *page) 253 260 {

Configure Feed

Configure Feed