blk-throttle: Split the blkthrotl queue

This patch splits the single queue into separate bps and iops queues. Now,
an IO request must first pass through the bps queue, then the iops queue,
and finally be dispatched. Due to the queue splitting, we need to modify
the throtl add/peek/pop function.

Additionally, the patch modifies the logic related to tg_dispatch_time().
If bio needs to wait for bps, function directly returns the bps wait time;
otherwise, it charges bps and returns the iops wait time so that bio can be
directly placed into the iops queue afterward. Note that this may lead to
more frequent updates to disptime, but the overhead is negligible for the
slow path.

Signed-off-by: Zizhi Wo <wozizhi@huawei.com>
Reviewed-by: Yu Kuai <yukuai3@huawei.com>
Signed-off-by: Zizhi Wo <wozizhi@huaweicloud.com>
Link: https://lore.kernel.org/r/20250506020935.655574-6-wozizhi@huaweicloud.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Zizhi Wo and committed by

Jens Axboe 1 year ago f2c4902b c4da7bf5

+36 -16

2 changed files

expand all

block

blk-throttle.c

blk-throttle.h

+34 -15

block/blk-throttle.c

··· 143 143 static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg) 144 144 { 145 145 INIT_LIST_HEAD(&qn->node); 146 - bio_list_init(&qn->bios); 146 + bio_list_init(&qn->bios_bps); 147 + bio_list_init(&qn->bios_iops); 147 148 qn->tg = tg; 148 149 } 149 150 ··· 161 160 static void throtl_qnode_add_bio(struct bio *bio, struct throtl_qnode *qn, 162 161 struct list_head *queued) 163 162 { 164 - bio_list_add(&qn->bios, bio); 163 + if (bio_flagged(bio, BIO_TG_BPS_THROTTLED)) 164 + bio_list_add(&qn->bios_iops, bio); 165 + else 166 + bio_list_add(&qn->bios_bps, bio); 167 + 165 168 if (list_empty(&qn->node)) { 166 169 list_add_tail(&qn->node, queued); 167 170 blkg_get(tg_to_blkg(qn->tg)); ··· 175 170 /** 176 171 * throtl_peek_queued - peek the first bio on a qnode list 177 172 * @queued: the qnode list to peek 173 + * 174 + * Always take a bio from the head of the iops queue first. If the queue is 175 + * empty, we then take it from the bps queue to maintain the overall idea of 176 + * fetching bios from the head. 178 177 */ 179 178 static struct bio *throtl_peek_queued(struct list_head *queued) 180 179 { ··· 189 180 return NULL; 190 181 191 182 qn = list_first_entry(queued, struct throtl_qnode, node); 192 - bio = bio_list_peek(&qn->bios); 183 + bio = bio_list_peek(&qn->bios_iops); 184 + if (!bio) 185 + bio = bio_list_peek(&qn->bios_bps); 193 186 WARN_ON_ONCE(!bio); 194 187 return bio; 195 188 } ··· 201 190 * @queued: the qnode list to pop a bio from 202 191 * @tg_to_put: optional out argument for throtl_grp to put 203 192 * 204 - * Pop the first bio from the qnode list @queued. After popping, the first 205 - * qnode is removed from @queued if empty or moved to the end of @queued so 206 - * that the popping order is round-robin. 193 + * Pop the first bio from the qnode list @queued. Note that we firstly focus on 194 + * the iops list because bios are ultimately dispatched from it. After popping, 195 + * the first qnode is removed from @queued if empty or moved to the end of 196 + * @queued so that the popping order is round-robin. 207 197 * 208 198 * When the first qnode is removed, its associated throtl_grp should be put 209 199 * too. If @tg_to_put is NULL, this function automatically puts it; ··· 221 209 return NULL; 222 210 223 211 qn = list_first_entry(queued, struct throtl_qnode, node); 224 - bio = bio_list_pop(&qn->bios); 212 + bio = bio_list_pop(&qn->bios_iops); 213 + if (!bio) 214 + bio = bio_list_pop(&qn->bios_bps); 225 215 WARN_ON_ONCE(!bio); 226 216 227 - if (bio_list_empty(&qn->bios)) { 217 + if (bio_list_empty(&qn->bios_bps) && bio_list_empty(&qn->bios_iops)) { 228 218 list_del_init(&qn->node); 229 219 if (tg_to_put) 230 220 *tg_to_put = qn->tg; ··· 870 856 871 857 /* 872 858 * Returns approx number of jiffies to wait before this bio is with-in IO rate 873 - * and can be dispatched. 859 + * and can be moved to other queue or dispatched. 874 860 */ 875 861 static unsigned long tg_dispatch_time(struct throtl_grp *tg, struct bio *bio) 876 862 { 877 863 bool rw = bio_data_dir(bio); 878 - unsigned long bps_wait, iops_wait; 864 + unsigned long wait; 879 865 880 866 /* 881 867 * Currently whole state machine of group depends on first bio ··· 886 872 BUG_ON(tg->service_queue.nr_queued[rw] && 887 873 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); 888 874 889 - bps_wait = tg_dispatch_bps_time(tg, bio); 890 - iops_wait = tg_dispatch_iops_time(tg, bio); 875 + wait = tg_dispatch_bps_time(tg, bio); 876 + if (wait != 0) 877 + return wait; 891 878 892 - return max(bps_wait, iops_wait); 879 + /* 880 + * Charge bps here because @bio will be directly placed into the 881 + * iops queue afterward. 882 + */ 883 + throtl_charge_bps_bio(tg, bio); 884 + 885 + return tg_dispatch_iops_time(tg, bio); 893 886 } 894 887 895 888 /** ··· 985 964 bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put); 986 965 sq->nr_queued[rw]--; 987 966 988 - throtl_charge_bps_bio(tg, bio); 989 967 throtl_charge_iops_bio(tg, bio); 990 968 991 969 /* ··· 1712 1692 while (true) { 1713 1693 if (tg_within_limit(tg, bio, rw)) { 1714 1694 /* within limits, let's charge and dispatch directly */ 1715 - throtl_charge_bps_bio(tg, bio); 1716 1695 throtl_charge_iops_bio(tg, bio); 1717 1696 1718 1697 /*

+2 -1

block/blk-throttle.h

··· 29 29 */ 30 30 struct throtl_qnode { 31 31 struct list_head node; /* service_queue->queued[] */ 32 - struct bio_list bios; /* queued bios */ 32 + struct bio_list bios_bps; /* queued bios for bps limit */ 33 + struct bio_list bios_iops; /* queued bios for iops limit */ 33 34 struct throtl_grp *tg; /* tg this qnode belongs to */ 34 35 }; 35 36

Configure Feed

Configure Feed