Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
block, bfq: turn bfqq_data into an array in bfq_io_cq
When a bfq_queue Q is merged with another queue, several pieces of
information are saved about Q. These pieces are stored in the
bfqq_data field in the bfq_io_cq data structure of the process
associated with Q.
Yet, with a multi-actuator drive, a process may get associated with
multiple bfq_queues: one queue for each of the N actuators. Each of
these queues may undergo a merge. So, the bfq_io_cq data structure
must be able to accommodate the above information for N queues.
This commit solves this problem by turning the bfqq_data scalar field
into an array of N elements (and by changing code so as to handle
this array).
This solution is written under the assumption that bfq_queues
associated with different actuators cannot be cross-merged. This
assumption holds naturally with basic queue merging: the latter is
triggered by spatial locality, and sectors for different actuators are
not close to each other (apart from the corner case of the last
sectors served by a given actuator and the first sectors served by the
next actuator). As for stable cross-merging, the assumption here is
that it is disabled.
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Signed-off-by: Gabriele Felici <felicigb@gmail.com>
Signed-off-by: Gianmarco Lusvardi <glusvardi@posteo.net>
Signed-off-by: Giulio Barabino <giuliobarabino99@gmail.com>
Signed-off-by: Emiliano Maccaferri <inbox@emilianomaccaferri.com>
Signed-off-by: Paolo Valente <paolo.valente@linaro.org>
Link: https://lore.kernel.org/r/20230103145503.71712-5-paolo.valente@linaro.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Paolo Valente
and committed by
Jens Axboe
fd571df0
a6123047
+67
-45
+59
-41
block/bfq-iosched.c
+59
-41
block/bfq-iosched.c
···
408
408
* we cancel the stable merge if
409
409
* bic->stable_merge_bfqq == bfqq.
410
410
*/
411
-
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
411
+
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[actuator_idx];
412
412
413
413
/* Clear bic pointer if bfqq is detached from this bic */
414
414
if (old_bfqq && old_bfqq->bic == bic)
···
1135
1135
bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd,
1136
1136
struct bfq_io_cq *bic, bool bfq_already_existing)
1137
1137
{
1138
-
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
1139
1138
unsigned int old_wr_coeff = 1;
1140
1139
bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq);
1140
+
unsigned int a_idx = bfqq->actuator_idx;
1141
+
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
1141
1142
1142
1143
if (bfqq_data->saved_has_short_ttime)
1143
1144
bfq_mark_bfqq_has_short_ttime(bfqq);
···
1819
1818
arrived_in_time = ktime_get_ns() <=
1820
1819
bfqq->ttime.last_end_request +
1821
1820
bfqd->bfq_slice_idle * 3;
1822
-
1821
+
unsigned int act_idx = bfq_actuator_index(bfqd, rq->bio);
1822
+
bool bfqq_non_merged_or_stably_merged =
1823
+
bfqq->bic || RQ_BIC(rq)->bfqq_data[act_idx].stably_merged;
1823
1824
1824
1825
/*
1825
1826
* bfqq deserves to be weight-raised if:
···
1855
1852
*/
1856
1853
wr_or_deserves_wr = bfqd->low_latency &&
1857
1854
(bfqq->wr_coeff > 1 ||
1858
-
(bfq_bfqq_sync(bfqq) &&
1859
-
(bfqq->bic || RQ_BIC(rq)->bfqq_data.stably_merged) &&
1860
-
(*interactive || soft_rt)));
1855
+
(bfq_bfqq_sync(bfqq) && bfqq_non_merged_or_stably_merged &&
1856
+
(*interactive || soft_rt)));
1861
1857
1862
1858
/*
1863
1859
* Using the last flag, update budget and check whether bfqq
···
2830
2828
static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd,
2831
2829
struct bfq_queue *bfqq);
2832
2830
2831
+
static struct bfq_queue *
2832
+
bfq_setup_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq,
2833
+
struct bfq_queue *stable_merge_bfqq,
2834
+
struct bfq_iocq_bfqq_data *bfqq_data)
2835
+
{
2836
+
int proc_ref = min(bfqq_process_refs(bfqq),
2837
+
bfqq_process_refs(stable_merge_bfqq));
2838
+
struct bfq_queue *new_bfqq;
2839
+
2840
+
if (idling_boosts_thr_without_issues(bfqd, bfqq) ||
2841
+
proc_ref == 0)
2842
+
return NULL;
2843
+
2844
+
/* next function will take at least one ref */
2845
+
new_bfqq = bfq_setup_merge(bfqq, stable_merge_bfqq);
2846
+
2847
+
if (new_bfqq) {
2848
+
bfqq_data->stably_merged = true;
2849
+
if (new_bfqq->bic) {
2850
+
unsigned int new_a_idx = new_bfqq->actuator_idx;
2851
+
struct bfq_iocq_bfqq_data *new_bfqq_data =
2852
+
&new_bfqq->bic->bfqq_data[new_a_idx];
2853
+
2854
+
new_bfqq_data->stably_merged = true;
2855
+
}
2856
+
}
2857
+
return new_bfqq;
2858
+
}
2859
+
2833
2860
/*
2834
2861
* Attempt to schedule a merge of bfqq with the currently in-service
2835
2862
* queue or with a close queue among the scheduled queues. Return
···
2884
2853
void *io_struct, bool request, struct bfq_io_cq *bic)
2885
2854
{
2886
2855
struct bfq_queue *in_service_bfqq, *new_bfqq;
2887
-
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
2856
+
unsigned int a_idx = bfqq->actuator_idx;
2857
+
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
2888
2858
2889
2859
/* if a merge has already been setup, then proceed with that first */
2890
2860
if (bfqq->new_bfqq)
···
2915
2883
msecs_to_jiffies(bfq_late_stable_merging))) {
2916
2884
struct bfq_queue *stable_merge_bfqq =
2917
2885
bfqq_data->stable_merge_bfqq;
2918
-
int proc_ref = min(bfqq_process_refs(bfqq),
2919
-
bfqq_process_refs(stable_merge_bfqq));
2920
2886
2921
2887
/* deschedule stable merge, because done or aborted here */
2922
2888
bfq_put_stable_ref(stable_merge_bfqq);
2923
2889
2924
2890
bfqq_data->stable_merge_bfqq = NULL;
2925
2891
2926
-
if (!idling_boosts_thr_without_issues(bfqd, bfqq) &&
2927
-
proc_ref > 0) {
2928
-
/* next function will take at least one ref */
2929
-
struct bfq_queue *new_bfqq =
2930
-
bfq_setup_merge(bfqq, stable_merge_bfqq);
2931
-
2932
-
if (new_bfqq) {
2933
-
bfqq_data->stably_merged = true;
2934
-
if (new_bfqq->bic)
2935
-
new_bfqq->bic->bfqq_data.stably_merged =
2936
-
true;
2937
-
}
2938
-
return new_bfqq;
2939
-
} else
2940
-
return NULL;
2892
+
return bfq_setup_stable_merge(bfqd, bfqq,
2893
+
stable_merge_bfqq,
2894
+
bfqq_data);
2941
2895
}
2942
2896
}
2943
2897
···
3018
3000
static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
3019
3001
{
3020
3002
struct bfq_io_cq *bic = bfqq->bic;
3021
-
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
3003
+
unsigned int a_idx = bfqq->actuator_idx;
3004
+
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
3022
3005
3023
3006
/*
3024
3007
* If !bfqq->bic, the queue is already shared or its requests
···
3030
3011
return;
3031
3012
3032
3013
bfqq_data->saved_last_serv_time_ns = bfqq->last_serv_time_ns;
3033
-
bfqq_data->saved_inject_limit = bfqq->inject_limit;
3014
+
bfqq_data->saved_inject_limit = bfqq->inject_limit;
3034
3015
bfqq_data->saved_decrease_time_jif = bfqq->decrease_time_jif;
3035
3016
3036
3017
bfqq_data->saved_weight = bfqq->entity.orig_weight;
···
5396
5377
* therefore on its unused per-actuator fields being NULL.
5397
5378
*/
5398
5379
unsigned int num_actuators = BFQ_MAX_ACTUATORS;
5399
-
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
5380
+
struct bfq_iocq_bfqq_data *bfqq_data = bic->bfqq_data;
5400
5381
5401
5382
/*
5402
5383
* bfqd is NULL if scheduler already exited, and in that case
···
5407
5388
num_actuators = bfqd->num_actuators;
5408
5389
}
5409
5390
5410
-
if (bfqq_data->stable_merge_bfqq)
5411
-
bfq_put_stable_ref(bfqq_data->stable_merge_bfqq);
5412
-
5413
5391
for (act_idx = 0; act_idx < num_actuators; act_idx++) {
5392
+
if (bfqq_data[act_idx].stable_merge_bfqq)
5393
+
bfq_put_stable_ref(bfqq_data[act_idx].stable_merge_bfqq);
5394
+
5414
5395
bfq_exit_icq_bfqq(bic, true, act_idx);
5415
5396
bfq_exit_icq_bfqq(bic, false, act_idx);
5416
5397
}
···
5597
5578
struct bfq_io_cq *bic,
5598
5579
struct bfq_queue *last_bfqq_created)
5599
5580
{
5581
+
unsigned int a_idx = last_bfqq_created->actuator_idx;
5600
5582
struct bfq_queue *new_bfqq =
5601
5583
bfq_setup_merge(bfqq, last_bfqq_created);
5602
-
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
5603
5584
5604
5585
if (!new_bfqq)
5605
5586
return bfqq;
5606
5587
5607
5588
if (new_bfqq->bic)
5608
-
new_bfqq->bic->bfqq_data.stably_merged = true;
5609
-
bfqq_data->stably_merged = true;
5589
+
new_bfqq->bic->bfqq_data[a_idx].stably_merged = true;
5590
+
bic->bfqq_data[a_idx].stably_merged = true;
5610
5591
5611
5592
/*
5612
5593
* Reusing merge functions. This implies that
···
5675
5656
&bfqd->last_bfqq_created;
5676
5657
5677
5658
struct bfq_queue *last_bfqq_created = *source_bfqq;
5678
-
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
5679
5659
5680
5660
/*
5681
5661
* If last_bfqq_created has not been set yet, then init it. If
···
5736
5718
/*
5737
5719
* Record the bfqq to merge to.
5738
5720
*/
5739
-
bfqq_data->stable_merge_bfqq = last_bfqq_created;
5721
+
bic->bfqq_data[last_bfqq_created->actuator_idx].stable_merge_bfqq =
5722
+
last_bfqq_created;
5740
5723
}
5741
5724
}
5742
5725
···
6658
6639
{
6659
6640
unsigned int act_idx = bfq_actuator_index(bfqd, bio);
6660
6641
struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, act_idx);
6661
-
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
6642
+
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[act_idx];
6662
6643
6663
6644
if (likely(bfqq && bfqq != &bfqd->oom_bfqq))
6664
6645
return bfqq;
···
6766
6747
struct bfq_queue *bfqq;
6767
6748
bool new_queue = false;
6768
6749
bool bfqq_already_existing = false, split = false;
6769
-
struct bfq_iocq_bfqq_data *bfqq_data;
6750
+
unsigned int a_idx = bfq_actuator_index(bfqd, bio);
6770
6751
6771
6752
if (unlikely(!rq->elv.icq))
6772
6753
return NULL;
···
6790
6771
bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,
6791
6772
&new_queue);
6792
6773
6793
-
bfqq_data = &bic->bfqq_data;
6794
-
6795
6774
if (likely(!new_queue)) {
6796
6775
/* If the queue was seeky for too long, break it apart. */
6797
6776
if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq) &&
6798
-
!bfqq_data->stably_merged) {
6777
+
!bic->bfqq_data[a_idx].stably_merged) {
6799
6778
struct bfq_queue *old_bfqq = bfqq;
6800
6779
6801
6780
/* Update bic before losing reference to bfqq */
6802
6781
if (bfq_bfqq_in_large_burst(bfqq))
6803
-
bfqq_data->saved_in_large_burst = true;
6782
+
bic->bfqq_data[a_idx].saved_in_large_burst =
6783
+
true;
6804
6784
6805
6785
bfqq = bfq_split_bfqq(bic, bfqq);
6806
6786
split = true;
+8
-4
block/bfq-iosched.h
+8
-4
block/bfq-iosched.h
···
418
418
struct bfq_iocq_bfqq_data {
419
419
/*
420
420
* Snapshot of the has_short_time flag before merging; taken
421
-
* to remember its value while the queue is merged, so as to
421
+
* to remember its values while the queue is merged, so as to
422
422
* be able to restore it in case of split.
423
423
*/
424
424
bool saved_has_short_ttime;
···
432
432
u64 saved_tot_idle_time;
433
433
434
434
/*
435
-
* Same purpose as the previous fields for the value of the
435
+
* Same purpose as the previous fields for the values of the
436
436
* field keeping the queue's belonging to a large burst
437
437
*/
438
438
bool saved_in_large_burst;
···
495
495
uint64_t blkcg_serial_nr; /* the current blkcg serial */
496
496
#endif
497
497
498
-
/* persistent data for associated synchronous process queue */
499
-
struct bfq_iocq_bfqq_data bfqq_data;
498
+
/*
499
+
* Persistent data for associated synchronous process queues
500
+
* (one queue per actuator, see field bfqq above). In
501
+
* particular, each of these queues may undergo a merge.
502
+
*/
503
+
struct bfq_iocq_bfqq_data bfqq_data[BFQ_MAX_ACTUATORS];
500
504
501
505
unsigned int requests; /* Number of requests this process has in flight */
502
506
};