block, bfq: move io_cq-persistent bfqq data into a dedicated struct

+109 -79

2 changed files

expand all

block

bfq-iosched.c

bfq-iosched.h

+78 -58

block/bfq-iosched.c

··· 395 395 { 396 396 struct bfq_queue *old_bfqq = bic->bfqq[is_sync][actuator_idx]; 397 397 398 - /* Clear bic pointer if bfqq is detached from this bic */ 399 - if (old_bfqq && old_bfqq->bic == bic) 400 - old_bfqq->bic = NULL; 401 - 402 398 /* 403 399 * If bfqq != NULL, then a non-stable queue merge between 404 400 * bic->bfqq and bfqq is happening here. This causes troubles ··· 408 412 * we cancel the stable merge if 409 413 * bic->stable_merge_bfqq == bfqq. 410 414 */ 415 + struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data; 416 + 417 + /* Clear bic pointer if bfqq is detached from this bic */ 418 + if (old_bfqq && old_bfqq->bic == bic) 419 + old_bfqq->bic = NULL; 420 + 411 421 if (is_sync) 412 422 bic->bfqq[1][actuator_idx] = bfqq; 413 423 else 414 424 bic->bfqq[0][actuator_idx] = bfqq; 415 425 416 - if (bfqq && bic->stable_merge_bfqq == bfqq) { 426 + if (bfqq && bfqq_data->stable_merge_bfqq == bfqq) { 417 427 /* 418 428 * Actually, these same instructions are executed also 419 429 * in bfq_setup_cooperator, in case of abort or actual ··· 428 426 * did so, we would nest even more complexity in this 429 427 * function. 430 428 */ 431 - bfq_put_stable_ref(bic->stable_merge_bfqq); 429 + bfq_put_stable_ref(bfqq_data->stable_merge_bfqq); 432 430 433 - bic->stable_merge_bfqq = NULL; 431 + bfqq_data->stable_merge_bfqq = NULL; 434 432 } 435 433 } 436 434 ··· 1135 1133 bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, 1136 1134 struct bfq_io_cq *bic, bool bfq_already_existing) 1137 1135 { 1136 + struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data; 1138 1137 unsigned int old_wr_coeff = 1; 1139 1138 bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq); 1140 1139 1141 - if (bic->saved_has_short_ttime) 1140 + if (bfqq_data->saved_has_short_ttime) 1142 1141 bfq_mark_bfqq_has_short_ttime(bfqq); 1143 1142 else 1144 1143 bfq_clear_bfqq_has_short_ttime(bfqq); 1145 1144 1146 - if (bic->saved_IO_bound) 1145 + if (bfqq_data->saved_IO_bound) 1147 1146 bfq_mark_bfqq_IO_bound(bfqq); 1148 1147 else 1149 1148 bfq_clear_bfqq_IO_bound(bfqq); 1150 1149 1151 - bfqq->last_serv_time_ns = bic->saved_last_serv_time_ns; 1152 - bfqq->inject_limit = bic->saved_inject_limit; 1153 - bfqq->decrease_time_jif = bic->saved_decrease_time_jif; 1150 + bfqq->last_serv_time_ns = bfqq_data->saved_last_serv_time_ns; 1151 + bfqq->inject_limit = bfqq_data->saved_inject_limit; 1152 + bfqq->decrease_time_jif = bfqq_data->saved_decrease_time_jif; 1154 1153 1155 - bfqq->entity.new_weight = bic->saved_weight; 1156 - bfqq->ttime = bic->saved_ttime; 1157 - bfqq->io_start_time = bic->saved_io_start_time; 1158 - bfqq->tot_idle_time = bic->saved_tot_idle_time; 1154 + bfqq->entity.new_weight = bfqq_data->saved_weight; 1155 + bfqq->ttime = bfqq_data->saved_ttime; 1156 + bfqq->io_start_time = bfqq_data->saved_io_start_time; 1157 + bfqq->tot_idle_time = bfqq_data->saved_tot_idle_time; 1159 1158 /* 1160 1159 * Restore weight coefficient only if low_latency is on 1161 1160 */ 1162 1161 if (bfqd->low_latency) { 1163 1162 old_wr_coeff = bfqq->wr_coeff; 1164 - bfqq->wr_coeff = bic->saved_wr_coeff; 1163 + bfqq->wr_coeff = bfqq_data->saved_wr_coeff; 1165 1164 } 1166 - bfqq->service_from_wr = bic->saved_service_from_wr; 1167 - bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt; 1168 - bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish; 1169 - bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time; 1165 + bfqq->service_from_wr = bfqq_data->saved_service_from_wr; 1166 + bfqq->wr_start_at_switch_to_srt = 1167 + bfqq_data->saved_wr_start_at_switch_to_srt; 1168 + bfqq->last_wr_start_finish = bfqq_data->saved_last_wr_start_finish; 1169 + bfqq->wr_cur_max_time = bfqq_data->saved_wr_cur_max_time; 1170 1170 1171 1171 if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) || 1172 1172 time_is_before_jiffies(bfqq->last_wr_start_finish + ··· 1853 1849 wr_or_deserves_wr = bfqd->low_latency && 1854 1850 (bfqq->wr_coeff > 1 || 1855 1851 (bfq_bfqq_sync(bfqq) && 1856 - (bfqq->bic || RQ_BIC(rq)->stably_merged) && 1852 + (bfqq->bic || RQ_BIC(rq)->bfqq_data.stably_merged) && 1857 1853 (*interactive || soft_rt))); 1858 1854 1859 1855 /* ··· 2853 2849 void *io_struct, bool request, struct bfq_io_cq *bic) 2854 2850 { 2855 2851 struct bfq_queue *in_service_bfqq, *new_bfqq; 2852 + struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data; 2856 2853 2857 2854 /* if a merge has already been setup, then proceed with that first */ 2858 2855 if (bfqq->new_bfqq) ··· 2875 2870 * stable merging) also if bic is associated with a 2876 2871 * sync queue, but this bfqq is async 2877 2872 */ 2878 - if (bfq_bfqq_sync(bfqq) && bic->stable_merge_bfqq && 2873 + if (bfq_bfqq_sync(bfqq) && bfqq_data->stable_merge_bfqq && 2879 2874 !bfq_bfqq_just_created(bfqq) && 2880 2875 time_is_before_jiffies(bfqq->split_time + 2881 2876 msecs_to_jiffies(bfq_late_stable_merging)) && 2882 2877 time_is_before_jiffies(bfqq->creation_time + 2883 2878 msecs_to_jiffies(bfq_late_stable_merging))) { 2884 2879 struct bfq_queue *stable_merge_bfqq = 2885 - bic->stable_merge_bfqq; 2880 + bfqq_data->stable_merge_bfqq; 2886 2881 int proc_ref = min(bfqq_process_refs(bfqq), 2887 2882 bfqq_process_refs(stable_merge_bfqq)); 2888 2883 2889 2884 /* deschedule stable merge, because done or aborted here */ 2890 2885 bfq_put_stable_ref(stable_merge_bfqq); 2891 2886 2892 - bic->stable_merge_bfqq = NULL; 2887 + bfqq_data->stable_merge_bfqq = NULL; 2893 2888 2894 2889 if (!idling_boosts_thr_without_issues(bfqd, bfqq) && 2895 2890 proc_ref > 0) { ··· 2898 2893 bfq_setup_merge(bfqq, stable_merge_bfqq); 2899 2894 2900 2895 if (new_bfqq) { 2901 - bic->stably_merged = true; 2896 + bfqq_data->stably_merged = true; 2902 2897 if (new_bfqq->bic) 2903 - new_bfqq->bic->stably_merged = 2904 - true; 2898 + new_bfqq->bic->bfqq_data.stably_merged = 2899 + true; 2905 2900 } 2906 2901 return new_bfqq; 2907 2902 } else ··· 3000 2995 static void bfq_bfqq_save_state(struct bfq_queue *bfqq) 3001 2996 { 3002 2997 struct bfq_io_cq *bic = bfqq->bic; 2998 + struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data; 3003 2999 3004 3000 /* 3005 3001 * If !bfqq->bic, the queue is already shared or its requests ··· 3010 3004 if (!bic) 3011 3005 return; 3012 3006 3013 - bic->saved_last_serv_time_ns = bfqq->last_serv_time_ns; 3014 - bic->saved_inject_limit = bfqq->inject_limit; 3015 - bic->saved_decrease_time_jif = bfqq->decrease_time_jif; 3007 + bfqq_data->saved_last_serv_time_ns = bfqq->last_serv_time_ns; 3008 + bfqq_data->saved_inject_limit = bfqq->inject_limit; 3009 + bfqq_data->saved_decrease_time_jif = bfqq->decrease_time_jif; 3016 3010 3017 - bic->saved_weight = bfqq->entity.orig_weight; 3018 - bic->saved_ttime = bfqq->ttime; 3019 - bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq); 3020 - bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); 3021 - bic->saved_io_start_time = bfqq->io_start_time; 3022 - bic->saved_tot_idle_time = bfqq->tot_idle_time; 3023 - bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); 3024 - bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); 3011 + bfqq_data->saved_weight = bfqq->entity.orig_weight; 3012 + bfqq_data->saved_ttime = bfqq->ttime; 3013 + bfqq_data->saved_has_short_ttime = 3014 + bfq_bfqq_has_short_ttime(bfqq); 3015 + bfqq_data->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); 3016 + bfqq_data->saved_io_start_time = bfqq->io_start_time; 3017 + bfqq_data->saved_tot_idle_time = bfqq->tot_idle_time; 3018 + bfqq_data->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); 3019 + bfqq_data->was_in_burst_list = 3020 + !hlist_unhashed(&bfqq->burst_list_node); 3021 + 3025 3022 if (unlikely(bfq_bfqq_just_created(bfqq) && 3026 3023 !bfq_bfqq_in_large_burst(bfqq) && 3027 3024 bfqq->bfqd->low_latency)) { ··· 3037 3028 * to bfqq, so that to avoid that bfqq unjustly fails 3038 3029 * to enjoy weight raising if split soon. 3039 3030 */ 3040 - bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff; 3041 - bic->saved_wr_start_at_switch_to_srt = bfq_smallest_from_now(); 3042 - bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd); 3043 - bic->saved_last_wr_start_finish = jiffies; 3031 + bfqq_data->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff; 3032 + bfqq_data->saved_wr_start_at_switch_to_srt = 3033 + bfq_smallest_from_now(); 3034 + bfqq_data->saved_wr_cur_max_time = 3035 + bfq_wr_duration(bfqq->bfqd); 3036 + bfqq_data->saved_last_wr_start_finish = jiffies; 3044 3037 } else { 3045 - bic->saved_wr_coeff = bfqq->wr_coeff; 3046 - bic->saved_wr_start_at_switch_to_srt = 3038 + bfqq_data->saved_wr_coeff = bfqq->wr_coeff; 3039 + bfqq_data->saved_wr_start_at_switch_to_srt = 3047 3040 bfqq->wr_start_at_switch_to_srt; 3048 - bic->saved_service_from_wr = bfqq->service_from_wr; 3049 - bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish; 3050 - bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time; 3041 + bfqq_data->saved_service_from_wr = 3042 + bfqq->service_from_wr; 3043 + bfqq_data->saved_last_wr_start_finish = 3044 + bfqq->last_wr_start_finish; 3045 + bfqq_data->saved_wr_cur_max_time = bfqq->wr_cur_max_time; 3051 3046 } 3052 3047 } 3053 3048 ··· 5377 5364 * therefore on its unused per-actuator fields being NULL. 5378 5365 */ 5379 5366 unsigned int num_actuators = BFQ_MAX_ACTUATORS; 5367 + struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data; 5380 5368 5381 5369 /* 5382 5370 * bfqd is NULL if scheduler already exited, and in that case ··· 5388 5374 num_actuators = bfqd->num_actuators; 5389 5375 } 5390 5376 5391 - if (bic->stable_merge_bfqq) 5392 - bfq_put_stable_ref(bic->stable_merge_bfqq); 5377 + if (bfqq_data->stable_merge_bfqq) 5378 + bfq_put_stable_ref(bfqq_data->stable_merge_bfqq); 5393 5379 5394 5380 for (act_idx = 0; act_idx < num_actuators; act_idx++) { 5395 5381 bfq_exit_icq_bfqq(bic, true, act_idx); ··· 5580 5566 { 5581 5567 struct bfq_queue *new_bfqq = 5582 5568 bfq_setup_merge(bfqq, last_bfqq_created); 5569 + struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data; 5583 5570 5584 5571 if (!new_bfqq) 5585 5572 return bfqq; 5586 5573 5587 5574 if (new_bfqq->bic) 5588 - new_bfqq->bic->stably_merged = true; 5589 - bic->stably_merged = true; 5575 + new_bfqq->bic->bfqq_data.stably_merged = true; 5576 + bfqq_data->stably_merged = true; 5590 5577 5591 5578 /* 5592 5579 * Reusing merge functions. This implies that ··· 5656 5641 &bfqd->last_bfqq_created; 5657 5642 5658 5643 struct bfq_queue *last_bfqq_created = *source_bfqq; 5644 + struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data; 5659 5645 5660 5646 /* 5661 5647 * If last_bfqq_created has not been set yet, then init it. If ··· 5718 5702 /* 5719 5703 * Record the bfqq to merge to. 5720 5704 */ 5721 - bic->stable_merge_bfqq = last_bfqq_created; 5705 + bfqq_data->stable_merge_bfqq = last_bfqq_created; 5722 5706 } 5723 5707 } 5724 5708 ··· 6639 6623 { 6640 6624 unsigned int act_idx = bfq_actuator_index(bfqd, bio); 6641 6625 struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, act_idx); 6626 + struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data; 6642 6627 6643 6628 if (likely(bfqq && bfqq != &bfqd->oom_bfqq)) 6644 6629 return bfqq; ··· 6653 6636 6654 6637 bic_set_bfqq(bic, bfqq, is_sync, act_idx); 6655 6638 if (split && is_sync) { 6656 - if ((bic->was_in_burst_list && bfqd->large_burst) || 6657 - bic->saved_in_large_burst) 6639 + if ((bfqq_data->was_in_burst_list && bfqd->large_burst) || 6640 + bfqq_data->saved_in_large_burst) 6658 6641 bfq_mark_bfqq_in_large_burst(bfqq); 6659 6642 else { 6660 6643 bfq_clear_bfqq_in_large_burst(bfqq); 6661 - if (bic->was_in_burst_list) 6644 + if (bfqq_data->was_in_burst_list) 6662 6645 /* 6663 6646 * If bfqq was in the current 6664 6647 * burst list before being ··· 6747 6730 struct bfq_queue *bfqq; 6748 6731 bool new_queue = false; 6749 6732 bool bfqq_already_existing = false, split = false; 6733 + struct bfq_iocq_bfqq_data *bfqq_data; 6750 6734 6751 6735 if (unlikely(!rq->elv.icq)) 6752 6736 return NULL; ··· 6771 6753 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync, 6772 6754 &new_queue); 6773 6755 6756 + bfqq_data = &bic->bfqq_data; 6757 + 6774 6758 if (likely(!new_queue)) { 6775 6759 /* If the queue was seeky for too long, break it apart. */ 6776 6760 if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq) && 6777 - !bic->stably_merged) { 6761 + !bfqq_data->stably_merged) { 6778 6762 struct bfq_queue *old_bfqq = bfqq; 6779 6763 6780 6764 /* Update bic before losing reference to bfqq */ 6781 6765 if (bfq_bfqq_in_large_burst(bfqq)) 6782 - bic->saved_in_large_burst = true; 6766 + bfqq_data->saved_in_large_burst = true; 6783 6767 6784 6768 bfqq = bfq_split_bfqq(bic, bfqq); 6785 6769 split = true;

+31 -21

block/bfq-iosched.h

··· 413 413 }; 414 414 415 415 /** 416 - * struct bfq_io_cq - per (request_queue, io_context) structure. 417 - */ 418 - struct bfq_io_cq { 419 - /* associated io_cq structure */ 420 - struct io_cq icq; /* must be the first member */ 421 - /* 422 - * Matrix of associated process queues: first row for async 423 - * queues, second row sync queues. Each row contains one 424 - * column for each actuator. An I/O request generated by the 425 - * process is inserted into the queue pointed by bfqq[i][j] if 426 - * the request is to be served by the j-th actuator of the 427 - * drive, where i==0 or i==1, depending on whether the request 428 - * is async or sync. So there is a distinct queue for each 429 - * actuator. 430 - */ 431 - struct bfq_queue *bfqq[2][BFQ_MAX_ACTUATORS]; 432 - /* per (request_queue, blkcg) ioprio */ 433 - int ioprio; 434 - #ifdef CONFIG_BFQ_GROUP_IOSCHED 435 - uint64_t blkcg_serial_nr; /* the current blkcg serial */ 436 - #endif 416 + * struct bfq_data - bfqq data unique and persistent for associated bfq_io_cq 417 + */ 418 + struct bfq_iocq_bfqq_data { 437 419 /* 438 420 * Snapshot of the has_short_time flag before merging; taken 439 421 * to remember its value while the queue is merged, so as to ··· 470 488 struct bfq_queue *stable_merge_bfqq; 471 489 472 490 bool stably_merged; /* non splittable if true */ 491 + }; 492 + 493 + /** 494 + * struct bfq_io_cq - per (request_queue, io_context) structure. 495 + */ 496 + struct bfq_io_cq { 497 + /* associated io_cq structure */ 498 + struct io_cq icq; /* must be the first member */ 499 + /* 500 + * Matrix of associated process queues: first row for async 501 + * queues, second row sync queues. Each row contains one 502 + * column for each actuator. An I/O request generated by the 503 + * process is inserted into the queue pointed by bfqq[i][j] if 504 + * the request is to be served by the j-th actuator of the 505 + * drive, where i==0 or i==1, depending on whether the request 506 + * is async or sync. So there is a distinct queue for each 507 + * actuator. 508 + */ 509 + struct bfq_queue *bfqq[2][BFQ_MAX_ACTUATORS]; 510 + /* per (request_queue, blkcg) ioprio */ 511 + int ioprio; 512 + #ifdef CONFIG_BFQ_GROUP_IOSCHED 513 + uint64_t blkcg_serial_nr; /* the current blkcg serial */ 514 + #endif 515 + 516 + /* persistent data for associated synchronous process queue */ 517 + struct bfq_iocq_bfqq_data bfqq_data; 518 + 473 519 unsigned int requests; /* Number of requests this process has in flight */ 474 520 }; 475 521

Configure Feed

Configure Feed