tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

sched/fair: Merge select_idle_core/cpu()

Both select_idle_core() and select_idle_cpu() do a loop over the same
cpumask. Observe that by clearing the already visited CPUs, we can
fold the iteration and iterate a core at a time.

All we need to do is remember any non-idle CPU we encountered while
scanning for an idle core. This way we'll only iterate every CPU once.

Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lkml.kernel.org/r/20210127135203.19633-5-mgorman@techsingularity.net

authored by

Mel Gorman and committed by
Ingo Molnar 9fe1f127 6cd56ef1

+59 -40
+59 -40
kernel/sched/fair.c
··· 6019 6019 return new_cpu; 6020 6020 } 6021 6021 6022 + static inline int __select_idle_cpu(int cpu) 6023 + { 6024 + if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 6025 + return cpu; 6026 + 6027 + return -1; 6028 + } 6029 + 6022 6030 #ifdef CONFIG_SCHED_SMT 6023 6031 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 6024 6032 EXPORT_SYMBOL_GPL(sched_smt_present); ··· 6085 6077 * there are no idle cores left in the system; tracked through 6086 6078 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 6087 6079 */ 6088 - static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) 6080 + static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 6089 6081 { 6090 - struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); 6091 - int core, cpu; 6082 + bool idle = true; 6083 + int cpu; 6092 6084 6093 6085 if (!static_branch_likely(&sched_smt_present)) 6094 - return -1; 6086 + return __select_idle_cpu(core); 6095 6087 6096 - if (!test_idle_cores(target, false)) 6097 - return -1; 6098 - 6099 - cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 6100 - 6101 - for_each_cpu_wrap(core, cpus, target) { 6102 - bool idle = true; 6103 - 6104 - for_each_cpu(cpu, cpu_smt_mask(core)) { 6105 - if (!available_idle_cpu(cpu)) { 6106 - idle = false; 6107 - break; 6088 + for_each_cpu(cpu, cpu_smt_mask(core)) { 6089 + if (!available_idle_cpu(cpu)) { 6090 + idle = false; 6091 + if (*idle_cpu == -1) { 6092 + if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, p->cpus_ptr)) { 6093 + *idle_cpu = cpu; 6094 + break; 6095 + } 6096 + continue; 6108 6097 } 6098 + break; 6109 6099 } 6110 - 6111 - if (idle) 6112 - return core; 6113 - 6114 - cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 6100 + if (*idle_cpu == -1 && cpumask_test_cpu(cpu, p->cpus_ptr)) 6101 + *idle_cpu = cpu; 6115 6102 } 6116 6103 6117 - /* 6118 - * Failed to find an idle core; stop looking for one. 6119 - */ 6120 - set_idle_cores(target, 0); 6104 + if (idle) 6105 + return core; 6121 6106 6107 + cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 6122 6108 return -1; 6123 6109 } 6124 6110 6125 6111 #else /* CONFIG_SCHED_SMT */ 6126 6112 6127 - static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) 6113 + static inline void set_idle_cores(int cpu, int val) 6128 6114 { 6129 - return -1; 6115 + } 6116 + 6117 + static inline bool test_idle_cores(int cpu, bool def) 6118 + { 6119 + return def; 6120 + } 6121 + 6122 + static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 6123 + { 6124 + return __select_idle_cpu(core); 6130 6125 } 6131 6126 6132 6127 #endif /* CONFIG_SCHED_SMT */ ··· 6142 6131 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int target) 6143 6132 { 6144 6133 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); 6134 + int i, cpu, idle_cpu = -1, nr = INT_MAX; 6135 + bool smt = test_idle_cores(target, false); 6136 + int this = smp_processor_id(); 6145 6137 struct sched_domain *this_sd; 6146 6138 u64 time; 6147 - int this = smp_processor_id(); 6148 - int cpu, nr = INT_MAX; 6149 6139 6150 6140 this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); 6151 6141 if (!this_sd) ··· 6154 6142 6155 6143 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 6156 6144 6157 - if (sched_feat(SIS_PROP)) { 6145 + if (sched_feat(SIS_PROP) && !smt) { 6158 6146 u64 avg_cost, avg_idle, span_avg; 6159 6147 6160 6148 /* ··· 6174 6162 } 6175 6163 6176 6164 for_each_cpu_wrap(cpu, cpus, target) { 6177 - if (!--nr) 6178 - return -1; 6179 - if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 6180 - break; 6165 + if (smt) { 6166 + i = select_idle_core(p, cpu, cpus, &idle_cpu); 6167 + if ((unsigned int)i < nr_cpumask_bits) 6168 + return i; 6169 + 6170 + } else { 6171 + if (!--nr) 6172 + return -1; 6173 + idle_cpu = __select_idle_cpu(cpu); 6174 + if ((unsigned int)idle_cpu < nr_cpumask_bits) 6175 + break; 6176 + } 6181 6177 } 6182 6178 6183 - if (sched_feat(SIS_PROP)) { 6179 + if (smt) 6180 + set_idle_cores(this, false); 6181 + 6182 + if (sched_feat(SIS_PROP) && !smt) { 6184 6183 time = cpu_clock(this) - time; 6185 6184 update_avg(&this_sd->avg_scan_cost, time); 6186 6185 } 6187 6186 6188 - return cpu; 6187 + return idle_cpu; 6189 6188 } 6190 6189 6191 6190 /* ··· 6324 6301 sd = rcu_dereference(per_cpu(sd_llc, target)); 6325 6302 if (!sd) 6326 6303 return target; 6327 - 6328 - i = select_idle_core(p, sd, target); 6329 - if ((unsigned)i < nr_cpumask_bits) 6330 - return i; 6331 6304 6332 6305 i = select_idle_cpu(p, sd, target); 6333 6306 if ((unsigned)i < nr_cpumask_bits)