Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * intel_pstate.c: Native P state management for Intel processors
4 *
5 * (C) Copyright 2012 Intel Corporation
6 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
7 */
8
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/kernel_stat.h>
13#include <linux/module.h>
14#include <linux/ktime.h>
15#include <linux/hrtimer.h>
16#include <linux/tick.h>
17#include <linux/slab.h>
18#include <linux/sched/cpufreq.h>
19#include <linux/sched/smt.h>
20#include <linux/list.h>
21#include <linux/cpu.h>
22#include <linux/cpufreq.h>
23#include <linux/sysfs.h>
24#include <linux/types.h>
25#include <linux/fs.h>
26#include <linux/acpi.h>
27#include <linux/vmalloc.h>
28#include <linux/pm_qos.h>
29#include <linux/bitfield.h>
30#include <trace/events/power.h>
31#include <linux/units.h>
32
33#include <asm/cpu.h>
34#include <asm/div64.h>
35#include <asm/msr.h>
36#include <asm/cpu_device_id.h>
37#include <asm/cpufeature.h>
38#include <asm/intel-family.h>
39#include "../drivers/thermal/intel/thermal_interrupt.h"
40
41#define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
42
43#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
44#define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000
45#define INTEL_CPUFREQ_TRANSITION_DELAY 500
46
47#ifdef CONFIG_ACPI
48#include <acpi/processor.h>
49#include <acpi/cppc_acpi.h>
50#endif
51
52#define FRAC_BITS 8
53#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
54#define fp_toint(X) ((X) >> FRAC_BITS)
55
56#define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3))
57
58#define EXT_BITS 6
59#define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
60#define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
61#define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
62
63static inline int32_t mul_fp(int32_t x, int32_t y)
64{
65 return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
66}
67
68static inline int32_t div_fp(s64 x, s64 y)
69{
70 return div64_s64((int64_t)x << FRAC_BITS, y);
71}
72
73static inline int ceiling_fp(int32_t x)
74{
75 int mask, ret;
76
77 ret = fp_toint(x);
78 mask = (1 << FRAC_BITS) - 1;
79 if (x & mask)
80 ret += 1;
81 return ret;
82}
83
84static inline u64 mul_ext_fp(u64 x, u64 y)
85{
86 return (x * y) >> EXT_FRAC_BITS;
87}
88
89static inline u64 div_ext_fp(u64 x, u64 y)
90{
91 return div64_u64(x << EXT_FRAC_BITS, y);
92}
93
94/**
95 * struct sample - Store performance sample
96 * @core_avg_perf: Ratio of APERF/MPERF which is the actual average
97 * performance during last sample period
98 * @busy_scaled: Scaled busy value which is used to calculate next
99 * P state. This can be different than core_avg_perf
100 * to account for cpu idle period
101 * @aperf: Difference of actual performance frequency clock count
102 * read from APERF MSR between last and current sample
103 * @mperf: Difference of maximum performance frequency clock count
104 * read from MPERF MSR between last and current sample
105 * @tsc: Difference of time stamp counter between last and
106 * current sample
107 * @time: Current time from scheduler
108 *
109 * This structure is used in the cpudata structure to store performance sample
110 * data for choosing next P State.
111 */
112struct sample {
113 int32_t core_avg_perf;
114 int32_t busy_scaled;
115 u64 aperf;
116 u64 mperf;
117 u64 tsc;
118 u64 time;
119};
120
121/**
122 * struct pstate_data - Store P state data
123 * @current_pstate: Current requested P state
124 * @min_pstate: Min P state possible for this platform
125 * @max_pstate: Max P state possible for this platform
126 * @max_pstate_physical:This is physical Max P state for a processor
127 * This can be higher than the max_pstate which can
128 * be limited by platform thermal design power limits
129 * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor
130 * @scaling: Scaling factor between performance and frequency
131 * @turbo_pstate: Max Turbo P state possible for this platform
132 * @min_freq: @min_pstate frequency in cpufreq units
133 * @max_freq: @max_pstate frequency in cpufreq units
134 * @turbo_freq: @turbo_pstate frequency in cpufreq units
135 *
136 * Stores the per cpu model P state limits and current P state.
137 */
138struct pstate_data {
139 int current_pstate;
140 int min_pstate;
141 int max_pstate;
142 int max_pstate_physical;
143 int perf_ctl_scaling;
144 int scaling;
145 int turbo_pstate;
146 unsigned int min_freq;
147 unsigned int max_freq;
148 unsigned int turbo_freq;
149};
150
151/**
152 * struct vid_data - Stores voltage information data
153 * @min: VID data for this platform corresponding to
154 * the lowest P state
155 * @max: VID data corresponding to the highest P State.
156 * @turbo: VID data for turbo P state
157 * @ratio: Ratio of (vid max - vid min) /
158 * (max P state - Min P State)
159 *
160 * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
161 * This data is used in Atom platforms, where in addition to target P state,
162 * the voltage data needs to be specified to select next P State.
163 */
164struct vid_data {
165 int min;
166 int max;
167 int turbo;
168 int32_t ratio;
169};
170
171/**
172 * struct global_params - Global parameters, mostly tunable via sysfs.
173 * @no_turbo: Whether or not to use turbo P-states.
174 * @turbo_disabled: Whether or not turbo P-states are available at all,
175 * based on the MSR_IA32_MISC_ENABLE value and whether or
176 * not the maximum reported turbo P-state is different from
177 * the maximum reported non-turbo one.
178 * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
179 * P-state capacity.
180 * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
181 * P-state capacity.
182 */
183struct global_params {
184 bool no_turbo;
185 bool turbo_disabled;
186 int max_perf_pct;
187 int min_perf_pct;
188};
189
190/**
191 * struct cpudata - Per CPU instance data storage
192 * @cpu: CPU number for this instance data
193 * @policy: CPUFreq policy value
194 * @update_util: CPUFreq utility callback information
195 * @update_util_set: CPUFreq utility callback is set
196 * @iowait_boost: iowait-related boost fraction
197 * @last_update: Time of the last update.
198 * @pstate: Stores P state limits for this CPU
199 * @vid: Stores VID limits for this CPU
200 * @last_sample_time: Last Sample time
201 * @aperf_mperf_shift: APERF vs MPERF counting frequency difference
202 * @prev_aperf: Last APERF value read from APERF MSR
203 * @prev_mperf: Last MPERF value read from MPERF MSR
204 * @prev_tsc: Last timestamp counter (TSC) value
205 * @sample: Storage for storing last Sample data
206 * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios
207 * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios
208 * @acpi_perf_data: Stores ACPI perf information read from _PSS
209 * @valid_pss_table: Set to true for valid ACPI _PSS entries found
210 * @epp_powersave: Last saved HWP energy performance preference
211 * (EPP) or energy performance bias (EPB),
212 * when policy switched to performance
213 * @epp_policy: Last saved policy used to set EPP/EPB
214 * @epp_default: Power on default HWP energy performance
215 * preference/bias
216 * @epp_cached: Cached HWP energy-performance preference value
217 * @hwp_req_cached: Cached value of the last HWP Request MSR
218 * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR
219 * @last_io_update: Last time when IO wake flag was set
220 * @capacity_perf: Highest perf used for scale invariance
221 * @sched_flags: Store scheduler flags for possible cross CPU update
222 * @hwp_boost_min: Last HWP boosted min performance
223 * @suspended: Whether or not the driver has been suspended.
224 * @pd_registered: Set when a perf domain is registered for this CPU.
225 * @hwp_notify_work: workqueue for HWP notifications.
226 *
227 * This structure stores per CPU instance data for all CPUs.
228 */
229struct cpudata {
230 int cpu;
231
232 unsigned int policy;
233 struct update_util_data update_util;
234 bool update_util_set;
235
236 struct pstate_data pstate;
237 struct vid_data vid;
238
239 u64 last_update;
240 u64 last_sample_time;
241 u64 aperf_mperf_shift;
242 u64 prev_aperf;
243 u64 prev_mperf;
244 u64 prev_tsc;
245 struct sample sample;
246 int32_t min_perf_ratio;
247 int32_t max_perf_ratio;
248#ifdef CONFIG_ACPI
249 struct acpi_processor_performance acpi_perf_data;
250 bool valid_pss_table;
251#endif
252 unsigned int iowait_boost;
253 s16 epp_powersave;
254 s16 epp_policy;
255 s16 epp_default;
256 s16 epp_cached;
257 u64 hwp_req_cached;
258 u64 hwp_cap_cached;
259 u64 last_io_update;
260 unsigned int capacity_perf;
261 unsigned int sched_flags;
262 u32 hwp_boost_min;
263 bool suspended;
264#ifdef CONFIG_ENERGY_MODEL
265 bool pd_registered;
266#endif
267 struct delayed_work hwp_notify_work;
268};
269
270static struct cpudata **all_cpu_data;
271
272/**
273 * struct pstate_funcs - Per CPU model specific callbacks
274 * @get_max: Callback to get maximum non turbo effective P state
275 * @get_max_physical: Callback to get maximum non turbo physical P state
276 * @get_min: Callback to get minimum P state
277 * @get_turbo: Callback to get turbo P state
278 * @get_scaling: Callback to get frequency scaling factor
279 * @get_cpu_scaling: Get frequency scaling factor for a given cpu
280 * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference
281 * @get_val: Callback to convert P state to actual MSR write value
282 * @get_vid: Callback to get VID data for Atom platforms
283 *
284 * Core and Atom CPU models have different way to get P State limits. This
285 * structure is used to store those callbacks.
286 */
287struct pstate_funcs {
288 int (*get_max)(int cpu);
289 int (*get_max_physical)(int cpu);
290 int (*get_min)(int cpu);
291 int (*get_turbo)(int cpu);
292 int (*get_scaling)(void);
293 int (*get_cpu_scaling)(int cpu);
294 int (*get_aperf_mperf_shift)(void);
295 u64 (*get_val)(struct cpudata*, int pstate);
296 void (*get_vid)(struct cpudata *);
297};
298
299static struct pstate_funcs pstate_funcs __read_mostly;
300
301static bool hwp_active __ro_after_init;
302static int hwp_mode_bdw __ro_after_init;
303static bool per_cpu_limits __ro_after_init;
304static bool hwp_forced __ro_after_init;
305static bool hwp_boost __read_mostly;
306static bool hwp_is_hybrid;
307
308static struct cpufreq_driver *intel_pstate_driver __read_mostly;
309
310#define INTEL_PSTATE_CORE_SCALING 100000
311#define HYBRID_SCALING_FACTOR_ADL 78741
312#define HYBRID_SCALING_FACTOR_MTL 80000
313#define HYBRID_SCALING_FACTOR_LNL 86957
314
315static int hybrid_scaling_factor;
316
317static inline int core_get_scaling(void)
318{
319 return INTEL_PSTATE_CORE_SCALING;
320}
321
322#ifdef CONFIG_ACPI
323static bool acpi_ppc;
324#endif
325
326static struct global_params global;
327
328static DEFINE_MUTEX(intel_pstate_driver_lock);
329static DEFINE_MUTEX(intel_pstate_limits_lock);
330
331#ifdef CONFIG_ACPI
332
333static bool intel_pstate_acpi_pm_profile_server(void)
334{
335 if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
336 acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
337 return true;
338
339 return false;
340}
341
342static bool intel_pstate_get_ppc_enable_status(void)
343{
344 if (intel_pstate_acpi_pm_profile_server())
345 return true;
346
347 return acpi_ppc;
348}
349
350#ifdef CONFIG_ACPI_CPPC_LIB
351
352/* The work item is needed to avoid CPU hotplug locking issues */
353static void intel_pstste_sched_itmt_work_fn(struct work_struct *work)
354{
355 sched_set_itmt_support();
356}
357
358static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn);
359
360#define CPPC_MAX_PERF U8_MAX
361
362static void intel_pstate_set_itmt_prio(int cpu)
363{
364 struct cppc_perf_caps cppc_perf;
365 static u32 max_highest_perf = 0, min_highest_perf = U32_MAX;
366 int ret;
367
368 ret = cppc_get_perf_caps(cpu, &cppc_perf);
369 /*
370 * If CPPC is not available, fall back to MSR_HWP_CAPABILITIES bits [8:0].
371 *
372 * Also, on some systems with overclocking enabled, CPPC.highest_perf is
373 * hardcoded to 0xff, so CPPC.highest_perf cannot be used to enable ITMT.
374 * Fall back to MSR_HWP_CAPABILITIES then too.
375 */
376 if (ret || cppc_perf.highest_perf == CPPC_MAX_PERF)
377 cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached));
378
379 /*
380 * The priorities can be set regardless of whether or not
381 * sched_set_itmt_support(true) has been called and it is valid to
382 * update them at any time after it has been called.
383 */
384 sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu);
385
386 if (max_highest_perf <= min_highest_perf) {
387 if (cppc_perf.highest_perf > max_highest_perf)
388 max_highest_perf = cppc_perf.highest_perf;
389
390 if (cppc_perf.highest_perf < min_highest_perf)
391 min_highest_perf = cppc_perf.highest_perf;
392
393 if (max_highest_perf > min_highest_perf) {
394 /*
395 * This code can be run during CPU online under the
396 * CPU hotplug locks, so sched_set_itmt_support()
397 * cannot be called from here. Queue up a work item
398 * to invoke it.
399 */
400 schedule_work(&sched_itmt_work);
401 }
402 }
403}
404
405static int intel_pstate_get_cppc_guaranteed(int cpu)
406{
407 struct cppc_perf_caps cppc_perf;
408 int ret;
409
410 ret = cppc_get_perf_caps(cpu, &cppc_perf);
411 if (ret)
412 return ret;
413
414 if (cppc_perf.guaranteed_perf)
415 return cppc_perf.guaranteed_perf;
416
417 return cppc_perf.nominal_perf;
418}
419
420static int intel_pstate_cppc_get_scaling(int cpu)
421{
422 struct cppc_perf_caps cppc_perf;
423
424 /*
425 * Compute the perf-to-frequency scaling factor for the given CPU if
426 * possible, unless it would be 0.
427 */
428 if (!cppc_get_perf_caps(cpu, &cppc_perf) &&
429 cppc_perf.nominal_perf && cppc_perf.nominal_freq)
430 return div_u64(cppc_perf.nominal_freq * KHZ_PER_MHZ,
431 cppc_perf.nominal_perf);
432
433 return core_get_scaling();
434}
435
436#else /* CONFIG_ACPI_CPPC_LIB */
437static inline void intel_pstate_set_itmt_prio(int cpu)
438{
439}
440#endif /* CONFIG_ACPI_CPPC_LIB */
441
442static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
443{
444 struct cpudata *cpu;
445 int ret;
446 int i;
447
448 if (hwp_active) {
449 intel_pstate_set_itmt_prio(policy->cpu);
450 return;
451 }
452
453 if (!intel_pstate_get_ppc_enable_status())
454 return;
455
456 cpu = all_cpu_data[policy->cpu];
457
458 ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
459 policy->cpu);
460 if (ret)
461 return;
462
463 /*
464 * Check if the control value in _PSS is for PERF_CTL MSR, which should
465 * guarantee that the states returned by it map to the states in our
466 * list directly.
467 */
468 if (cpu->acpi_perf_data.control_register.space_id !=
469 ACPI_ADR_SPACE_FIXED_HARDWARE)
470 goto err;
471
472 /*
473 * If there is only one entry _PSS, simply ignore _PSS and continue as
474 * usual without taking _PSS into account
475 */
476 if (cpu->acpi_perf_data.state_count < 2)
477 goto err;
478
479 pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu);
480 for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
481 pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n",
482 (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
483 (u32) cpu->acpi_perf_data.states[i].core_frequency,
484 (u32) cpu->acpi_perf_data.states[i].power,
485 (u32) cpu->acpi_perf_data.states[i].control);
486 }
487
488 cpu->valid_pss_table = true;
489 pr_debug("_PPC limits will be enforced\n");
490
491 return;
492
493 err:
494 cpu->valid_pss_table = false;
495 acpi_processor_unregister_performance(policy->cpu);
496}
497
498static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
499{
500 struct cpudata *cpu;
501
502 cpu = all_cpu_data[policy->cpu];
503 if (!cpu->valid_pss_table)
504 return;
505
506 acpi_processor_unregister_performance(policy->cpu);
507}
508#else /* CONFIG_ACPI */
509static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
510{
511}
512
513static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
514{
515}
516
517static inline bool intel_pstate_acpi_pm_profile_server(void)
518{
519 return false;
520}
521#endif /* CONFIG_ACPI */
522
523#ifndef CONFIG_ACPI_CPPC_LIB
524static inline int intel_pstate_get_cppc_guaranteed(int cpu)
525{
526 return -ENOTSUPP;
527}
528
529static int intel_pstate_cppc_get_scaling(int cpu)
530{
531 return core_get_scaling();
532}
533#endif /* CONFIG_ACPI_CPPC_LIB */
534
535static int intel_pstate_freq_to_hwp_rel(struct cpudata *cpu, int freq,
536 unsigned int relation)
537{
538 if (freq == cpu->pstate.turbo_freq)
539 return cpu->pstate.turbo_pstate;
540
541 if (freq == cpu->pstate.max_freq)
542 return cpu->pstate.max_pstate;
543
544 switch (relation) {
545 case CPUFREQ_RELATION_H:
546 return freq / cpu->pstate.scaling;
547 case CPUFREQ_RELATION_C:
548 return DIV_ROUND_CLOSEST(freq, cpu->pstate.scaling);
549 }
550
551 return DIV_ROUND_UP(freq, cpu->pstate.scaling);
552}
553
554static int intel_pstate_freq_to_hwp(struct cpudata *cpu, int freq)
555{
556 return intel_pstate_freq_to_hwp_rel(cpu, freq, CPUFREQ_RELATION_L);
557}
558
559/**
560 * intel_pstate_hybrid_hwp_adjust - Calibrate HWP performance levels.
561 * @cpu: Target CPU.
562 *
563 * On hybrid processors, HWP may expose more performance levels than there are
564 * P-states accessible through the PERF_CTL interface. If that happens, the
565 * scaling factor between HWP performance levels and CPU frequency will be less
566 * than the scaling factor between P-state values and CPU frequency.
567 *
568 * In that case, adjust the CPU parameters used in computations accordingly.
569 */
570static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu)
571{
572 int perf_ctl_max_phys = cpu->pstate.max_pstate_physical;
573 int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
574 int perf_ctl_turbo = pstate_funcs.get_turbo(cpu->cpu);
575 int scaling = cpu->pstate.scaling;
576 int freq;
577
578 pr_debug("CPU%d: PERF_CTL max_phys = %d\n", cpu->cpu, perf_ctl_max_phys);
579 pr_debug("CPU%d: PERF_CTL turbo = %d\n", cpu->cpu, perf_ctl_turbo);
580 pr_debug("CPU%d: PERF_CTL scaling = %d\n", cpu->cpu, perf_ctl_scaling);
581 pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate);
582 pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu->cpu, cpu->pstate.turbo_pstate);
583 pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu->cpu, scaling);
584
585 if (scaling == perf_ctl_scaling)
586 return;
587
588 hwp_is_hybrid = true;
589
590 cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_pstate * scaling,
591 perf_ctl_scaling);
592 cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling,
593 perf_ctl_scaling);
594
595 freq = perf_ctl_max_phys * perf_ctl_scaling;
596 cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq);
597
598 freq = cpu->pstate.min_pstate * perf_ctl_scaling;
599 cpu->pstate.min_freq = freq;
600 /*
601 * Cast the min P-state value retrieved via pstate_funcs.get_min() to
602 * the effective range of HWP performance levels.
603 */
604 cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq);
605}
606
607static bool turbo_is_disabled(void)
608{
609 u64 misc_en;
610
611 rdmsrq(MSR_IA32_MISC_ENABLE, misc_en);
612
613 return !!(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
614}
615
616static int min_perf_pct_min(void)
617{
618 struct cpudata *cpu = all_cpu_data[0];
619 int turbo_pstate = cpu->pstate.turbo_pstate;
620
621 return turbo_pstate ?
622 (cpu->pstate.min_pstate * 100 / turbo_pstate) : 0;
623}
624
625static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)
626{
627 s16 epp = -EOPNOTSUPP;
628
629 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
630 /*
631 * When hwp_req_data is 0, means that caller didn't read
632 * MSR_HWP_REQUEST, so need to read and get EPP.
633 */
634 if (!hwp_req_data) {
635 epp = rdmsrq_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST,
636 &hwp_req_data);
637 if (epp)
638 return epp;
639 }
640 epp = (hwp_req_data >> 24) & 0xff;
641 }
642
643 return epp;
644}
645
646/*
647 * EPP display strings corresponding to EPP index in the
648 * energy_perf_strings[]
649 * index String
650 *-------------------------------------
651 * 0 default
652 * 1 performance
653 * 2 balance_performance
654 * 3 balance_power
655 * 4 power
656 */
657
658enum energy_perf_value_index {
659 EPP_INDEX_DEFAULT = 0,
660 EPP_INDEX_PERFORMANCE,
661 EPP_INDEX_BALANCE_PERFORMANCE,
662 EPP_INDEX_BALANCE_POWERSAVE,
663 EPP_INDEX_POWERSAVE,
664};
665
666static const char * const energy_perf_strings[] = {
667 [EPP_INDEX_DEFAULT] = "default",
668 [EPP_INDEX_PERFORMANCE] = "performance",
669 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
670 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
671 [EPP_INDEX_POWERSAVE] = "power",
672 NULL
673};
674static unsigned int epp_values[] = {
675 [EPP_INDEX_DEFAULT] = 0, /* Unused index */
676 [EPP_INDEX_PERFORMANCE] = HWP_EPP_PERFORMANCE,
677 [EPP_INDEX_BALANCE_PERFORMANCE] = HWP_EPP_BALANCE_PERFORMANCE,
678 [EPP_INDEX_BALANCE_POWERSAVE] = HWP_EPP_BALANCE_POWERSAVE,
679 [EPP_INDEX_POWERSAVE] = HWP_EPP_POWERSAVE,
680};
681
682static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp)
683{
684 s16 epp;
685 int index = -EINVAL;
686
687 *raw_epp = 0;
688 epp = intel_pstate_get_epp(cpu_data, 0);
689 if (epp < 0)
690 return epp;
691
692 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
693 if (epp == epp_values[EPP_INDEX_PERFORMANCE])
694 return EPP_INDEX_PERFORMANCE;
695 if (epp == epp_values[EPP_INDEX_BALANCE_PERFORMANCE])
696 return EPP_INDEX_BALANCE_PERFORMANCE;
697 if (epp == epp_values[EPP_INDEX_BALANCE_POWERSAVE])
698 return EPP_INDEX_BALANCE_POWERSAVE;
699 if (epp == epp_values[EPP_INDEX_POWERSAVE])
700 return EPP_INDEX_POWERSAVE;
701 *raw_epp = epp;
702 return 0;
703 } else if (boot_cpu_has(X86_FEATURE_EPB)) {
704 /*
705 * Range:
706 * 0x00-0x03 : Performance
707 * 0x04-0x07 : Balance performance
708 * 0x08-0x0B : Balance power
709 * 0x0C-0x0F : Power
710 * The EPB is a 4 bit value, but our ranges restrict the
711 * value which can be set. Here only using top two bits
712 * effectively.
713 */
714 index = (epp >> 2) + 1;
715 }
716
717 return index;
718}
719
720static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)
721{
722 int ret;
723
724 /*
725 * Use the cached HWP Request MSR value, because in the active mode the
726 * register itself may be updated by intel_pstate_hwp_boost_up() or
727 * intel_pstate_hwp_boost_down() at any time.
728 */
729 u64 value = READ_ONCE(cpu->hwp_req_cached);
730
731 value &= ~GENMASK_ULL(31, 24);
732 value |= (u64)epp << 24;
733 /*
734 * The only other updater of hwp_req_cached in the active mode,
735 * intel_pstate_hwp_set(), is called under the same lock as this
736 * function, so it cannot run in parallel with the update below.
737 */
738 WRITE_ONCE(cpu->hwp_req_cached, value);
739 ret = wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
740 if (!ret)
741 cpu->epp_cached = epp;
742
743 return ret;
744}
745
746static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
747 int pref_index, bool use_raw,
748 u32 raw_epp)
749{
750 int epp = -EINVAL;
751 int ret = -EOPNOTSUPP;
752
753 if (!pref_index)
754 epp = cpu_data->epp_default;
755
756 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
757 if (use_raw)
758 epp = raw_epp;
759 else if (epp == -EINVAL)
760 epp = epp_values[pref_index];
761
762 /*
763 * To avoid confusion, refuse to set EPP to any values different
764 * from 0 (performance) if the current policy is "performance",
765 * because those values would be overridden.
766 */
767 if (epp > 0 && cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
768 return -EBUSY;
769
770 ret = intel_pstate_set_epp(cpu_data, epp);
771 }
772
773 return ret;
774}
775
776static ssize_t show_energy_performance_available_preferences(
777 struct cpufreq_policy *policy, char *buf)
778{
779 int i = 0;
780 int ret = 0;
781
782 while (energy_perf_strings[i] != NULL)
783 ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]);
784
785 ret += sprintf(&buf[ret], "\n");
786
787 return ret;
788}
789
790cpufreq_freq_attr_ro(energy_performance_available_preferences);
791
792static struct cpufreq_driver intel_pstate;
793
794static ssize_t store_energy_performance_preference(
795 struct cpufreq_policy *policy, const char *buf, size_t count)
796{
797 struct cpudata *cpu = all_cpu_data[policy->cpu];
798 char str_preference[21];
799 bool raw = false;
800 ssize_t ret;
801 u32 epp = 0;
802
803 ret = sscanf(buf, "%20s", str_preference);
804 if (ret != 1)
805 return -EINVAL;
806
807 ret = match_string(energy_perf_strings, -1, str_preference);
808 if (ret < 0) {
809 if (!boot_cpu_has(X86_FEATURE_HWP_EPP))
810 return ret;
811
812 ret = kstrtouint(buf, 10, &epp);
813 if (ret)
814 return ret;
815
816 if (epp > 255)
817 return -EINVAL;
818
819 raw = true;
820 }
821
822 /*
823 * This function runs with the policy R/W semaphore held, which
824 * guarantees that the driver pointer will not change while it is
825 * running.
826 */
827 if (!intel_pstate_driver)
828 return -EAGAIN;
829
830 mutex_lock(&intel_pstate_limits_lock);
831
832 if (intel_pstate_driver == &intel_pstate) {
833 ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp);
834 } else {
835 /*
836 * In the passive mode the governor needs to be stopped on the
837 * target CPU before the EPP update and restarted after it,
838 * which is super-heavy-weight, so make sure it is worth doing
839 * upfront.
840 */
841 if (!raw)
842 epp = ret ? epp_values[ret] : cpu->epp_default;
843
844 if (cpu->epp_cached != epp) {
845 int err;
846
847 cpufreq_stop_governor(policy);
848 ret = intel_pstate_set_epp(cpu, epp);
849 err = cpufreq_start_governor(policy);
850 if (!ret)
851 ret = err;
852 } else {
853 ret = 0;
854 }
855 }
856
857 mutex_unlock(&intel_pstate_limits_lock);
858
859 return ret ?: count;
860}
861
862static ssize_t show_energy_performance_preference(
863 struct cpufreq_policy *policy, char *buf)
864{
865 struct cpudata *cpu_data = all_cpu_data[policy->cpu];
866 int preference, raw_epp;
867
868 preference = intel_pstate_get_energy_pref_index(cpu_data, &raw_epp);
869 if (preference < 0)
870 return preference;
871
872 if (raw_epp)
873 return sprintf(buf, "%d\n", raw_epp);
874 else
875 return sprintf(buf, "%s\n", energy_perf_strings[preference]);
876}
877
878cpufreq_freq_attr_rw(energy_performance_preference);
879
880static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf)
881{
882 struct cpudata *cpu = all_cpu_data[policy->cpu];
883 int ratio, freq;
884
885 ratio = intel_pstate_get_cppc_guaranteed(policy->cpu);
886 if (ratio <= 0) {
887 u64 cap;
888
889 rdmsrq_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap);
890 ratio = HWP_GUARANTEED_PERF(cap);
891 }
892
893 freq = ratio * cpu->pstate.scaling;
894 if (cpu->pstate.scaling != cpu->pstate.perf_ctl_scaling)
895 freq = rounddown(freq, cpu->pstate.perf_ctl_scaling);
896
897 return sprintf(buf, "%d\n", freq);
898}
899
900cpufreq_freq_attr_ro(base_frequency);
901
902enum hwp_cpufreq_attr_index {
903 HWP_BASE_FREQUENCY_INDEX = 0,
904 HWP_PERFORMANCE_PREFERENCE_INDEX,
905 HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX,
906 HWP_CPUFREQ_ATTR_COUNT,
907};
908
909static struct freq_attr *hwp_cpufreq_attrs[] = {
910 [HWP_BASE_FREQUENCY_INDEX] = &base_frequency,
911 [HWP_PERFORMANCE_PREFERENCE_INDEX] = &energy_performance_preference,
912 [HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX] =
913 &energy_performance_available_preferences,
914 [HWP_CPUFREQ_ATTR_COUNT] = NULL,
915};
916
917static u8 hybrid_get_cpu_type(unsigned int cpu)
918{
919 return cpu_data(cpu).topo.intel_type;
920}
921
922static bool no_cas __ro_after_init;
923
924static struct cpudata *hybrid_max_perf_cpu __read_mostly;
925/*
926 * Protects hybrid_max_perf_cpu, the capacity_perf fields in struct cpudata,
927 * and the x86 arch scale-invariance information from concurrent updates.
928 */
929static DEFINE_MUTEX(hybrid_capacity_lock);
930
931#ifdef CONFIG_ENERGY_MODEL
932#define HYBRID_EM_STATE_COUNT 4
933
934static int hybrid_active_power(struct device *dev, unsigned long *power,
935 unsigned long *freq)
936{
937 /*
938 * Create four "states" corresponding to 40%, 60%, 80%, and 100% of the
939 * full capacity.
940 *
941 * For this purpose, return the "frequency" of 2 for the first
942 * performance level and otherwise leave the value set by the caller.
943 */
944 if (!*freq)
945 *freq = 2;
946
947 /* No power information. */
948 *power = EM_MAX_POWER;
949
950 return 0;
951}
952
953static bool hybrid_has_l3(unsigned int cpu)
954{
955 struct cpu_cacheinfo *cacheinfo = get_cpu_cacheinfo(cpu);
956 unsigned int i;
957
958 if (!cacheinfo)
959 return false;
960
961 for (i = 0; i < cacheinfo->num_leaves; i++) {
962 if (cacheinfo->info_list[i].level == 3)
963 return true;
964 }
965
966 return false;
967}
968
969static int hybrid_get_cost(struct device *dev, unsigned long freq,
970 unsigned long *cost)
971{
972 /* Facilitate load balancing between CPUs of the same type. */
973 *cost = freq;
974 /*
975 * Adjust the cost depending on CPU type.
976 *
977 * The idea is to start loading up LPE-cores before E-cores and start
978 * to populate E-cores when LPE-cores are utilized above 60% of the
979 * capacity. Similarly, P-cores start to be populated when E-cores are
980 * utilized above 60% of the capacity.
981 */
982 if (hybrid_get_cpu_type(dev->id) == INTEL_CPU_TYPE_ATOM) {
983 if (hybrid_has_l3(dev->id)) /* E-core */
984 *cost += 1;
985 } else { /* P-core */
986 *cost += 2;
987 }
988
989 return 0;
990}
991
992static bool hybrid_register_perf_domain(unsigned int cpu)
993{
994 static const struct em_data_callback cb
995 = EM_ADV_DATA_CB(hybrid_active_power, hybrid_get_cost);
996 struct cpudata *cpudata = all_cpu_data[cpu];
997 struct device *cpu_dev;
998
999 /*
1000 * Registering EM perf domains without enabling asymmetric CPU capacity
1001 * support is not really useful and one domain should not be registered
1002 * more than once.
1003 */
1004 if (!hybrid_max_perf_cpu || cpudata->pd_registered)
1005 return false;
1006
1007 cpu_dev = get_cpu_device(cpu);
1008 if (!cpu_dev)
1009 return false;
1010
1011 if (em_dev_register_pd_no_update(cpu_dev, HYBRID_EM_STATE_COUNT, &cb,
1012 cpumask_of(cpu), false))
1013 return false;
1014
1015 cpudata->pd_registered = true;
1016
1017 return true;
1018}
1019
1020static void hybrid_register_all_perf_domains(void)
1021{
1022 unsigned int cpu;
1023
1024 for_each_online_cpu(cpu)
1025 hybrid_register_perf_domain(cpu);
1026}
1027
1028static void hybrid_update_perf_domain(struct cpudata *cpu)
1029{
1030 if (cpu->pd_registered)
1031 em_adjust_cpu_capacity(cpu->cpu);
1032}
1033#else /* !CONFIG_ENERGY_MODEL */
1034static inline bool hybrid_register_perf_domain(unsigned int cpu) { return false; }
1035static inline void hybrid_register_all_perf_domains(void) {}
1036static inline void hybrid_update_perf_domain(struct cpudata *cpu) {}
1037#endif /* CONFIG_ENERGY_MODEL */
1038
1039static void hybrid_set_cpu_capacity(struct cpudata *cpu)
1040{
1041 arch_set_cpu_capacity(cpu->cpu, cpu->capacity_perf,
1042 hybrid_max_perf_cpu->capacity_perf,
1043 cpu->capacity_perf,
1044 cpu->pstate.max_pstate_physical);
1045 hybrid_update_perf_domain(cpu);
1046
1047 topology_set_cpu_scale(cpu->cpu, arch_scale_cpu_capacity(cpu->cpu));
1048
1049 pr_debug("CPU%d: capacity perf = %u, base perf = %u, sys max perf = %u\n",
1050 cpu->cpu, cpu->capacity_perf, cpu->pstate.max_pstate_physical,
1051 hybrid_max_perf_cpu->capacity_perf);
1052}
1053
1054static void hybrid_clear_cpu_capacity(unsigned int cpunum)
1055{
1056 arch_set_cpu_capacity(cpunum, 1, 1, 1, 1);
1057}
1058
1059static void hybrid_get_capacity_perf(struct cpudata *cpu)
1060{
1061 if (READ_ONCE(global.no_turbo)) {
1062 cpu->capacity_perf = cpu->pstate.max_pstate_physical;
1063 return;
1064 }
1065
1066 cpu->capacity_perf = HWP_HIGHEST_PERF(READ_ONCE(cpu->hwp_cap_cached));
1067}
1068
1069static void hybrid_set_capacity_of_cpus(void)
1070{
1071 int cpunum;
1072
1073 for_each_online_cpu(cpunum) {
1074 struct cpudata *cpu = all_cpu_data[cpunum];
1075
1076 if (cpu)
1077 hybrid_set_cpu_capacity(cpu);
1078 }
1079}
1080
1081static void hybrid_update_cpu_capacity_scaling(void)
1082{
1083 struct cpudata *max_perf_cpu = NULL;
1084 unsigned int max_cap_perf = 0;
1085 int cpunum;
1086
1087 for_each_online_cpu(cpunum) {
1088 struct cpudata *cpu = all_cpu_data[cpunum];
1089
1090 if (!cpu)
1091 continue;
1092
1093 /*
1094 * During initialization, CPU performance at full capacity needs
1095 * to be determined.
1096 */
1097 if (!hybrid_max_perf_cpu)
1098 hybrid_get_capacity_perf(cpu);
1099
1100 /*
1101 * If hybrid_max_perf_cpu is not NULL at this point, it is
1102 * being replaced, so don't take it into account when looking
1103 * for the new one.
1104 */
1105 if (cpu == hybrid_max_perf_cpu)
1106 continue;
1107
1108 if (cpu->capacity_perf > max_cap_perf) {
1109 max_cap_perf = cpu->capacity_perf;
1110 max_perf_cpu = cpu;
1111 }
1112 }
1113
1114 if (max_perf_cpu) {
1115 hybrid_max_perf_cpu = max_perf_cpu;
1116 hybrid_set_capacity_of_cpus();
1117 } else {
1118 pr_info("Found no CPUs with nonzero maximum performance\n");
1119 /* Revert to the flat CPU capacity structure. */
1120 for_each_online_cpu(cpunum)
1121 hybrid_clear_cpu_capacity(cpunum);
1122 }
1123}
1124
1125static void __hybrid_refresh_cpu_capacity_scaling(void)
1126{
1127 hybrid_max_perf_cpu = NULL;
1128 hybrid_update_cpu_capacity_scaling();
1129}
1130
1131static void hybrid_refresh_cpu_capacity_scaling(void)
1132{
1133 guard(mutex)(&hybrid_capacity_lock);
1134
1135 __hybrid_refresh_cpu_capacity_scaling();
1136 /*
1137 * Perf domains are not registered before setting hybrid_max_perf_cpu,
1138 * so register them all after setting up CPU capacity scaling.
1139 */
1140 hybrid_register_all_perf_domains();
1141}
1142
1143static void hybrid_init_cpu_capacity_scaling(bool refresh)
1144{
1145 /* Bail out if enabling capacity-aware scheduling is prohibited. */
1146 if (no_cas)
1147 return;
1148
1149 /*
1150 * If hybrid_max_perf_cpu is set at this point, the hybrid CPU capacity
1151 * scaling has been enabled already and the driver is just changing the
1152 * operation mode.
1153 */
1154 if (refresh) {
1155 hybrid_refresh_cpu_capacity_scaling();
1156 return;
1157 }
1158
1159 /*
1160 * On hybrid systems, use asym capacity instead of ITMT, but because
1161 * the capacity of SMT threads is not deterministic even approximately,
1162 * do not do that when SMT is in use.
1163 */
1164 if (hwp_is_hybrid && !cpu_smt_possible() && arch_enable_hybrid_capacity_scale()) {
1165 hybrid_refresh_cpu_capacity_scaling();
1166 /*
1167 * Disabling ITMT causes sched domains to be rebuilt to disable asym
1168 * packing and enable asym capacity and EAS.
1169 */
1170 sched_clear_itmt_support();
1171 }
1172}
1173
1174static bool hybrid_clear_max_perf_cpu(void)
1175{
1176 bool ret;
1177
1178 guard(mutex)(&hybrid_capacity_lock);
1179
1180 ret = !!hybrid_max_perf_cpu;
1181 hybrid_max_perf_cpu = NULL;
1182
1183 return ret;
1184}
1185
1186static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)
1187{
1188 u64 cap;
1189
1190 rdmsrq_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap);
1191 WRITE_ONCE(cpu->hwp_cap_cached, cap);
1192 cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(cap);
1193 cpu->pstate.turbo_pstate = HWP_HIGHEST_PERF(cap);
1194}
1195
1196static void intel_pstate_get_hwp_cap(struct cpudata *cpu)
1197{
1198 int scaling = cpu->pstate.scaling;
1199
1200 __intel_pstate_get_hwp_cap(cpu);
1201
1202 cpu->pstate.max_freq = cpu->pstate.max_pstate * scaling;
1203 cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling;
1204 if (scaling != cpu->pstate.perf_ctl_scaling) {
1205 int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
1206
1207 cpu->pstate.max_freq = rounddown(cpu->pstate.max_freq,
1208 perf_ctl_scaling);
1209 cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_freq,
1210 perf_ctl_scaling);
1211 }
1212}
1213
1214static void hybrid_update_capacity(struct cpudata *cpu)
1215{
1216 unsigned int max_cap_perf;
1217
1218 mutex_lock(&hybrid_capacity_lock);
1219
1220 if (!hybrid_max_perf_cpu)
1221 goto unlock;
1222
1223 /*
1224 * The maximum performance of the CPU may have changed, but assume
1225 * that the performance of the other CPUs has not changed.
1226 */
1227 max_cap_perf = hybrid_max_perf_cpu->capacity_perf;
1228
1229 intel_pstate_get_hwp_cap(cpu);
1230
1231 hybrid_get_capacity_perf(cpu);
1232 /* Should hybrid_max_perf_cpu be replaced by this CPU? */
1233 if (cpu->capacity_perf > max_cap_perf) {
1234 hybrid_max_perf_cpu = cpu;
1235 hybrid_set_capacity_of_cpus();
1236 goto unlock;
1237 }
1238
1239 /* If this CPU is hybrid_max_perf_cpu, should it be replaced? */
1240 if (cpu == hybrid_max_perf_cpu && cpu->capacity_perf < max_cap_perf) {
1241 hybrid_update_cpu_capacity_scaling();
1242 goto unlock;
1243 }
1244
1245 hybrid_set_cpu_capacity(cpu);
1246 /*
1247 * If the CPU was offline to start with and it is going online for the
1248 * first time, a perf domain needs to be registered for it if hybrid
1249 * capacity scaling has been enabled already. In that case, sched
1250 * domains need to be rebuilt to take the new perf domain into account.
1251 */
1252 if (hybrid_register_perf_domain(cpu->cpu))
1253 em_rebuild_sched_domains();
1254
1255unlock:
1256 mutex_unlock(&hybrid_capacity_lock);
1257}
1258
1259static void intel_pstate_hwp_set(unsigned int cpu)
1260{
1261 struct cpudata *cpu_data = all_cpu_data[cpu];
1262 int max, min;
1263 u64 value;
1264 s16 epp;
1265
1266 max = cpu_data->max_perf_ratio;
1267 min = cpu_data->min_perf_ratio;
1268
1269 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
1270 min = max;
1271
1272 rdmsrq_on_cpu(cpu, MSR_HWP_REQUEST, &value);
1273
1274 value &= ~HWP_MIN_PERF(~0L);
1275 value |= HWP_MIN_PERF(min);
1276
1277 value &= ~HWP_MAX_PERF(~0L);
1278 value |= HWP_MAX_PERF(max);
1279
1280 if (cpu_data->epp_policy == cpu_data->policy)
1281 goto skip_epp;
1282
1283 cpu_data->epp_policy = cpu_data->policy;
1284
1285 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
1286 epp = intel_pstate_get_epp(cpu_data, value);
1287 cpu_data->epp_powersave = epp;
1288 /* If EPP read was failed, then don't try to write */
1289 if (epp < 0)
1290 goto skip_epp;
1291
1292 epp = 0;
1293 } else {
1294 /* skip setting EPP, when saved value is invalid */
1295 if (cpu_data->epp_powersave < 0)
1296 goto skip_epp;
1297
1298 /*
1299 * No need to restore EPP when it is not zero. This
1300 * means:
1301 * - Policy is not changed
1302 * - user has manually changed
1303 * - Error reading EPB
1304 */
1305 epp = intel_pstate_get_epp(cpu_data, value);
1306 if (epp)
1307 goto skip_epp;
1308
1309 epp = cpu_data->epp_powersave;
1310 }
1311 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
1312 value &= ~GENMASK_ULL(31, 24);
1313 value |= (u64)epp << 24;
1314 }
1315
1316skip_epp:
1317 WRITE_ONCE(cpu_data->hwp_req_cached, value);
1318 wrmsrq_on_cpu(cpu, MSR_HWP_REQUEST, value);
1319}
1320
1321static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata);
1322
1323static void intel_pstate_hwp_offline(struct cpudata *cpu)
1324{
1325 u64 value = READ_ONCE(cpu->hwp_req_cached);
1326 int min_perf;
1327
1328 intel_pstate_disable_hwp_interrupt(cpu);
1329
1330 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
1331 /*
1332 * In case the EPP has been set to "performance" by the
1333 * active mode "performance" scaling algorithm, replace that
1334 * temporary value with the cached EPP one.
1335 */
1336 value &= ~GENMASK_ULL(31, 24);
1337 value |= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached);
1338 /*
1339 * However, make sure that EPP will be set to "performance" when
1340 * the CPU is brought back online again and the "performance"
1341 * scaling algorithm is still in effect.
1342 */
1343 cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN;
1344 }
1345
1346 /*
1347 * Clear the desired perf field in the cached HWP request value to
1348 * prevent nonzero desired values from being leaked into the active
1349 * mode.
1350 */
1351 value &= ~HWP_DESIRED_PERF(~0L);
1352 WRITE_ONCE(cpu->hwp_req_cached, value);
1353
1354 value &= ~GENMASK_ULL(31, 0);
1355 min_perf = HWP_LOWEST_PERF(READ_ONCE(cpu->hwp_cap_cached));
1356
1357 /* Set hwp_max = hwp_min */
1358 value |= HWP_MAX_PERF(min_perf);
1359 value |= HWP_MIN_PERF(min_perf);
1360
1361 /* Set EPP to min */
1362 if (boot_cpu_has(X86_FEATURE_HWP_EPP))
1363 value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE);
1364
1365 wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
1366
1367 mutex_lock(&hybrid_capacity_lock);
1368
1369 if (!hybrid_max_perf_cpu) {
1370 mutex_unlock(&hybrid_capacity_lock);
1371
1372 return;
1373 }
1374
1375 if (hybrid_max_perf_cpu == cpu)
1376 hybrid_update_cpu_capacity_scaling();
1377
1378 mutex_unlock(&hybrid_capacity_lock);
1379
1380 /* Reset the capacity of the CPU going offline to the initial value. */
1381 hybrid_clear_cpu_capacity(cpu->cpu);
1382}
1383
1384#define POWER_CTL_EE_ENABLE 1
1385#define POWER_CTL_EE_DISABLE 2
1386
1387/* Enable bit for Dynamic Efficiency Control (DEC) */
1388#define POWER_CTL_DEC_ENABLE 27
1389
1390static int power_ctl_ee_state;
1391
1392static void set_power_ctl_ee_state(bool input)
1393{
1394 u64 power_ctl;
1395
1396 guard(mutex)(&intel_pstate_driver_lock);
1397
1398 rdmsrq(MSR_IA32_POWER_CTL, power_ctl);
1399 if (input) {
1400 power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE);
1401 power_ctl_ee_state = POWER_CTL_EE_ENABLE;
1402 } else {
1403 power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE);
1404 power_ctl_ee_state = POWER_CTL_EE_DISABLE;
1405 }
1406 wrmsrq(MSR_IA32_POWER_CTL, power_ctl);
1407}
1408
1409static void intel_pstate_hwp_enable(struct cpudata *cpudata);
1410
1411static void intel_pstate_hwp_reenable(struct cpudata *cpu)
1412{
1413 intel_pstate_hwp_enable(cpu);
1414 wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached));
1415}
1416
1417static int intel_pstate_suspend(struct cpufreq_policy *policy)
1418{
1419 struct cpudata *cpu = all_cpu_data[policy->cpu];
1420
1421 pr_debug("CPU %d suspending\n", cpu->cpu);
1422
1423 cpu->suspended = true;
1424
1425 /* disable HWP interrupt and cancel any pending work */
1426 intel_pstate_disable_hwp_interrupt(cpu);
1427
1428 return 0;
1429}
1430
1431static int intel_pstate_resume(struct cpufreq_policy *policy)
1432{
1433 struct cpudata *cpu = all_cpu_data[policy->cpu];
1434
1435 pr_debug("CPU %d resuming\n", cpu->cpu);
1436
1437 /* Only restore if the system default is changed */
1438 if (power_ctl_ee_state == POWER_CTL_EE_ENABLE)
1439 set_power_ctl_ee_state(true);
1440 else if (power_ctl_ee_state == POWER_CTL_EE_DISABLE)
1441 set_power_ctl_ee_state(false);
1442
1443 if (cpu->suspended && hwp_active) {
1444 mutex_lock(&intel_pstate_limits_lock);
1445
1446 /* Re-enable HWP, because "online" has not done that. */
1447 intel_pstate_hwp_reenable(cpu);
1448
1449 mutex_unlock(&intel_pstate_limits_lock);
1450 }
1451
1452 cpu->suspended = false;
1453
1454 return 0;
1455}
1456
1457static void intel_pstate_update_policies(void)
1458{
1459 int cpu;
1460
1461 for_each_possible_cpu(cpu)
1462 cpufreq_update_policy(cpu);
1463}
1464
1465static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy,
1466 struct cpudata *cpudata)
1467{
1468 guard(cpufreq_policy_write)(policy);
1469
1470 if (hwp_active)
1471 intel_pstate_get_hwp_cap(cpudata);
1472
1473 policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ?
1474 cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;
1475
1476 refresh_frequency_limits(policy);
1477}
1478
1479static bool intel_pstate_update_max_freq(int cpu)
1480{
1481 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1482 if (!policy)
1483 return false;
1484
1485 __intel_pstate_update_max_freq(policy, all_cpu_data[cpu]);
1486
1487 return true;
1488}
1489
1490static void intel_pstate_update_limits(struct cpufreq_policy *policy)
1491{
1492 struct cpudata *cpudata = all_cpu_data[policy->cpu];
1493
1494 __intel_pstate_update_max_freq(policy, cpudata);
1495
1496 hybrid_update_capacity(cpudata);
1497}
1498
1499static void intel_pstate_update_limits_for_all(void)
1500{
1501 int cpu;
1502
1503 for_each_possible_cpu(cpu)
1504 intel_pstate_update_max_freq(cpu);
1505
1506 mutex_lock(&hybrid_capacity_lock);
1507
1508 if (hybrid_max_perf_cpu)
1509 __hybrid_refresh_cpu_capacity_scaling();
1510
1511 mutex_unlock(&hybrid_capacity_lock);
1512}
1513
1514/************************** sysfs begin ************************/
1515#define show_one(file_name, object) \
1516 static ssize_t show_##file_name \
1517 (struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
1518 { \
1519 return sprintf(buf, "%u\n", global.object); \
1520 }
1521
1522static ssize_t intel_pstate_show_status(char *buf);
1523static int intel_pstate_update_status(const char *buf, size_t size);
1524
1525static ssize_t show_status(struct kobject *kobj,
1526 struct kobj_attribute *attr, char *buf)
1527{
1528 guard(mutex)(&intel_pstate_driver_lock);
1529
1530 return intel_pstate_show_status(buf);
1531}
1532
1533static ssize_t store_status(struct kobject *a, struct kobj_attribute *b,
1534 const char *buf, size_t count)
1535{
1536 char *p = memchr(buf, '\n', count);
1537 int ret;
1538
1539 guard(mutex)(&intel_pstate_driver_lock);
1540
1541 ret = intel_pstate_update_status(buf, p ? p - buf : count);
1542 if (ret < 0)
1543 return ret;
1544
1545 return count;
1546}
1547
1548static ssize_t show_turbo_pct(struct kobject *kobj,
1549 struct kobj_attribute *attr, char *buf)
1550{
1551 struct cpudata *cpu;
1552 int total, no_turbo, turbo_pct;
1553 uint32_t turbo_fp;
1554
1555 guard(mutex)(&intel_pstate_driver_lock);
1556
1557 if (!intel_pstate_driver)
1558 return -EAGAIN;
1559
1560 cpu = all_cpu_data[0];
1561
1562 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
1563 no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
1564 turbo_fp = div_fp(no_turbo, total);
1565 turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
1566
1567 return sprintf(buf, "%u\n", turbo_pct);
1568}
1569
1570static ssize_t show_num_pstates(struct kobject *kobj,
1571 struct kobj_attribute *attr, char *buf)
1572{
1573 struct cpudata *cpu;
1574 int total;
1575
1576 guard(mutex)(&intel_pstate_driver_lock);
1577
1578 if (!intel_pstate_driver)
1579 return -EAGAIN;
1580
1581 cpu = all_cpu_data[0];
1582 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
1583
1584 return sprintf(buf, "%u\n", total);
1585}
1586
1587static ssize_t show_no_turbo(struct kobject *kobj,
1588 struct kobj_attribute *attr, char *buf)
1589{
1590 guard(mutex)(&intel_pstate_driver_lock);
1591
1592 if (!intel_pstate_driver)
1593 return -EAGAIN;
1594
1595 return sprintf(buf, "%u\n", global.no_turbo);
1596}
1597
1598static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b,
1599 const char *buf, size_t count)
1600{
1601 unsigned int input;
1602 bool no_turbo;
1603
1604 if (sscanf(buf, "%u", &input) != 1)
1605 return -EINVAL;
1606
1607 guard(mutex)(&intel_pstate_driver_lock);
1608
1609 if (!intel_pstate_driver)
1610 return -EAGAIN;
1611
1612 no_turbo = !!clamp_t(int, input, 0, 1);
1613
1614 WRITE_ONCE(global.turbo_disabled, turbo_is_disabled());
1615 if (global.turbo_disabled && !no_turbo) {
1616 pr_notice("Turbo disabled by BIOS or unavailable on processor\n");
1617 if (global.no_turbo)
1618 return -EPERM;
1619
1620 no_turbo = 1;
1621 }
1622
1623 if (no_turbo == global.no_turbo)
1624 return count;
1625
1626 WRITE_ONCE(global.no_turbo, no_turbo);
1627
1628 mutex_lock(&intel_pstate_limits_lock);
1629
1630 if (no_turbo) {
1631 struct cpudata *cpu = all_cpu_data[0];
1632 int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate;
1633
1634 /* Squash the global minimum into the permitted range. */
1635 if (global.min_perf_pct > pct)
1636 global.min_perf_pct = pct;
1637 }
1638
1639 mutex_unlock(&intel_pstate_limits_lock);
1640
1641 intel_pstate_update_limits_for_all();
1642 arch_set_max_freq_ratio(no_turbo);
1643
1644 return count;
1645}
1646
1647static void update_cpu_qos_request(int cpu, enum freq_qos_req_type type)
1648{
1649 struct cpudata *cpudata = all_cpu_data[cpu];
1650 struct freq_qos_request *req;
1651 unsigned int freq;
1652
1653 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1654 if (!policy)
1655 return;
1656
1657 req = policy->driver_data;
1658 if (!req)
1659 return;
1660
1661 if (hwp_active)
1662 intel_pstate_get_hwp_cap(cpudata);
1663
1664 freq = cpudata->pstate.turbo_freq;
1665
1666 if (type == FREQ_QOS_MIN) {
1667 freq = DIV_ROUND_UP(freq * global.min_perf_pct, 100);
1668 } else {
1669 req++;
1670 freq = (freq * global.max_perf_pct) / 100;
1671 }
1672
1673 if (freq_qos_update_request(req, freq) < 0)
1674 pr_warn("Failed to update freq constraint: CPU%d\n", cpu);
1675}
1676
1677static void update_qos_requests(enum freq_qos_req_type type)
1678{
1679 int i;
1680
1681 for_each_possible_cpu(i)
1682 update_cpu_qos_request(i, type);
1683}
1684
1685static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b,
1686 const char *buf, size_t count)
1687{
1688 unsigned int input;
1689 int ret;
1690
1691 ret = sscanf(buf, "%u", &input);
1692 if (ret != 1)
1693 return -EINVAL;
1694
1695 guard(mutex)(&intel_pstate_driver_lock);
1696
1697 if (!intel_pstate_driver)
1698 return -EAGAIN;
1699
1700 mutex_lock(&intel_pstate_limits_lock);
1701
1702 global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100);
1703
1704 mutex_unlock(&intel_pstate_limits_lock);
1705
1706 if (intel_pstate_driver == &intel_pstate)
1707 intel_pstate_update_policies();
1708 else
1709 update_qos_requests(FREQ_QOS_MAX);
1710
1711 return count;
1712}
1713
1714static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b,
1715 const char *buf, size_t count)
1716{
1717 unsigned int input;
1718 int ret;
1719
1720 ret = sscanf(buf, "%u", &input);
1721 if (ret != 1)
1722 return -EINVAL;
1723
1724 guard(mutex)(&intel_pstate_driver_lock);
1725
1726 if (!intel_pstate_driver)
1727 return -EAGAIN;
1728
1729 mutex_lock(&intel_pstate_limits_lock);
1730
1731 global.min_perf_pct = clamp_t(int, input,
1732 min_perf_pct_min(), global.max_perf_pct);
1733
1734 mutex_unlock(&intel_pstate_limits_lock);
1735
1736 if (intel_pstate_driver == &intel_pstate)
1737 intel_pstate_update_policies();
1738 else
1739 update_qos_requests(FREQ_QOS_MIN);
1740
1741 return count;
1742}
1743
1744static ssize_t show_hwp_dynamic_boost(struct kobject *kobj,
1745 struct kobj_attribute *attr, char *buf)
1746{
1747 return sprintf(buf, "%u\n", hwp_boost);
1748}
1749
1750static ssize_t store_hwp_dynamic_boost(struct kobject *a,
1751 struct kobj_attribute *b,
1752 const char *buf, size_t count)
1753{
1754 unsigned int input;
1755 int ret;
1756
1757 ret = kstrtouint(buf, 10, &input);
1758 if (ret)
1759 return ret;
1760
1761 guard(mutex)(&intel_pstate_driver_lock);
1762
1763 hwp_boost = !!input;
1764 intel_pstate_update_policies();
1765
1766 return count;
1767}
1768
1769static ssize_t show_energy_efficiency(struct kobject *kobj, struct kobj_attribute *attr,
1770 char *buf)
1771{
1772 u64 power_ctl;
1773 int enable;
1774
1775 rdmsrq(MSR_IA32_POWER_CTL, power_ctl);
1776 enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE));
1777 return sprintf(buf, "%d\n", !enable);
1778}
1779
1780static ssize_t store_energy_efficiency(struct kobject *a, struct kobj_attribute *b,
1781 const char *buf, size_t count)
1782{
1783 bool input;
1784 int ret;
1785
1786 ret = kstrtobool(buf, &input);
1787 if (ret)
1788 return ret;
1789
1790 set_power_ctl_ee_state(input);
1791
1792 return count;
1793}
1794
1795show_one(max_perf_pct, max_perf_pct);
1796show_one(min_perf_pct, min_perf_pct);
1797
1798define_one_global_rw(status);
1799define_one_global_rw(no_turbo);
1800define_one_global_rw(max_perf_pct);
1801define_one_global_rw(min_perf_pct);
1802define_one_global_ro(turbo_pct);
1803define_one_global_ro(num_pstates);
1804define_one_global_rw(hwp_dynamic_boost);
1805define_one_global_rw(energy_efficiency);
1806
1807static struct attribute *intel_pstate_attributes[] = {
1808 &status.attr,
1809 &no_turbo.attr,
1810 NULL
1811};
1812
1813static const struct attribute_group intel_pstate_attr_group = {
1814 .attrs = intel_pstate_attributes,
1815};
1816
1817static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[];
1818
1819static struct kobject *intel_pstate_kobject;
1820
1821static void __init intel_pstate_sysfs_expose_params(void)
1822{
1823 struct device *dev_root = bus_get_dev_root(&cpu_subsys);
1824 int rc;
1825
1826 if (dev_root) {
1827 intel_pstate_kobject = kobject_create_and_add("intel_pstate", &dev_root->kobj);
1828 put_device(dev_root);
1829 }
1830 if (WARN_ON(!intel_pstate_kobject))
1831 return;
1832
1833 rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
1834 if (WARN_ON(rc))
1835 return;
1836
1837 if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
1838 rc = sysfs_create_file(intel_pstate_kobject, &turbo_pct.attr);
1839 WARN_ON(rc);
1840
1841 rc = sysfs_create_file(intel_pstate_kobject, &num_pstates.attr);
1842 WARN_ON(rc);
1843 }
1844
1845 /*
1846 * If per cpu limits are enforced there are no global limits, so
1847 * return without creating max/min_perf_pct attributes
1848 */
1849 if (per_cpu_limits)
1850 return;
1851
1852 rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr);
1853 WARN_ON(rc);
1854
1855 rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
1856 WARN_ON(rc);
1857
1858 if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) {
1859 rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr);
1860 WARN_ON(rc);
1861 }
1862}
1863
1864static void __init intel_pstate_sysfs_remove(void)
1865{
1866 if (!intel_pstate_kobject)
1867 return;
1868
1869 sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group);
1870
1871 if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
1872 sysfs_remove_file(intel_pstate_kobject, &num_pstates.attr);
1873 sysfs_remove_file(intel_pstate_kobject, &turbo_pct.attr);
1874 }
1875
1876 if (!per_cpu_limits) {
1877 sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr);
1878 sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr);
1879
1880 if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids))
1881 sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr);
1882 }
1883
1884 kobject_put(intel_pstate_kobject);
1885}
1886
1887static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void)
1888{
1889 int rc;
1890
1891 if (!hwp_active)
1892 return;
1893
1894 rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
1895 WARN_ON_ONCE(rc);
1896}
1897
1898static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void)
1899{
1900 if (!hwp_active)
1901 return;
1902
1903 sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
1904}
1905
1906/************************** sysfs end ************************/
1907
1908static void intel_pstate_notify_work(struct work_struct *work)
1909{
1910 struct cpudata *cpudata =
1911 container_of(to_delayed_work(work), struct cpudata, hwp_notify_work);
1912
1913 if (intel_pstate_update_max_freq(cpudata->cpu)) {
1914 /*
1915 * The driver will not be unregistered while this function is
1916 * running, so update the capacity without acquiring the driver
1917 * lock.
1918 */
1919 hybrid_update_capacity(cpudata);
1920 }
1921
1922 wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);
1923}
1924
1925static DEFINE_RAW_SPINLOCK(hwp_notify_lock);
1926static cpumask_t hwp_intr_enable_mask;
1927
1928#define HWP_GUARANTEED_PERF_CHANGE_STATUS BIT(0)
1929#define HWP_HIGHEST_PERF_CHANGE_STATUS BIT(3)
1930
1931void notify_hwp_interrupt(void)
1932{
1933 unsigned int this_cpu = smp_processor_id();
1934 u64 value, status_mask;
1935 unsigned long flags;
1936
1937 if (!hwp_active || !cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY))
1938 return;
1939
1940 status_mask = HWP_GUARANTEED_PERF_CHANGE_STATUS;
1941 if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE))
1942 status_mask |= HWP_HIGHEST_PERF_CHANGE_STATUS;
1943
1944 rdmsrq_safe(MSR_HWP_STATUS, &value);
1945 if (!(value & status_mask))
1946 return;
1947
1948 raw_spin_lock_irqsave(&hwp_notify_lock, flags);
1949
1950 if (!cpumask_test_cpu(this_cpu, &hwp_intr_enable_mask))
1951 goto ack_intr;
1952
1953 schedule_delayed_work(&all_cpu_data[this_cpu]->hwp_notify_work,
1954 msecs_to_jiffies(10));
1955
1956 raw_spin_unlock_irqrestore(&hwp_notify_lock, flags);
1957
1958 return;
1959
1960ack_intr:
1961 wrmsrq_safe(MSR_HWP_STATUS, 0);
1962 raw_spin_unlock_irqrestore(&hwp_notify_lock, flags);
1963}
1964
1965static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata)
1966{
1967 bool cancel_work;
1968
1969 if (!cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY))
1970 return;
1971
1972 /* wrmsrq_on_cpu has to be outside spinlock as this can result in IPC */
1973 wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
1974
1975 raw_spin_lock_irq(&hwp_notify_lock);
1976 cancel_work = cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask);
1977 raw_spin_unlock_irq(&hwp_notify_lock);
1978
1979 if (cancel_work)
1980 cancel_delayed_work_sync(&cpudata->hwp_notify_work);
1981}
1982
1983#define HWP_GUARANTEED_PERF_CHANGE_REQ BIT(0)
1984#define HWP_HIGHEST_PERF_CHANGE_REQ BIT(2)
1985
1986static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata)
1987{
1988 /* Enable HWP notification interrupt for performance change */
1989 if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY)) {
1990 u64 interrupt_mask = HWP_GUARANTEED_PERF_CHANGE_REQ;
1991
1992 raw_spin_lock_irq(&hwp_notify_lock);
1993 INIT_DELAYED_WORK(&cpudata->hwp_notify_work, intel_pstate_notify_work);
1994 cpumask_set_cpu(cpudata->cpu, &hwp_intr_enable_mask);
1995 raw_spin_unlock_irq(&hwp_notify_lock);
1996
1997 if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE))
1998 interrupt_mask |= HWP_HIGHEST_PERF_CHANGE_REQ;
1999
2000 /* wrmsrq_on_cpu has to be outside spinlock as this can result in IPC */
2001 wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, interrupt_mask);
2002 wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);
2003 }
2004}
2005
2006static void intel_pstate_update_epp_defaults(struct cpudata *cpudata)
2007{
2008 cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
2009
2010 /*
2011 * If the EPP is set by firmware, which means that firmware enabled HWP
2012 * - Is equal or less than 0x80 (default balance_perf EPP)
2013 * - But less performance oriented than performance EPP
2014 * then use this as new balance_perf EPP.
2015 */
2016 if (hwp_forced && cpudata->epp_default <= HWP_EPP_BALANCE_PERFORMANCE &&
2017 cpudata->epp_default > HWP_EPP_PERFORMANCE) {
2018 epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = cpudata->epp_default;
2019 return;
2020 }
2021
2022 /*
2023 * If this CPU gen doesn't call for change in balance_perf
2024 * EPP return.
2025 */
2026 if (epp_values[EPP_INDEX_BALANCE_PERFORMANCE] == HWP_EPP_BALANCE_PERFORMANCE)
2027 return;
2028
2029 /*
2030 * Use hard coded value per gen to update the balance_perf
2031 * and default EPP.
2032 */
2033 cpudata->epp_default = epp_values[EPP_INDEX_BALANCE_PERFORMANCE];
2034 intel_pstate_set_epp(cpudata, cpudata->epp_default);
2035}
2036
2037static void intel_pstate_hwp_enable(struct cpudata *cpudata)
2038{
2039 /* First disable HWP notification interrupt till we activate again */
2040 if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY))
2041 wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
2042
2043 wrmsrq_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
2044
2045 intel_pstate_enable_hwp_interrupt(cpudata);
2046
2047 if (cpudata->epp_default >= 0)
2048 return;
2049
2050 intel_pstate_update_epp_defaults(cpudata);
2051}
2052
2053static u64 get_perf_ctl_val(int pstate)
2054{
2055 u64 val;
2056
2057 val = (u64)pstate << 8;
2058 if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled) &&
2059 cpu_feature_enabled(X86_FEATURE_IDA))
2060 val |= (u64)1 << 32;
2061
2062 return val;
2063}
2064
2065static int atom_get_min_pstate(int not_used)
2066{
2067 u64 value;
2068
2069 rdmsrq(MSR_ATOM_CORE_RATIOS, value);
2070 return (value >> 8) & 0x7F;
2071}
2072
2073static int atom_get_max_pstate(int not_used)
2074{
2075 u64 value;
2076
2077 rdmsrq(MSR_ATOM_CORE_RATIOS, value);
2078 return (value >> 16) & 0x7F;
2079}
2080
2081static int atom_get_turbo_pstate(int not_used)
2082{
2083 u64 value;
2084
2085 rdmsrq(MSR_ATOM_CORE_TURBO_RATIOS, value);
2086 return value & 0x7F;
2087}
2088
2089static u64 atom_get_val(struct cpudata *cpudata, int pstate)
2090{
2091 u64 val = get_perf_ctl_val(pstate);
2092 int32_t vid_fp;
2093 u32 vid;
2094
2095 vid_fp = cpudata->vid.min + mul_fp(
2096 int_tofp(pstate - cpudata->pstate.min_pstate),
2097 cpudata->vid.ratio);
2098
2099 vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
2100 vid = ceiling_fp(vid_fp);
2101
2102 if (pstate > cpudata->pstate.max_pstate)
2103 vid = cpudata->vid.turbo;
2104
2105 return val | vid;
2106}
2107
2108static int silvermont_get_scaling(void)
2109{
2110 u64 value;
2111 int i;
2112 /* Defined in Table 35-6 from SDM (Sept 2015) */
2113 static int silvermont_freq_table[] = {
2114 83300, 100000, 133300, 116700, 80000};
2115
2116 rdmsrq(MSR_FSB_FREQ, value);
2117 i = value & 0x7;
2118 WARN_ON(i > 4);
2119
2120 return silvermont_freq_table[i];
2121}
2122
2123static int airmont_get_scaling(void)
2124{
2125 u64 value;
2126 int i;
2127 /* Defined in Table 35-10 from SDM (Sept 2015) */
2128 static int airmont_freq_table[] = {
2129 83300, 100000, 133300, 116700, 80000,
2130 93300, 90000, 88900, 87500};
2131
2132 rdmsrq(MSR_FSB_FREQ, value);
2133 i = value & 0xF;
2134 WARN_ON(i > 8);
2135
2136 return airmont_freq_table[i];
2137}
2138
2139static void atom_get_vid(struct cpudata *cpudata)
2140{
2141 u64 value;
2142
2143 rdmsrq(MSR_ATOM_CORE_VIDS, value);
2144 cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
2145 cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
2146 cpudata->vid.ratio = div_fp(
2147 cpudata->vid.max - cpudata->vid.min,
2148 int_tofp(cpudata->pstate.max_pstate -
2149 cpudata->pstate.min_pstate));
2150
2151 rdmsrq(MSR_ATOM_CORE_TURBO_VIDS, value);
2152 cpudata->vid.turbo = value & 0x7f;
2153}
2154
2155static int core_get_min_pstate(int cpu)
2156{
2157 u64 value;
2158
2159 rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &value);
2160 return (value >> 40) & 0xFF;
2161}
2162
2163static int core_get_max_pstate_physical(int cpu)
2164{
2165 u64 value;
2166
2167 rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &value);
2168 return (value >> 8) & 0xFF;
2169}
2170
2171static int core_get_tdp_ratio(int cpu, u64 plat_info)
2172{
2173 /* Check how many TDP levels present */
2174 if (plat_info & 0x600000000) {
2175 u64 tdp_ctrl;
2176 u64 tdp_ratio;
2177 int tdp_msr;
2178 int err;
2179
2180 /* Get the TDP level (0, 1, 2) to get ratios */
2181 err = rdmsrq_safe_on_cpu(cpu, MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
2182 if (err)
2183 return err;
2184
2185 /* TDP MSR are continuous starting at 0x648 */
2186 tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03);
2187 err = rdmsrq_safe_on_cpu(cpu, tdp_msr, &tdp_ratio);
2188 if (err)
2189 return err;
2190
2191 /* For level 1 and 2, bits[23:16] contain the ratio */
2192 if (tdp_ctrl & 0x03)
2193 tdp_ratio >>= 16;
2194
2195 tdp_ratio &= 0xff; /* ratios are only 8 bits long */
2196 pr_debug("tdp_ratio %x\n", (int)tdp_ratio);
2197
2198 return (int)tdp_ratio;
2199 }
2200
2201 return -ENXIO;
2202}
2203
2204static int core_get_max_pstate(int cpu)
2205{
2206 u64 tar;
2207 u64 plat_info;
2208 int max_pstate;
2209 int tdp_ratio;
2210 int err;
2211
2212 rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &plat_info);
2213 max_pstate = (plat_info >> 8) & 0xFF;
2214
2215 tdp_ratio = core_get_tdp_ratio(cpu, plat_info);
2216 if (tdp_ratio <= 0)
2217 return max_pstate;
2218
2219 if (hwp_active) {
2220 /* Turbo activation ratio is not used on HWP platforms */
2221 return tdp_ratio;
2222 }
2223
2224 err = rdmsrq_safe_on_cpu(cpu, MSR_TURBO_ACTIVATION_RATIO, &tar);
2225 if (!err) {
2226 int tar_levels;
2227
2228 /* Do some sanity checking for safety */
2229 tar_levels = tar & 0xff;
2230 if (tdp_ratio - 1 == tar_levels) {
2231 max_pstate = tar_levels;
2232 pr_debug("max_pstate=TAC %x\n", max_pstate);
2233 }
2234 }
2235
2236 return max_pstate;
2237}
2238
2239static int core_get_turbo_pstate(int cpu)
2240{
2241 u64 value;
2242 int nont, ret;
2243
2244 rdmsrq_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value);
2245 nont = core_get_max_pstate(cpu);
2246 ret = (value) & 255;
2247 if (ret <= nont)
2248 ret = nont;
2249 return ret;
2250}
2251
2252static u64 core_get_val(struct cpudata *cpudata, int pstate)
2253{
2254 return get_perf_ctl_val(pstate);
2255}
2256
2257static int knl_get_aperf_mperf_shift(void)
2258{
2259 return 10;
2260}
2261
2262static int knl_get_turbo_pstate(int cpu)
2263{
2264 u64 value;
2265 int nont, ret;
2266
2267 rdmsrq_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value);
2268 nont = core_get_max_pstate(cpu);
2269 ret = (((value) >> 8) & 0xFF);
2270 if (ret <= nont)
2271 ret = nont;
2272 return ret;
2273}
2274
2275static int hwp_get_cpu_scaling(int cpu)
2276{
2277 if (hybrid_scaling_factor) {
2278 /*
2279 * Return the hybrid scaling factor for P-cores and use the
2280 * default core scaling for E-cores.
2281 */
2282 if (hybrid_get_cpu_type(cpu) == INTEL_CPU_TYPE_CORE)
2283 return hybrid_scaling_factor;
2284
2285 return core_get_scaling();
2286 }
2287
2288 /* Use core scaling on non-hybrid systems. */
2289 if (!cpu_feature_enabled(X86_FEATURE_HYBRID_CPU))
2290 return core_get_scaling();
2291
2292 /*
2293 * The system is hybrid, but the hybrid scaling factor is not known or
2294 * the CPU type is not one of the above, so use CPPC to compute the
2295 * scaling factor for this CPU.
2296 */
2297 return intel_pstate_cppc_get_scaling(cpu);
2298}
2299
2300static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
2301{
2302 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
2303 cpu->pstate.current_pstate = pstate;
2304 /*
2305 * Generally, there is no guarantee that this code will always run on
2306 * the CPU being updated, so force the register update to run on the
2307 * right CPU.
2308 */
2309 wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
2310 pstate_funcs.get_val(cpu, pstate));
2311}
2312
2313static void intel_pstate_set_min_pstate(struct cpudata *cpu)
2314{
2315 intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
2316}
2317
2318static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
2319{
2320 int perf_ctl_scaling = pstate_funcs.get_scaling();
2321
2322 cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(cpu->cpu);
2323 cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu);
2324 cpu->pstate.perf_ctl_scaling = perf_ctl_scaling;
2325
2326 if (hwp_active && !hwp_mode_bdw) {
2327 __intel_pstate_get_hwp_cap(cpu);
2328
2329 if (pstate_funcs.get_cpu_scaling) {
2330 cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpu->cpu);
2331 intel_pstate_hybrid_hwp_adjust(cpu);
2332 } else {
2333 cpu->pstate.scaling = perf_ctl_scaling;
2334 }
2335 /*
2336 * If the CPU is going online for the first time and it was
2337 * offline initially, asym capacity scaling needs to be updated.
2338 */
2339 hybrid_update_capacity(cpu);
2340 } else {
2341 cpu->pstate.scaling = perf_ctl_scaling;
2342 cpu->pstate.max_pstate = pstate_funcs.get_max(cpu->cpu);
2343 cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu);
2344 }
2345
2346 if (cpu->pstate.scaling == perf_ctl_scaling) {
2347 cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling;
2348 cpu->pstate.max_freq = cpu->pstate.max_pstate * perf_ctl_scaling;
2349 cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * perf_ctl_scaling;
2350 }
2351
2352 if (pstate_funcs.get_aperf_mperf_shift)
2353 cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift();
2354
2355 if (pstate_funcs.get_vid)
2356 pstate_funcs.get_vid(cpu);
2357
2358 intel_pstate_set_min_pstate(cpu);
2359}
2360
2361/*
2362 * Long hold time will keep high perf limits for long time,
2363 * which negatively impacts perf/watt for some workloads,
2364 * like specpower. 3ms is based on experiements on some
2365 * workoads.
2366 */
2367static int hwp_boost_hold_time_ns = 3 * NSEC_PER_MSEC;
2368
2369static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)
2370{
2371 u64 hwp_req = READ_ONCE(cpu->hwp_req_cached);
2372 u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached);
2373 u32 max_limit = (hwp_req & 0xff00) >> 8;
2374 u32 min_limit = (hwp_req & 0xff);
2375 u32 boost_level1;
2376
2377 /*
2378 * Cases to consider (User changes via sysfs or boot time):
2379 * If, P0 (Turbo max) = P1 (Guaranteed max) = min:
2380 * No boost, return.
2381 * If, P0 (Turbo max) > P1 (Guaranteed max) = min:
2382 * Should result in one level boost only for P0.
2383 * If, P0 (Turbo max) = P1 (Guaranteed max) > min:
2384 * Should result in two level boost:
2385 * (min + p1)/2 and P1.
2386 * If, P0 (Turbo max) > P1 (Guaranteed max) > min:
2387 * Should result in three level boost:
2388 * (min + p1)/2, P1 and P0.
2389 */
2390
2391 /* If max and min are equal or already at max, nothing to boost */
2392 if (max_limit == min_limit || cpu->hwp_boost_min >= max_limit)
2393 return;
2394
2395 if (!cpu->hwp_boost_min)
2396 cpu->hwp_boost_min = min_limit;
2397
2398 /* level at half way mark between min and guranteed */
2399 boost_level1 = (HWP_GUARANTEED_PERF(hwp_cap) + min_limit) >> 1;
2400
2401 if (cpu->hwp_boost_min < boost_level1)
2402 cpu->hwp_boost_min = boost_level1;
2403 else if (cpu->hwp_boost_min < HWP_GUARANTEED_PERF(hwp_cap))
2404 cpu->hwp_boost_min = HWP_GUARANTEED_PERF(hwp_cap);
2405 else if (cpu->hwp_boost_min == HWP_GUARANTEED_PERF(hwp_cap) &&
2406 max_limit != HWP_GUARANTEED_PERF(hwp_cap))
2407 cpu->hwp_boost_min = max_limit;
2408 else
2409 return;
2410
2411 hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min;
2412 wrmsrq(MSR_HWP_REQUEST, hwp_req);
2413 cpu->last_update = cpu->sample.time;
2414}
2415
2416static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)
2417{
2418 if (cpu->hwp_boost_min) {
2419 bool expired;
2420
2421 /* Check if we are idle for hold time to boost down */
2422 expired = time_after64(cpu->sample.time, cpu->last_update +
2423 hwp_boost_hold_time_ns);
2424 if (expired) {
2425 wrmsrq(MSR_HWP_REQUEST, cpu->hwp_req_cached);
2426 cpu->hwp_boost_min = 0;
2427 }
2428 }
2429 cpu->last_update = cpu->sample.time;
2430}
2431
2432static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu,
2433 u64 time)
2434{
2435 cpu->sample.time = time;
2436
2437 if (cpu->sched_flags & SCHED_CPUFREQ_IOWAIT) {
2438 bool do_io = false;
2439
2440 cpu->sched_flags = 0;
2441 /*
2442 * Set iowait_boost flag and update time. Since IO WAIT flag
2443 * is set all the time, we can't just conclude that there is
2444 * some IO bound activity is scheduled on this CPU with just
2445 * one occurrence. If we receive at least two in two
2446 * consecutive ticks, then we treat as boost candidate.
2447 */
2448 if (time_before64(time, cpu->last_io_update + 2 * TICK_NSEC))
2449 do_io = true;
2450
2451 cpu->last_io_update = time;
2452
2453 if (do_io)
2454 intel_pstate_hwp_boost_up(cpu);
2455
2456 } else {
2457 intel_pstate_hwp_boost_down(cpu);
2458 }
2459}
2460
2461static inline void intel_pstate_update_util_hwp(struct update_util_data *data,
2462 u64 time, unsigned int flags)
2463{
2464 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
2465
2466 cpu->sched_flags |= flags;
2467
2468 if (smp_processor_id() == cpu->cpu)
2469 intel_pstate_update_util_hwp_local(cpu, time);
2470}
2471
2472static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
2473{
2474 struct sample *sample = &cpu->sample;
2475
2476 sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf);
2477}
2478
2479static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
2480{
2481 u64 aperf, mperf;
2482 unsigned long flags;
2483 u64 tsc;
2484
2485 local_irq_save(flags);
2486 rdmsrq(MSR_IA32_APERF, aperf);
2487 rdmsrq(MSR_IA32_MPERF, mperf);
2488 tsc = rdtsc();
2489 if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) {
2490 local_irq_restore(flags);
2491 return false;
2492 }
2493 local_irq_restore(flags);
2494
2495 cpu->last_sample_time = cpu->sample.time;
2496 cpu->sample.time = time;
2497 cpu->sample.aperf = aperf;
2498 cpu->sample.mperf = mperf;
2499 cpu->sample.tsc = tsc;
2500 cpu->sample.aperf -= cpu->prev_aperf;
2501 cpu->sample.mperf -= cpu->prev_mperf;
2502 cpu->sample.tsc -= cpu->prev_tsc;
2503
2504 cpu->prev_aperf = aperf;
2505 cpu->prev_mperf = mperf;
2506 cpu->prev_tsc = tsc;
2507 /*
2508 * First time this function is invoked in a given cycle, all of the
2509 * previous sample data fields are equal to zero or stale and they must
2510 * be populated with meaningful numbers for things to work, so assume
2511 * that sample.time will always be reset before setting the utilization
2512 * update hook and make the caller skip the sample then.
2513 */
2514 if (likely(cpu->last_sample_time)) {
2515 intel_pstate_calc_avg_perf(cpu);
2516 return true;
2517 }
2518 return false;
2519}
2520
2521static inline int32_t get_avg_frequency(struct cpudata *cpu)
2522{
2523 return mul_ext_fp(cpu->sample.core_avg_perf, cpu_khz);
2524}
2525
2526static inline int32_t get_avg_pstate(struct cpudata *cpu)
2527{
2528 return mul_ext_fp(cpu->pstate.max_pstate_physical,
2529 cpu->sample.core_avg_perf);
2530}
2531
2532static inline int32_t get_target_pstate(struct cpudata *cpu)
2533{
2534 struct sample *sample = &cpu->sample;
2535 int32_t busy_frac;
2536 int target, avg_pstate;
2537
2538 busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift,
2539 sample->tsc);
2540
2541 if (busy_frac < cpu->iowait_boost)
2542 busy_frac = cpu->iowait_boost;
2543
2544 sample->busy_scaled = busy_frac * 100;
2545
2546 target = READ_ONCE(global.no_turbo) ?
2547 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
2548 target += target >> 2;
2549 target = mul_fp(target, busy_frac);
2550 if (target < cpu->pstate.min_pstate)
2551 target = cpu->pstate.min_pstate;
2552
2553 /*
2554 * If the average P-state during the previous cycle was higher than the
2555 * current target, add 50% of the difference to the target to reduce
2556 * possible performance oscillations and offset possible performance
2557 * loss related to moving the workload from one CPU to another within
2558 * a package/module.
2559 */
2560 avg_pstate = get_avg_pstate(cpu);
2561 if (avg_pstate > target)
2562 target += (avg_pstate - target) >> 1;
2563
2564 return target;
2565}
2566
2567static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
2568{
2569 int min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio);
2570 int max_pstate = max(min_pstate, cpu->max_perf_ratio);
2571
2572 return clamp_t(int, pstate, min_pstate, max_pstate);
2573}
2574
2575static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
2576{
2577 if (pstate == cpu->pstate.current_pstate)
2578 return;
2579
2580 cpu->pstate.current_pstate = pstate;
2581 wrmsrq(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
2582}
2583
2584static void intel_pstate_adjust_pstate(struct cpudata *cpu)
2585{
2586 int from = cpu->pstate.current_pstate;
2587 struct sample *sample;
2588 int target_pstate;
2589
2590 target_pstate = get_target_pstate(cpu);
2591 target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
2592 trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu);
2593 intel_pstate_update_pstate(cpu, target_pstate);
2594
2595 sample = &cpu->sample;
2596 trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf),
2597 fp_toint(sample->busy_scaled),
2598 from,
2599 cpu->pstate.current_pstate,
2600 sample->mperf,
2601 sample->aperf,
2602 sample->tsc,
2603 get_avg_frequency(cpu),
2604 fp_toint(cpu->iowait_boost * 100));
2605}
2606
2607static void intel_pstate_update_util(struct update_util_data *data, u64 time,
2608 unsigned int flags)
2609{
2610 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
2611 u64 delta_ns;
2612
2613 /* Don't allow remote callbacks */
2614 if (smp_processor_id() != cpu->cpu)
2615 return;
2616
2617 delta_ns = time - cpu->last_update;
2618 if (flags & SCHED_CPUFREQ_IOWAIT) {
2619 /* Start over if the CPU may have been idle. */
2620 if (delta_ns > TICK_NSEC) {
2621 cpu->iowait_boost = ONE_EIGHTH_FP;
2622 } else if (cpu->iowait_boost >= ONE_EIGHTH_FP) {
2623 cpu->iowait_boost <<= 1;
2624 if (cpu->iowait_boost > int_tofp(1))
2625 cpu->iowait_boost = int_tofp(1);
2626 } else {
2627 cpu->iowait_boost = ONE_EIGHTH_FP;
2628 }
2629 } else if (cpu->iowait_boost) {
2630 /* Clear iowait_boost if the CPU may have been idle. */
2631 if (delta_ns > TICK_NSEC)
2632 cpu->iowait_boost = 0;
2633 else
2634 cpu->iowait_boost >>= 1;
2635 }
2636 cpu->last_update = time;
2637 delta_ns = time - cpu->sample.time;
2638 if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL)
2639 return;
2640
2641 if (intel_pstate_sample(cpu, time))
2642 intel_pstate_adjust_pstate(cpu);
2643}
2644
2645static struct pstate_funcs core_funcs = {
2646 .get_max = core_get_max_pstate,
2647 .get_max_physical = core_get_max_pstate_physical,
2648 .get_min = core_get_min_pstate,
2649 .get_turbo = core_get_turbo_pstate,
2650 .get_scaling = core_get_scaling,
2651 .get_val = core_get_val,
2652};
2653
2654static const struct pstate_funcs silvermont_funcs = {
2655 .get_max = atom_get_max_pstate,
2656 .get_max_physical = atom_get_max_pstate,
2657 .get_min = atom_get_min_pstate,
2658 .get_turbo = atom_get_turbo_pstate,
2659 .get_val = atom_get_val,
2660 .get_scaling = silvermont_get_scaling,
2661 .get_vid = atom_get_vid,
2662};
2663
2664static const struct pstate_funcs airmont_funcs = {
2665 .get_max = atom_get_max_pstate,
2666 .get_max_physical = atom_get_max_pstate,
2667 .get_min = atom_get_min_pstate,
2668 .get_turbo = atom_get_turbo_pstate,
2669 .get_val = atom_get_val,
2670 .get_scaling = airmont_get_scaling,
2671 .get_vid = atom_get_vid,
2672};
2673
2674static const struct pstate_funcs knl_funcs = {
2675 .get_max = core_get_max_pstate,
2676 .get_max_physical = core_get_max_pstate_physical,
2677 .get_min = core_get_min_pstate,
2678 .get_turbo = knl_get_turbo_pstate,
2679 .get_aperf_mperf_shift = knl_get_aperf_mperf_shift,
2680 .get_scaling = core_get_scaling,
2681 .get_val = core_get_val,
2682};
2683
2684#define X86_MATCH(vfm, policy) \
2685 X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_APERFMPERF, &policy)
2686
2687static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
2688 X86_MATCH(INTEL_SANDYBRIDGE, core_funcs),
2689 X86_MATCH(INTEL_SANDYBRIDGE_X, core_funcs),
2690 X86_MATCH(INTEL_ATOM_SILVERMONT, silvermont_funcs),
2691 X86_MATCH(INTEL_IVYBRIDGE, core_funcs),
2692 X86_MATCH(INTEL_HASWELL, core_funcs),
2693 X86_MATCH(INTEL_BROADWELL, core_funcs),
2694 X86_MATCH(INTEL_IVYBRIDGE_X, core_funcs),
2695 X86_MATCH(INTEL_HASWELL_X, core_funcs),
2696 X86_MATCH(INTEL_HASWELL_L, core_funcs),
2697 X86_MATCH(INTEL_HASWELL_G, core_funcs),
2698 X86_MATCH(INTEL_BROADWELL_G, core_funcs),
2699 X86_MATCH(INTEL_ATOM_AIRMONT, airmont_funcs),
2700 X86_MATCH(INTEL_SKYLAKE_L, core_funcs),
2701 X86_MATCH(INTEL_BROADWELL_X, core_funcs),
2702 X86_MATCH(INTEL_SKYLAKE, core_funcs),
2703 X86_MATCH(INTEL_BROADWELL_D, core_funcs),
2704 X86_MATCH(INTEL_XEON_PHI_KNL, knl_funcs),
2705 X86_MATCH(INTEL_XEON_PHI_KNM, knl_funcs),
2706 X86_MATCH(INTEL_ATOM_GOLDMONT, core_funcs),
2707 X86_MATCH(INTEL_ATOM_GOLDMONT_PLUS, core_funcs),
2708 X86_MATCH(INTEL_SKYLAKE_X, core_funcs),
2709 X86_MATCH(INTEL_COMETLAKE, core_funcs),
2710 X86_MATCH(INTEL_ICELAKE_X, core_funcs),
2711 X86_MATCH(INTEL_TIGERLAKE, core_funcs),
2712 X86_MATCH(INTEL_SAPPHIRERAPIDS_X, core_funcs),
2713 X86_MATCH(INTEL_EMERALDRAPIDS_X, core_funcs),
2714 X86_MATCH(INTEL_GRANITERAPIDS_D, core_funcs),
2715 X86_MATCH(INTEL_GRANITERAPIDS_X, core_funcs),
2716 {}
2717};
2718MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
2719
2720#ifdef CONFIG_ACPI
2721static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
2722 X86_MATCH(INTEL_BROADWELL_D, core_funcs),
2723 X86_MATCH(INTEL_BROADWELL_X, core_funcs),
2724 X86_MATCH(INTEL_SKYLAKE_X, core_funcs),
2725 X86_MATCH(INTEL_ICELAKE_X, core_funcs),
2726 X86_MATCH(INTEL_SAPPHIRERAPIDS_X, core_funcs),
2727 X86_MATCH(INTEL_EMERALDRAPIDS_X, core_funcs),
2728 X86_MATCH(INTEL_GRANITERAPIDS_D, core_funcs),
2729 X86_MATCH(INTEL_GRANITERAPIDS_X, core_funcs),
2730 X86_MATCH(INTEL_ATOM_CRESTMONT, core_funcs),
2731 X86_MATCH(INTEL_ATOM_CRESTMONT_X, core_funcs),
2732 X86_MATCH(INTEL_ATOM_DARKMONT_X, core_funcs),
2733 X86_MATCH(INTEL_DIAMONDRAPIDS_X, core_funcs),
2734 {}
2735};
2736#endif
2737
2738static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
2739 X86_MATCH(INTEL_KABYLAKE, core_funcs),
2740 {}
2741};
2742
2743static int intel_pstate_init_cpu(unsigned int cpunum)
2744{
2745 struct cpudata *cpu;
2746
2747 cpu = all_cpu_data[cpunum];
2748
2749 if (!cpu) {
2750 cpu = kzalloc_obj(*cpu);
2751 if (!cpu)
2752 return -ENOMEM;
2753
2754 WRITE_ONCE(all_cpu_data[cpunum], cpu);
2755
2756 cpu->cpu = cpunum;
2757
2758 cpu->epp_default = -EINVAL;
2759
2760 if (hwp_active) {
2761 intel_pstate_hwp_enable(cpu);
2762
2763 if (intel_pstate_acpi_pm_profile_server())
2764 hwp_boost = true;
2765 }
2766 } else if (hwp_active) {
2767 /*
2768 * Re-enable HWP in case this happens after a resume from ACPI
2769 * S3 if the CPU was offline during the whole system/resume
2770 * cycle.
2771 */
2772 intel_pstate_hwp_reenable(cpu);
2773 }
2774
2775 cpu->epp_powersave = -EINVAL;
2776 cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN;
2777
2778 intel_pstate_get_cpu_pstates(cpu);
2779
2780 pr_debug("controlling: cpu %d\n", cpunum);
2781
2782 return 0;
2783}
2784
2785static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
2786{
2787 struct cpudata *cpu = all_cpu_data[cpu_num];
2788
2789 if (hwp_active && !hwp_boost)
2790 return;
2791
2792 if (cpu->update_util_set)
2793 return;
2794
2795 /* Prevent intel_pstate_update_util() from using stale data. */
2796 cpu->sample.time = 0;
2797 cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
2798 (hwp_active ?
2799 intel_pstate_update_util_hwp :
2800 intel_pstate_update_util));
2801 cpu->update_util_set = true;
2802}
2803
2804static void intel_pstate_clear_update_util_hook(unsigned int cpu)
2805{
2806 struct cpudata *cpu_data = all_cpu_data[cpu];
2807
2808 if (!cpu_data->update_util_set)
2809 return;
2810
2811 cpufreq_remove_update_util_hook(cpu);
2812 cpu_data->update_util_set = false;
2813 synchronize_rcu();
2814}
2815
2816static int intel_pstate_get_max_freq(struct cpudata *cpu)
2817{
2818 return READ_ONCE(global.no_turbo) ?
2819 cpu->pstate.max_freq : cpu->pstate.turbo_freq;
2820}
2821
2822static void intel_pstate_update_perf_limits(struct cpudata *cpu,
2823 unsigned int policy_min,
2824 unsigned int policy_max)
2825{
2826 int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
2827 int32_t max_policy_perf, min_policy_perf;
2828
2829 max_policy_perf = policy_max / perf_ctl_scaling;
2830 if (policy_max == policy_min) {
2831 min_policy_perf = max_policy_perf;
2832 } else {
2833 min_policy_perf = policy_min / perf_ctl_scaling;
2834 min_policy_perf = clamp_t(int32_t, min_policy_perf,
2835 0, max_policy_perf);
2836 }
2837
2838 /*
2839 * HWP needs some special consideration, because HWP_REQUEST uses
2840 * abstract values to represent performance rather than pure ratios.
2841 */
2842 if (hwp_active && cpu->pstate.scaling != perf_ctl_scaling) {
2843 int freq;
2844
2845 freq = max_policy_perf * perf_ctl_scaling;
2846 max_policy_perf = intel_pstate_freq_to_hwp(cpu, freq);
2847 freq = min_policy_perf * perf_ctl_scaling;
2848 min_policy_perf = intel_pstate_freq_to_hwp(cpu, freq);
2849 }
2850
2851 pr_debug("cpu:%d min_policy_perf:%d max_policy_perf:%d\n",
2852 cpu->cpu, min_policy_perf, max_policy_perf);
2853
2854 /* Normalize user input to [min_perf, max_perf] */
2855 if (per_cpu_limits) {
2856 cpu->min_perf_ratio = min_policy_perf;
2857 cpu->max_perf_ratio = max_policy_perf;
2858 } else {
2859 int turbo_max = cpu->pstate.turbo_pstate;
2860 int32_t global_min, global_max;
2861
2862 /* Global limits are in percent of the maximum turbo P-state. */
2863 global_max = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100);
2864 global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
2865 global_min = clamp_t(int32_t, global_min, 0, global_max);
2866
2867 pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu->cpu,
2868 global_min, global_max);
2869
2870 cpu->min_perf_ratio = max(min_policy_perf, global_min);
2871 cpu->min_perf_ratio = min(cpu->min_perf_ratio, max_policy_perf);
2872 cpu->max_perf_ratio = min(max_policy_perf, global_max);
2873 cpu->max_perf_ratio = max(min_policy_perf, cpu->max_perf_ratio);
2874
2875 /* Make sure min_perf <= max_perf */
2876 cpu->min_perf_ratio = min(cpu->min_perf_ratio,
2877 cpu->max_perf_ratio);
2878
2879 }
2880 pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu->cpu,
2881 cpu->max_perf_ratio,
2882 cpu->min_perf_ratio);
2883}
2884
2885static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2886{
2887 struct cpudata *cpu;
2888
2889 if (!policy->cpuinfo.max_freq)
2890 return -ENODEV;
2891
2892 pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
2893 policy->cpuinfo.max_freq, policy->max);
2894
2895 cpu = all_cpu_data[policy->cpu];
2896 cpu->policy = policy->policy;
2897
2898 mutex_lock(&intel_pstate_limits_lock);
2899
2900 intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
2901
2902 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
2903 int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio);
2904
2905 /*
2906 * NOHZ_FULL CPUs need this as the governor callback may not
2907 * be invoked on them.
2908 */
2909 intel_pstate_clear_update_util_hook(policy->cpu);
2910 intel_pstate_set_pstate(cpu, pstate);
2911 } else {
2912 intel_pstate_set_update_util_hook(policy->cpu);
2913 }
2914
2915 if (hwp_active) {
2916 /*
2917 * When hwp_boost was active before and dynamically it
2918 * was turned off, in that case we need to clear the
2919 * update util hook.
2920 */
2921 if (!hwp_boost)
2922 intel_pstate_clear_update_util_hook(policy->cpu);
2923 intel_pstate_hwp_set(policy->cpu);
2924 }
2925 /*
2926 * policy->cur is never updated with the intel_pstate driver, but it
2927 * is used as a stale frequency value. So, keep it within limits.
2928 */
2929 policy->cur = policy->min;
2930
2931 mutex_unlock(&intel_pstate_limits_lock);
2932
2933 return 0;
2934}
2935
2936static void intel_pstate_adjust_policy_max(struct cpudata *cpu,
2937 struct cpufreq_policy_data *policy)
2938{
2939 if (!hwp_active &&
2940 cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
2941 policy->max < policy->cpuinfo.max_freq &&
2942 policy->max > cpu->pstate.max_freq) {
2943 pr_debug("policy->max > max non turbo frequency\n");
2944 policy->max = policy->cpuinfo.max_freq;
2945 }
2946}
2947
2948static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,
2949 struct cpufreq_policy_data *policy)
2950{
2951 int max_freq;
2952
2953 if (hwp_active) {
2954 intel_pstate_get_hwp_cap(cpu);
2955 max_freq = READ_ONCE(global.no_turbo) ?
2956 cpu->pstate.max_freq : cpu->pstate.turbo_freq;
2957 } else {
2958 max_freq = intel_pstate_get_max_freq(cpu);
2959 }
2960 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, max_freq);
2961
2962 intel_pstate_adjust_policy_max(cpu, policy);
2963}
2964
2965static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
2966{
2967 intel_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy);
2968
2969 return 0;
2970}
2971
2972static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy)
2973{
2974 struct cpudata *cpu = all_cpu_data[policy->cpu];
2975
2976 pr_debug("CPU %d going offline\n", cpu->cpu);
2977
2978 if (cpu->suspended)
2979 return 0;
2980
2981 /*
2982 * If the CPU is an SMT thread and it goes offline with the performance
2983 * settings different from the minimum, it will prevent its sibling
2984 * from getting to lower performance levels, so force the minimum
2985 * performance on CPU offline to prevent that from happening.
2986 */
2987 if (hwp_active)
2988 intel_pstate_hwp_offline(cpu);
2989 else
2990 intel_pstate_set_min_pstate(cpu);
2991
2992 intel_pstate_exit_perf_limits(policy);
2993
2994 return 0;
2995}
2996
2997static int intel_pstate_cpu_online(struct cpufreq_policy *policy)
2998{
2999 struct cpudata *cpu = all_cpu_data[policy->cpu];
3000
3001 pr_debug("CPU %d going online\n", cpu->cpu);
3002
3003 intel_pstate_init_acpi_perf_limits(policy);
3004
3005 if (hwp_active) {
3006 /*
3007 * Re-enable HWP and clear the "suspended" flag to let "resume"
3008 * know that it need not do that.
3009 */
3010 intel_pstate_hwp_reenable(cpu);
3011 cpu->suspended = false;
3012
3013 hybrid_update_capacity(cpu);
3014 }
3015
3016 return 0;
3017}
3018
3019static int intel_pstate_cpu_offline(struct cpufreq_policy *policy)
3020{
3021 intel_pstate_clear_update_util_hook(policy->cpu);
3022
3023 return intel_cpufreq_cpu_offline(policy);
3024}
3025
3026static void intel_pstate_cpu_exit(struct cpufreq_policy *policy)
3027{
3028 pr_debug("CPU %d exiting\n", policy->cpu);
3029
3030 policy->fast_switch_possible = false;
3031}
3032
3033static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
3034{
3035 struct cpudata *cpu;
3036 int rc;
3037
3038 rc = intel_pstate_init_cpu(policy->cpu);
3039 if (rc)
3040 return rc;
3041
3042 cpu = all_cpu_data[policy->cpu];
3043
3044 cpu->max_perf_ratio = 0xFF;
3045 cpu->min_perf_ratio = 0;
3046
3047 /* cpuinfo and default policy values */
3048 policy->cpuinfo.min_freq = cpu->pstate.min_freq;
3049 policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ?
3050 cpu->pstate.max_freq : cpu->pstate.turbo_freq;
3051
3052 policy->min = policy->cpuinfo.min_freq;
3053 policy->max = policy->cpuinfo.max_freq;
3054
3055 intel_pstate_init_acpi_perf_limits(policy);
3056
3057 policy->fast_switch_possible = true;
3058
3059 return 0;
3060}
3061
3062static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
3063{
3064 int ret = __intel_pstate_cpu_init(policy);
3065
3066 if (ret)
3067 return ret;
3068
3069 /*
3070 * Set the policy to powersave to provide a valid fallback value in case
3071 * the default cpufreq governor is neither powersave nor performance.
3072 */
3073 policy->policy = CPUFREQ_POLICY_POWERSAVE;
3074
3075 if (hwp_active) {
3076 struct cpudata *cpu = all_cpu_data[policy->cpu];
3077
3078 cpu->epp_cached = intel_pstate_get_epp(cpu, 0);
3079 }
3080
3081 return 0;
3082}
3083
3084static struct cpufreq_driver intel_pstate = {
3085 .flags = CPUFREQ_CONST_LOOPS,
3086 .verify = intel_pstate_verify_policy,
3087 .setpolicy = intel_pstate_set_policy,
3088 .suspend = intel_pstate_suspend,
3089 .resume = intel_pstate_resume,
3090 .init = intel_pstate_cpu_init,
3091 .exit = intel_pstate_cpu_exit,
3092 .offline = intel_pstate_cpu_offline,
3093 .online = intel_pstate_cpu_online,
3094 .update_limits = intel_pstate_update_limits,
3095 .name = "intel_pstate",
3096};
3097
3098static int intel_cpufreq_verify_policy(struct cpufreq_policy_data *policy)
3099{
3100 struct cpudata *cpu = all_cpu_data[policy->cpu];
3101
3102 intel_pstate_verify_cpu_policy(cpu, policy);
3103 intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
3104
3105 return 0;
3106}
3107
3108/* Use of trace in passive mode:
3109 *
3110 * In passive mode the trace core_busy field (also known as the
3111 * performance field, and lablelled as such on the graphs; also known as
3112 * core_avg_perf) is not needed and so is re-assigned to indicate if the
3113 * driver call was via the normal or fast switch path. Various graphs
3114 * output from the intel_pstate_tracer.py utility that include core_busy
3115 * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%,
3116 * so we use 10 to indicate the normal path through the driver, and
3117 * 90 to indicate the fast switch path through the driver.
3118 * The scaled_busy field is not used, and is set to 0.
3119 */
3120
3121#define INTEL_PSTATE_TRACE_TARGET 10
3122#define INTEL_PSTATE_TRACE_FAST_SWITCH 90
3123
3124static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)
3125{
3126 struct sample *sample;
3127
3128 if (!trace_pstate_sample_enabled())
3129 return;
3130
3131 if (!intel_pstate_sample(cpu, ktime_get()))
3132 return;
3133
3134 sample = &cpu->sample;
3135 trace_call__pstate_sample(trace_type,
3136 0,
3137 old_pstate,
3138 cpu->pstate.current_pstate,
3139 sample->mperf,
3140 sample->aperf,
3141 sample->tsc,
3142 get_avg_frequency(cpu),
3143 fp_toint(cpu->iowait_boost * 100));
3144}
3145
3146static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max,
3147 u32 desired, bool fast_switch)
3148{
3149 u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev;
3150
3151 value &= ~HWP_MIN_PERF(~0L);
3152 value |= HWP_MIN_PERF(min);
3153
3154 value &= ~HWP_MAX_PERF(~0L);
3155 value |= HWP_MAX_PERF(max);
3156
3157 value &= ~HWP_DESIRED_PERF(~0L);
3158 value |= HWP_DESIRED_PERF(desired);
3159
3160 if (value == prev)
3161 return;
3162
3163 WRITE_ONCE(cpu->hwp_req_cached, value);
3164 if (fast_switch)
3165 wrmsrq(MSR_HWP_REQUEST, value);
3166 else
3167 wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
3168}
3169
3170static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu,
3171 u32 target_pstate, bool fast_switch)
3172{
3173 if (fast_switch)
3174 wrmsrq(MSR_IA32_PERF_CTL,
3175 pstate_funcs.get_val(cpu, target_pstate));
3176 else
3177 wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
3178 pstate_funcs.get_val(cpu, target_pstate));
3179}
3180
3181static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy,
3182 int target_pstate, bool fast_switch)
3183{
3184 struct cpudata *cpu = all_cpu_data[policy->cpu];
3185 int old_pstate = cpu->pstate.current_pstate;
3186
3187 target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
3188 if (hwp_active) {
3189 int max_pstate = policy->strict_target ?
3190 target_pstate : cpu->max_perf_ratio;
3191
3192 intel_cpufreq_hwp_update(cpu, target_pstate, max_pstate,
3193 target_pstate, fast_switch);
3194 } else if (target_pstate != old_pstate) {
3195 intel_cpufreq_perf_ctl_update(cpu, target_pstate, fast_switch);
3196 }
3197
3198 cpu->pstate.current_pstate = target_pstate;
3199
3200 intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH :
3201 INTEL_PSTATE_TRACE_TARGET, old_pstate);
3202
3203 return target_pstate;
3204}
3205
3206static int intel_cpufreq_target(struct cpufreq_policy *policy,
3207 unsigned int target_freq,
3208 unsigned int relation)
3209{
3210 struct cpudata *cpu = all_cpu_data[policy->cpu];
3211 struct cpufreq_freqs freqs;
3212 int target_pstate;
3213
3214 freqs.old = policy->cur;
3215 freqs.new = target_freq;
3216
3217 cpufreq_freq_transition_begin(policy, &freqs);
3218
3219 target_pstate = intel_pstate_freq_to_hwp_rel(cpu, freqs.new, relation);
3220 target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, false);
3221
3222 freqs.new = target_pstate * cpu->pstate.scaling;
3223
3224 cpufreq_freq_transition_end(policy, &freqs, false);
3225
3226 return 0;
3227}
3228
3229static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
3230 unsigned int target_freq)
3231{
3232 struct cpudata *cpu = all_cpu_data[policy->cpu];
3233 int target_pstate;
3234
3235 target_pstate = intel_pstate_freq_to_hwp(cpu, target_freq);
3236
3237 target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true);
3238
3239 return target_pstate * cpu->pstate.scaling;
3240}
3241
3242static void intel_cpufreq_adjust_perf(struct cpufreq_policy *policy,
3243 unsigned long min_perf,
3244 unsigned long target_perf,
3245 unsigned long capacity)
3246{
3247 struct cpudata *cpu = all_cpu_data[policy->cpu];
3248 u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached);
3249 int old_pstate = cpu->pstate.current_pstate;
3250 int cap_pstate, min_pstate, max_pstate, target_pstate;
3251
3252 cap_pstate = READ_ONCE(global.no_turbo) ?
3253 HWP_GUARANTEED_PERF(hwp_cap) :
3254 HWP_HIGHEST_PERF(hwp_cap);
3255
3256 /* Optimization: Avoid unnecessary divisions. */
3257
3258 target_pstate = cap_pstate;
3259 if (target_perf < capacity)
3260 target_pstate = DIV_ROUND_UP(cap_pstate * target_perf, capacity);
3261
3262 min_pstate = cap_pstate;
3263 if (min_perf < capacity)
3264 min_pstate = DIV_ROUND_UP(cap_pstate * min_perf, capacity);
3265
3266 if (min_pstate < cpu->pstate.min_pstate)
3267 min_pstate = cpu->pstate.min_pstate;
3268
3269 if (min_pstate < cpu->min_perf_ratio)
3270 min_pstate = cpu->min_perf_ratio;
3271
3272 if (min_pstate > cpu->max_perf_ratio)
3273 min_pstate = cpu->max_perf_ratio;
3274
3275 max_pstate = min(cap_pstate, cpu->max_perf_ratio);
3276 if (max_pstate < min_pstate)
3277 max_pstate = min_pstate;
3278
3279 target_pstate = clamp_t(int, target_pstate, min_pstate, max_pstate);
3280
3281 intel_cpufreq_hwp_update(cpu, min_pstate, max_pstate, target_pstate, true);
3282
3283 cpu->pstate.current_pstate = target_pstate;
3284 intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate);
3285}
3286
3287static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
3288{
3289 struct freq_qos_request *req;
3290 struct cpudata *cpu;
3291 struct device *dev;
3292 int ret, freq;
3293
3294 dev = get_cpu_device(policy->cpu);
3295 if (!dev)
3296 return -ENODEV;
3297
3298 ret = __intel_pstate_cpu_init(policy);
3299 if (ret)
3300 return ret;
3301
3302 policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
3303 /* This reflects the intel_pstate_get_cpu_pstates() setting. */
3304 policy->cur = policy->cpuinfo.min_freq;
3305
3306 req = kzalloc_objs(*req, 2);
3307 if (!req) {
3308 ret = -ENOMEM;
3309 goto pstate_exit;
3310 }
3311
3312 cpu = all_cpu_data[policy->cpu];
3313
3314 if (hwp_active) {
3315 u64 value;
3316
3317 policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP;
3318
3319 intel_pstate_get_hwp_cap(cpu);
3320
3321 rdmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value);
3322 WRITE_ONCE(cpu->hwp_req_cached, value);
3323
3324 cpu->epp_cached = intel_pstate_get_epp(cpu, value);
3325 } else {
3326 policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
3327 }
3328
3329 freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100);
3330
3331 ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MIN,
3332 freq);
3333 if (ret < 0) {
3334 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
3335 goto free_req;
3336 }
3337
3338 freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.max_perf_pct, 100);
3339
3340 ret = freq_qos_add_request(&policy->constraints, req + 1, FREQ_QOS_MAX,
3341 freq);
3342 if (ret < 0) {
3343 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
3344 goto remove_min_req;
3345 }
3346
3347 policy->driver_data = req;
3348
3349 return 0;
3350
3351remove_min_req:
3352 freq_qos_remove_request(req);
3353free_req:
3354 kfree(req);
3355pstate_exit:
3356 intel_pstate_exit_perf_limits(policy);
3357
3358 return ret;
3359}
3360
3361static void intel_cpufreq_cpu_exit(struct cpufreq_policy *policy)
3362{
3363 struct freq_qos_request *req;
3364
3365 req = policy->driver_data;
3366
3367 freq_qos_remove_request(req + 1);
3368 freq_qos_remove_request(req);
3369 kfree(req);
3370
3371 intel_pstate_cpu_exit(policy);
3372}
3373
3374static int intel_cpufreq_suspend(struct cpufreq_policy *policy)
3375{
3376 intel_pstate_suspend(policy);
3377
3378 if (hwp_active) {
3379 struct cpudata *cpu = all_cpu_data[policy->cpu];
3380 u64 value = READ_ONCE(cpu->hwp_req_cached);
3381
3382 /*
3383 * Clear the desired perf field in MSR_HWP_REQUEST in case
3384 * intel_cpufreq_adjust_perf() is in use and the last value
3385 * written by it may not be suitable.
3386 */
3387 value &= ~HWP_DESIRED_PERF(~0L);
3388 wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
3389 WRITE_ONCE(cpu->hwp_req_cached, value);
3390 }
3391
3392 return 0;
3393}
3394
3395static struct cpufreq_driver intel_cpufreq = {
3396 .flags = CPUFREQ_CONST_LOOPS,
3397 .verify = intel_cpufreq_verify_policy,
3398 .target = intel_cpufreq_target,
3399 .fast_switch = intel_cpufreq_fast_switch,
3400 .init = intel_cpufreq_cpu_init,
3401 .exit = intel_cpufreq_cpu_exit,
3402 .offline = intel_cpufreq_cpu_offline,
3403 .online = intel_pstate_cpu_online,
3404 .suspend = intel_cpufreq_suspend,
3405 .resume = intel_pstate_resume,
3406 .update_limits = intel_pstate_update_limits,
3407 .name = "intel_cpufreq",
3408};
3409
3410static struct cpufreq_driver *default_driver;
3411
3412static void intel_pstate_driver_cleanup(void)
3413{
3414 unsigned int cpu;
3415
3416 cpus_read_lock();
3417 for_each_online_cpu(cpu) {
3418 if (all_cpu_data[cpu]) {
3419 if (intel_pstate_driver == &intel_pstate)
3420 intel_pstate_clear_update_util_hook(cpu);
3421
3422 kfree(all_cpu_data[cpu]);
3423 WRITE_ONCE(all_cpu_data[cpu], NULL);
3424 }
3425 }
3426 cpus_read_unlock();
3427
3428 intel_pstate_driver = NULL;
3429}
3430
3431static int intel_pstate_register_driver(struct cpufreq_driver *driver)
3432{
3433 bool refresh_cpu_cap_scaling;
3434 int ret;
3435
3436 if (driver == &intel_pstate)
3437 intel_pstate_sysfs_expose_hwp_dynamic_boost();
3438
3439 memset(&global, 0, sizeof(global));
3440 global.max_perf_pct = 100;
3441 global.turbo_disabled = turbo_is_disabled();
3442 global.no_turbo = global.turbo_disabled;
3443
3444 arch_set_max_freq_ratio(global.turbo_disabled);
3445
3446 refresh_cpu_cap_scaling = hybrid_clear_max_perf_cpu();
3447
3448 intel_pstate_driver = driver;
3449 ret = cpufreq_register_driver(intel_pstate_driver);
3450 if (ret) {
3451 intel_pstate_driver_cleanup();
3452 return ret;
3453 }
3454
3455 global.min_perf_pct = min_perf_pct_min();
3456
3457 hybrid_init_cpu_capacity_scaling(refresh_cpu_cap_scaling);
3458
3459 return 0;
3460}
3461
3462static ssize_t intel_pstate_show_status(char *buf)
3463{
3464 if (!intel_pstate_driver)
3465 return sprintf(buf, "off\n");
3466
3467 return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
3468 "active" : "passive");
3469}
3470
3471static int intel_pstate_update_status(const char *buf, size_t size)
3472{
3473 if (size == 3 && !strncmp(buf, "off", size)) {
3474 if (!intel_pstate_driver)
3475 return 0;
3476
3477 if (hwp_active)
3478 return -EBUSY;
3479
3480 cpufreq_unregister_driver(intel_pstate_driver);
3481 intel_pstate_driver_cleanup();
3482 return 0;
3483 }
3484
3485 if (size == 6 && !strncmp(buf, "active", size)) {
3486 if (intel_pstate_driver) {
3487 if (intel_pstate_driver == &intel_pstate)
3488 return 0;
3489
3490 cpufreq_unregister_driver(intel_pstate_driver);
3491 }
3492
3493 return intel_pstate_register_driver(&intel_pstate);
3494 }
3495
3496 if (size == 7 && !strncmp(buf, "passive", size)) {
3497 if (intel_pstate_driver) {
3498 if (intel_pstate_driver == &intel_cpufreq)
3499 return 0;
3500
3501 cpufreq_unregister_driver(intel_pstate_driver);
3502 intel_pstate_sysfs_hide_hwp_dynamic_boost();
3503 }
3504
3505 return intel_pstate_register_driver(&intel_cpufreq);
3506 }
3507
3508 return -EINVAL;
3509}
3510
3511static int no_load __initdata;
3512static int no_hwp __initdata;
3513static int hwp_only __initdata;
3514static unsigned int force_load __initdata;
3515
3516static int __init intel_pstate_msrs_not_valid(void)
3517{
3518 if (!pstate_funcs.get_max(0) ||
3519 !pstate_funcs.get_min(0) ||
3520 !pstate_funcs.get_turbo(0))
3521 return -ENODEV;
3522
3523 return 0;
3524}
3525
3526static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
3527{
3528 pstate_funcs.get_max = funcs->get_max;
3529 pstate_funcs.get_max_physical = funcs->get_max_physical;
3530 pstate_funcs.get_min = funcs->get_min;
3531 pstate_funcs.get_turbo = funcs->get_turbo;
3532 pstate_funcs.get_scaling = funcs->get_scaling;
3533 pstate_funcs.get_val = funcs->get_val;
3534 pstate_funcs.get_vid = funcs->get_vid;
3535 pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift;
3536}
3537
3538#ifdef CONFIG_ACPI
3539
3540static bool __init intel_pstate_no_acpi_pss(void)
3541{
3542 int i;
3543
3544 for_each_possible_cpu(i) {
3545 acpi_status status;
3546 union acpi_object *pss;
3547 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
3548 struct acpi_processor *pr = per_cpu(processors, i);
3549
3550 if (!pr)
3551 continue;
3552
3553 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
3554 if (ACPI_FAILURE(status))
3555 continue;
3556
3557 pss = buffer.pointer;
3558 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
3559 kfree(pss);
3560 return false;
3561 }
3562
3563 kfree(pss);
3564 }
3565
3566 pr_debug("ACPI _PSS not found\n");
3567 return true;
3568}
3569
3570static bool __init intel_pstate_no_acpi_pcch(void)
3571{
3572 acpi_status status;
3573 acpi_handle handle;
3574
3575 status = acpi_get_handle(NULL, "\\_SB", &handle);
3576 if (ACPI_FAILURE(status))
3577 goto not_found;
3578
3579 if (acpi_has_method(handle, "PCCH"))
3580 return false;
3581
3582not_found:
3583 pr_debug("ACPI PCCH not found\n");
3584 return true;
3585}
3586
3587static bool __init intel_pstate_has_acpi_ppc(void)
3588{
3589 int i;
3590
3591 for_each_possible_cpu(i) {
3592 struct acpi_processor *pr = per_cpu(processors, i);
3593
3594 if (!pr)
3595 continue;
3596 if (acpi_has_method(pr->handle, "_PPC"))
3597 return true;
3598 }
3599 pr_debug("ACPI _PPC not found\n");
3600 return false;
3601}
3602
3603enum {
3604 PSS,
3605 PPC,
3606};
3607
3608/* Hardware vendor-specific info that has its own power management modes */
3609static struct acpi_platform_list plat_info[] __initdata = {
3610 {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, NULL, PSS},
3611 {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3612 {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3613 {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3614 {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3615 {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3616 {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3617 {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3618 {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3619 {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3620 {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3621 {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3622 {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3623 {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3624 {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
3625 { } /* End */
3626};
3627
3628#define BITMASK_OOB (BIT(8) | BIT(18))
3629
3630static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
3631{
3632 const struct x86_cpu_id *id;
3633 u64 misc_pwr;
3634 int idx;
3635
3636 id = x86_match_cpu(intel_pstate_cpu_oob_ids);
3637 if (id) {
3638 rdmsrq(MSR_MISC_PWR_MGMT, misc_pwr);
3639 if (misc_pwr & BITMASK_OOB) {
3640 pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n");
3641 pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n");
3642 return true;
3643 }
3644 }
3645
3646 idx = acpi_match_platform_list(plat_info);
3647 if (idx < 0)
3648 return false;
3649
3650 switch (plat_info[idx].data) {
3651 case PSS:
3652 if (!intel_pstate_no_acpi_pss())
3653 return false;
3654
3655 return intel_pstate_no_acpi_pcch();
3656 case PPC:
3657 return intel_pstate_has_acpi_ppc() && !force_load;
3658 }
3659
3660 return false;
3661}
3662
3663static void intel_pstate_request_control_from_smm(void)
3664{
3665 /*
3666 * It may be unsafe to request P-states control from SMM if _PPC support
3667 * has not been enabled.
3668 */
3669 if (acpi_ppc)
3670 acpi_processor_pstate_control();
3671}
3672#else /* CONFIG_ACPI not enabled */
3673static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
3674static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
3675static inline void intel_pstate_request_control_from_smm(void) {}
3676#endif /* CONFIG_ACPI */
3677
3678#define INTEL_PSTATE_HWP_BROADWELL 0x01
3679
3680#define X86_MATCH_HWP(vfm, hwp_mode) \
3681 X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_HWP, hwp_mode)
3682
3683static const struct x86_cpu_id hwp_support_ids[] __initconst = {
3684 X86_MATCH_HWP(INTEL_BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
3685 X86_MATCH_HWP(INTEL_BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL),
3686 X86_MATCH_HWP(INTEL_ANY, 0),
3687 {}
3688};
3689
3690static bool intel_pstate_hwp_is_enabled(void)
3691{
3692 u64 value;
3693
3694 rdmsrq(MSR_PM_ENABLE, value);
3695 return !!(value & 0x1);
3696}
3697
3698#define POWERSAVE_MASK GENMASK(7, 0)
3699#define BALANCE_POWER_MASK GENMASK(15, 8)
3700#define BALANCE_PERFORMANCE_MASK GENMASK(23, 16)
3701#define PERFORMANCE_MASK GENMASK(31, 24)
3702
3703#define HWP_SET_EPP_VALUES(powersave, balance_power, balance_perf, performance) \
3704 (FIELD_PREP_CONST(POWERSAVE_MASK, powersave) |\
3705 FIELD_PREP_CONST(BALANCE_POWER_MASK, balance_power) |\
3706 FIELD_PREP_CONST(BALANCE_PERFORMANCE_MASK, balance_perf) |\
3707 FIELD_PREP_CONST(PERFORMANCE_MASK, performance))
3708
3709#define HWP_SET_DEF_BALANCE_PERF_EPP(balance_perf) \
3710 (HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, HWP_EPP_BALANCE_POWERSAVE,\
3711 balance_perf, HWP_EPP_PERFORMANCE))
3712
3713static const struct x86_cpu_id intel_epp_default[] = {
3714 /*
3715 * Set EPP value as 102, this is the max suggested EPP
3716 * which can result in one core turbo frequency for
3717 * AlderLake Mobile CPUs.
3718 */
3719 X86_MATCH_VFM(INTEL_ALDERLAKE_L, HWP_SET_DEF_BALANCE_PERF_EPP(102)),
3720 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
3721 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
3722 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
3723 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
3724 X86_MATCH_VFM(INTEL_METEORLAKE_L, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE,
3725 179, 64, 16)),
3726 X86_MATCH_VFM(INTEL_ARROWLAKE, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE,
3727 179, 64, 16)),
3728 {}
3729};
3730
3731static const struct x86_cpu_id intel_hybrid_scaling_factor[] = {
3732 X86_MATCH_VFM(INTEL_ALDERLAKE, HYBRID_SCALING_FACTOR_ADL),
3733 X86_MATCH_VFM(INTEL_ALDERLAKE_L, HYBRID_SCALING_FACTOR_ADL),
3734 X86_MATCH_VFM(INTEL_RAPTORLAKE, HYBRID_SCALING_FACTOR_ADL),
3735 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, HYBRID_SCALING_FACTOR_ADL),
3736 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, HYBRID_SCALING_FACTOR_ADL),
3737 X86_MATCH_VFM(INTEL_METEORLAKE_L, HYBRID_SCALING_FACTOR_MTL),
3738 X86_MATCH_VFM(INTEL_LUNARLAKE_M, HYBRID_SCALING_FACTOR_LNL),
3739 {}
3740};
3741
3742static bool hwp_check_epp(void)
3743{
3744 if (boot_cpu_has(X86_FEATURE_HWP_EPP))
3745 return true;
3746
3747 /* Without EPP support, don't expose EPP-related sysfs attributes. */
3748 hwp_cpufreq_attrs[HWP_PERFORMANCE_PREFERENCE_INDEX] = NULL;
3749 hwp_cpufreq_attrs[HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX] = NULL;
3750
3751 return false;
3752}
3753
3754static bool hwp_check_dec(void)
3755{
3756 u64 power_ctl;
3757
3758 rdmsrq(MSR_IA32_POWER_CTL, power_ctl);
3759 return !!(power_ctl & BIT(POWER_CTL_DEC_ENABLE));
3760}
3761
3762static int __init intel_pstate_init(void)
3763{
3764 static struct cpudata **_all_cpu_data;
3765 const struct x86_cpu_id *id;
3766 int rc;
3767
3768 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
3769 return -ENODEV;
3770
3771 /*
3772 * The Intel pstate driver will be ignored if the platform
3773 * firmware has its own power management modes.
3774 */
3775 if (intel_pstate_platform_pwr_mgmt_exists()) {
3776 pr_info("P-states controlled by the platform\n");
3777 return -ENODEV;
3778 }
3779
3780 id = x86_match_cpu(hwp_support_ids);
3781 if (id) {
3782 bool epp_present = hwp_check_epp();
3783
3784 /*
3785 * If HWP is enabled already, there is no choice but to deal
3786 * with it.
3787 */
3788 hwp_forced = intel_pstate_hwp_is_enabled();
3789 if (hwp_forced) {
3790 pr_info("HWP enabled by BIOS\n");
3791 no_hwp = 0;
3792 } else if (no_load) {
3793 return -ENODEV;
3794 } else if (!epp_present && !hwp_check_dec()) {
3795 /*
3796 * Avoid enabling HWP for processors without EPP support
3797 * unless the Dynamic Efficiency Control (DEC) enable
3798 * bit (MSR_IA32_POWER_CTL, bit 27) is set because that
3799 * means incomplete HWP implementation which is a corner
3800 * case and supporting it is generally problematic.
3801 */
3802 no_hwp = 1;
3803 }
3804
3805 copy_cpu_funcs(&core_funcs);
3806
3807 if (!no_hwp) {
3808 hwp_active = true;
3809 hwp_mode_bdw = id->driver_data;
3810 intel_pstate.attr = hwp_cpufreq_attrs;
3811 intel_cpufreq.attr = hwp_cpufreq_attrs;
3812 intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS;
3813 intel_cpufreq.adjust_perf = intel_cpufreq_adjust_perf;
3814 if (!default_driver)
3815 default_driver = &intel_pstate;
3816
3817 pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling;
3818
3819 goto hwp_cpu_matched;
3820 }
3821 pr_info("HWP not enabled\n");
3822 } else {
3823 if (no_load)
3824 return -ENODEV;
3825
3826 id = x86_match_cpu(intel_pstate_cpu_ids);
3827 if (!id) {
3828 pr_info("CPU model not supported\n");
3829 return -ENODEV;
3830 }
3831
3832 copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
3833 }
3834
3835 if (intel_pstate_msrs_not_valid()) {
3836 pr_info("Invalid MSRs\n");
3837 return -ENODEV;
3838 }
3839 /* Without HWP start in the passive mode. */
3840 if (!default_driver)
3841 default_driver = &intel_cpufreq;
3842
3843hwp_cpu_matched:
3844 if (!hwp_active && hwp_only)
3845 return -ENOTSUPP;
3846
3847 pr_info("Intel P-state driver initializing\n");
3848
3849 _all_cpu_data = vzalloc(array_size(sizeof(void *), num_possible_cpus()));
3850 if (!_all_cpu_data)
3851 return -ENOMEM;
3852
3853 WRITE_ONCE(all_cpu_data, _all_cpu_data);
3854
3855 intel_pstate_request_control_from_smm();
3856
3857 intel_pstate_sysfs_expose_params();
3858
3859 if (hwp_active) {
3860 const struct x86_cpu_id *id = x86_match_cpu(intel_epp_default);
3861 const struct x86_cpu_id *hybrid_id = x86_match_cpu(intel_hybrid_scaling_factor);
3862
3863 if (id) {
3864 epp_values[EPP_INDEX_POWERSAVE] =
3865 FIELD_GET(POWERSAVE_MASK, id->driver_data);
3866 epp_values[EPP_INDEX_BALANCE_POWERSAVE] =
3867 FIELD_GET(BALANCE_POWER_MASK, id->driver_data);
3868 epp_values[EPP_INDEX_BALANCE_PERFORMANCE] =
3869 FIELD_GET(BALANCE_PERFORMANCE_MASK, id->driver_data);
3870 epp_values[EPP_INDEX_PERFORMANCE] =
3871 FIELD_GET(PERFORMANCE_MASK, id->driver_data);
3872 pr_debug("Updated EPPs powersave:%x balanced power:%x balanced perf:%x performance:%x\n",
3873 epp_values[EPP_INDEX_POWERSAVE],
3874 epp_values[EPP_INDEX_BALANCE_POWERSAVE],
3875 epp_values[EPP_INDEX_BALANCE_PERFORMANCE],
3876 epp_values[EPP_INDEX_PERFORMANCE]);
3877 }
3878
3879 if (hybrid_id) {
3880 hybrid_scaling_factor = hybrid_id->driver_data;
3881 pr_debug("hybrid scaling factor: %d\n", hybrid_scaling_factor);
3882 }
3883
3884 }
3885
3886 scoped_guard(mutex, &intel_pstate_driver_lock) {
3887 rc = intel_pstate_register_driver(default_driver);
3888 }
3889 if (rc) {
3890 intel_pstate_sysfs_remove();
3891 return rc;
3892 }
3893
3894 if (hwp_active) {
3895 const struct x86_cpu_id *id;
3896
3897 id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
3898 if (id) {
3899 set_power_ctl_ee_state(false);
3900 pr_info("Disabling energy efficiency optimization\n");
3901 }
3902
3903 pr_info("HWP enabled\n");
3904 } else if (boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
3905 pr_warn("Problematic setup: Hybrid processor with disabled HWP\n");
3906 }
3907
3908 return 0;
3909}
3910device_initcall(intel_pstate_init);
3911
3912static int __init intel_pstate_setup(char *str)
3913{
3914 if (!str)
3915 return -EINVAL;
3916
3917 if (!strcmp(str, "disable"))
3918 no_load = 1;
3919 else if (!strcmp(str, "active"))
3920 default_driver = &intel_pstate;
3921 else if (!strcmp(str, "passive"))
3922 default_driver = &intel_cpufreq;
3923
3924 if (!strcmp(str, "no_hwp"))
3925 no_hwp = 1;
3926
3927 if (!strcmp(str, "no_cas"))
3928 no_cas = true;
3929
3930 if (!strcmp(str, "force"))
3931 force_load = 1;
3932 if (!strcmp(str, "hwp_only"))
3933 hwp_only = 1;
3934 if (!strcmp(str, "per_cpu_perf_limits"))
3935 per_cpu_limits = true;
3936
3937#ifdef CONFIG_ACPI
3938 if (!strcmp(str, "support_acpi_ppc"))
3939 acpi_ppc = true;
3940#endif
3941
3942 return 0;
3943}
3944early_param("intel_pstate", intel_pstate_setup);
3945
3946MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
3947MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");