Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * intel_rapl_tpmi: Intel RAPL driver via TPMI interface
4 *
5 * Copyright (c) 2023, Intel Corporation.
6 * All Rights Reserved.
7 *
8 */
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/auxiliary_bus.h>
12#include <linux/bits.h>
13#include <linux/intel_rapl.h>
14#include <linux/intel_tpmi.h>
15#include <linux/intel_vsec.h>
16#include <linux/io.h>
17#include <linux/module.h>
18#include <linux/slab.h>
19#include <linux/units.h>
20
21#define TPMI_RAPL_MAJOR_VERSION 0
22#define TPMI_RAPL_MINOR_VERSION 1
23
24/* 1 header + 10 registers + 5 reserved. 8 bytes for each. */
25#define TPMI_RAPL_DOMAIN_SIZE 128
26
27enum tpmi_rapl_domain_type {
28 TPMI_RAPL_DOMAIN_INVALID,
29 TPMI_RAPL_DOMAIN_SYSTEM,
30 TPMI_RAPL_DOMAIN_PACKAGE,
31 TPMI_RAPL_DOMAIN_RESERVED,
32 TPMI_RAPL_DOMAIN_MEMORY,
33 TPMI_RAPL_DOMAIN_MAX,
34};
35
36enum tpmi_rapl_register {
37 TPMI_RAPL_REG_HEADER,
38 TPMI_RAPL_REG_UNIT,
39 TPMI_RAPL_REG_PL1,
40 TPMI_RAPL_REG_PL2,
41 TPMI_RAPL_REG_PL3,
42 TPMI_RAPL_REG_PL4,
43 TPMI_RAPL_REG_RESERVED,
44 TPMI_RAPL_REG_ENERGY_STATUS,
45 TPMI_RAPL_REG_PERF_STATUS,
46 TPMI_RAPL_REG_POWER_INFO,
47 TPMI_RAPL_REG_DOMAIN_INFO,
48 TPMI_RAPL_REG_INTERRUPT,
49 TPMI_RAPL_REG_MAX = 15,
50};
51
52struct tpmi_rapl_package {
53 struct rapl_if_priv priv;
54 struct oobmsm_plat_info *tpmi_info;
55 struct rapl_package *rp;
56 void __iomem *base;
57 struct list_head node;
58};
59
60static LIST_HEAD(tpmi_rapl_packages);
61static DEFINE_MUTEX(tpmi_rapl_lock);
62
63static struct powercap_control_type *tpmi_control_type;
64
65/* bitmasks for RAPL TPMI, used by primitive access functions */
66#define TPMI_POWER_LIMIT_MASK GENMASK_ULL(17, 0)
67#define TPMI_POWER_LIMIT_ENABLE BIT_ULL(62)
68#define TPMI_POWER_HIGH_LOCK BIT_ULL(63)
69#define TPMI_TIME_WINDOW_MASK GENMASK_ULL(24, 18)
70#define TPMI_INFO_SPEC_MASK GENMASK_ULL(17, 0)
71#define TPMI_INFO_MIN_MASK GENMASK_ULL(35, 18)
72#define TPMI_INFO_MAX_MASK GENMASK_ULL(53, 36)
73#define TPMI_INFO_MAX_TIME_WIN_MASK GENMASK_ULL(60, 54)
74#define TPMI_ENERGY_STATUS_MASK GENMASK(31, 0)
75#define TPMI_PERF_STATUS_THROTTLE_TIME_MASK GENMASK(31, 0)
76
77/* RAPL primitives for TPMI I/F */
78static struct rapl_primitive_info rpi_tpmi[NR_RAPL_PRIMITIVES] = {
79 /* name, mask, shift, msr index, unit divisor */
80 [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, TPMI_POWER_LIMIT_MASK, 0,
81 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
82 [POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, TPMI_POWER_LIMIT_MASK, 0,
83 RAPL_DOMAIN_REG_PL2, POWER_UNIT, 0),
84 [POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, TPMI_POWER_LIMIT_MASK, 0,
85 RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),
86 [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, TPMI_ENERGY_STATUS_MASK, 0,
87 RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
88 [PL1_LOCK] = PRIMITIVE_INFO_INIT(PL1_LOCK, TPMI_POWER_HIGH_LOCK, 63,
89 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
90 [PL2_LOCK] = PRIMITIVE_INFO_INIT(PL2_LOCK, TPMI_POWER_HIGH_LOCK, 63,
91 RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),
92 [PL4_LOCK] = PRIMITIVE_INFO_INIT(PL4_LOCK, TPMI_POWER_HIGH_LOCK, 63,
93 RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
94 [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
95 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
96 [PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
97 RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),
98 [PL4_ENABLE] = PRIMITIVE_INFO_INIT(PL4_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
99 RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
100 [TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TPMI_TIME_WINDOW_MASK, 18,
101 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
102 [TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TPMI_TIME_WINDOW_MASK, 18,
103 RAPL_DOMAIN_REG_PL2, TIME_UNIT, 0),
104 [THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, TPMI_INFO_SPEC_MASK, 0,
105 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
106 [MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, TPMI_INFO_MAX_MASK, 36,
107 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
108 [MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, TPMI_INFO_MIN_MASK, 18,
109 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
110 [MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, TPMI_INFO_MAX_TIME_WIN_MASK,
111 54, RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
112 [THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME,
113 TPMI_PERF_STATUS_THROTTLE_TIME_MASK,
114 0, RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
115};
116
117static int tpmi_rapl_read_raw(int id, struct reg_action *ra, bool atomic)
118{
119 if (!ra->reg.mmio)
120 return -EINVAL;
121
122 ra->value = readq(ra->reg.mmio);
123
124 ra->value &= ra->mask;
125 return 0;
126}
127
128static int tpmi_rapl_write_raw(int id, struct reg_action *ra)
129{
130 u64 val;
131
132 if (!ra->reg.mmio)
133 return -EINVAL;
134
135 val = readq(ra->reg.mmio);
136
137 val &= ~ra->mask;
138 val |= ra->value;
139
140 writeq(val, ra->reg.mmio);
141 return 0;
142}
143
144static struct tpmi_rapl_package *trp_alloc(int pkg_id)
145{
146 struct tpmi_rapl_package *trp;
147 int ret;
148
149 mutex_lock(&tpmi_rapl_lock);
150
151 if (list_empty(&tpmi_rapl_packages)) {
152 tpmi_control_type = powercap_register_control_type(NULL, "intel-rapl", NULL);
153 if (IS_ERR(tpmi_control_type)) {
154 ret = PTR_ERR(tpmi_control_type);
155 goto err_unlock;
156 }
157 }
158
159 trp = kzalloc_obj(*trp);
160 if (!trp) {
161 ret = -ENOMEM;
162 goto err_del_powercap;
163 }
164
165 list_add(&trp->node, &tpmi_rapl_packages);
166
167 mutex_unlock(&tpmi_rapl_lock);
168 return trp;
169
170err_del_powercap:
171 if (list_empty(&tpmi_rapl_packages))
172 powercap_unregister_control_type(tpmi_control_type);
173err_unlock:
174 mutex_unlock(&tpmi_rapl_lock);
175 return ERR_PTR(ret);
176}
177
178static void trp_release(struct tpmi_rapl_package *trp)
179{
180 mutex_lock(&tpmi_rapl_lock);
181 list_del(&trp->node);
182
183 if (list_empty(&tpmi_rapl_packages))
184 powercap_unregister_control_type(tpmi_control_type);
185
186 kfree(trp);
187 mutex_unlock(&tpmi_rapl_lock);
188}
189
190/*
191 * Bit 0 of TPMI_RAPL_REG_DOMAIN_INFO indicates if the current package is a domain
192 * root or not. Only domain root packages can enumerate System (Psys) Domain.
193 */
194#define TPMI_RAPL_DOMAIN_ROOT BIT(0)
195
196static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset)
197{
198 u8 tpmi_domain_version;
199 enum rapl_domain_type domain_type;
200 enum tpmi_rapl_domain_type tpmi_domain_type;
201 enum tpmi_rapl_register reg_index;
202 enum rapl_domain_reg_id reg_id;
203 int tpmi_domain_size, tpmi_domain_flags;
204 u64 tpmi_domain_header = readq(trp->base + offset);
205 u64 tpmi_domain_info;
206
207 /* Domain Parent bits are ignored for now */
208 tpmi_domain_version = tpmi_domain_header & 0xff;
209 tpmi_domain_type = tpmi_domain_header >> 8 & 0xff;
210 tpmi_domain_size = tpmi_domain_header >> 16 & 0xff;
211 tpmi_domain_flags = tpmi_domain_header >> 32 & 0xffff;
212
213 if (tpmi_domain_version == TPMI_VERSION_INVALID) {
214 pr_debug("Invalid version, other instances may be valid\n");
215 return -ENODEV;
216 }
217
218 if (TPMI_MAJOR_VERSION(tpmi_domain_version) != TPMI_RAPL_MAJOR_VERSION) {
219 pr_warn(FW_BUG "Unsupported major version:%ld\n",
220 TPMI_MAJOR_VERSION(tpmi_domain_version));
221 return -ENODEV;
222 }
223
224 if (TPMI_MINOR_VERSION(tpmi_domain_version) > TPMI_RAPL_MINOR_VERSION)
225 pr_info("Ignore: Unsupported minor version:%ld\n",
226 TPMI_MINOR_VERSION(tpmi_domain_version));
227
228 /* Domain size: in unit of 128 Bytes */
229 if (tpmi_domain_size != 1) {
230 pr_warn(FW_BUG "Invalid Domain size %d\n", tpmi_domain_size);
231 return -EINVAL;
232 }
233
234 /* Unit register and Energy Status register are mandatory for each domain */
235 if (!(tpmi_domain_flags & BIT(TPMI_RAPL_REG_UNIT)) ||
236 !(tpmi_domain_flags & BIT(TPMI_RAPL_REG_ENERGY_STATUS))) {
237 pr_warn(FW_BUG "Invalid Domain flag 0x%x\n", tpmi_domain_flags);
238 return -EINVAL;
239 }
240
241 switch (tpmi_domain_type) {
242 case TPMI_RAPL_DOMAIN_PACKAGE:
243 domain_type = RAPL_DOMAIN_PACKAGE;
244 break;
245 case TPMI_RAPL_DOMAIN_SYSTEM:
246 if (!(tpmi_domain_flags & BIT(TPMI_RAPL_REG_DOMAIN_INFO))) {
247 pr_warn(FW_BUG "System domain must support Domain Info register\n");
248 return -ENODEV;
249 }
250 tpmi_domain_info = readq(trp->base + offset + TPMI_RAPL_REG_DOMAIN_INFO * 8);
251 if (!(tpmi_domain_info & TPMI_RAPL_DOMAIN_ROOT))
252 return 0;
253 domain_type = RAPL_DOMAIN_PLATFORM;
254 break;
255 case TPMI_RAPL_DOMAIN_MEMORY:
256 domain_type = RAPL_DOMAIN_DRAM;
257 break;
258 default:
259 pr_warn(FW_BUG "Unsupported Domain type %d\n", tpmi_domain_type);
260 return -EINVAL;
261 }
262
263 if (trp->priv.regs[domain_type][RAPL_DOMAIN_REG_UNIT].mmio) {
264 pr_warn(FW_BUG "Duplicate Domain type %d\n", tpmi_domain_type);
265 return -EINVAL;
266 }
267
268 reg_index = TPMI_RAPL_REG_HEADER;
269 while (++reg_index != TPMI_RAPL_REG_MAX) {
270 if (!(tpmi_domain_flags & BIT(reg_index)))
271 continue;
272
273 switch (reg_index) {
274 case TPMI_RAPL_REG_UNIT:
275 reg_id = RAPL_DOMAIN_REG_UNIT;
276 break;
277 case TPMI_RAPL_REG_PL1:
278 reg_id = RAPL_DOMAIN_REG_LIMIT;
279 trp->priv.limits[domain_type] |= BIT(POWER_LIMIT1);
280 break;
281 case TPMI_RAPL_REG_PL2:
282 reg_id = RAPL_DOMAIN_REG_PL2;
283 trp->priv.limits[domain_type] |= BIT(POWER_LIMIT2);
284 break;
285 case TPMI_RAPL_REG_PL4:
286 reg_id = RAPL_DOMAIN_REG_PL4;
287 trp->priv.limits[domain_type] |= BIT(POWER_LIMIT4);
288 break;
289 case TPMI_RAPL_REG_ENERGY_STATUS:
290 reg_id = RAPL_DOMAIN_REG_STATUS;
291 break;
292 case TPMI_RAPL_REG_PERF_STATUS:
293 reg_id = RAPL_DOMAIN_REG_PERF;
294 break;
295 case TPMI_RAPL_REG_POWER_INFO:
296 reg_id = RAPL_DOMAIN_REG_INFO;
297 break;
298 default:
299 continue;
300 }
301 trp->priv.regs[domain_type][reg_id].mmio = trp->base + offset + reg_index * 8;
302 }
303
304 return 0;
305}
306
307/* TPMI Unit register has different layout */
308#define TPMI_ENERGY_UNIT_SCALE 1000
309#define TPMI_POWER_UNIT_OFFSET 0x00
310#define TPMI_POWER_UNIT_MASK GENMASK(3, 0)
311#define TPMI_ENERGY_UNIT_OFFSET 0x06
312#define TPMI_ENERGY_UNIT_MASK GENMASK_ULL(10, 6)
313#define TPMI_TIME_UNIT_OFFSET 0x0C
314#define TPMI_TIME_UNIT_MASK GENMASK_ULL(15, 12)
315
316static int rapl_check_unit_tpmi(struct rapl_domain *rd)
317{
318 struct reg_action ra;
319 u32 value;
320
321 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
322 ra.mask = ~0;
323 if (tpmi_rapl_read_raw(rd->rp->id, &ra, false)) {
324 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
325 ra.reg.val, rd->rp->name, rd->name);
326 return -ENODEV;
327 }
328
329 value = (ra.value & TPMI_ENERGY_UNIT_MASK) >> TPMI_ENERGY_UNIT_OFFSET;
330 rd->energy_unit = (TPMI_ENERGY_UNIT_SCALE * MICROJOULE_PER_JOULE) >> value;
331
332 value = (ra.value & TPMI_POWER_UNIT_MASK) >> TPMI_POWER_UNIT_OFFSET;
333 rd->power_unit = MICROWATT_PER_WATT >> value;
334
335 value = (ra.value & TPMI_TIME_UNIT_MASK) >> TPMI_TIME_UNIT_OFFSET;
336 rd->time_unit = USEC_PER_SEC >> value;
337
338 pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",
339 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
340
341 return 0;
342}
343
344static const struct rapl_defaults defaults_tpmi = {
345 .check_unit = rapl_check_unit_tpmi,
346 /* Reuse existing logic, ignore the PL_CLAMP failures and enable all Power Limits */
347 .set_floor_freq = rapl_default_set_floor_freq,
348 .compute_time_window = rapl_default_compute_time_window,
349};
350
351static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev,
352 const struct auxiliary_device_id *id)
353{
354 struct tpmi_rapl_package *trp;
355 struct oobmsm_plat_info *info;
356 struct resource *res;
357 u32 offset;
358 int ret;
359
360 info = tpmi_get_platform_data(auxdev);
361 if (!info)
362 return -ENODEV;
363
364 trp = trp_alloc(info->package_id);
365 if (IS_ERR(trp))
366 return PTR_ERR(trp);
367
368 if (tpmi_get_resource_count(auxdev) > 1) {
369 dev_err(&auxdev->dev, "does not support multiple resources\n");
370 ret = -EINVAL;
371 goto err;
372 }
373
374 res = tpmi_get_resource_at_index(auxdev, 0);
375 if (!res) {
376 dev_err(&auxdev->dev, "can't fetch device resource info\n");
377 ret = -EIO;
378 goto err;
379 }
380
381 trp->base = devm_ioremap_resource(&auxdev->dev, res);
382 if (IS_ERR(trp->base)) {
383 ret = PTR_ERR(trp->base);
384 goto err;
385 }
386
387 for (offset = 0; offset < resource_size(res); offset += TPMI_RAPL_DOMAIN_SIZE) {
388 ret = parse_one_domain(trp, offset);
389 if (ret)
390 goto err;
391 }
392
393 trp->tpmi_info = info;
394 trp->priv.type = RAPL_IF_TPMI;
395 trp->priv.read_raw = tpmi_rapl_read_raw;
396 trp->priv.write_raw = tpmi_rapl_write_raw;
397 trp->priv.control_type = tpmi_control_type;
398 trp->priv.defaults = &defaults_tpmi;
399 trp->priv.rpi = rpi_tpmi;
400
401 /* RAPL TPMI I/F is per physical package */
402 trp->rp = rapl_find_package_domain(info->package_id, &trp->priv, false);
403 if (trp->rp) {
404 dev_err(&auxdev->dev, "Domain for Package%d already exists\n", info->package_id);
405 ret = -EEXIST;
406 goto err;
407 }
408
409 trp->rp = rapl_add_package(info->package_id, &trp->priv, false);
410 if (IS_ERR(trp->rp)) {
411 dev_err(&auxdev->dev, "Failed to add RAPL Domain for Package%d, %ld\n",
412 info->package_id, PTR_ERR(trp->rp));
413 ret = PTR_ERR(trp->rp);
414 goto err;
415 }
416
417 rapl_package_add_pmu(trp->rp);
418
419 auxiliary_set_drvdata(auxdev, trp);
420
421 return 0;
422err:
423 trp_release(trp);
424 return ret;
425}
426
427static void intel_rapl_tpmi_remove(struct auxiliary_device *auxdev)
428{
429 struct tpmi_rapl_package *trp = auxiliary_get_drvdata(auxdev);
430
431 rapl_package_remove_pmu(trp->rp);
432 rapl_remove_package(trp->rp);
433 trp_release(trp);
434}
435
436static const struct auxiliary_device_id intel_rapl_tpmi_ids[] = {
437 {.name = "intel_vsec.tpmi-rapl" },
438 { }
439};
440
441MODULE_DEVICE_TABLE(auxiliary, intel_rapl_tpmi_ids);
442
443static struct auxiliary_driver intel_rapl_tpmi_driver = {
444 .probe = intel_rapl_tpmi_probe,
445 .remove = intel_rapl_tpmi_remove,
446 .id_table = intel_rapl_tpmi_ids,
447};
448
449module_auxiliary_driver(intel_rapl_tpmi_driver)
450
451MODULE_IMPORT_NS("INTEL_RAPL");
452MODULE_IMPORT_NS("INTEL_TPMI");
453
454MODULE_DESCRIPTION("Intel RAPL TPMI Driver");
455MODULE_LICENSE("GPL");