Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * System Control and Management Interface (SCMI) Performance Protocol
4 *
5 * Copyright (C) 2018-2023 ARM Ltd.
6 */
7
8#define pr_fmt(fmt) "SCMI Notifications PERF - " fmt
9
10#include <linux/bits.h>
11#include <linux/hashtable.h>
12#include <linux/io.h>
13#include <linux/log2.h>
14#include <linux/module.h>
15#include <linux/of.h>
16#include <linux/platform_device.h>
17#include <linux/pm_opp.h>
18#include <linux/scmi_protocol.h>
19#include <linux/sort.h>
20#include <linux/xarray.h>
21
22#include <trace/events/scmi.h>
23
24#include "protocols.h"
25#include "notify.h"
26
27/* Updated only after ALL the mandatory features for that version are merged */
28#define SCMI_PROTOCOL_SUPPORTED_VERSION 0x40000
29
30#define MAX_OPPS 64
31
32enum scmi_performance_protocol_cmd {
33 PERF_DOMAIN_ATTRIBUTES = 0x3,
34 PERF_DESCRIBE_LEVELS = 0x4,
35 PERF_LIMITS_SET = 0x5,
36 PERF_LIMITS_GET = 0x6,
37 PERF_LEVEL_SET = 0x7,
38 PERF_LEVEL_GET = 0x8,
39 PERF_NOTIFY_LIMITS = 0x9,
40 PERF_NOTIFY_LEVEL = 0xa,
41 PERF_DESCRIBE_FASTCHANNEL = 0xb,
42 PERF_DOMAIN_NAME_GET = 0xc,
43};
44
45enum {
46 PERF_FC_LEVEL,
47 PERF_FC_LIMIT,
48 PERF_FC_MAX,
49};
50
51struct scmi_opp {
52 u32 perf;
53 u32 power;
54 u32 trans_latency_us;
55 u32 indicative_freq;
56 u32 level_index;
57 struct hlist_node hash;
58};
59
60struct scmi_msg_resp_perf_attributes {
61 __le16 num_domains;
62 __le16 flags;
63#define POWER_SCALE_IN_MILLIWATT(x) ((x) & BIT(0))
64#define POWER_SCALE_IN_MICROWATT(x) ((x) & BIT(1))
65 __le32 stats_addr_low;
66 __le32 stats_addr_high;
67 __le32 stats_size;
68};
69
70struct scmi_msg_resp_perf_domain_attributes {
71 __le32 flags;
72#define SUPPORTS_SET_LIMITS(x) ((x) & BIT(31))
73#define SUPPORTS_SET_PERF_LVL(x) ((x) & BIT(30))
74#define SUPPORTS_PERF_LIMIT_NOTIFY(x) ((x) & BIT(29))
75#define SUPPORTS_PERF_LEVEL_NOTIFY(x) ((x) & BIT(28))
76#define SUPPORTS_PERF_FASTCHANNELS(x) ((x) & BIT(27))
77#define SUPPORTS_EXTENDED_NAMES(x) ((x) & BIT(26))
78#define SUPPORTS_LEVEL_INDEXING(x) ((x) & BIT(25))
79 __le32 rate_limit_us;
80 __le32 sustained_freq_khz;
81 __le32 sustained_perf_level;
82 u8 name[SCMI_SHORT_NAME_MAX_SIZE];
83};
84
85struct scmi_msg_perf_describe_levels {
86 __le32 domain;
87 __le32 level_index;
88};
89
90struct scmi_perf_set_limits {
91 __le32 domain;
92 __le32 max_level;
93 __le32 min_level;
94};
95
96struct scmi_perf_get_limits {
97 __le32 max_level;
98 __le32 min_level;
99};
100
101struct scmi_perf_set_level {
102 __le32 domain;
103 __le32 level;
104};
105
106struct scmi_perf_notify_level_or_limits {
107 __le32 domain;
108 __le32 notify_enable;
109};
110
111struct scmi_perf_limits_notify_payld {
112 __le32 agent_id;
113 __le32 domain_id;
114 __le32 range_max;
115 __le32 range_min;
116};
117
118struct scmi_perf_level_notify_payld {
119 __le32 agent_id;
120 __le32 domain_id;
121 __le32 performance_level;
122};
123
124struct scmi_msg_resp_perf_describe_levels {
125 __le16 num_returned;
126 __le16 num_remaining;
127 struct {
128 __le32 perf_val;
129 __le32 power;
130 __le16 transition_latency_us;
131 __le16 reserved;
132 } opp[];
133};
134
135struct scmi_msg_resp_perf_describe_levels_v4 {
136 __le16 num_returned;
137 __le16 num_remaining;
138 struct {
139 __le32 perf_val;
140 __le32 power;
141 __le16 transition_latency_us;
142 __le16 reserved;
143 __le32 indicative_freq;
144 __le32 level_index;
145 } opp[];
146};
147
148struct perf_dom_info {
149 u32 id;
150 bool set_limits;
151 bool perf_limit_notify;
152 bool perf_level_notify;
153 bool perf_fastchannels;
154 bool level_indexing_mode;
155 u32 opp_count;
156 u32 rate_limit_us;
157 u32 sustained_freq_khz;
158 u32 sustained_perf_level;
159 unsigned long mult_factor;
160 struct scmi_perf_domain_info info;
161 struct scmi_opp opp[MAX_OPPS];
162 struct scmi_fc_info *fc_info;
163 struct xarray opps_by_idx;
164 struct xarray opps_by_lvl;
165 DECLARE_HASHTABLE(opps_by_freq, ilog2(MAX_OPPS));
166};
167
168#define LOOKUP_BY_FREQ(__htp, __freq) \
169({ \
170 /* u32 cast is needed to pick right hash func */ \
171 u32 f_ = (u32)(__freq); \
172 struct scmi_opp *_opp; \
173 \
174 hash_for_each_possible((__htp), _opp, hash, f_) \
175 if (_opp->indicative_freq == f_) \
176 break; \
177 _opp; \
178})
179
180struct scmi_perf_info {
181 u16 num_domains;
182 enum scmi_power_scale power_scale;
183 u64 stats_addr;
184 u32 stats_size;
185 bool notify_lvl_cmd;
186 bool notify_lim_cmd;
187 struct perf_dom_info *dom_info;
188};
189
190static enum scmi_performance_protocol_cmd evt_2_cmd[] = {
191 PERF_NOTIFY_LIMITS,
192 PERF_NOTIFY_LEVEL,
193};
194
195static int scmi_perf_attributes_get(const struct scmi_protocol_handle *ph,
196 struct scmi_perf_info *pi)
197{
198 int ret;
199 struct scmi_xfer *t;
200 struct scmi_msg_resp_perf_attributes *attr;
201
202 ret = ph->xops->xfer_get_init(ph, PROTOCOL_ATTRIBUTES, 0,
203 sizeof(*attr), &t);
204 if (ret)
205 return ret;
206
207 attr = t->rx.buf;
208
209 ret = ph->xops->do_xfer(ph, t);
210 if (!ret) {
211 u16 flags = le16_to_cpu(attr->flags);
212
213 pi->num_domains = le16_to_cpu(attr->num_domains);
214
215 if (POWER_SCALE_IN_MILLIWATT(flags))
216 pi->power_scale = SCMI_POWER_MILLIWATTS;
217 if (PROTOCOL_REV_MAJOR(ph->version) >= 0x3)
218 if (POWER_SCALE_IN_MICROWATT(flags))
219 pi->power_scale = SCMI_POWER_MICROWATTS;
220
221 pi->stats_addr = le32_to_cpu(attr->stats_addr_low) |
222 (u64)le32_to_cpu(attr->stats_addr_high) << 32;
223 pi->stats_size = le32_to_cpu(attr->stats_size);
224 }
225
226 ph->xops->xfer_put(ph, t);
227
228 if (!ret) {
229 if (!ph->hops->protocol_msg_check(ph, PERF_NOTIFY_LEVEL, NULL))
230 pi->notify_lvl_cmd = true;
231
232 if (!ph->hops->protocol_msg_check(ph, PERF_NOTIFY_LIMITS, NULL))
233 pi->notify_lim_cmd = true;
234 }
235
236 return ret;
237}
238
239static void scmi_perf_xa_destroy(void *data)
240{
241 int domain;
242 struct scmi_perf_info *pinfo = data;
243
244 for (domain = 0; domain < pinfo->num_domains; domain++) {
245 xa_destroy(&((pinfo->dom_info + domain)->opps_by_idx));
246 xa_destroy(&((pinfo->dom_info + domain)->opps_by_lvl));
247 }
248}
249
250static int
251scmi_perf_domain_attributes_get(const struct scmi_protocol_handle *ph,
252 struct perf_dom_info *dom_info,
253 bool notify_lim_cmd, bool notify_lvl_cmd)
254{
255 int ret;
256 u32 flags;
257 struct scmi_xfer *t;
258 struct scmi_msg_resp_perf_domain_attributes *attr;
259
260 ret = ph->xops->xfer_get_init(ph, PERF_DOMAIN_ATTRIBUTES,
261 sizeof(dom_info->id), sizeof(*attr), &t);
262 if (ret)
263 return ret;
264
265 put_unaligned_le32(dom_info->id, t->tx.buf);
266 attr = t->rx.buf;
267
268 ret = ph->xops->do_xfer(ph, t);
269 if (!ret) {
270 flags = le32_to_cpu(attr->flags);
271
272 dom_info->set_limits = SUPPORTS_SET_LIMITS(flags);
273 dom_info->info.set_perf = SUPPORTS_SET_PERF_LVL(flags);
274 if (notify_lim_cmd)
275 dom_info->perf_limit_notify =
276 SUPPORTS_PERF_LIMIT_NOTIFY(flags);
277 if (notify_lvl_cmd)
278 dom_info->perf_level_notify =
279 SUPPORTS_PERF_LEVEL_NOTIFY(flags);
280 dom_info->perf_fastchannels = SUPPORTS_PERF_FASTCHANNELS(flags);
281 if (PROTOCOL_REV_MAJOR(ph->version) >= 0x4)
282 dom_info->level_indexing_mode =
283 SUPPORTS_LEVEL_INDEXING(flags);
284 dom_info->rate_limit_us = le32_to_cpu(attr->rate_limit_us) &
285 GENMASK(19, 0);
286 dom_info->sustained_freq_khz =
287 le32_to_cpu(attr->sustained_freq_khz);
288 dom_info->sustained_perf_level =
289 le32_to_cpu(attr->sustained_perf_level);
290 /*
291 * sustained_freq_khz = mult_factor * sustained_perf_level
292 * mult_factor must be non zero positive integer(not fraction)
293 */
294 if (!dom_info->sustained_freq_khz ||
295 !dom_info->sustained_perf_level ||
296 dom_info->level_indexing_mode) {
297 /* CPUFreq converts to kHz, hence default 1000 */
298 dom_info->mult_factor = 1000;
299 } else {
300 dom_info->mult_factor =
301 (dom_info->sustained_freq_khz * 1000UL)
302 / dom_info->sustained_perf_level;
303 if ((dom_info->sustained_freq_khz * 1000UL) %
304 dom_info->sustained_perf_level)
305 dev_warn(ph->dev,
306 "multiplier for domain %d rounded\n",
307 dom_info->id);
308 }
309 if (!dom_info->mult_factor)
310 dev_warn(ph->dev,
311 "Wrong sustained perf/frequency(domain %d)\n",
312 dom_info->id);
313
314 strscpy(dom_info->info.name, attr->name,
315 SCMI_SHORT_NAME_MAX_SIZE);
316 }
317
318 ph->xops->xfer_put(ph, t);
319
320 /*
321 * If supported overwrite short name with the extended one;
322 * on error just carry on and use already provided short name.
323 */
324 if (!ret && PROTOCOL_REV_MAJOR(ph->version) >= 0x3 &&
325 SUPPORTS_EXTENDED_NAMES(flags))
326 ph->hops->extended_name_get(ph, PERF_DOMAIN_NAME_GET,
327 dom_info->id, NULL, dom_info->info.name,
328 SCMI_MAX_STR_SIZE);
329
330 xa_init(&dom_info->opps_by_lvl);
331 if (dom_info->level_indexing_mode) {
332 xa_init(&dom_info->opps_by_idx);
333 hash_init(dom_info->opps_by_freq);
334 }
335
336 return ret;
337}
338
339static int opp_cmp_func(const void *opp1, const void *opp2)
340{
341 const struct scmi_opp *t1 = opp1, *t2 = opp2;
342
343 return t1->perf - t2->perf;
344}
345
346static void iter_perf_levels_prepare_message(void *message,
347 unsigned int desc_index,
348 const void *priv)
349{
350 struct scmi_msg_perf_describe_levels *msg = message;
351 const struct perf_dom_info *perf_dom = priv;
352
353 msg->domain = cpu_to_le32(perf_dom->id);
354 /* Set the number of OPPs to be skipped/already read */
355 msg->level_index = cpu_to_le32(desc_index);
356}
357
358static int iter_perf_levels_update_state(struct scmi_iterator_state *st,
359 const void *response, void *priv)
360{
361 const struct scmi_msg_resp_perf_describe_levels *r = response;
362
363 st->num_returned = le16_to_cpu(r->num_returned);
364 st->num_remaining = le16_to_cpu(r->num_remaining);
365
366 return 0;
367}
368
369static inline int
370process_response_opp(struct device *dev, struct perf_dom_info *dom,
371 struct scmi_opp *opp, unsigned int loop_idx,
372 const struct scmi_msg_resp_perf_describe_levels *r)
373{
374 int ret;
375
376 opp->perf = le32_to_cpu(r->opp[loop_idx].perf_val);
377 opp->power = le32_to_cpu(r->opp[loop_idx].power);
378 opp->trans_latency_us =
379 le16_to_cpu(r->opp[loop_idx].transition_latency_us);
380
381 ret = xa_insert(&dom->opps_by_lvl, opp->perf, opp, GFP_KERNEL);
382 if (ret) {
383 dev_info(dev, FW_BUG "Failed to add opps_by_lvl at %d for %s - ret:%d\n",
384 opp->perf, dom->info.name, ret);
385 return ret;
386 }
387
388 return 0;
389}
390
391static inline int
392process_response_opp_v4(struct device *dev, struct perf_dom_info *dom,
393 struct scmi_opp *opp, unsigned int loop_idx,
394 const struct scmi_msg_resp_perf_describe_levels_v4 *r)
395{
396 int ret;
397
398 opp->perf = le32_to_cpu(r->opp[loop_idx].perf_val);
399 opp->power = le32_to_cpu(r->opp[loop_idx].power);
400 opp->trans_latency_us =
401 le16_to_cpu(r->opp[loop_idx].transition_latency_us);
402
403 ret = xa_insert(&dom->opps_by_lvl, opp->perf, opp, GFP_KERNEL);
404 if (ret) {
405 dev_info(dev, FW_BUG "Failed to add opps_by_lvl at %d for %s - ret:%d\n",
406 opp->perf, dom->info.name, ret);
407 return ret;
408 }
409
410 /* Note that PERF v4 reports always five 32-bit words */
411 opp->indicative_freq = le32_to_cpu(r->opp[loop_idx].indicative_freq);
412 if (dom->level_indexing_mode) {
413 opp->level_index = le32_to_cpu(r->opp[loop_idx].level_index);
414
415 ret = xa_insert(&dom->opps_by_idx, opp->level_index, opp,
416 GFP_KERNEL);
417 if (ret) {
418 dev_warn(dev,
419 "Failed to add opps_by_idx at %d for %s - ret:%d\n",
420 opp->level_index, dom->info.name, ret);
421
422 /* Cleanup by_lvl too */
423 xa_erase(&dom->opps_by_lvl, opp->perf);
424
425 return ret;
426 }
427
428 hash_add(dom->opps_by_freq, &opp->hash, opp->indicative_freq);
429 }
430
431 return 0;
432}
433
434static int
435iter_perf_levels_process_response(const struct scmi_protocol_handle *ph,
436 const void *response,
437 struct scmi_iterator_state *st, void *priv)
438{
439 int ret;
440 struct scmi_opp *opp;
441 struct perf_dom_info *perf_dom = priv;
442
443 opp = &perf_dom->opp[perf_dom->opp_count];
444 if (PROTOCOL_REV_MAJOR(ph->version) <= 0x3)
445 ret = process_response_opp(ph->dev, perf_dom, opp,
446 st->loop_idx, response);
447 else
448 ret = process_response_opp_v4(ph->dev, perf_dom, opp,
449 st->loop_idx, response);
450
451 /* Skip BAD duplicates received from firmware */
452 if (ret)
453 return ret == -EBUSY ? 0 : ret;
454
455 perf_dom->opp_count++;
456
457 dev_dbg(ph->dev, "Level %d Power %d Latency %dus Ifreq %d Index %d\n",
458 opp->perf, opp->power, opp->trans_latency_us,
459 opp->indicative_freq, opp->level_index);
460
461 return 0;
462}
463
464static int
465scmi_perf_describe_levels_get(const struct scmi_protocol_handle *ph,
466 struct perf_dom_info *perf_dom)
467{
468 int ret;
469 void *iter;
470 struct scmi_iterator_ops ops = {
471 .prepare_message = iter_perf_levels_prepare_message,
472 .update_state = iter_perf_levels_update_state,
473 .process_response = iter_perf_levels_process_response,
474 };
475
476 iter = ph->hops->iter_response_init(ph, &ops, MAX_OPPS,
477 PERF_DESCRIBE_LEVELS,
478 sizeof(struct scmi_msg_perf_describe_levels),
479 perf_dom);
480 if (IS_ERR(iter))
481 return PTR_ERR(iter);
482
483 ret = ph->hops->iter_response_run(iter);
484 if (ret)
485 return ret;
486
487 if (perf_dom->opp_count)
488 sort(perf_dom->opp, perf_dom->opp_count,
489 sizeof(struct scmi_opp), opp_cmp_func, NULL);
490
491 return ret;
492}
493
494static int scmi_perf_num_domains_get(const struct scmi_protocol_handle *ph)
495{
496 struct scmi_perf_info *pi = ph->get_priv(ph);
497
498 return pi->num_domains;
499}
500
501static inline struct perf_dom_info *
502scmi_perf_domain_lookup(const struct scmi_protocol_handle *ph, u32 domain)
503{
504 struct scmi_perf_info *pi = ph->get_priv(ph);
505
506 if (domain >= pi->num_domains)
507 return ERR_PTR(-EINVAL);
508
509 return pi->dom_info + domain;
510}
511
512static const struct scmi_perf_domain_info *
513scmi_perf_info_get(const struct scmi_protocol_handle *ph, u32 domain)
514{
515 struct perf_dom_info *dom;
516
517 dom = scmi_perf_domain_lookup(ph, domain);
518 if (IS_ERR(dom))
519 return ERR_PTR(-EINVAL);
520
521 return &dom->info;
522}
523
524static int scmi_perf_msg_limits_set(const struct scmi_protocol_handle *ph,
525 u32 domain, u32 max_perf, u32 min_perf)
526{
527 int ret;
528 struct scmi_xfer *t;
529 struct scmi_perf_set_limits *limits;
530
531 ret = ph->xops->xfer_get_init(ph, PERF_LIMITS_SET,
532 sizeof(*limits), 0, &t);
533 if (ret)
534 return ret;
535
536 limits = t->tx.buf;
537 limits->domain = cpu_to_le32(domain);
538 limits->max_level = cpu_to_le32(max_perf);
539 limits->min_level = cpu_to_le32(min_perf);
540
541 ret = ph->xops->do_xfer(ph, t);
542
543 ph->xops->xfer_put(ph, t);
544 return ret;
545}
546
547static int __scmi_perf_limits_set(const struct scmi_protocol_handle *ph,
548 struct perf_dom_info *dom, u32 max_perf,
549 u32 min_perf)
550{
551 if (dom->fc_info && dom->fc_info[PERF_FC_LIMIT].set_addr) {
552 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LIMIT];
553
554 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LIMITS_SET,
555 dom->id, min_perf, max_perf);
556 iowrite32(max_perf, fci->set_addr);
557 iowrite32(min_perf, fci->set_addr + 4);
558 ph->hops->fastchannel_db_ring(fci->set_db);
559 return 0;
560 }
561
562 return scmi_perf_msg_limits_set(ph, dom->id, max_perf, min_perf);
563}
564
565static int scmi_perf_limits_set(const struct scmi_protocol_handle *ph,
566 u32 domain, u32 max_perf, u32 min_perf)
567{
568 struct perf_dom_info *dom;
569
570 dom = scmi_perf_domain_lookup(ph, domain);
571 if (IS_ERR(dom))
572 return PTR_ERR(dom);
573
574 if (!dom->set_limits)
575 return -EOPNOTSUPP;
576
577 if (PROTOCOL_REV_MAJOR(ph->version) >= 0x3 && !max_perf && !min_perf)
578 return -EINVAL;
579
580 if (dom->level_indexing_mode) {
581 struct scmi_opp *opp;
582
583 if (min_perf) {
584 opp = xa_load(&dom->opps_by_lvl, min_perf);
585 if (!opp)
586 return -EIO;
587
588 min_perf = opp->level_index;
589 }
590
591 if (max_perf) {
592 opp = xa_load(&dom->opps_by_lvl, max_perf);
593 if (!opp)
594 return -EIO;
595
596 max_perf = opp->level_index;
597 }
598 }
599
600 return __scmi_perf_limits_set(ph, dom, max_perf, min_perf);
601}
602
603static int scmi_perf_msg_limits_get(const struct scmi_protocol_handle *ph,
604 u32 domain, u32 *max_perf, u32 *min_perf)
605{
606 int ret;
607 struct scmi_xfer *t;
608 struct scmi_perf_get_limits *limits;
609
610 ret = ph->xops->xfer_get_init(ph, PERF_LIMITS_GET,
611 sizeof(__le32), 0, &t);
612 if (ret)
613 return ret;
614
615 put_unaligned_le32(domain, t->tx.buf);
616
617 ret = ph->xops->do_xfer(ph, t);
618 if (!ret) {
619 limits = t->rx.buf;
620
621 *max_perf = le32_to_cpu(limits->max_level);
622 *min_perf = le32_to_cpu(limits->min_level);
623 }
624
625 ph->xops->xfer_put(ph, t);
626 return ret;
627}
628
629static int __scmi_perf_limits_get(const struct scmi_protocol_handle *ph,
630 struct perf_dom_info *dom, u32 *max_perf,
631 u32 *min_perf)
632{
633 if (dom->fc_info && dom->fc_info[PERF_FC_LIMIT].get_addr) {
634 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LIMIT];
635
636 *max_perf = ioread32(fci->get_addr);
637 *min_perf = ioread32(fci->get_addr + 4);
638 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LIMITS_GET,
639 dom->id, *min_perf, *max_perf);
640 return 0;
641 }
642
643 return scmi_perf_msg_limits_get(ph, dom->id, max_perf, min_perf);
644}
645
646static int scmi_perf_limits_get(const struct scmi_protocol_handle *ph,
647 u32 domain, u32 *max_perf, u32 *min_perf)
648{
649 int ret;
650 struct perf_dom_info *dom;
651
652 dom = scmi_perf_domain_lookup(ph, domain);
653 if (IS_ERR(dom))
654 return PTR_ERR(dom);
655
656 ret = __scmi_perf_limits_get(ph, dom, max_perf, min_perf);
657 if (ret)
658 return ret;
659
660 if (dom->level_indexing_mode) {
661 struct scmi_opp *opp;
662
663 opp = xa_load(&dom->opps_by_idx, *min_perf);
664 if (!opp)
665 return -EIO;
666
667 *min_perf = opp->perf;
668
669 opp = xa_load(&dom->opps_by_idx, *max_perf);
670 if (!opp)
671 return -EIO;
672
673 *max_perf = opp->perf;
674 }
675
676 return 0;
677}
678
679static int scmi_perf_msg_level_set(const struct scmi_protocol_handle *ph,
680 u32 domain, u32 level, bool poll)
681{
682 int ret;
683 struct scmi_xfer *t;
684 struct scmi_perf_set_level *lvl;
685
686 ret = ph->xops->xfer_get_init(ph, PERF_LEVEL_SET, sizeof(*lvl), 0, &t);
687 if (ret)
688 return ret;
689
690 t->hdr.poll_completion = poll;
691 lvl = t->tx.buf;
692 lvl->domain = cpu_to_le32(domain);
693 lvl->level = cpu_to_le32(level);
694
695 ret = ph->xops->do_xfer(ph, t);
696
697 ph->xops->xfer_put(ph, t);
698 return ret;
699}
700
701static int __scmi_perf_level_set(const struct scmi_protocol_handle *ph,
702 struct perf_dom_info *dom, u32 level,
703 bool poll)
704{
705 if (dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr) {
706 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LEVEL];
707
708 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LEVEL_SET,
709 dom->id, level, 0);
710 iowrite32(level, fci->set_addr);
711 ph->hops->fastchannel_db_ring(fci->set_db);
712 return 0;
713 }
714
715 return scmi_perf_msg_level_set(ph, dom->id, level, poll);
716}
717
718static int scmi_perf_level_set(const struct scmi_protocol_handle *ph,
719 u32 domain, u32 level, bool poll)
720{
721 struct perf_dom_info *dom;
722
723 dom = scmi_perf_domain_lookup(ph, domain);
724 if (IS_ERR(dom))
725 return PTR_ERR(dom);
726
727 if (!dom->info.set_perf)
728 return -EOPNOTSUPP;
729
730 if (dom->level_indexing_mode) {
731 struct scmi_opp *opp;
732
733 opp = xa_load(&dom->opps_by_lvl, level);
734 if (!opp)
735 return -EIO;
736
737 level = opp->level_index;
738 }
739
740 return __scmi_perf_level_set(ph, dom, level, poll);
741}
742
743static int scmi_perf_msg_level_get(const struct scmi_protocol_handle *ph,
744 u32 domain, u32 *level, bool poll)
745{
746 int ret;
747 struct scmi_xfer *t;
748
749 ret = ph->xops->xfer_get_init(ph, PERF_LEVEL_GET,
750 sizeof(u32), sizeof(u32), &t);
751 if (ret)
752 return ret;
753
754 t->hdr.poll_completion = poll;
755 put_unaligned_le32(domain, t->tx.buf);
756
757 ret = ph->xops->do_xfer(ph, t);
758 if (!ret)
759 *level = get_unaligned_le32(t->rx.buf);
760
761 ph->xops->xfer_put(ph, t);
762 return ret;
763}
764
765static int __scmi_perf_level_get(const struct scmi_protocol_handle *ph,
766 struct perf_dom_info *dom, u32 *level,
767 bool poll)
768{
769 if (dom->fc_info && dom->fc_info[PERF_FC_LEVEL].get_addr) {
770 *level = ioread32(dom->fc_info[PERF_FC_LEVEL].get_addr);
771 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LEVEL_GET,
772 dom->id, *level, 0);
773 return 0;
774 }
775
776 return scmi_perf_msg_level_get(ph, dom->id, level, poll);
777}
778
779static int scmi_perf_level_get(const struct scmi_protocol_handle *ph,
780 u32 domain, u32 *level, bool poll)
781{
782 int ret;
783 struct perf_dom_info *dom;
784
785 dom = scmi_perf_domain_lookup(ph, domain);
786 if (IS_ERR(dom))
787 return PTR_ERR(dom);
788
789 ret = __scmi_perf_level_get(ph, dom, level, poll);
790 if (ret)
791 return ret;
792
793 if (dom->level_indexing_mode) {
794 struct scmi_opp *opp;
795
796 opp = xa_load(&dom->opps_by_idx, *level);
797 if (!opp)
798 return -EIO;
799
800 *level = opp->perf;
801 }
802
803 return 0;
804}
805
806static int scmi_perf_level_limits_notify(const struct scmi_protocol_handle *ph,
807 u32 domain, int message_id,
808 bool enable)
809{
810 int ret;
811 struct scmi_xfer *t;
812 struct scmi_perf_notify_level_or_limits *notify;
813
814 ret = ph->xops->xfer_get_init(ph, message_id, sizeof(*notify), 0, &t);
815 if (ret)
816 return ret;
817
818 notify = t->tx.buf;
819 notify->domain = cpu_to_le32(domain);
820 notify->notify_enable = enable ? cpu_to_le32(BIT(0)) : 0;
821
822 ret = ph->xops->do_xfer(ph, t);
823
824 ph->xops->xfer_put(ph, t);
825 return ret;
826}
827
828static void scmi_perf_domain_init_fc(const struct scmi_protocol_handle *ph,
829 struct perf_dom_info *dom)
830{
831 struct scmi_fc_info *fc;
832
833 fc = devm_kcalloc(ph->dev, PERF_FC_MAX, sizeof(*fc), GFP_KERNEL);
834 if (!fc)
835 return;
836
837 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
838 PERF_LEVEL_GET, 4, dom->id,
839 &fc[PERF_FC_LEVEL].get_addr, NULL,
840 &fc[PERF_FC_LEVEL].rate_limit);
841
842 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
843 PERF_LIMITS_GET, 8, dom->id,
844 &fc[PERF_FC_LIMIT].get_addr, NULL,
845 &fc[PERF_FC_LIMIT].rate_limit);
846
847 if (dom->info.set_perf)
848 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
849 PERF_LEVEL_SET, 4, dom->id,
850 &fc[PERF_FC_LEVEL].set_addr,
851 &fc[PERF_FC_LEVEL].set_db,
852 &fc[PERF_FC_LEVEL].rate_limit);
853
854 if (dom->set_limits)
855 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
856 PERF_LIMITS_SET, 8, dom->id,
857 &fc[PERF_FC_LIMIT].set_addr,
858 &fc[PERF_FC_LIMIT].set_db,
859 &fc[PERF_FC_LIMIT].rate_limit);
860
861 dom->fc_info = fc;
862}
863
864static int scmi_dvfs_device_opps_add(const struct scmi_protocol_handle *ph,
865 struct device *dev, u32 domain)
866{
867 int idx, ret;
868 unsigned long freq;
869 struct dev_pm_opp_data data = {};
870 struct perf_dom_info *dom;
871
872 dom = scmi_perf_domain_lookup(ph, domain);
873 if (IS_ERR(dom))
874 return PTR_ERR(dom);
875
876 for (idx = 0; idx < dom->opp_count; idx++) {
877 if (!dom->level_indexing_mode)
878 freq = dom->opp[idx].perf * dom->mult_factor;
879 else
880 freq = dom->opp[idx].indicative_freq * dom->mult_factor;
881
882 /* All OPPs above the sustained frequency are treated as turbo */
883 data.turbo = freq > dom->sustained_freq_khz * 1000UL;
884
885 data.level = dom->opp[idx].perf;
886 data.freq = freq;
887
888 ret = dev_pm_opp_add_dynamic(dev, &data);
889 if (ret) {
890 dev_warn(dev, "[%d][%s]: Failed to add OPP[%d] %lu\n",
891 domain, dom->info.name, idx, freq);
892 dev_pm_opp_remove_all_dynamic(dev);
893 return ret;
894 }
895
896 dev_dbg(dev, "[%d][%s]:: Registered OPP[%d] %lu\n",
897 domain, dom->info.name, idx, freq);
898 }
899 return 0;
900}
901
902static int
903scmi_dvfs_transition_latency_get(const struct scmi_protocol_handle *ph,
904 u32 domain)
905{
906 struct perf_dom_info *dom;
907
908 dom = scmi_perf_domain_lookup(ph, domain);
909 if (IS_ERR(dom))
910 return PTR_ERR(dom);
911
912 /* uS to nS */
913 return dom->opp[dom->opp_count - 1].trans_latency_us * 1000;
914}
915
916static int
917scmi_dvfs_rate_limit_get(const struct scmi_protocol_handle *ph,
918 u32 domain, u32 *rate_limit)
919{
920 struct perf_dom_info *dom;
921
922 if (!rate_limit)
923 return -EINVAL;
924
925 dom = scmi_perf_domain_lookup(ph, domain);
926 if (IS_ERR(dom))
927 return PTR_ERR(dom);
928
929 *rate_limit = dom->rate_limit_us;
930 return 0;
931}
932
933static int scmi_dvfs_freq_set(const struct scmi_protocol_handle *ph, u32 domain,
934 unsigned long freq, bool poll)
935{
936 unsigned int level;
937 struct perf_dom_info *dom;
938
939 dom = scmi_perf_domain_lookup(ph, domain);
940 if (IS_ERR(dom))
941 return PTR_ERR(dom);
942
943 if (!dom->level_indexing_mode) {
944 level = freq / dom->mult_factor;
945 } else {
946 struct scmi_opp *opp;
947
948 opp = LOOKUP_BY_FREQ(dom->opps_by_freq,
949 freq / dom->mult_factor);
950 if (!opp)
951 return -EIO;
952
953 level = opp->level_index;
954 }
955
956 return __scmi_perf_level_set(ph, dom, level, poll);
957}
958
959static int scmi_dvfs_freq_get(const struct scmi_protocol_handle *ph, u32 domain,
960 unsigned long *freq, bool poll)
961{
962 int ret;
963 u32 level;
964 struct perf_dom_info *dom;
965
966 dom = scmi_perf_domain_lookup(ph, domain);
967 if (IS_ERR(dom))
968 return PTR_ERR(dom);
969
970 ret = __scmi_perf_level_get(ph, dom, &level, poll);
971 if (ret)
972 return ret;
973
974 if (!dom->level_indexing_mode) {
975 *freq = level * dom->mult_factor;
976 } else {
977 struct scmi_opp *opp;
978
979 opp = xa_load(&dom->opps_by_idx, level);
980 if (!opp)
981 return -EIO;
982
983 *freq = opp->indicative_freq * dom->mult_factor;
984 }
985
986 return ret;
987}
988
989static int scmi_dvfs_est_power_get(const struct scmi_protocol_handle *ph,
990 u32 domain, unsigned long *freq,
991 unsigned long *power)
992{
993 struct perf_dom_info *dom;
994 unsigned long opp_freq;
995 int idx, ret = -EINVAL;
996 struct scmi_opp *opp;
997
998 dom = scmi_perf_domain_lookup(ph, domain);
999 if (IS_ERR(dom))
1000 return PTR_ERR(dom);
1001
1002 for (opp = dom->opp, idx = 0; idx < dom->opp_count; idx++, opp++) {
1003 if (!dom->level_indexing_mode)
1004 opp_freq = opp->perf * dom->mult_factor;
1005 else
1006 opp_freq = opp->indicative_freq * dom->mult_factor;
1007
1008 if (opp_freq < *freq)
1009 continue;
1010
1011 *freq = opp_freq;
1012 *power = opp->power;
1013 ret = 0;
1014 break;
1015 }
1016
1017 return ret;
1018}
1019
1020static bool scmi_fast_switch_possible(const struct scmi_protocol_handle *ph,
1021 u32 domain)
1022{
1023 struct perf_dom_info *dom;
1024
1025 dom = scmi_perf_domain_lookup(ph, domain);
1026 if (IS_ERR(dom))
1027 return false;
1028
1029 return dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr;
1030}
1031
1032static int scmi_fast_switch_rate_limit(const struct scmi_protocol_handle *ph,
1033 u32 domain, u32 *rate_limit)
1034{
1035 struct perf_dom_info *dom;
1036
1037 if (!rate_limit)
1038 return -EINVAL;
1039
1040 dom = scmi_perf_domain_lookup(ph, domain);
1041 if (IS_ERR(dom))
1042 return PTR_ERR(dom);
1043
1044 if (!dom->fc_info)
1045 return -EINVAL;
1046
1047 *rate_limit = dom->fc_info[PERF_FC_LEVEL].rate_limit;
1048 return 0;
1049}
1050
1051static enum scmi_power_scale
1052scmi_power_scale_get(const struct scmi_protocol_handle *ph)
1053{
1054 struct scmi_perf_info *pi = ph->get_priv(ph);
1055
1056 return pi->power_scale;
1057}
1058
1059static const struct scmi_perf_proto_ops perf_proto_ops = {
1060 .num_domains_get = scmi_perf_num_domains_get,
1061 .info_get = scmi_perf_info_get,
1062 .limits_set = scmi_perf_limits_set,
1063 .limits_get = scmi_perf_limits_get,
1064 .level_set = scmi_perf_level_set,
1065 .level_get = scmi_perf_level_get,
1066 .transition_latency_get = scmi_dvfs_transition_latency_get,
1067 .rate_limit_get = scmi_dvfs_rate_limit_get,
1068 .device_opps_add = scmi_dvfs_device_opps_add,
1069 .freq_set = scmi_dvfs_freq_set,
1070 .freq_get = scmi_dvfs_freq_get,
1071 .est_power_get = scmi_dvfs_est_power_get,
1072 .fast_switch_possible = scmi_fast_switch_possible,
1073 .fast_switch_rate_limit = scmi_fast_switch_rate_limit,
1074 .power_scale_get = scmi_power_scale_get,
1075};
1076
1077static bool scmi_perf_notify_supported(const struct scmi_protocol_handle *ph,
1078 u8 evt_id, u32 src_id)
1079{
1080 bool supported;
1081 struct perf_dom_info *dom;
1082
1083 if (evt_id >= ARRAY_SIZE(evt_2_cmd))
1084 return false;
1085
1086 dom = scmi_perf_domain_lookup(ph, src_id);
1087 if (IS_ERR(dom))
1088 return false;
1089
1090 if (evt_id == SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED)
1091 supported = dom->perf_limit_notify;
1092 else
1093 supported = dom->perf_level_notify;
1094
1095 return supported;
1096}
1097
1098static int scmi_perf_set_notify_enabled(const struct scmi_protocol_handle *ph,
1099 u8 evt_id, u32 src_id, bool enable)
1100{
1101 int ret, cmd_id;
1102
1103 if (evt_id >= ARRAY_SIZE(evt_2_cmd))
1104 return -EINVAL;
1105
1106 cmd_id = evt_2_cmd[evt_id];
1107 ret = scmi_perf_level_limits_notify(ph, src_id, cmd_id, enable);
1108 if (ret)
1109 pr_debug("FAIL_ENABLED - evt[%X] dom[%d] - ret:%d\n",
1110 evt_id, src_id, ret);
1111
1112 return ret;
1113}
1114
1115static int
1116scmi_perf_xlate_opp_to_freq(struct perf_dom_info *dom,
1117 unsigned int index, unsigned long *freq)
1118{
1119 struct scmi_opp *opp;
1120
1121 if (!dom || !freq)
1122 return -EINVAL;
1123
1124 if (!dom->level_indexing_mode) {
1125 opp = xa_load(&dom->opps_by_lvl, index);
1126 if (!opp)
1127 return -ENODEV;
1128
1129 *freq = opp->perf * dom->mult_factor;
1130 } else {
1131 opp = xa_load(&dom->opps_by_idx, index);
1132 if (!opp)
1133 return -ENODEV;
1134
1135 *freq = opp->indicative_freq * dom->mult_factor;
1136 }
1137
1138 return 0;
1139}
1140
1141static void *scmi_perf_fill_custom_report(const struct scmi_protocol_handle *ph,
1142 u8 evt_id, ktime_t timestamp,
1143 const void *payld, size_t payld_sz,
1144 void *report, u32 *src_id)
1145{
1146 int ret;
1147 void *rep = NULL;
1148 struct perf_dom_info *dom;
1149
1150 switch (evt_id) {
1151 case SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED:
1152 {
1153 const struct scmi_perf_limits_notify_payld *p = payld;
1154 struct scmi_perf_limits_report *r = report;
1155 unsigned long freq_min, freq_max;
1156
1157 if (sizeof(*p) != payld_sz)
1158 break;
1159
1160 r->timestamp = timestamp;
1161 r->agent_id = le32_to_cpu(p->agent_id);
1162 r->domain_id = le32_to_cpu(p->domain_id);
1163 r->range_max = le32_to_cpu(p->range_max);
1164 r->range_min = le32_to_cpu(p->range_min);
1165 /* Check if the reported domain exist at all */
1166 dom = scmi_perf_domain_lookup(ph, r->domain_id);
1167 if (IS_ERR(dom))
1168 break;
1169 /*
1170 * Event will be reported from this point on...
1171 * ...even if, later, xlated frequencies were not retrieved.
1172 */
1173 *src_id = r->domain_id;
1174 rep = r;
1175
1176 ret = scmi_perf_xlate_opp_to_freq(dom, r->range_max, &freq_max);
1177 if (ret)
1178 break;
1179
1180 ret = scmi_perf_xlate_opp_to_freq(dom, r->range_min, &freq_min);
1181 if (ret)
1182 break;
1183
1184 /* Report translated freqs ONLY if both available */
1185 r->range_max_freq = freq_max;
1186 r->range_min_freq = freq_min;
1187
1188 break;
1189 }
1190 case SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED:
1191 {
1192 const struct scmi_perf_level_notify_payld *p = payld;
1193 struct scmi_perf_level_report *r = report;
1194 unsigned long freq;
1195
1196 if (sizeof(*p) != payld_sz)
1197 break;
1198
1199 r->timestamp = timestamp;
1200 r->agent_id = le32_to_cpu(p->agent_id);
1201 r->domain_id = le32_to_cpu(p->domain_id);
1202 /* Report translated freqs ONLY if available */
1203 r->performance_level = le32_to_cpu(p->performance_level);
1204 /* Check if the reported domain exist at all */
1205 dom = scmi_perf_domain_lookup(ph, r->domain_id);
1206 if (IS_ERR(dom))
1207 break;
1208 /*
1209 * Event will be reported from this point on...
1210 * ...even if, later, xlated frequencies were not retrieved.
1211 */
1212 *src_id = r->domain_id;
1213 rep = r;
1214
1215 /* Report translated freqs ONLY if available */
1216 ret = scmi_perf_xlate_opp_to_freq(dom, r->performance_level,
1217 &freq);
1218 if (ret)
1219 break;
1220
1221 r->performance_level_freq = freq;
1222
1223 break;
1224 }
1225 default:
1226 break;
1227 }
1228
1229 return rep;
1230}
1231
1232static int scmi_perf_get_num_sources(const struct scmi_protocol_handle *ph)
1233{
1234 struct scmi_perf_info *pi = ph->get_priv(ph);
1235
1236 if (!pi)
1237 return -EINVAL;
1238
1239 return pi->num_domains;
1240}
1241
1242static const struct scmi_event perf_events[] = {
1243 {
1244 .id = SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED,
1245 .max_payld_sz = sizeof(struct scmi_perf_limits_notify_payld),
1246 .max_report_sz = sizeof(struct scmi_perf_limits_report),
1247 },
1248 {
1249 .id = SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED,
1250 .max_payld_sz = sizeof(struct scmi_perf_level_notify_payld),
1251 .max_report_sz = sizeof(struct scmi_perf_level_report),
1252 },
1253};
1254
1255static const struct scmi_event_ops perf_event_ops = {
1256 .is_notify_supported = scmi_perf_notify_supported,
1257 .get_num_sources = scmi_perf_get_num_sources,
1258 .set_notify_enabled = scmi_perf_set_notify_enabled,
1259 .fill_custom_report = scmi_perf_fill_custom_report,
1260};
1261
1262static const struct scmi_protocol_events perf_protocol_events = {
1263 .queue_sz = SCMI_PROTO_QUEUE_SZ,
1264 .ops = &perf_event_ops,
1265 .evts = perf_events,
1266 .num_events = ARRAY_SIZE(perf_events),
1267};
1268
1269static int scmi_perf_protocol_init(const struct scmi_protocol_handle *ph)
1270{
1271 int domain, ret;
1272 struct scmi_perf_info *pinfo;
1273
1274 dev_dbg(ph->dev, "Performance Version %d.%d\n",
1275 PROTOCOL_REV_MAJOR(ph->version), PROTOCOL_REV_MINOR(ph->version));
1276
1277 pinfo = devm_kzalloc(ph->dev, sizeof(*pinfo), GFP_KERNEL);
1278 if (!pinfo)
1279 return -ENOMEM;
1280
1281 ret = scmi_perf_attributes_get(ph, pinfo);
1282 if (ret)
1283 return ret;
1284
1285 pinfo->dom_info = devm_kcalloc(ph->dev, pinfo->num_domains,
1286 sizeof(*pinfo->dom_info), GFP_KERNEL);
1287 if (!pinfo->dom_info)
1288 return -ENOMEM;
1289
1290 for (domain = 0; domain < pinfo->num_domains; domain++) {
1291 struct perf_dom_info *dom = pinfo->dom_info + domain;
1292
1293 dom->id = domain;
1294 scmi_perf_domain_attributes_get(ph, dom, pinfo->notify_lim_cmd,
1295 pinfo->notify_lvl_cmd);
1296 scmi_perf_describe_levels_get(ph, dom);
1297
1298 if (dom->perf_fastchannels)
1299 scmi_perf_domain_init_fc(ph, dom);
1300 }
1301
1302 ret = devm_add_action_or_reset(ph->dev, scmi_perf_xa_destroy, pinfo);
1303 if (ret)
1304 return ret;
1305
1306 return ph->set_priv(ph, pinfo);
1307}
1308
1309static const struct scmi_protocol scmi_perf = {
1310 .id = SCMI_PROTOCOL_PERF,
1311 .owner = THIS_MODULE,
1312 .instance_init = &scmi_perf_protocol_init,
1313 .ops = &perf_proto_ops,
1314 .events = &perf_protocol_events,
1315 .supported_version = SCMI_PROTOCOL_SUPPORTED_VERSION,
1316};
1317
1318DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(perf, scmi_perf)