Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * ARM DynamIQ Shared Unit (DSU) PMU driver
4 *
5 * Copyright (C) ARM Limited, 2017.
6 *
7 * Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers.
8 */
9
10#define PMUNAME "arm_dsu"
11#define DRVNAME PMUNAME "_pmu"
12#define pr_fmt(fmt) DRVNAME ": " fmt
13
14#include <linux/acpi.h>
15#include <linux/bitmap.h>
16#include <linux/bitops.h>
17#include <linux/bug.h>
18#include <linux/cpumask.h>
19#include <linux/device.h>
20#include <linux/interrupt.h>
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/of.h>
24#include <linux/perf_event.h>
25#include <linux/platform_device.h>
26#include <linux/spinlock.h>
27#include <linux/smp.h>
28#include <linux/sysfs.h>
29#include <linux/types.h>
30
31#include <asm/arm_dsu_pmu.h>
32#include <asm/local64.h>
33
34/* PMU event codes */
35#define DSU_PMU_EVT_CYCLES 0x11
36#define DSU_PMU_EVT_CHAIN 0x1e
37
38#define DSU_PMU_MAX_COMMON_EVENTS 0x40
39
40#define DSU_PMU_MAX_HW_CNTRS 32
41#define DSU_PMU_HW_COUNTER_MASK (DSU_PMU_MAX_HW_CNTRS - 1)
42
43#define CLUSTERPMCR_E BIT(0)
44#define CLUSTERPMCR_P BIT(1)
45#define CLUSTERPMCR_C BIT(2)
46#define CLUSTERPMCR_N_SHIFT 11
47#define CLUSTERPMCR_N_MASK 0x1f
48#define CLUSTERPMCR_IDCODE_SHIFT 16
49#define CLUSTERPMCR_IDCODE_MASK 0xff
50#define CLUSTERPMCR_IMP_SHIFT 24
51#define CLUSTERPMCR_IMP_MASK 0xff
52#define CLUSTERPMCR_RES_MASK 0x7e8
53#define CLUSTERPMCR_RES_VAL 0x40
54
55#define DSU_ACTIVE_CPU_MASK 0x0
56#define DSU_ASSOCIATED_CPU_MASK 0x1
57
58/*
59 * We use the index of the counters as they appear in the counter
60 * bit maps in the PMU registers (e.g CLUSTERPMSELR).
61 * i.e,
62 * counter 0 - Bit 0
63 * counter 1 - Bit 1
64 * ...
65 * Cycle counter - Bit 31
66 */
67#define DSU_PMU_IDX_CYCLE_COUNTER 31
68
69#define DSU_EXT_ATTR(_name, _func, _config) \
70 (&((struct dev_ext_attribute[]) { \
71 { \
72 .attr = __ATTR(_name, 0444, _func, NULL), \
73 .var = (void *)_config \
74 } \
75 })[0].attr.attr)
76
77#define DSU_EVENT_ATTR(_name, _config) \
78 DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config)
79
80#define DSU_FORMAT_ATTR(_name, _config) \
81 DSU_EXT_ATTR(_name, device_show_string, _config)
82
83#define DSU_CPUMASK_ATTR(_name, _config) \
84 DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config)
85
86struct dsu_hw_events {
87 DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS);
88 struct perf_event *events[DSU_PMU_MAX_HW_CNTRS];
89};
90
91/*
92 * struct dsu_pmu - DSU PMU descriptor
93 *
94 * @pmu_lock : Protects accesses to DSU PMU register from normal vs
95 * interrupt handler contexts.
96 * @hw_events : Holds the event counter state.
97 * @associated_cpus : CPUs attached to the DSU.
98 * @active_cpu : CPU to which the PMU is bound for accesses.
99 * @cpuhp_node : Node for CPU hotplug notifier link.
100 * @num_counters : Number of event counters implemented by the PMU,
101 * excluding the cycle counter.
102 * @irq : Interrupt line for counter overflow.
103 * @has_32b_pmevcntr : Are the non-cycle counters only 32-bit?
104 * @has_pmccntr : Do we even have a dedicated cycle counter?
105 * @cpmceid_bitmap : Bitmap for the availability of architected common
106 * events (event_code < 0x40).
107 */
108struct dsu_pmu {
109 struct pmu pmu;
110 struct device *dev;
111 raw_spinlock_t pmu_lock;
112 struct dsu_hw_events hw_events;
113 cpumask_t associated_cpus;
114 cpumask_t active_cpu;
115 struct hlist_node cpuhp_node;
116 s8 num_counters;
117 int irq;
118 bool has_32b_pmevcntr;
119 bool has_pmccntr;
120 DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS);
121};
122
123static unsigned long dsu_pmu_cpuhp_state;
124
125static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu)
126{
127 return container_of(pmu, struct dsu_pmu, pmu);
128}
129
130static ssize_t dsu_pmu_sysfs_event_show(struct device *dev,
131 struct device_attribute *attr,
132 char *buf)
133{
134 struct dev_ext_attribute *eattr = container_of(attr,
135 struct dev_ext_attribute, attr);
136 return sysfs_emit(buf, "event=0x%lx\n", (unsigned long)eattr->var);
137}
138
139static ssize_t dsu_pmu_cpumask_show(struct device *dev,
140 struct device_attribute *attr,
141 char *buf)
142{
143 struct pmu *pmu = dev_get_drvdata(dev);
144 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
145 struct dev_ext_attribute *eattr = container_of(attr,
146 struct dev_ext_attribute, attr);
147 unsigned long mask_id = (unsigned long)eattr->var;
148 const cpumask_t *cpumask;
149
150 switch (mask_id) {
151 case DSU_ACTIVE_CPU_MASK:
152 cpumask = &dsu_pmu->active_cpu;
153 break;
154 case DSU_ASSOCIATED_CPU_MASK:
155 cpumask = &dsu_pmu->associated_cpus;
156 break;
157 default:
158 return 0;
159 }
160 return cpumap_print_to_pagebuf(true, buf, cpumask);
161}
162
163static struct attribute *dsu_pmu_format_attrs[] = {
164 DSU_FORMAT_ATTR(event, "config:0-31"),
165 NULL,
166};
167
168static const struct attribute_group dsu_pmu_format_attr_group = {
169 .name = "format",
170 .attrs = dsu_pmu_format_attrs,
171};
172
173static struct attribute *dsu_pmu_event_attrs[] = {
174 DSU_EVENT_ATTR(cycles, 0x11),
175 DSU_EVENT_ATTR(bus_access, 0x19),
176 DSU_EVENT_ATTR(memory_error, 0x1a),
177 DSU_EVENT_ATTR(bus_cycles, 0x1d),
178 DSU_EVENT_ATTR(l3d_cache_allocate, 0x29),
179 DSU_EVENT_ATTR(l3d_cache_refill, 0x2a),
180 DSU_EVENT_ATTR(l3d_cache, 0x2b),
181 DSU_EVENT_ATTR(l3d_cache_wb, 0x2c),
182 NULL,
183};
184
185static umode_t
186dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr,
187 int unused)
188{
189 struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
190 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
191 struct dev_ext_attribute *eattr = container_of(attr,
192 struct dev_ext_attribute, attr.attr);
193 unsigned long evt = (unsigned long)eattr->var;
194
195 return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0;
196}
197
198static const struct attribute_group dsu_pmu_events_attr_group = {
199 .name = "events",
200 .attrs = dsu_pmu_event_attrs,
201 .is_visible = dsu_pmu_event_attr_is_visible,
202};
203
204static struct attribute *dsu_pmu_cpumask_attrs[] = {
205 DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK),
206 DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK),
207 NULL,
208};
209
210static const struct attribute_group dsu_pmu_cpumask_attr_group = {
211 .attrs = dsu_pmu_cpumask_attrs,
212};
213
214static const struct attribute_group *dsu_pmu_attr_groups[] = {
215 &dsu_pmu_cpumask_attr_group,
216 &dsu_pmu_events_attr_group,
217 &dsu_pmu_format_attr_group,
218 NULL,
219};
220
221static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx)
222{
223 return (idx < dsu_pmu->num_counters) ||
224 (idx == DSU_PMU_IDX_CYCLE_COUNTER);
225}
226
227static inline u64 dsu_pmu_read_counter(struct perf_event *event)
228{
229 u64 val;
230 unsigned long flags;
231 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
232 int idx = event->hw.idx;
233
234 if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
235 &dsu_pmu->associated_cpus)))
236 return 0;
237
238 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
239 dev_err(event->pmu->dev,
240 "Trying reading invalid counter %d\n", idx);
241 return 0;
242 }
243
244 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
245 if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
246 val = __dsu_pmu_read_pmccntr();
247 else
248 val = __dsu_pmu_read_counter(idx);
249 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
250
251 return val;
252}
253
254static void dsu_pmu_write_counter(struct perf_event *event, u64 val)
255{
256 unsigned long flags;
257 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
258 int idx = event->hw.idx;
259
260 if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
261 &dsu_pmu->associated_cpus)))
262 return;
263
264 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
265 dev_err(event->pmu->dev,
266 "writing to invalid counter %d\n", idx);
267 return;
268 }
269
270 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
271 if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
272 __dsu_pmu_write_pmccntr(val);
273 else
274 __dsu_pmu_write_counter(idx, val);
275 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
276}
277
278static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events,
279 struct perf_event *event)
280{
281 int idx;
282 unsigned long evtype = event->attr.config;
283 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
284 unsigned long *used_mask = hw_events->used_mask;
285
286 if (evtype == DSU_PMU_EVT_CYCLES && dsu_pmu->has_pmccntr) {
287 if (!test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask))
288 return DSU_PMU_IDX_CYCLE_COUNTER;
289 }
290
291 idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters);
292 if (idx >= dsu_pmu->num_counters)
293 return -EAGAIN;
294 set_bit(idx, hw_events->used_mask);
295 return idx;
296}
297
298static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx)
299{
300 __dsu_pmu_counter_interrupt_enable(idx);
301 __dsu_pmu_enable_counter(idx);
302}
303
304static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx)
305{
306 __dsu_pmu_disable_counter(idx);
307 __dsu_pmu_counter_interrupt_disable(idx);
308}
309
310static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu,
311 struct perf_event *event)
312{
313 int idx = event->hw.idx;
314 unsigned long flags;
315
316 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
317 dev_err(event->pmu->dev,
318 "Trying to set invalid counter %d\n", idx);
319 return;
320 }
321
322 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
323 __dsu_pmu_set_event(idx, event->hw.config_base);
324 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
325}
326
327static u64 dsu_pmu_counter_mask(struct hw_perf_event *hw)
328{
329 return (hw->flags && hw->idx != DSU_PMU_IDX_CYCLE_COUNTER) ? U32_MAX : U64_MAX;
330}
331
332static void dsu_pmu_event_update(struct perf_event *event)
333{
334 struct hw_perf_event *hwc = &event->hw;
335 u64 delta, prev_count, new_count;
336
337 do {
338 /* We may also be called from the irq handler */
339 prev_count = local64_read(&hwc->prev_count);
340 new_count = dsu_pmu_read_counter(event);
341 } while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) !=
342 prev_count);
343 delta = (new_count - prev_count) & dsu_pmu_counter_mask(hwc);
344 local64_add(delta, &event->count);
345}
346
347static void dsu_pmu_read(struct perf_event *event)
348{
349 dsu_pmu_event_update(event);
350}
351
352static inline u32 dsu_pmu_get_reset_overflow(void)
353{
354 return __dsu_pmu_get_reset_overflow();
355}
356
357/*
358 * dsu_pmu_set_event_period: Set the period for the counter.
359 *
360 * All DSU PMU event counters, except the cycle counter are 32bit
361 * counters. To handle cases of extreme interrupt latency, we program
362 * the counter with half of the max count for the counters.
363 */
364static void dsu_pmu_set_event_period(struct perf_event *event)
365{
366 u64 val = dsu_pmu_counter_mask(&event->hw) >> 1;
367
368 local64_set(&event->hw.prev_count, val);
369 dsu_pmu_write_counter(event, val);
370}
371
372static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev)
373{
374 int i;
375 bool handled = false;
376 struct dsu_pmu *dsu_pmu = dev;
377 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
378 unsigned long overflow;
379
380 overflow = dsu_pmu_get_reset_overflow();
381 if (!overflow)
382 return IRQ_NONE;
383
384 for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) {
385 struct perf_event *event = hw_events->events[i];
386
387 if (!event)
388 continue;
389 dsu_pmu_event_update(event);
390 dsu_pmu_set_event_period(event);
391 handled = true;
392 }
393
394 return IRQ_RETVAL(handled);
395}
396
397static void dsu_pmu_start(struct perf_event *event, int pmu_flags)
398{
399 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
400
401 /* We always reprogram the counter */
402 if (pmu_flags & PERF_EF_RELOAD)
403 WARN_ON(!(event->hw.state & PERF_HES_UPTODATE));
404 dsu_pmu_set_event_period(event);
405 if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER)
406 dsu_pmu_set_event(dsu_pmu, event);
407 event->hw.state = 0;
408 dsu_pmu_enable_counter(dsu_pmu, event->hw.idx);
409}
410
411static void dsu_pmu_stop(struct perf_event *event, int pmu_flags)
412{
413 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
414
415 if (event->hw.state & PERF_HES_STOPPED)
416 return;
417 dsu_pmu_disable_counter(dsu_pmu, event->hw.idx);
418 dsu_pmu_event_update(event);
419 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
420}
421
422static int dsu_pmu_add(struct perf_event *event, int flags)
423{
424 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
425 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
426 struct hw_perf_event *hwc = &event->hw;
427 int idx;
428
429 if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
430 &dsu_pmu->associated_cpus)))
431 return -ENOENT;
432
433 idx = dsu_pmu_get_event_idx(hw_events, event);
434 if (idx < 0)
435 return idx;
436
437 hwc->idx = idx;
438 hw_events->events[idx] = event;
439 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
440
441 if (flags & PERF_EF_START)
442 dsu_pmu_start(event, PERF_EF_RELOAD);
443
444 perf_event_update_userpage(event);
445 return 0;
446}
447
448static void dsu_pmu_del(struct perf_event *event, int flags)
449{
450 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
451 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
452 struct hw_perf_event *hwc = &event->hw;
453 int idx = hwc->idx;
454
455 dsu_pmu_stop(event, PERF_EF_UPDATE);
456 hw_events->events[idx] = NULL;
457 clear_bit(idx, hw_events->used_mask);
458 perf_event_update_userpage(event);
459}
460
461static void dsu_pmu_enable(struct pmu *pmu)
462{
463 u32 pmcr;
464 unsigned long flags;
465 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
466
467 /* If no counters are added, skip enabling the PMU */
468 if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS))
469 return;
470
471 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
472 pmcr = __dsu_pmu_read_pmcr();
473 pmcr |= CLUSTERPMCR_E;
474 __dsu_pmu_write_pmcr(pmcr);
475 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
476}
477
478static void dsu_pmu_disable(struct pmu *pmu)
479{
480 u32 pmcr;
481 unsigned long flags;
482 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
483
484 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
485 pmcr = __dsu_pmu_read_pmcr();
486 pmcr &= ~CLUSTERPMCR_E;
487 __dsu_pmu_write_pmcr(pmcr);
488 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
489}
490
491static bool dsu_pmu_validate_event(struct pmu *pmu,
492 struct dsu_hw_events *hw_events,
493 struct perf_event *event)
494{
495 if (is_software_event(event))
496 return true;
497 /* Reject groups spanning multiple HW PMUs. */
498 if (event->pmu != pmu)
499 return false;
500 return dsu_pmu_get_event_idx(hw_events, event) >= 0;
501}
502
503/*
504 * Make sure the group of events can be scheduled at once
505 * on the PMU.
506 */
507static bool dsu_pmu_validate_group(struct perf_event *event)
508{
509 struct perf_event *sibling, *leader = event->group_leader;
510 struct dsu_hw_events fake_hw;
511
512 if (event->group_leader == event)
513 return true;
514
515 memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask));
516 if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader))
517 return false;
518 for_each_sibling_event(sibling, leader) {
519 if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling))
520 return false;
521 }
522 return dsu_pmu_validate_event(event->pmu, &fake_hw, event);
523}
524
525static int dsu_pmu_event_init(struct perf_event *event)
526{
527 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
528
529 if (event->attr.type != event->pmu->type)
530 return -ENOENT;
531
532 /* We don't support sampling */
533 if (is_sampling_event(event)) {
534 dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n");
535 return -EOPNOTSUPP;
536 }
537
538 /* We cannot support task bound events */
539 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
540 dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n");
541 return -EINVAL;
542 }
543
544 if (has_branch_stack(event)) {
545 dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n");
546 return -EINVAL;
547 }
548
549 if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) {
550 dev_dbg(dsu_pmu->pmu.dev,
551 "Requested cpu is not associated with the DSU\n");
552 return -EINVAL;
553 }
554 /*
555 * Choose the current active CPU to read the events. We don't want
556 * to migrate the event contexts, irq handling etc to the requested
557 * CPU. As long as the requested CPU is within the same DSU, we
558 * are fine.
559 */
560 event->cpu = cpumask_first(&dsu_pmu->active_cpu);
561 if (event->cpu >= nr_cpu_ids)
562 return -EINVAL;
563 if (!dsu_pmu_validate_group(event))
564 return -EINVAL;
565
566 event->hw.config_base = event->attr.config;
567 event->hw.flags = dsu_pmu->has_32b_pmevcntr;
568 return 0;
569}
570
571static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev)
572{
573 struct dsu_pmu *dsu_pmu;
574
575 dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL);
576 if (!dsu_pmu)
577 return ERR_PTR(-ENOMEM);
578
579 raw_spin_lock_init(&dsu_pmu->pmu_lock);
580 /*
581 * Initialise the number of counters to -1, until we probe
582 * the real number on a connected CPU.
583 */
584 dsu_pmu->num_counters = -1;
585 return dsu_pmu;
586}
587
588/*
589 * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster
590 * from device tree.
591 */
592static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask)
593{
594 int i = 0, n, cpu;
595 struct device_node *cpu_node;
596
597 n = of_count_phandle_with_args(dev->of_node, "cpus", NULL);
598 if (n <= 0)
599 return -ENODEV;
600 for (; i < n; i++) {
601 cpu_node = of_parse_phandle(dev->of_node, "cpus", i);
602 if (!cpu_node)
603 break;
604 cpu = of_cpu_node_to_id(cpu_node);
605 of_node_put(cpu_node);
606 /*
607 * We have to ignore the failures here and continue scanning
608 * the list to handle cases where the nr_cpus could be capped
609 * in the running kernel.
610 */
611 if (cpu < 0)
612 continue;
613 cpumask_set_cpu(cpu, mask);
614 }
615 return 0;
616}
617
618/*
619 * dsu_pmu_acpi_get_cpus: Get the list of CPUs in the cluster
620 * from ACPI.
621 */
622static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask)
623{
624#ifdef CONFIG_ACPI
625 struct acpi_device *parent_adev = acpi_dev_parent(ACPI_COMPANION(dev));
626 int cpu;
627
628 /*
629 * A dsu pmu node is inside a cluster parent node along with cpu nodes.
630 * We need to find out all cpus that have the same parent with this pmu.
631 */
632 for_each_possible_cpu(cpu) {
633 struct acpi_device *acpi_dev;
634 struct device *cpu_dev = get_cpu_device(cpu);
635
636 if (!cpu_dev)
637 continue;
638
639 acpi_dev = ACPI_COMPANION(cpu_dev);
640 if (acpi_dev && acpi_dev_parent(acpi_dev) == parent_adev)
641 cpumask_set_cpu(cpu, mask);
642 }
643#endif
644
645 return 0;
646}
647
648/*
649 * dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster.
650 */
651static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu)
652{
653 u64 num_counters;
654 u32 cpmceid[2];
655
656 num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) &
657 CLUSTERPMCR_N_MASK;
658 /* We can only support up to 31 independent counters */
659 if (WARN_ON(num_counters > 31))
660 num_counters = 31;
661 dsu_pmu->num_counters = num_counters;
662 if (!dsu_pmu->num_counters)
663 return;
664 cpmceid[0] = __dsu_pmu_read_pmceid(0);
665 cpmceid[1] = __dsu_pmu_read_pmceid(1);
666 bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid,
667 DSU_PMU_MAX_COMMON_EVENTS);
668 /* Newer DSUs have 64-bit counters */
669 __dsu_pmu_write_counter(0, U64_MAX);
670 if (__dsu_pmu_read_counter(0) != U64_MAX)
671 dsu_pmu->has_32b_pmevcntr = true;
672 /* On even newer DSUs, PMCCNTR is RAZ/WI */
673 __dsu_pmu_write_pmccntr(U64_MAX);
674 if (__dsu_pmu_read_pmccntr() == U64_MAX)
675 dsu_pmu->has_pmccntr = true;
676}
677
678static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu)
679{
680 cpumask_set_cpu(cpu, &dsu_pmu->active_cpu);
681 if (irq_set_affinity(dsu_pmu->irq, &dsu_pmu->active_cpu))
682 pr_warn("Failed to set irq affinity to %d\n", cpu);
683}
684
685/*
686 * dsu_pmu_init_pmu: Initialise the DSU PMU configurations if
687 * we haven't done it already.
688 */
689static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu)
690{
691 if (dsu_pmu->num_counters == -1)
692 dsu_pmu_probe_pmu(dsu_pmu);
693 /* Reset the interrupt overflow mask */
694 dsu_pmu_get_reset_overflow();
695}
696
697static int dsu_pmu_device_probe(struct platform_device *pdev)
698{
699 int irq, rc;
700 struct dsu_pmu *dsu_pmu;
701 struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev);
702 char *name;
703 static atomic_t pmu_idx = ATOMIC_INIT(-1);
704
705 dsu_pmu = dsu_pmu_alloc(pdev);
706 if (IS_ERR(dsu_pmu))
707 return PTR_ERR(dsu_pmu);
708
709 if (is_of_node(fwnode))
710 rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
711 else if (is_acpi_device_node(fwnode))
712 rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
713 else
714 return -ENOENT;
715
716 if (rc) {
717 dev_warn(&pdev->dev, "Failed to parse the CPUs\n");
718 return rc;
719 }
720
721 irq = platform_get_irq(pdev, 0);
722 if (irq < 0)
723 return -EINVAL;
724
725 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
726 PMUNAME, atomic_inc_return(&pmu_idx));
727 if (!name)
728 return -ENOMEM;
729 rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq,
730 IRQF_NOBALANCING, name, dsu_pmu);
731 if (rc) {
732 dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq);
733 return rc;
734 }
735
736 dsu_pmu->irq = irq;
737 platform_set_drvdata(pdev, dsu_pmu);
738 rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state,
739 &dsu_pmu->cpuhp_node);
740 if (rc)
741 return rc;
742
743 dsu_pmu->pmu = (struct pmu) {
744 .task_ctx_nr = perf_invalid_context,
745 .parent = &pdev->dev,
746 .module = THIS_MODULE,
747 .pmu_enable = dsu_pmu_enable,
748 .pmu_disable = dsu_pmu_disable,
749 .event_init = dsu_pmu_event_init,
750 .add = dsu_pmu_add,
751 .del = dsu_pmu_del,
752 .start = dsu_pmu_start,
753 .stop = dsu_pmu_stop,
754 .read = dsu_pmu_read,
755
756 .attr_groups = dsu_pmu_attr_groups,
757 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
758 };
759
760 rc = perf_pmu_register(&dsu_pmu->pmu, name, -1);
761 if (rc) {
762 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state,
763 &dsu_pmu->cpuhp_node);
764 }
765
766 return rc;
767}
768
769static void dsu_pmu_device_remove(struct platform_device *pdev)
770{
771 struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev);
772
773 perf_pmu_unregister(&dsu_pmu->pmu);
774 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node);
775}
776
777static const struct of_device_id dsu_pmu_of_match[] = {
778 { .compatible = "arm,dsu-pmu", },
779 {},
780};
781MODULE_DEVICE_TABLE(of, dsu_pmu_of_match);
782
783#ifdef CONFIG_ACPI
784static const struct acpi_device_id dsu_pmu_acpi_match[] = {
785 { "ARMHD500", 0},
786 {},
787};
788MODULE_DEVICE_TABLE(acpi, dsu_pmu_acpi_match);
789#endif
790
791static struct platform_driver dsu_pmu_driver = {
792 .driver = {
793 .name = DRVNAME,
794 .of_match_table = of_match_ptr(dsu_pmu_of_match),
795 .acpi_match_table = ACPI_PTR(dsu_pmu_acpi_match),
796 .suppress_bind_attrs = true,
797 },
798 .probe = dsu_pmu_device_probe,
799 .remove = dsu_pmu_device_remove,
800};
801
802static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
803{
804 struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
805 cpuhp_node);
806
807 if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus))
808 return 0;
809
810 /* If the PMU is already managed, there is nothing to do */
811 if (!cpumask_empty(&dsu_pmu->active_cpu))
812 return 0;
813
814 dsu_pmu_init_pmu(dsu_pmu);
815 dsu_pmu_set_active_cpu(cpu, dsu_pmu);
816
817 return 0;
818}
819
820static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
821{
822 struct dsu_pmu *dsu_pmu;
823 unsigned int dst;
824
825 dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, cpuhp_node);
826
827 if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu))
828 return 0;
829
830 dst = cpumask_any_and_but(&dsu_pmu->associated_cpus,
831 cpu_online_mask, cpu);
832 /* If there are no active CPUs in the DSU, leave IRQ disabled */
833 if (dst >= nr_cpu_ids)
834 return 0;
835
836 perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst);
837 dsu_pmu_set_active_cpu(dst, dsu_pmu);
838
839 return 0;
840}
841
842static int __init dsu_pmu_init(void)
843{
844 int ret;
845
846 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
847 DRVNAME,
848 dsu_pmu_cpu_online,
849 dsu_pmu_cpu_teardown);
850 if (ret < 0)
851 return ret;
852 dsu_pmu_cpuhp_state = ret;
853 ret = platform_driver_register(&dsu_pmu_driver);
854 if (ret)
855 cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
856
857 return ret;
858}
859
860static void __exit dsu_pmu_exit(void)
861{
862 platform_driver_unregister(&dsu_pmu_driver);
863 cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
864}
865
866module_init(dsu_pmu_init);
867module_exit(dsu_pmu_exit);
868
869MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit");
870MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>");
871MODULE_LICENSE("GPL v2");