Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * ARM DynamIQ Shared Unit (DSU) PMU driver
4 *
5 * Copyright (C) ARM Limited, 2017.
6 *
7 * Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers.
8 */
9
10#define PMUNAME "arm_dsu"
11#define DRVNAME PMUNAME "_pmu"
12#define pr_fmt(fmt) DRVNAME ": " fmt
13
14#include <linux/acpi.h>
15#include <linux/bitmap.h>
16#include <linux/bitops.h>
17#include <linux/bug.h>
18#include <linux/cpumask.h>
19#include <linux/device.h>
20#include <linux/interrupt.h>
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/of.h>
24#include <linux/perf_event.h>
25#include <linux/platform_device.h>
26#include <linux/spinlock.h>
27#include <linux/smp.h>
28#include <linux/sysfs.h>
29#include <linux/types.h>
30
31#include <asm/arm_dsu_pmu.h>
32#include <asm/local64.h>
33
34/* PMU event codes */
35#define DSU_PMU_EVT_CYCLES 0x11
36#define DSU_PMU_EVT_CHAIN 0x1e
37
38#define DSU_PMU_MAX_COMMON_EVENTS 0x40
39
40#define DSU_PMU_MAX_HW_CNTRS 32
41#define DSU_PMU_HW_COUNTER_MASK (DSU_PMU_MAX_HW_CNTRS - 1)
42
43#define CLUSTERPMCR_E BIT(0)
44#define CLUSTERPMCR_P BIT(1)
45#define CLUSTERPMCR_C BIT(2)
46#define CLUSTERPMCR_N_SHIFT 11
47#define CLUSTERPMCR_N_MASK 0x1f
48#define CLUSTERPMCR_IDCODE_SHIFT 16
49#define CLUSTERPMCR_IDCODE_MASK 0xff
50#define CLUSTERPMCR_IMP_SHIFT 24
51#define CLUSTERPMCR_IMP_MASK 0xff
52#define CLUSTERPMCR_RES_MASK 0x7e8
53#define CLUSTERPMCR_RES_VAL 0x40
54
55#define DSU_ACTIVE_CPU_MASK 0x0
56#define DSU_ASSOCIATED_CPU_MASK 0x1
57
58/*
59 * We use the index of the counters as they appear in the counter
60 * bit maps in the PMU registers (e.g CLUSTERPMSELR).
61 * i.e,
62 * counter 0 - Bit 0
63 * counter 1 - Bit 1
64 * ...
65 * Cycle counter - Bit 31
66 */
67#define DSU_PMU_IDX_CYCLE_COUNTER 31
68
69/* All event counters are 32bit, with a 64bit Cycle counter */
70#define DSU_PMU_COUNTER_WIDTH(idx) \
71 (((idx) == DSU_PMU_IDX_CYCLE_COUNTER) ? 64 : 32)
72
73#define DSU_PMU_COUNTER_MASK(idx) \
74 GENMASK_ULL((DSU_PMU_COUNTER_WIDTH((idx)) - 1), 0)
75
76#define DSU_EXT_ATTR(_name, _func, _config) \
77 (&((struct dev_ext_attribute[]) { \
78 { \
79 .attr = __ATTR(_name, 0444, _func, NULL), \
80 .var = (void *)_config \
81 } \
82 })[0].attr.attr)
83
84#define DSU_EVENT_ATTR(_name, _config) \
85 DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config)
86
87#define DSU_FORMAT_ATTR(_name, _config) \
88 DSU_EXT_ATTR(_name, device_show_string, _config)
89
90#define DSU_CPUMASK_ATTR(_name, _config) \
91 DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config)
92
93struct dsu_hw_events {
94 DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS);
95 struct perf_event *events[DSU_PMU_MAX_HW_CNTRS];
96};
97
98/*
99 * struct dsu_pmu - DSU PMU descriptor
100 *
101 * @pmu_lock : Protects accesses to DSU PMU register from normal vs
102 * interrupt handler contexts.
103 * @hw_events : Holds the event counter state.
104 * @associated_cpus : CPUs attached to the DSU.
105 * @active_cpu : CPU to which the PMU is bound for accesses.
106 * @cpuhp_node : Node for CPU hotplug notifier link.
107 * @num_counters : Number of event counters implemented by the PMU,
108 * excluding the cycle counter.
109 * @irq : Interrupt line for counter overflow.
110 * @cpmceid_bitmap : Bitmap for the availability of architected common
111 * events (event_code < 0x40).
112 */
113struct dsu_pmu {
114 struct pmu pmu;
115 struct device *dev;
116 raw_spinlock_t pmu_lock;
117 struct dsu_hw_events hw_events;
118 cpumask_t associated_cpus;
119 cpumask_t active_cpu;
120 struct hlist_node cpuhp_node;
121 s8 num_counters;
122 int irq;
123 DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS);
124};
125
126static unsigned long dsu_pmu_cpuhp_state;
127
128static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu)
129{
130 return container_of(pmu, struct dsu_pmu, pmu);
131}
132
133static ssize_t dsu_pmu_sysfs_event_show(struct device *dev,
134 struct device_attribute *attr,
135 char *buf)
136{
137 struct dev_ext_attribute *eattr = container_of(attr,
138 struct dev_ext_attribute, attr);
139 return sysfs_emit(buf, "event=0x%lx\n", (unsigned long)eattr->var);
140}
141
142static ssize_t dsu_pmu_cpumask_show(struct device *dev,
143 struct device_attribute *attr,
144 char *buf)
145{
146 struct pmu *pmu = dev_get_drvdata(dev);
147 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
148 struct dev_ext_attribute *eattr = container_of(attr,
149 struct dev_ext_attribute, attr);
150 unsigned long mask_id = (unsigned long)eattr->var;
151 const cpumask_t *cpumask;
152
153 switch (mask_id) {
154 case DSU_ACTIVE_CPU_MASK:
155 cpumask = &dsu_pmu->active_cpu;
156 break;
157 case DSU_ASSOCIATED_CPU_MASK:
158 cpumask = &dsu_pmu->associated_cpus;
159 break;
160 default:
161 return 0;
162 }
163 return cpumap_print_to_pagebuf(true, buf, cpumask);
164}
165
166static struct attribute *dsu_pmu_format_attrs[] = {
167 DSU_FORMAT_ATTR(event, "config:0-31"),
168 NULL,
169};
170
171static const struct attribute_group dsu_pmu_format_attr_group = {
172 .name = "format",
173 .attrs = dsu_pmu_format_attrs,
174};
175
176static struct attribute *dsu_pmu_event_attrs[] = {
177 DSU_EVENT_ATTR(cycles, 0x11),
178 DSU_EVENT_ATTR(bus_access, 0x19),
179 DSU_EVENT_ATTR(memory_error, 0x1a),
180 DSU_EVENT_ATTR(bus_cycles, 0x1d),
181 DSU_EVENT_ATTR(l3d_cache_allocate, 0x29),
182 DSU_EVENT_ATTR(l3d_cache_refill, 0x2a),
183 DSU_EVENT_ATTR(l3d_cache, 0x2b),
184 DSU_EVENT_ATTR(l3d_cache_wb, 0x2c),
185 NULL,
186};
187
188static umode_t
189dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr,
190 int unused)
191{
192 struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
193 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
194 struct dev_ext_attribute *eattr = container_of(attr,
195 struct dev_ext_attribute, attr.attr);
196 unsigned long evt = (unsigned long)eattr->var;
197
198 return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0;
199}
200
201static const struct attribute_group dsu_pmu_events_attr_group = {
202 .name = "events",
203 .attrs = dsu_pmu_event_attrs,
204 .is_visible = dsu_pmu_event_attr_is_visible,
205};
206
207static struct attribute *dsu_pmu_cpumask_attrs[] = {
208 DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK),
209 DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK),
210 NULL,
211};
212
213static const struct attribute_group dsu_pmu_cpumask_attr_group = {
214 .attrs = dsu_pmu_cpumask_attrs,
215};
216
217static const struct attribute_group *dsu_pmu_attr_groups[] = {
218 &dsu_pmu_cpumask_attr_group,
219 &dsu_pmu_events_attr_group,
220 &dsu_pmu_format_attr_group,
221 NULL,
222};
223
224static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx)
225{
226 return (idx < dsu_pmu->num_counters) ||
227 (idx == DSU_PMU_IDX_CYCLE_COUNTER);
228}
229
230static inline u64 dsu_pmu_read_counter(struct perf_event *event)
231{
232 u64 val;
233 unsigned long flags;
234 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
235 int idx = event->hw.idx;
236
237 if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
238 &dsu_pmu->associated_cpus)))
239 return 0;
240
241 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
242 dev_err(event->pmu->dev,
243 "Trying reading invalid counter %d\n", idx);
244 return 0;
245 }
246
247 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
248 if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
249 val = __dsu_pmu_read_pmccntr();
250 else
251 val = __dsu_pmu_read_counter(idx);
252 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
253
254 return val;
255}
256
257static void dsu_pmu_write_counter(struct perf_event *event, u64 val)
258{
259 unsigned long flags;
260 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
261 int idx = event->hw.idx;
262
263 if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
264 &dsu_pmu->associated_cpus)))
265 return;
266
267 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
268 dev_err(event->pmu->dev,
269 "writing to invalid counter %d\n", idx);
270 return;
271 }
272
273 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
274 if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
275 __dsu_pmu_write_pmccntr(val);
276 else
277 __dsu_pmu_write_counter(idx, val);
278 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
279}
280
281static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events,
282 struct perf_event *event)
283{
284 int idx;
285 unsigned long evtype = event->attr.config;
286 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
287 unsigned long *used_mask = hw_events->used_mask;
288
289 if (evtype == DSU_PMU_EVT_CYCLES) {
290 if (test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask))
291 return -EAGAIN;
292 return DSU_PMU_IDX_CYCLE_COUNTER;
293 }
294
295 idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters);
296 if (idx >= dsu_pmu->num_counters)
297 return -EAGAIN;
298 set_bit(idx, hw_events->used_mask);
299 return idx;
300}
301
302static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx)
303{
304 __dsu_pmu_counter_interrupt_enable(idx);
305 __dsu_pmu_enable_counter(idx);
306}
307
308static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx)
309{
310 __dsu_pmu_disable_counter(idx);
311 __dsu_pmu_counter_interrupt_disable(idx);
312}
313
314static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu,
315 struct perf_event *event)
316{
317 int idx = event->hw.idx;
318 unsigned long flags;
319
320 if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
321 dev_err(event->pmu->dev,
322 "Trying to set invalid counter %d\n", idx);
323 return;
324 }
325
326 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
327 __dsu_pmu_set_event(idx, event->hw.config_base);
328 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
329}
330
331static void dsu_pmu_event_update(struct perf_event *event)
332{
333 struct hw_perf_event *hwc = &event->hw;
334 u64 delta, prev_count, new_count;
335
336 do {
337 /* We may also be called from the irq handler */
338 prev_count = local64_read(&hwc->prev_count);
339 new_count = dsu_pmu_read_counter(event);
340 } while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) !=
341 prev_count);
342 delta = (new_count - prev_count) & DSU_PMU_COUNTER_MASK(hwc->idx);
343 local64_add(delta, &event->count);
344}
345
346static void dsu_pmu_read(struct perf_event *event)
347{
348 dsu_pmu_event_update(event);
349}
350
351static inline u32 dsu_pmu_get_reset_overflow(void)
352{
353 return __dsu_pmu_get_reset_overflow();
354}
355
356/*
357 * dsu_pmu_set_event_period: Set the period for the counter.
358 *
359 * All DSU PMU event counters, except the cycle counter are 32bit
360 * counters. To handle cases of extreme interrupt latency, we program
361 * the counter with half of the max count for the counters.
362 */
363static void dsu_pmu_set_event_period(struct perf_event *event)
364{
365 int idx = event->hw.idx;
366 u64 val = DSU_PMU_COUNTER_MASK(idx) >> 1;
367
368 local64_set(&event->hw.prev_count, val);
369 dsu_pmu_write_counter(event, val);
370}
371
372static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev)
373{
374 int i;
375 bool handled = false;
376 struct dsu_pmu *dsu_pmu = dev;
377 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
378 unsigned long overflow;
379
380 overflow = dsu_pmu_get_reset_overflow();
381 if (!overflow)
382 return IRQ_NONE;
383
384 for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) {
385 struct perf_event *event = hw_events->events[i];
386
387 if (!event)
388 continue;
389 dsu_pmu_event_update(event);
390 dsu_pmu_set_event_period(event);
391 handled = true;
392 }
393
394 return IRQ_RETVAL(handled);
395}
396
397static void dsu_pmu_start(struct perf_event *event, int pmu_flags)
398{
399 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
400
401 /* We always reprogram the counter */
402 if (pmu_flags & PERF_EF_RELOAD)
403 WARN_ON(!(event->hw.state & PERF_HES_UPTODATE));
404 dsu_pmu_set_event_period(event);
405 if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER)
406 dsu_pmu_set_event(dsu_pmu, event);
407 event->hw.state = 0;
408 dsu_pmu_enable_counter(dsu_pmu, event->hw.idx);
409}
410
411static void dsu_pmu_stop(struct perf_event *event, int pmu_flags)
412{
413 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
414
415 if (event->hw.state & PERF_HES_STOPPED)
416 return;
417 dsu_pmu_disable_counter(dsu_pmu, event->hw.idx);
418 dsu_pmu_event_update(event);
419 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
420}
421
422static int dsu_pmu_add(struct perf_event *event, int flags)
423{
424 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
425 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
426 struct hw_perf_event *hwc = &event->hw;
427 int idx;
428
429 if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
430 &dsu_pmu->associated_cpus)))
431 return -ENOENT;
432
433 idx = dsu_pmu_get_event_idx(hw_events, event);
434 if (idx < 0)
435 return idx;
436
437 hwc->idx = idx;
438 hw_events->events[idx] = event;
439 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
440
441 if (flags & PERF_EF_START)
442 dsu_pmu_start(event, PERF_EF_RELOAD);
443
444 perf_event_update_userpage(event);
445 return 0;
446}
447
448static void dsu_pmu_del(struct perf_event *event, int flags)
449{
450 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
451 struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
452 struct hw_perf_event *hwc = &event->hw;
453 int idx = hwc->idx;
454
455 dsu_pmu_stop(event, PERF_EF_UPDATE);
456 hw_events->events[idx] = NULL;
457 clear_bit(idx, hw_events->used_mask);
458 perf_event_update_userpage(event);
459}
460
461static void dsu_pmu_enable(struct pmu *pmu)
462{
463 u32 pmcr;
464 unsigned long flags;
465 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
466
467 /* If no counters are added, skip enabling the PMU */
468 if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS))
469 return;
470
471 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
472 pmcr = __dsu_pmu_read_pmcr();
473 pmcr |= CLUSTERPMCR_E;
474 __dsu_pmu_write_pmcr(pmcr);
475 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
476}
477
478static void dsu_pmu_disable(struct pmu *pmu)
479{
480 u32 pmcr;
481 unsigned long flags;
482 struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
483
484 raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
485 pmcr = __dsu_pmu_read_pmcr();
486 pmcr &= ~CLUSTERPMCR_E;
487 __dsu_pmu_write_pmcr(pmcr);
488 raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
489}
490
491static bool dsu_pmu_validate_event(struct pmu *pmu,
492 struct dsu_hw_events *hw_events,
493 struct perf_event *event)
494{
495 if (is_software_event(event))
496 return true;
497 /* Reject groups spanning multiple HW PMUs. */
498 if (event->pmu != pmu)
499 return false;
500 return dsu_pmu_get_event_idx(hw_events, event) >= 0;
501}
502
503/*
504 * Make sure the group of events can be scheduled at once
505 * on the PMU.
506 */
507static bool dsu_pmu_validate_group(struct perf_event *event)
508{
509 struct perf_event *sibling, *leader = event->group_leader;
510 struct dsu_hw_events fake_hw;
511
512 if (event->group_leader == event)
513 return true;
514
515 memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask));
516 if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader))
517 return false;
518 for_each_sibling_event(sibling, leader) {
519 if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling))
520 return false;
521 }
522 return dsu_pmu_validate_event(event->pmu, &fake_hw, event);
523}
524
525static int dsu_pmu_event_init(struct perf_event *event)
526{
527 struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
528
529 if (event->attr.type != event->pmu->type)
530 return -ENOENT;
531
532 /* We don't support sampling */
533 if (is_sampling_event(event)) {
534 dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n");
535 return -EOPNOTSUPP;
536 }
537
538 /* We cannot support task bound events */
539 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
540 dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n");
541 return -EINVAL;
542 }
543
544 if (has_branch_stack(event)) {
545 dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n");
546 return -EINVAL;
547 }
548
549 if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) {
550 dev_dbg(dsu_pmu->pmu.dev,
551 "Requested cpu is not associated with the DSU\n");
552 return -EINVAL;
553 }
554 /*
555 * Choose the current active CPU to read the events. We don't want
556 * to migrate the event contexts, irq handling etc to the requested
557 * CPU. As long as the requested CPU is within the same DSU, we
558 * are fine.
559 */
560 event->cpu = cpumask_first(&dsu_pmu->active_cpu);
561 if (event->cpu >= nr_cpu_ids)
562 return -EINVAL;
563 if (!dsu_pmu_validate_group(event))
564 return -EINVAL;
565
566 event->hw.config_base = event->attr.config;
567 return 0;
568}
569
570static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev)
571{
572 struct dsu_pmu *dsu_pmu;
573
574 dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL);
575 if (!dsu_pmu)
576 return ERR_PTR(-ENOMEM);
577
578 raw_spin_lock_init(&dsu_pmu->pmu_lock);
579 /*
580 * Initialise the number of counters to -1, until we probe
581 * the real number on a connected CPU.
582 */
583 dsu_pmu->num_counters = -1;
584 return dsu_pmu;
585}
586
587/*
588 * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster
589 * from device tree.
590 */
591static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask)
592{
593 int i = 0, n, cpu;
594 struct device_node *cpu_node;
595
596 n = of_count_phandle_with_args(dev->of_node, "cpus", NULL);
597 if (n <= 0)
598 return -ENODEV;
599 for (; i < n; i++) {
600 cpu_node = of_parse_phandle(dev->of_node, "cpus", i);
601 if (!cpu_node)
602 break;
603 cpu = of_cpu_node_to_id(cpu_node);
604 of_node_put(cpu_node);
605 /*
606 * We have to ignore the failures here and continue scanning
607 * the list to handle cases where the nr_cpus could be capped
608 * in the running kernel.
609 */
610 if (cpu < 0)
611 continue;
612 cpumask_set_cpu(cpu, mask);
613 }
614 return 0;
615}
616
617/*
618 * dsu_pmu_acpi_get_cpus: Get the list of CPUs in the cluster
619 * from ACPI.
620 */
621static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask)
622{
623#ifdef CONFIG_ACPI
624 struct acpi_device *parent_adev = acpi_dev_parent(ACPI_COMPANION(dev));
625 int cpu;
626
627 /*
628 * A dsu pmu node is inside a cluster parent node along with cpu nodes.
629 * We need to find out all cpus that have the same parent with this pmu.
630 */
631 for_each_possible_cpu(cpu) {
632 struct acpi_device *acpi_dev;
633 struct device *cpu_dev = get_cpu_device(cpu);
634
635 if (!cpu_dev)
636 continue;
637
638 acpi_dev = ACPI_COMPANION(cpu_dev);
639 if (acpi_dev && acpi_dev_parent(acpi_dev) == parent_adev)
640 cpumask_set_cpu(cpu, mask);
641 }
642#endif
643
644 return 0;
645}
646
647/*
648 * dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster.
649 */
650static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu)
651{
652 u64 num_counters;
653 u32 cpmceid[2];
654
655 num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) &
656 CLUSTERPMCR_N_MASK;
657 /* We can only support up to 31 independent counters */
658 if (WARN_ON(num_counters > 31))
659 num_counters = 31;
660 dsu_pmu->num_counters = num_counters;
661 if (!dsu_pmu->num_counters)
662 return;
663 cpmceid[0] = __dsu_pmu_read_pmceid(0);
664 cpmceid[1] = __dsu_pmu_read_pmceid(1);
665 bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid,
666 DSU_PMU_MAX_COMMON_EVENTS);
667}
668
669static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu)
670{
671 cpumask_set_cpu(cpu, &dsu_pmu->active_cpu);
672 if (irq_set_affinity(dsu_pmu->irq, &dsu_pmu->active_cpu))
673 pr_warn("Failed to set irq affinity to %d\n", cpu);
674}
675
676/*
677 * dsu_pmu_init_pmu: Initialise the DSU PMU configurations if
678 * we haven't done it already.
679 */
680static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu)
681{
682 if (dsu_pmu->num_counters == -1)
683 dsu_pmu_probe_pmu(dsu_pmu);
684 /* Reset the interrupt overflow mask */
685 dsu_pmu_get_reset_overflow();
686}
687
688static int dsu_pmu_device_probe(struct platform_device *pdev)
689{
690 int irq, rc;
691 struct dsu_pmu *dsu_pmu;
692 struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev);
693 char *name;
694 static atomic_t pmu_idx = ATOMIC_INIT(-1);
695
696 dsu_pmu = dsu_pmu_alloc(pdev);
697 if (IS_ERR(dsu_pmu))
698 return PTR_ERR(dsu_pmu);
699
700 if (is_of_node(fwnode))
701 rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
702 else if (is_acpi_device_node(fwnode))
703 rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
704 else
705 return -ENOENT;
706
707 if (rc) {
708 dev_warn(&pdev->dev, "Failed to parse the CPUs\n");
709 return rc;
710 }
711
712 irq = platform_get_irq(pdev, 0);
713 if (irq < 0)
714 return -EINVAL;
715
716 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
717 PMUNAME, atomic_inc_return(&pmu_idx));
718 if (!name)
719 return -ENOMEM;
720 rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq,
721 IRQF_NOBALANCING, name, dsu_pmu);
722 if (rc) {
723 dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq);
724 return rc;
725 }
726
727 dsu_pmu->irq = irq;
728 platform_set_drvdata(pdev, dsu_pmu);
729 rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state,
730 &dsu_pmu->cpuhp_node);
731 if (rc)
732 return rc;
733
734 dsu_pmu->pmu = (struct pmu) {
735 .task_ctx_nr = perf_invalid_context,
736 .parent = &pdev->dev,
737 .module = THIS_MODULE,
738 .pmu_enable = dsu_pmu_enable,
739 .pmu_disable = dsu_pmu_disable,
740 .event_init = dsu_pmu_event_init,
741 .add = dsu_pmu_add,
742 .del = dsu_pmu_del,
743 .start = dsu_pmu_start,
744 .stop = dsu_pmu_stop,
745 .read = dsu_pmu_read,
746
747 .attr_groups = dsu_pmu_attr_groups,
748 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
749 };
750
751 rc = perf_pmu_register(&dsu_pmu->pmu, name, -1);
752 if (rc) {
753 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state,
754 &dsu_pmu->cpuhp_node);
755 }
756
757 return rc;
758}
759
760static void dsu_pmu_device_remove(struct platform_device *pdev)
761{
762 struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev);
763
764 perf_pmu_unregister(&dsu_pmu->pmu);
765 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node);
766}
767
768static const struct of_device_id dsu_pmu_of_match[] = {
769 { .compatible = "arm,dsu-pmu", },
770 {},
771};
772MODULE_DEVICE_TABLE(of, dsu_pmu_of_match);
773
774#ifdef CONFIG_ACPI
775static const struct acpi_device_id dsu_pmu_acpi_match[] = {
776 { "ARMHD500", 0},
777 {},
778};
779MODULE_DEVICE_TABLE(acpi, dsu_pmu_acpi_match);
780#endif
781
782static struct platform_driver dsu_pmu_driver = {
783 .driver = {
784 .name = DRVNAME,
785 .of_match_table = of_match_ptr(dsu_pmu_of_match),
786 .acpi_match_table = ACPI_PTR(dsu_pmu_acpi_match),
787 .suppress_bind_attrs = true,
788 },
789 .probe = dsu_pmu_device_probe,
790 .remove = dsu_pmu_device_remove,
791};
792
793static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
794{
795 struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
796 cpuhp_node);
797
798 if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus))
799 return 0;
800
801 /* If the PMU is already managed, there is nothing to do */
802 if (!cpumask_empty(&dsu_pmu->active_cpu))
803 return 0;
804
805 dsu_pmu_init_pmu(dsu_pmu);
806 dsu_pmu_set_active_cpu(cpu, dsu_pmu);
807
808 return 0;
809}
810
811static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
812{
813 struct dsu_pmu *dsu_pmu;
814 unsigned int dst;
815
816 dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, cpuhp_node);
817
818 if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu))
819 return 0;
820
821 dst = cpumask_any_and_but(&dsu_pmu->associated_cpus,
822 cpu_online_mask, cpu);
823 /* If there are no active CPUs in the DSU, leave IRQ disabled */
824 if (dst >= nr_cpu_ids)
825 return 0;
826
827 perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst);
828 dsu_pmu_set_active_cpu(dst, dsu_pmu);
829
830 return 0;
831}
832
833static int __init dsu_pmu_init(void)
834{
835 int ret;
836
837 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
838 DRVNAME,
839 dsu_pmu_cpu_online,
840 dsu_pmu_cpu_teardown);
841 if (ret < 0)
842 return ret;
843 dsu_pmu_cpuhp_state = ret;
844 ret = platform_driver_register(&dsu_pmu_driver);
845 if (ret)
846 cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
847
848 return ret;
849}
850
851static void __exit dsu_pmu_exit(void)
852{
853 platform_driver_unregister(&dsu_pmu_driver);
854 cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
855}
856
857module_init(dsu_pmu_init);
858module_exit(dsu_pmu_exit);
859
860MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit");
861MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>");
862MODULE_LICENSE("GPL v2");