1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * ARM DynamIQ Shared Unit (DSU) PMU driver
  4 *
  5 * Copyright (C) ARM Limited, 2017.
  6 *
  7 * Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers.
  8 */
  9
 10#define PMUNAME		"arm_dsu"
 11#define DRVNAME		PMUNAME "_pmu"
 12#define pr_fmt(fmt)	DRVNAME ": " fmt
 13
 14#include <linux/bitmap.h>
 15#include <linux/bitops.h>
 16#include <linux/bug.h>
 17#include <linux/cpumask.h>
 18#include <linux/device.h>
 19#include <linux/interrupt.h>
 20#include <linux/kernel.h>
 21#include <linux/module.h>
 22#include <linux/of_device.h>
 23#include <linux/perf_event.h>
 24#include <linux/platform_device.h>
 25#include <linux/spinlock.h>
 26#include <linux/smp.h>
 27#include <linux/sysfs.h>
 28#include <linux/types.h>
 29
 30#include <asm/arm_dsu_pmu.h>
 31#include <asm/local64.h>
 32
 33/* PMU event codes */
 34#define DSU_PMU_EVT_CYCLES		0x11
 35#define DSU_PMU_EVT_CHAIN		0x1e
 36
 37#define DSU_PMU_MAX_COMMON_EVENTS	0x40
 38
 39#define DSU_PMU_MAX_HW_CNTRS		32
 40#define DSU_PMU_HW_COUNTER_MASK		(DSU_PMU_MAX_HW_CNTRS - 1)
 41
 42#define CLUSTERPMCR_E			BIT(0)
 43#define CLUSTERPMCR_P			BIT(1)
 44#define CLUSTERPMCR_C			BIT(2)
 45#define CLUSTERPMCR_N_SHIFT		11
 46#define CLUSTERPMCR_N_MASK		0x1f
 47#define CLUSTERPMCR_IDCODE_SHIFT	16
 48#define CLUSTERPMCR_IDCODE_MASK		0xff
 49#define CLUSTERPMCR_IMP_SHIFT		24
 50#define CLUSTERPMCR_IMP_MASK		0xff
 51#define CLUSTERPMCR_RES_MASK		0x7e8
 52#define CLUSTERPMCR_RES_VAL		0x40
 53
 54#define DSU_ACTIVE_CPU_MASK		0x0
 55#define DSU_ASSOCIATED_CPU_MASK		0x1
 56
 57/*
 58 * We use the index of the counters as they appear in the counter
 59 * bit maps in the PMU registers (e.g CLUSTERPMSELR).
 60 * i.e,
 61 *	counter 0	- Bit 0
 62 *	counter 1	- Bit 1
 63 *	...
 64 *	Cycle counter	- Bit 31
 65 */
 66#define DSU_PMU_IDX_CYCLE_COUNTER	31
 67
 68/* All event counters are 32bit, with a 64bit Cycle counter */
 69#define DSU_PMU_COUNTER_WIDTH(idx)	\
 70	(((idx) == DSU_PMU_IDX_CYCLE_COUNTER) ? 64 : 32)
 71
 72#define DSU_PMU_COUNTER_MASK(idx)	\
 73	GENMASK_ULL((DSU_PMU_COUNTER_WIDTH((idx)) - 1), 0)
 74
 75#define DSU_EXT_ATTR(_name, _func, _config)		\
 76	(&((struct dev_ext_attribute[]) {				\
 77		{							\
 78			.attr = __ATTR(_name, 0444, _func, NULL),	\
 79			.var = (void *)_config				\
 80		}							\
 81	})[0].attr.attr)
 82
 83#define DSU_EVENT_ATTR(_name, _config)		\
 84	DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config)
 85
 86#define DSU_FORMAT_ATTR(_name, _config)		\
 87	DSU_EXT_ATTR(_name, dsu_pmu_sysfs_format_show, (char *)_config)
 88
 89#define DSU_CPUMASK_ATTR(_name, _config)	\
 90	DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config)
 91
 92struct dsu_hw_events {
 93	DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS);
 94	struct perf_event	*events[DSU_PMU_MAX_HW_CNTRS];
 95};
 96
 97/*
 98 * struct dsu_pmu	- DSU PMU descriptor
 99 *
100 * @pmu_lock		: Protects accesses to DSU PMU register from normal vs
101 *			  interrupt handler contexts.
102 * @hw_events		: Holds the event counter state.
103 * @associated_cpus	: CPUs attached to the DSU.
104 * @active_cpu		: CPU to which the PMU is bound for accesses.
105 * @cpuhp_node		: Node for CPU hotplug notifier link.
106 * @num_counters	: Number of event counters implemented by the PMU,
107 *			  excluding the cycle counter.
108 * @irq			: Interrupt line for counter overflow.
109 * @cpmceid_bitmap	: Bitmap for the availability of architected common
110 *			  events (event_code < 0x40).
111 */
112struct dsu_pmu {
113	struct pmu			pmu;
114	struct device			*dev;
115	raw_spinlock_t			pmu_lock;
116	struct dsu_hw_events		hw_events;
117	cpumask_t			associated_cpus;
118	cpumask_t			active_cpu;
119	struct hlist_node		cpuhp_node;
120	s8				num_counters;
121	int				irq;
122	DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS);
123};
124
125static unsigned long dsu_pmu_cpuhp_state;
126
127static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu)
128{
129	return container_of(pmu, struct dsu_pmu, pmu);
130}
131
132static ssize_t dsu_pmu_sysfs_event_show(struct device *dev,
133					struct device_attribute *attr,
134					char *buf)
135{
136	struct dev_ext_attribute *eattr = container_of(attr,
137					struct dev_ext_attribute, attr);
138	return snprintf(buf, PAGE_SIZE, "event=0x%lx\n",
139					 (unsigned long)eattr->var);
140}
141
142static ssize_t dsu_pmu_sysfs_format_show(struct device *dev,
143					 struct device_attribute *attr,
144					 char *buf)
145{
146	struct dev_ext_attribute *eattr = container_of(attr,
147					struct dev_ext_attribute, attr);
148	return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var);
149}
150
151static ssize_t dsu_pmu_cpumask_show(struct device *dev,
152				    struct device_attribute *attr,
153				    char *buf)
154{
155	struct pmu *pmu = dev_get_drvdata(dev);
156	struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
157	struct dev_ext_attribute *eattr = container_of(attr,
158					struct dev_ext_attribute, attr);
159	unsigned long mask_id = (unsigned long)eattr->var;
160	const cpumask_t *cpumask;
161
162	switch (mask_id) {
163	case DSU_ACTIVE_CPU_MASK:
164		cpumask = &dsu_pmu->active_cpu;
165		break;
166	case DSU_ASSOCIATED_CPU_MASK:
167		cpumask = &dsu_pmu->associated_cpus;
168		break;
169	default:
170		return 0;
171	}
172	return cpumap_print_to_pagebuf(true, buf, cpumask);
173}
174
175static struct attribute *dsu_pmu_format_attrs[] = {
176	DSU_FORMAT_ATTR(event, "config:0-31"),
177	NULL,
178};
179
180static const struct attribute_group dsu_pmu_format_attr_group = {
181	.name = "format",
182	.attrs = dsu_pmu_format_attrs,
183};
184
185static struct attribute *dsu_pmu_event_attrs[] = {
186	DSU_EVENT_ATTR(cycles, 0x11),
187	DSU_EVENT_ATTR(bus_access, 0x19),
188	DSU_EVENT_ATTR(memory_error, 0x1a),
189	DSU_EVENT_ATTR(bus_cycles, 0x1d),
190	DSU_EVENT_ATTR(l3d_cache_allocate, 0x29),
191	DSU_EVENT_ATTR(l3d_cache_refill, 0x2a),
192	DSU_EVENT_ATTR(l3d_cache, 0x2b),
193	DSU_EVENT_ATTR(l3d_cache_wb, 0x2c),
194	NULL,
195};
196
197static umode_t
198dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr,
199				int unused)
200{
201	struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
202	struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
203	struct dev_ext_attribute *eattr = container_of(attr,
204					struct dev_ext_attribute, attr.attr);
205	unsigned long evt = (unsigned long)eattr->var;
206
207	return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0;
208}
209
210static const struct attribute_group dsu_pmu_events_attr_group = {
211	.name = "events",
212	.attrs = dsu_pmu_event_attrs,
213	.is_visible = dsu_pmu_event_attr_is_visible,
214};
215
216static struct attribute *dsu_pmu_cpumask_attrs[] = {
217	DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK),
218	DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK),
219	NULL,
220};
221
222static const struct attribute_group dsu_pmu_cpumask_attr_group = {
223	.attrs = dsu_pmu_cpumask_attrs,
224};
225
226static const struct attribute_group *dsu_pmu_attr_groups[] = {
227	&dsu_pmu_cpumask_attr_group,
228	&dsu_pmu_events_attr_group,
229	&dsu_pmu_format_attr_group,
230	NULL,
231};
232
233static int dsu_pmu_get_online_cpu_any_but(struct dsu_pmu *dsu_pmu, int cpu)
234{
235	struct cpumask online_supported;
236
237	cpumask_and(&online_supported,
238			 &dsu_pmu->associated_cpus, cpu_online_mask);
239	return cpumask_any_but(&online_supported, cpu);
240}
241
242static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx)
243{
244	return (idx < dsu_pmu->num_counters) ||
245	       (idx == DSU_PMU_IDX_CYCLE_COUNTER);
246}
247
248static inline u64 dsu_pmu_read_counter(struct perf_event *event)
249{
250	u64 val;
251	unsigned long flags;
252	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
253	int idx = event->hw.idx;
254
255	if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
256				 &dsu_pmu->associated_cpus)))
257		return 0;
258
259	if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
260		dev_err(event->pmu->dev,
261			"Trying reading invalid counter %d\n", idx);
262		return 0;
263	}
264
265	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
266	if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
267		val = __dsu_pmu_read_pmccntr();
268	else
269		val = __dsu_pmu_read_counter(idx);
270	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
271
272	return val;
273}
274
275static void dsu_pmu_write_counter(struct perf_event *event, u64 val)
276{
277	unsigned long flags;
278	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
279	int idx = event->hw.idx;
280
281	if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
282			 &dsu_pmu->associated_cpus)))
283		return;
284
285	if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
286		dev_err(event->pmu->dev,
287			"writing to invalid counter %d\n", idx);
288		return;
289	}
290
291	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
292	if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
293		__dsu_pmu_write_pmccntr(val);
294	else
295		__dsu_pmu_write_counter(idx, val);
296	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
297}
298
299static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events,
300				 struct perf_event *event)
301{
302	int idx;
303	unsigned long evtype = event->attr.config;
304	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
305	unsigned long *used_mask = hw_events->used_mask;
306
307	if (evtype == DSU_PMU_EVT_CYCLES) {
308		if (test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask))
309			return -EAGAIN;
310		return DSU_PMU_IDX_CYCLE_COUNTER;
311	}
312
313	idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters);
314	if (idx >= dsu_pmu->num_counters)
315		return -EAGAIN;
316	set_bit(idx, hw_events->used_mask);
317	return idx;
318}
319
320static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx)
321{
322	__dsu_pmu_counter_interrupt_enable(idx);
323	__dsu_pmu_enable_counter(idx);
324}
325
326static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx)
327{
328	__dsu_pmu_disable_counter(idx);
329	__dsu_pmu_counter_interrupt_disable(idx);
330}
331
332static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu,
333					struct perf_event *event)
334{
335	int idx = event->hw.idx;
336	unsigned long flags;
337
338	if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
339		dev_err(event->pmu->dev,
340			"Trying to set invalid counter %d\n", idx);
341		return;
342	}
343
344	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
345	__dsu_pmu_set_event(idx, event->hw.config_base);
346	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
347}
348
349static void dsu_pmu_event_update(struct perf_event *event)
350{
351	struct hw_perf_event *hwc = &event->hw;
352	u64 delta, prev_count, new_count;
353
354	do {
355		/* We may also be called from the irq handler */
356		prev_count = local64_read(&hwc->prev_count);
357		new_count = dsu_pmu_read_counter(event);
358	} while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) !=
359			prev_count);
360	delta = (new_count - prev_count) & DSU_PMU_COUNTER_MASK(hwc->idx);
361	local64_add(delta, &event->count);
362}
363
364static void dsu_pmu_read(struct perf_event *event)
365{
366	dsu_pmu_event_update(event);
367}
368
369static inline u32 dsu_pmu_get_reset_overflow(void)
370{
371	return __dsu_pmu_get_reset_overflow();
372}
373
374/**
375 * dsu_pmu_set_event_period: Set the period for the counter.
376 *
377 * All DSU PMU event counters, except the cycle counter are 32bit
378 * counters. To handle cases of extreme interrupt latency, we program
379 * the counter with half of the max count for the counters.
380 */
381static void dsu_pmu_set_event_period(struct perf_event *event)
382{
383	int idx = event->hw.idx;
384	u64 val = DSU_PMU_COUNTER_MASK(idx) >> 1;
385
386	local64_set(&event->hw.prev_count, val);
387	dsu_pmu_write_counter(event, val);
388}
389
390static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev)
391{
392	int i;
393	bool handled = false;
394	struct dsu_pmu *dsu_pmu = dev;
395	struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
396	unsigned long overflow;
397
398	overflow = dsu_pmu_get_reset_overflow();
399	if (!overflow)
400		return IRQ_NONE;
401
402	for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) {
403		struct perf_event *event = hw_events->events[i];
404
405		if (!event)
406			continue;
407		dsu_pmu_event_update(event);
408		dsu_pmu_set_event_period(event);
409		handled = true;
410	}
411
412	return IRQ_RETVAL(handled);
413}
414
415static void dsu_pmu_start(struct perf_event *event, int pmu_flags)
416{
417	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
418
419	/* We always reprogram the counter */
420	if (pmu_flags & PERF_EF_RELOAD)
421		WARN_ON(!(event->hw.state & PERF_HES_UPTODATE));
422	dsu_pmu_set_event_period(event);
423	if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER)
424		dsu_pmu_set_event(dsu_pmu, event);
425	event->hw.state = 0;
426	dsu_pmu_enable_counter(dsu_pmu, event->hw.idx);
427}
428
429static void dsu_pmu_stop(struct perf_event *event, int pmu_flags)
430{
431	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
432
433	if (event->hw.state & PERF_HES_STOPPED)
434		return;
435	dsu_pmu_disable_counter(dsu_pmu, event->hw.idx);
436	dsu_pmu_event_update(event);
437	event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
438}
439
440static int dsu_pmu_add(struct perf_event *event, int flags)
441{
442	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
443	struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
444	struct hw_perf_event *hwc = &event->hw;
445	int idx;
446
447	if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
448					   &dsu_pmu->associated_cpus)))
449		return -ENOENT;
450
451	idx = dsu_pmu_get_event_idx(hw_events, event);
452	if (idx < 0)
453		return idx;
454
455	hwc->idx = idx;
456	hw_events->events[idx] = event;
457	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
458
459	if (flags & PERF_EF_START)
460		dsu_pmu_start(event, PERF_EF_RELOAD);
461
462	perf_event_update_userpage(event);
463	return 0;
464}
465
466static void dsu_pmu_del(struct perf_event *event, int flags)
467{
468	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
469	struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
470	struct hw_perf_event *hwc = &event->hw;
471	int idx = hwc->idx;
472
473	dsu_pmu_stop(event, PERF_EF_UPDATE);
474	hw_events->events[idx] = NULL;
475	clear_bit(idx, hw_events->used_mask);
476	perf_event_update_userpage(event);
477}
478
479static void dsu_pmu_enable(struct pmu *pmu)
480{
481	u32 pmcr;
482	unsigned long flags;
483	struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
484
485	/* If no counters are added, skip enabling the PMU */
486	if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS))
487		return;
488
489	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
490	pmcr = __dsu_pmu_read_pmcr();
491	pmcr |= CLUSTERPMCR_E;
492	__dsu_pmu_write_pmcr(pmcr);
493	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
494}
495
496static void dsu_pmu_disable(struct pmu *pmu)
497{
498	u32 pmcr;
499	unsigned long flags;
500	struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
501
502	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
503	pmcr = __dsu_pmu_read_pmcr();
504	pmcr &= ~CLUSTERPMCR_E;
505	__dsu_pmu_write_pmcr(pmcr);
506	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
507}
508
509static bool dsu_pmu_validate_event(struct pmu *pmu,
510				  struct dsu_hw_events *hw_events,
511				  struct perf_event *event)
512{
513	if (is_software_event(event))
514		return true;
515	/* Reject groups spanning multiple HW PMUs. */
516	if (event->pmu != pmu)
517		return false;
518	return dsu_pmu_get_event_idx(hw_events, event) >= 0;
519}
520
521/*
522 * Make sure the group of events can be scheduled at once
523 * on the PMU.
524 */
525static bool dsu_pmu_validate_group(struct perf_event *event)
526{
527	struct perf_event *sibling, *leader = event->group_leader;
528	struct dsu_hw_events fake_hw;
529
530	if (event->group_leader == event)
531		return true;
532
533	memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask));
534	if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader))
535		return false;
536	for_each_sibling_event(sibling, leader) {
537		if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling))
538			return false;
539	}
540	return dsu_pmu_validate_event(event->pmu, &fake_hw, event);
541}
542
543static int dsu_pmu_event_init(struct perf_event *event)
544{
545	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
546
547	if (event->attr.type != event->pmu->type)
548		return -ENOENT;
549
550	/* We don't support sampling */
551	if (is_sampling_event(event)) {
552		dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n");
553		return -EOPNOTSUPP;
554	}
555
556	/* We cannot support task bound events */
557	if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
558		dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n");
559		return -EINVAL;
560	}
561
562	if (has_branch_stack(event)) {
563		dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n");
564		return -EINVAL;
565	}
566
567	if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) {
568		dev_dbg(dsu_pmu->pmu.dev,
569			 "Requested cpu is not associated with the DSU\n");
570		return -EINVAL;
571	}
572	/*
573	 * Choose the current active CPU to read the events. We don't want
574	 * to migrate the event contexts, irq handling etc to the requested
575	 * CPU. As long as the requested CPU is within the same DSU, we
576	 * are fine.
577	 */
578	event->cpu = cpumask_first(&dsu_pmu->active_cpu);
579	if (event->cpu >= nr_cpu_ids)
580		return -EINVAL;
581	if (!dsu_pmu_validate_group(event))
582		return -EINVAL;
583
584	event->hw.config_base = event->attr.config;
585	return 0;
586}
587
588static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev)
589{
590	struct dsu_pmu *dsu_pmu;
591
592	dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL);
593	if (!dsu_pmu)
594		return ERR_PTR(-ENOMEM);
595
596	raw_spin_lock_init(&dsu_pmu->pmu_lock);
597	/*
598	 * Initialise the number of counters to -1, until we probe
599	 * the real number on a connected CPU.
600	 */
601	dsu_pmu->num_counters = -1;
602	return dsu_pmu;
603}
604
605/**
606 * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster.
607 */
608static int dsu_pmu_dt_get_cpus(struct device_node *dev, cpumask_t *mask)
609{
610	int i = 0, n, cpu;
611	struct device_node *cpu_node;
612
613	n = of_count_phandle_with_args(dev, "cpus", NULL);
614	if (n <= 0)
615		return -ENODEV;
616	for (; i < n; i++) {
617		cpu_node = of_parse_phandle(dev, "cpus", i);
618		if (!cpu_node)
619			break;
620		cpu = of_cpu_node_to_id(cpu_node);
621		of_node_put(cpu_node);
622		/*
623		 * We have to ignore the failures here and continue scanning
624		 * the list to handle cases where the nr_cpus could be capped
625		 * in the running kernel.
626		 */
627		if (cpu < 0)
628			continue;
629		cpumask_set_cpu(cpu, mask);
630	}
631	return 0;
632}
633
634/*
635 * dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster.
636 */
637static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu)
638{
639	u64 num_counters;
640	u32 cpmceid[2];
641
642	num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) &
643						CLUSTERPMCR_N_MASK;
644	/* We can only support up to 31 independent counters */
645	if (WARN_ON(num_counters > 31))
646		num_counters = 31;
647	dsu_pmu->num_counters = num_counters;
648	if (!dsu_pmu->num_counters)
649		return;
650	cpmceid[0] = __dsu_pmu_read_pmceid(0);
651	cpmceid[1] = __dsu_pmu_read_pmceid(1);
652	bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid,
653			  DSU_PMU_MAX_COMMON_EVENTS);
654}
655
656static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu)
657{
658	cpumask_set_cpu(cpu, &dsu_pmu->active_cpu);
659	if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu))
660		pr_warn("Failed to set irq affinity to %d\n", cpu);
661}
662
663/*
664 * dsu_pmu_init_pmu: Initialise the DSU PMU configurations if
665 * we haven't done it already.
666 */
667static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu)
668{
669	if (dsu_pmu->num_counters == -1)
670		dsu_pmu_probe_pmu(dsu_pmu);
671	/* Reset the interrupt overflow mask */
672	dsu_pmu_get_reset_overflow();
673}
674
675static int dsu_pmu_device_probe(struct platform_device *pdev)
676{
677	int irq, rc;
678	struct dsu_pmu *dsu_pmu;
679	char *name;
680	static atomic_t pmu_idx = ATOMIC_INIT(-1);
681
682	dsu_pmu = dsu_pmu_alloc(pdev);
683	if (IS_ERR(dsu_pmu))
684		return PTR_ERR(dsu_pmu);
685
686	rc = dsu_pmu_dt_get_cpus(pdev->dev.of_node, &dsu_pmu->associated_cpus);
687	if (rc) {
688		dev_warn(&pdev->dev, "Failed to parse the CPUs\n");
689		return rc;
690	}
691
692	irq = platform_get_irq(pdev, 0);
693	if (irq < 0)
 
694		return -EINVAL;
 
695
696	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
697				PMUNAME, atomic_inc_return(&pmu_idx));
698	if (!name)
699		return -ENOMEM;
700	rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq,
701			      IRQF_NOBALANCING, name, dsu_pmu);
702	if (rc) {
703		dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq);
704		return rc;
705	}
706
707	dsu_pmu->irq = irq;
708	platform_set_drvdata(pdev, dsu_pmu);
709	rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state,
710						&dsu_pmu->cpuhp_node);
711	if (rc)
712		return rc;
713
714	dsu_pmu->pmu = (struct pmu) {
715		.task_ctx_nr	= perf_invalid_context,
716		.module		= THIS_MODULE,
717		.pmu_enable	= dsu_pmu_enable,
718		.pmu_disable	= dsu_pmu_disable,
719		.event_init	= dsu_pmu_event_init,
720		.add		= dsu_pmu_add,
721		.del		= dsu_pmu_del,
722		.start		= dsu_pmu_start,
723		.stop		= dsu_pmu_stop,
724		.read		= dsu_pmu_read,
725
726		.attr_groups	= dsu_pmu_attr_groups,
727		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
728	};
729
730	rc = perf_pmu_register(&dsu_pmu->pmu, name, -1);
731	if (rc) {
732		cpuhp_state_remove_instance(dsu_pmu_cpuhp_state,
733						 &dsu_pmu->cpuhp_node);
734		irq_set_affinity_hint(dsu_pmu->irq, NULL);
735	}
736
737	return rc;
738}
739
740static int dsu_pmu_device_remove(struct platform_device *pdev)
741{
742	struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev);
743
744	perf_pmu_unregister(&dsu_pmu->pmu);
745	cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node);
746	irq_set_affinity_hint(dsu_pmu->irq, NULL);
747
748	return 0;
749}
750
751static const struct of_device_id dsu_pmu_of_match[] = {
752	{ .compatible = "arm,dsu-pmu", },
753	{},
754};
755
756static struct platform_driver dsu_pmu_driver = {
757	.driver = {
758		.name	= DRVNAME,
759		.of_match_table = of_match_ptr(dsu_pmu_of_match),
760		.suppress_bind_attrs = true,
761	},
762	.probe = dsu_pmu_device_probe,
763	.remove = dsu_pmu_device_remove,
764};
765
766static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
767{
768	struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
769						   cpuhp_node);
770
771	if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus))
772		return 0;
773
774	/* If the PMU is already managed, there is nothing to do */
775	if (!cpumask_empty(&dsu_pmu->active_cpu))
776		return 0;
777
778	dsu_pmu_init_pmu(dsu_pmu);
779	dsu_pmu_set_active_cpu(cpu, dsu_pmu);
780
781	return 0;
782}
783
784static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
785{
786	int dst;
787	struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
788						   cpuhp_node);
789
790	if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu))
791		return 0;
792
793	dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu);
794	/* If there are no active CPUs in the DSU, leave IRQ disabled */
795	if (dst >= nr_cpu_ids) {
796		irq_set_affinity_hint(dsu_pmu->irq, NULL);
797		return 0;
798	}
799
800	perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst);
801	dsu_pmu_set_active_cpu(dst, dsu_pmu);
802
803	return 0;
804}
805
806static int __init dsu_pmu_init(void)
807{
808	int ret;
809
810	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
811					DRVNAME,
812					dsu_pmu_cpu_online,
813					dsu_pmu_cpu_teardown);
814	if (ret < 0)
815		return ret;
816	dsu_pmu_cpuhp_state = ret;
817	return platform_driver_register(&dsu_pmu_driver);
818}
819
820static void __exit dsu_pmu_exit(void)
821{
822	platform_driver_unregister(&dsu_pmu_driver);
823	cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
824}
825
826module_init(dsu_pmu_init);
827module_exit(dsu_pmu_exit);
828
829MODULE_DEVICE_TABLE(of, dsu_pmu_of_match);
830MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit");
831MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>");
832MODULE_LICENSE("GPL v2");