Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2// CCI Cache Coherent Interconnect PMU driver
3// Copyright (C) 2013-2018 Arm Ltd.
4// Author: Punit Agrawal <punit.agrawal@arm.com>, Suzuki Poulose <suzuki.poulose@arm.com>
5
6#include <linux/arm-cci.h>
7#include <linux/io.h>
8#include <linux/interrupt.h>
9#include <linux/module.h>
10#include <linux/of.h>
11#include <linux/perf_event.h>
12#include <linux/platform_device.h>
13#include <linux/slab.h>
14#include <linux/spinlock.h>
15
16#define DRIVER_NAME "ARM-CCI PMU"
17
18#define CCI_PMCR 0x0100
19#define CCI_PID2 0x0fe8
20
21#define CCI_PMCR_CEN 0x00000001
22#define CCI_PMCR_NCNT_MASK 0x0000f800
23#define CCI_PMCR_NCNT_SHIFT 11
24
25#define CCI_PID2_REV_MASK 0xf0
26#define CCI_PID2_REV_SHIFT 4
27
28#define CCI_PMU_EVT_SEL 0x000
29#define CCI_PMU_CNTR 0x004
30#define CCI_PMU_CNTR_CTRL 0x008
31#define CCI_PMU_OVRFLW 0x00c
32
33#define CCI_PMU_OVRFLW_FLAG 1
34
35#define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size)
36#define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model))
37#define CCI_PMU_CNTR_MASK ((1ULL << 32) - 1)
38#define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1)
39
40#define CCI_PMU_MAX_HW_CNTRS(model) \
41 ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs)
42
43/* Types of interfaces that can generate events */
44enum {
45 CCI_IF_SLAVE,
46 CCI_IF_MASTER,
47#ifdef CONFIG_ARM_CCI5xx_PMU
48 CCI_IF_GLOBAL,
49#endif
50 CCI_IF_MAX,
51};
52
53#define NUM_HW_CNTRS_CII_4XX 4
54#define NUM_HW_CNTRS_CII_5XX 8
55#define NUM_HW_CNTRS_MAX NUM_HW_CNTRS_CII_5XX
56
57#define FIXED_HW_CNTRS_CII_4XX 1
58#define FIXED_HW_CNTRS_CII_5XX 0
59#define FIXED_HW_CNTRS_MAX FIXED_HW_CNTRS_CII_4XX
60
61#define HW_CNTRS_MAX (NUM_HW_CNTRS_MAX + FIXED_HW_CNTRS_MAX)
62
63struct event_range {
64 u32 min;
65 u32 max;
66};
67
68struct cci_pmu_hw_events {
69 struct perf_event **events;
70 unsigned long *used_mask;
71 raw_spinlock_t pmu_lock;
72};
73
74struct cci_pmu;
75/*
76 * struct cci_pmu_model:
77 * @fixed_hw_cntrs - Number of fixed event counters
78 * @num_hw_cntrs - Maximum number of programmable event counters
79 * @cntr_size - Size of an event counter mapping
80 */
81struct cci_pmu_model {
82 char *name;
83 u32 fixed_hw_cntrs;
84 u32 num_hw_cntrs;
85 u32 cntr_size;
86 struct attribute **format_attrs;
87 struct attribute **event_attrs;
88 struct event_range event_ranges[CCI_IF_MAX];
89 int (*validate_hw_event)(struct cci_pmu *, unsigned long);
90 int (*get_event_idx)(struct cci_pmu *, struct cci_pmu_hw_events *, unsigned long);
91 void (*write_counters)(struct cci_pmu *, unsigned long *);
92};
93
94static struct cci_pmu_model cci_pmu_models[];
95
96struct cci_pmu {
97 void __iomem *base;
98 void __iomem *ctrl_base;
99 struct pmu pmu;
100 int cpu;
101 int nr_irqs;
102 int *irqs;
103 unsigned long active_irqs;
104 const struct cci_pmu_model *model;
105 struct cci_pmu_hw_events hw_events;
106 struct platform_device *plat_device;
107 int num_cntrs;
108 atomic_t active_events;
109 struct mutex reserve_mutex;
110};
111
112#define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu))
113
114static struct cci_pmu *g_cci_pmu;
115
116enum cci_models {
117#ifdef CONFIG_ARM_CCI400_PMU
118 CCI400_R0,
119 CCI400_R1,
120#endif
121#ifdef CONFIG_ARM_CCI5xx_PMU
122 CCI500_R0,
123 CCI550_R0,
124#endif
125 CCI_MODEL_MAX
126};
127
128static void pmu_write_counters(struct cci_pmu *cci_pmu,
129 unsigned long *mask);
130static ssize_t __maybe_unused cci_pmu_format_show(struct device *dev,
131 struct device_attribute *attr, char *buf);
132static ssize_t __maybe_unused cci_pmu_event_show(struct device *dev,
133 struct device_attribute *attr, char *buf);
134
135#define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \
136 &((struct dev_ext_attribute[]) { \
137 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \
138 })[0].attr.attr
139
140#define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \
141 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_format_show, (char *)_config)
142#define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \
143 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config)
144
145/* CCI400 PMU Specific definitions */
146
147#ifdef CONFIG_ARM_CCI400_PMU
148
149/* Port ids */
150#define CCI400_PORT_S0 0
151#define CCI400_PORT_S1 1
152#define CCI400_PORT_S2 2
153#define CCI400_PORT_S3 3
154#define CCI400_PORT_S4 4
155#define CCI400_PORT_M0 5
156#define CCI400_PORT_M1 6
157#define CCI400_PORT_M2 7
158
159#define CCI400_R1_PX 5
160
161/*
162 * Instead of an event id to monitor CCI cycles, a dedicated counter is
163 * provided. Use 0xff to represent CCI cycles and hope that no future revisions
164 * make use of this event in hardware.
165 */
166enum cci400_perf_events {
167 CCI400_PMU_CYCLES = 0xff
168};
169
170#define CCI400_PMU_CYCLE_CNTR_IDX 0
171#define CCI400_PMU_CNTR0_IDX 1
172
173/*
174 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8
175 * ports and bits 4:0 are event codes. There are different event codes
176 * associated with each port type.
177 *
178 * Additionally, the range of events associated with the port types changed
179 * between Rev0 and Rev1.
180 *
181 * The constants below define the range of valid codes for each port type for
182 * the different revisions and are used to validate the event to be monitored.
183 */
184
185#define CCI400_PMU_EVENT_MASK 0xffUL
186#define CCI400_PMU_EVENT_SOURCE_SHIFT 5
187#define CCI400_PMU_EVENT_SOURCE_MASK 0x7
188#define CCI400_PMU_EVENT_CODE_SHIFT 0
189#define CCI400_PMU_EVENT_CODE_MASK 0x1f
190#define CCI400_PMU_EVENT_SOURCE(event) \
191 ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \
192 CCI400_PMU_EVENT_SOURCE_MASK)
193#define CCI400_PMU_EVENT_CODE(event) \
194 ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK)
195
196#define CCI400_R0_SLAVE_PORT_MIN_EV 0x00
197#define CCI400_R0_SLAVE_PORT_MAX_EV 0x13
198#define CCI400_R0_MASTER_PORT_MIN_EV 0x14
199#define CCI400_R0_MASTER_PORT_MAX_EV 0x1a
200
201#define CCI400_R1_SLAVE_PORT_MIN_EV 0x00
202#define CCI400_R1_SLAVE_PORT_MAX_EV 0x14
203#define CCI400_R1_MASTER_PORT_MIN_EV 0x00
204#define CCI400_R1_MASTER_PORT_MAX_EV 0x11
205
206#define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \
207 CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \
208 (unsigned long)_config)
209
210static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
211 struct device_attribute *attr, char *buf);
212
213static struct attribute *cci400_pmu_format_attrs[] = {
214 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),
215 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-7"),
216 NULL
217};
218
219static struct attribute *cci400_r0_pmu_event_attrs[] = {
220 /* Slave events */
221 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),
222 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),
223 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),
224 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),
225 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),
226 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),
227 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),
228 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
229 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),
230 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),
231 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),
232 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),
233 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),
234 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),
235 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),
236 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),
237 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),
238 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),
239 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),
240 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),
241 /* Master events */
242 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x14),
243 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard, 0x15),
244 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard, 0x16),
245 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full, 0x17),
246 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x18),
247 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x19),
248 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full, 0x1A),
249 /* Special event for cycles counter */
250 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),
251 NULL
252};
253
254static struct attribute *cci400_r1_pmu_event_attrs[] = {
255 /* Slave events */
256 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),
257 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),
258 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),
259 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),
260 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),
261 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),
262 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),
263 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
264 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),
265 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),
266 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),
267 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),
268 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),
269 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),
270 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),
271 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),
272 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),
273 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),
274 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),
275 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),
276 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard, 0x14),
277 /* Master events */
278 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x0),
279 CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard, 0x1),
280 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard, 0x2),
281 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full, 0x3),
282 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x4),
283 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x5),
284 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full, 0x6),
285 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full, 0x7),
286 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full, 0x8),
287 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0, 0x9),
288 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1, 0xA),
289 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2, 0xB),
290 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3, 0xC),
291 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0, 0xD),
292 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1, 0xE),
293 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2, 0xF),
294 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3, 0x10),
295 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard, 0x11),
296 /* Special event for cycles counter */
297 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),
298 NULL
299};
300
301static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
302 struct device_attribute *attr, char *buf)
303{
304 struct dev_ext_attribute *eattr = container_of(attr,
305 struct dev_ext_attribute, attr);
306 return sysfs_emit(buf, "config=0x%lx\n", (unsigned long)eattr->var);
307}
308
309static int cci400_get_event_idx(struct cci_pmu *cci_pmu,
310 struct cci_pmu_hw_events *hw,
311 unsigned long cci_event)
312{
313 int idx;
314
315 /* cycles event idx is fixed */
316 if (cci_event == CCI400_PMU_CYCLES) {
317 if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX, hw->used_mask))
318 return -EAGAIN;
319
320 return CCI400_PMU_CYCLE_CNTR_IDX;
321 }
322
323 for (idx = CCI400_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx)
324 if (!test_and_set_bit(idx, hw->used_mask))
325 return idx;
326
327 /* No counters available */
328 return -EAGAIN;
329}
330
331static int cci400_validate_hw_event(struct cci_pmu *cci_pmu, unsigned long hw_event)
332{
333 u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event);
334 u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event);
335 int if_type;
336
337 if (hw_event & ~CCI400_PMU_EVENT_MASK)
338 return -ENOENT;
339
340 if (hw_event == CCI400_PMU_CYCLES)
341 return hw_event;
342
343 switch (ev_source) {
344 case CCI400_PORT_S0:
345 case CCI400_PORT_S1:
346 case CCI400_PORT_S2:
347 case CCI400_PORT_S3:
348 case CCI400_PORT_S4:
349 /* Slave Interface */
350 if_type = CCI_IF_SLAVE;
351 break;
352 case CCI400_PORT_M0:
353 case CCI400_PORT_M1:
354 case CCI400_PORT_M2:
355 /* Master Interface */
356 if_type = CCI_IF_MASTER;
357 break;
358 default:
359 return -ENOENT;
360 }
361
362 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
363 ev_code <= cci_pmu->model->event_ranges[if_type].max)
364 return hw_event;
365
366 return -ENOENT;
367}
368
369static int probe_cci400_revision(struct cci_pmu *cci_pmu)
370{
371 int rev;
372 rev = readl_relaxed(cci_pmu->ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK;
373 rev >>= CCI_PID2_REV_SHIFT;
374
375 if (rev < CCI400_R1_PX)
376 return CCI400_R0;
377 else
378 return CCI400_R1;
379}
380
381static const struct cci_pmu_model *probe_cci_model(struct cci_pmu *cci_pmu)
382{
383 if (platform_has_secure_cci_access())
384 return &cci_pmu_models[probe_cci400_revision(cci_pmu)];
385 return NULL;
386}
387#else /* !CONFIG_ARM_CCI400_PMU */
388static inline struct cci_pmu_model *probe_cci_model(struct cci_pmu *cci_pmu)
389{
390 return NULL;
391}
392#endif /* CONFIG_ARM_CCI400_PMU */
393
394#ifdef CONFIG_ARM_CCI5xx_PMU
395
396/*
397 * CCI5xx PMU event id is an 9-bit value made of two parts.
398 * bits [8:5] - Source for the event
399 * bits [4:0] - Event code (specific to type of interface)
400 *
401 *
402 */
403
404/* Port ids */
405#define CCI5xx_PORT_S0 0x0
406#define CCI5xx_PORT_S1 0x1
407#define CCI5xx_PORT_S2 0x2
408#define CCI5xx_PORT_S3 0x3
409#define CCI5xx_PORT_S4 0x4
410#define CCI5xx_PORT_S5 0x5
411#define CCI5xx_PORT_S6 0x6
412
413#define CCI5xx_PORT_M0 0x8
414#define CCI5xx_PORT_M1 0x9
415#define CCI5xx_PORT_M2 0xa
416#define CCI5xx_PORT_M3 0xb
417#define CCI5xx_PORT_M4 0xc
418#define CCI5xx_PORT_M5 0xd
419#define CCI5xx_PORT_M6 0xe
420
421#define CCI5xx_PORT_GLOBAL 0xf
422
423#define CCI5xx_PMU_EVENT_MASK 0x1ffUL
424#define CCI5xx_PMU_EVENT_SOURCE_SHIFT 0x5
425#define CCI5xx_PMU_EVENT_SOURCE_MASK 0xf
426#define CCI5xx_PMU_EVENT_CODE_SHIFT 0x0
427#define CCI5xx_PMU_EVENT_CODE_MASK 0x1f
428
429#define CCI5xx_PMU_EVENT_SOURCE(event) \
430 ((event >> CCI5xx_PMU_EVENT_SOURCE_SHIFT) & CCI5xx_PMU_EVENT_SOURCE_MASK)
431#define CCI5xx_PMU_EVENT_CODE(event) \
432 ((event >> CCI5xx_PMU_EVENT_CODE_SHIFT) & CCI5xx_PMU_EVENT_CODE_MASK)
433
434#define CCI5xx_SLAVE_PORT_MIN_EV 0x00
435#define CCI5xx_SLAVE_PORT_MAX_EV 0x1f
436#define CCI5xx_MASTER_PORT_MIN_EV 0x00
437#define CCI5xx_MASTER_PORT_MAX_EV 0x06
438#define CCI5xx_GLOBAL_PORT_MIN_EV 0x00
439#define CCI5xx_GLOBAL_PORT_MAX_EV 0x0f
440
441
442#define CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \
443 CCI_EXT_ATTR_ENTRY(_name, cci5xx_pmu_global_event_show, \
444 (unsigned long) _config)
445
446static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
447 struct device_attribute *attr, char *buf);
448
449static struct attribute *cci5xx_pmu_format_attrs[] = {
450 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),
451 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-8"),
452 NULL,
453};
454
455static struct attribute *cci5xx_pmu_event_attrs[] = {
456 /* Slave events */
457 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid, 0x0),
458 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev, 0x1),
459 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable, 0x2),
460 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc, 0x3),
461 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc, 0x4),
462 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate, 0x5),
463 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint, 0x6),
464 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
465 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval, 0x8),
466 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop, 0x9),
467 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid, 0xA),
468 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev, 0xB),
469 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable, 0xC),
470 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb, 0xD),
471 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu, 0xE),
472 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique, 0xF),
473 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict, 0x10),
474 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict, 0x11),
475 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat, 0x12),
476 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid, 0x13),
477 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read, 0x14),
478 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean, 0x15),
479 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low, 0x16),
480 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid, 0x17),
481 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall, 0x18),
482 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall, 0x19),
483 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall, 0x1A),
484 CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall, 0x1B),
485 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall, 0x1C),
486 CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall, 0x1D),
487 CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit, 0x1E),
488 CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit, 0x1F),
489
490 /* Master events */
491 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any, 0x0),
492 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any, 0x1),
493 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall, 0x2),
494 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall, 0x3),
495 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall, 0x4),
496 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall, 0x5),
497 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall, 0x6),
498
499 /* Global events */
500 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1, 0x0),
501 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3, 0x1),
502 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5, 0x2),
503 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7, 0x3),
504 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1, 0x4),
505 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3, 0x5),
506 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5, 0x6),
507 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7, 0x7),
508 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation, 0x8),
509 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy, 0x9),
510 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full, 0xA),
511 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB),
512 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC),
513 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD),
514 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_stall_tt_full, 0xE),
515 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF),
516 NULL
517};
518
519static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
520 struct device_attribute *attr, char *buf)
521{
522 struct dev_ext_attribute *eattr = container_of(attr,
523 struct dev_ext_attribute, attr);
524 /* Global events have single fixed source code */
525 return sysfs_emit(buf, "event=0x%lx,source=0x%x\n",
526 (unsigned long)eattr->var, CCI5xx_PORT_GLOBAL);
527}
528
529/*
530 * CCI500 provides 8 independent event counters that can count
531 * any of the events available.
532 * CCI500 PMU event source ids
533 * 0x0-0x6 - Slave interfaces
534 * 0x8-0xD - Master interfaces
535 * 0xf - Global Events
536 * 0x7,0xe - Reserved
537 */
538static int cci500_validate_hw_event(struct cci_pmu *cci_pmu,
539 unsigned long hw_event)
540{
541 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);
542 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);
543 int if_type;
544
545 if (hw_event & ~CCI5xx_PMU_EVENT_MASK)
546 return -ENOENT;
547
548 switch (ev_source) {
549 case CCI5xx_PORT_S0:
550 case CCI5xx_PORT_S1:
551 case CCI5xx_PORT_S2:
552 case CCI5xx_PORT_S3:
553 case CCI5xx_PORT_S4:
554 case CCI5xx_PORT_S5:
555 case CCI5xx_PORT_S6:
556 if_type = CCI_IF_SLAVE;
557 break;
558 case CCI5xx_PORT_M0:
559 case CCI5xx_PORT_M1:
560 case CCI5xx_PORT_M2:
561 case CCI5xx_PORT_M3:
562 case CCI5xx_PORT_M4:
563 case CCI5xx_PORT_M5:
564 if_type = CCI_IF_MASTER;
565 break;
566 case CCI5xx_PORT_GLOBAL:
567 if_type = CCI_IF_GLOBAL;
568 break;
569 default:
570 return -ENOENT;
571 }
572
573 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
574 ev_code <= cci_pmu->model->event_ranges[if_type].max)
575 return hw_event;
576
577 return -ENOENT;
578}
579
580/*
581 * CCI550 provides 8 independent event counters that can count
582 * any of the events available.
583 * CCI550 PMU event source ids
584 * 0x0-0x6 - Slave interfaces
585 * 0x8-0xe - Master interfaces
586 * 0xf - Global Events
587 * 0x7 - Reserved
588 */
589static int cci550_validate_hw_event(struct cci_pmu *cci_pmu,
590 unsigned long hw_event)
591{
592 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);
593 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);
594 int if_type;
595
596 if (hw_event & ~CCI5xx_PMU_EVENT_MASK)
597 return -ENOENT;
598
599 switch (ev_source) {
600 case CCI5xx_PORT_S0:
601 case CCI5xx_PORT_S1:
602 case CCI5xx_PORT_S2:
603 case CCI5xx_PORT_S3:
604 case CCI5xx_PORT_S4:
605 case CCI5xx_PORT_S5:
606 case CCI5xx_PORT_S6:
607 if_type = CCI_IF_SLAVE;
608 break;
609 case CCI5xx_PORT_M0:
610 case CCI5xx_PORT_M1:
611 case CCI5xx_PORT_M2:
612 case CCI5xx_PORT_M3:
613 case CCI5xx_PORT_M4:
614 case CCI5xx_PORT_M5:
615 case CCI5xx_PORT_M6:
616 if_type = CCI_IF_MASTER;
617 break;
618 case CCI5xx_PORT_GLOBAL:
619 if_type = CCI_IF_GLOBAL;
620 break;
621 default:
622 return -ENOENT;
623 }
624
625 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
626 ev_code <= cci_pmu->model->event_ranges[if_type].max)
627 return hw_event;
628
629 return -ENOENT;
630}
631
632#endif /* CONFIG_ARM_CCI5xx_PMU */
633
634/*
635 * Program the CCI PMU counters which have PERF_HES_ARCH set
636 * with the event period and mark them ready before we enable
637 * PMU.
638 */
639static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu)
640{
641 int i;
642 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
643 DECLARE_BITMAP(mask, HW_CNTRS_MAX);
644
645 bitmap_zero(mask, HW_CNTRS_MAX);
646 for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) {
647 struct perf_event *event = cci_hw->events[i];
648
649 if (WARN_ON(!event))
650 continue;
651
652 /* Leave the events which are not counting */
653 if (event->hw.state & PERF_HES_STOPPED)
654 continue;
655 if (event->hw.state & PERF_HES_ARCH) {
656 __set_bit(i, mask);
657 event->hw.state &= ~PERF_HES_ARCH;
658 }
659 }
660
661 pmu_write_counters(cci_pmu, mask);
662}
663
664/* Should be called with cci_pmu->hw_events->pmu_lock held */
665static void __cci_pmu_enable_nosync(struct cci_pmu *cci_pmu)
666{
667 u32 val;
668
669 /* Enable all the PMU counters. */
670 val = readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) | CCI_PMCR_CEN;
671 writel(val, cci_pmu->ctrl_base + CCI_PMCR);
672}
673
674/* Should be called with cci_pmu->hw_events->pmu_lock held */
675static void __cci_pmu_enable_sync(struct cci_pmu *cci_pmu)
676{
677 cci_pmu_sync_counters(cci_pmu);
678 __cci_pmu_enable_nosync(cci_pmu);
679}
680
681/* Should be called with cci_pmu->hw_events->pmu_lock held */
682static void __cci_pmu_disable(struct cci_pmu *cci_pmu)
683{
684 u32 val;
685
686 /* Disable all the PMU counters. */
687 val = readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN;
688 writel(val, cci_pmu->ctrl_base + CCI_PMCR);
689}
690
691static ssize_t cci_pmu_format_show(struct device *dev,
692 struct device_attribute *attr, char *buf)
693{
694 struct dev_ext_attribute *eattr = container_of(attr,
695 struct dev_ext_attribute, attr);
696 return sysfs_emit(buf, "%s\n", (char *)eattr->var);
697}
698
699static ssize_t cci_pmu_event_show(struct device *dev,
700 struct device_attribute *attr, char *buf)
701{
702 struct dev_ext_attribute *eattr = container_of(attr,
703 struct dev_ext_attribute, attr);
704 /* source parameter is mandatory for normal PMU events */
705 return sysfs_emit(buf, "source=?,event=0x%lx\n",
706 (unsigned long)eattr->var);
707}
708
709static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx)
710{
711 return 0 <= idx && idx <= CCI_PMU_CNTR_LAST(cci_pmu);
712}
713
714static u32 pmu_read_register(struct cci_pmu *cci_pmu, int idx, unsigned int offset)
715{
716 return readl_relaxed(cci_pmu->base +
717 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);
718}
719
720static void pmu_write_register(struct cci_pmu *cci_pmu, u32 value,
721 int idx, unsigned int offset)
722{
723 writel_relaxed(value, cci_pmu->base +
724 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);
725}
726
727static void pmu_disable_counter(struct cci_pmu *cci_pmu, int idx)
728{
729 pmu_write_register(cci_pmu, 0, idx, CCI_PMU_CNTR_CTRL);
730}
731
732static void pmu_enable_counter(struct cci_pmu *cci_pmu, int idx)
733{
734 pmu_write_register(cci_pmu, 1, idx, CCI_PMU_CNTR_CTRL);
735}
736
737static bool __maybe_unused
738pmu_counter_is_enabled(struct cci_pmu *cci_pmu, int idx)
739{
740 return (pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR_CTRL) & 0x1) != 0;
741}
742
743static void pmu_set_event(struct cci_pmu *cci_pmu, int idx, unsigned long event)
744{
745 pmu_write_register(cci_pmu, event, idx, CCI_PMU_EVT_SEL);
746}
747
748/*
749 * For all counters on the CCI-PMU, disable any 'enabled' counters,
750 * saving the changed counters in the mask, so that we can restore
751 * it later using pmu_restore_counters. The mask is private to the
752 * caller. We cannot rely on the used_mask maintained by the CCI_PMU
753 * as it only tells us if the counter is assigned to perf_event or not.
754 * The state of the perf_event cannot be locked by the PMU layer, hence
755 * we check the individual counter status (which can be locked by
756 * cci_pm->hw_events->pmu_lock).
757 *
758 * @mask should be initialised to empty by the caller.
759 */
760static void __maybe_unused
761pmu_save_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
762{
763 int i;
764
765 for (i = 0; i < cci_pmu->num_cntrs; i++) {
766 if (pmu_counter_is_enabled(cci_pmu, i)) {
767 set_bit(i, mask);
768 pmu_disable_counter(cci_pmu, i);
769 }
770 }
771}
772
773/*
774 * Restore the status of the counters. Reversal of the pmu_save_counters().
775 * For each counter set in the mask, enable the counter back.
776 */
777static void __maybe_unused
778pmu_restore_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
779{
780 int i;
781
782 for_each_set_bit(i, mask, cci_pmu->num_cntrs)
783 pmu_enable_counter(cci_pmu, i);
784}
785
786/*
787 * Returns the number of programmable counters actually implemented
788 * by the cci
789 */
790static u32 pmu_get_max_counters(struct cci_pmu *cci_pmu)
791{
792 return (readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) &
793 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT;
794}
795
796static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event)
797{
798 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
799 unsigned long cci_event = event->hw.config_base;
800 int idx;
801
802 if (cci_pmu->model->get_event_idx)
803 return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event);
804
805 /* Generic code to find an unused idx from the mask */
806 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++)
807 if (!test_and_set_bit(idx, hw->used_mask))
808 return idx;
809
810 /* No counters available */
811 return -EAGAIN;
812}
813
814static int pmu_map_event(struct perf_event *event)
815{
816 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
817
818 if (event->attr.type < PERF_TYPE_MAX ||
819 !cci_pmu->model->validate_hw_event)
820 return -ENOENT;
821
822 return cci_pmu->model->validate_hw_event(cci_pmu, event->attr.config);
823}
824
825static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler)
826{
827 int i;
828 struct platform_device *pmu_device = cci_pmu->plat_device;
829
830 if (unlikely(!pmu_device))
831 return -ENODEV;
832
833 if (cci_pmu->nr_irqs < 1) {
834 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n");
835 return -ENODEV;
836 }
837
838 /*
839 * Register all available CCI PMU interrupts. In the interrupt handler
840 * we iterate over the counters checking for interrupt source (the
841 * overflowing counter) and clear it.
842 *
843 * This should allow handling of non-unique interrupt for the counters.
844 */
845 for (i = 0; i < cci_pmu->nr_irqs; i++) {
846 int err = request_irq(cci_pmu->irqs[i], handler, IRQF_SHARED,
847 "arm-cci-pmu", cci_pmu);
848 if (err) {
849 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n",
850 cci_pmu->irqs[i]);
851 return err;
852 }
853
854 set_bit(i, &cci_pmu->active_irqs);
855 }
856
857 return 0;
858}
859
860static void pmu_free_irq(struct cci_pmu *cci_pmu)
861{
862 int i;
863
864 for (i = 0; i < cci_pmu->nr_irqs; i++) {
865 if (!test_and_clear_bit(i, &cci_pmu->active_irqs))
866 continue;
867
868 free_irq(cci_pmu->irqs[i], cci_pmu);
869 }
870}
871
872static u32 pmu_read_counter(struct perf_event *event)
873{
874 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
875 struct hw_perf_event *hw_counter = &event->hw;
876 int idx = hw_counter->idx;
877 u32 value;
878
879 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
880 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
881 return 0;
882 }
883 value = pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR);
884
885 return value;
886}
887
888static void pmu_write_counter(struct cci_pmu *cci_pmu, u32 value, int idx)
889{
890 pmu_write_register(cci_pmu, value, idx, CCI_PMU_CNTR);
891}
892
893static void __pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
894{
895 int i;
896 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
897
898 for_each_set_bit(i, mask, cci_pmu->num_cntrs) {
899 struct perf_event *event = cci_hw->events[i];
900
901 if (WARN_ON(!event))
902 continue;
903 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);
904 }
905}
906
907static void pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
908{
909 if (cci_pmu->model->write_counters)
910 cci_pmu->model->write_counters(cci_pmu, mask);
911 else
912 __pmu_write_counters(cci_pmu, mask);
913}
914
915#ifdef CONFIG_ARM_CCI5xx_PMU
916
917/*
918 * CCI-500/CCI-550 has advanced power saving policies, which could gate the
919 * clocks to the PMU counters, which makes the writes to them ineffective.
920 * The only way to write to those counters is when the global counters
921 * are enabled and the particular counter is enabled.
922 *
923 * So we do the following :
924 *
925 * 1) Disable all the PMU counters, saving their current state
926 * 2) Enable the global PMU profiling, now that all counters are
927 * disabled.
928 *
929 * For each counter to be programmed, repeat steps 3-7:
930 *
931 * 3) Write an invalid event code to the event control register for the
932 counter, so that the counters are not modified.
933 * 4) Enable the counter control for the counter.
934 * 5) Set the counter value
935 * 6) Disable the counter
936 * 7) Restore the event in the target counter
937 *
938 * 8) Disable the global PMU.
939 * 9) Restore the status of the rest of the counters.
940 *
941 * We choose an event which for CCI-5xx is guaranteed not to count.
942 * We use the highest possible event code (0x1f) for the master interface 0.
943 */
944#define CCI5xx_INVALID_EVENT ((CCI5xx_PORT_M0 << CCI5xx_PMU_EVENT_SOURCE_SHIFT) | \
945 (CCI5xx_PMU_EVENT_CODE_MASK << CCI5xx_PMU_EVENT_CODE_SHIFT))
946static void cci5xx_pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
947{
948 int i;
949 DECLARE_BITMAP(saved_mask, HW_CNTRS_MAX);
950
951 bitmap_zero(saved_mask, cci_pmu->num_cntrs);
952 pmu_save_counters(cci_pmu, saved_mask);
953
954 /*
955 * Now that all the counters are disabled, we can safely turn the PMU on,
956 * without syncing the status of the counters
957 */
958 __cci_pmu_enable_nosync(cci_pmu);
959
960 for_each_set_bit(i, mask, cci_pmu->num_cntrs) {
961 struct perf_event *event = cci_pmu->hw_events.events[i];
962
963 if (WARN_ON(!event))
964 continue;
965
966 pmu_set_event(cci_pmu, i, CCI5xx_INVALID_EVENT);
967 pmu_enable_counter(cci_pmu, i);
968 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);
969 pmu_disable_counter(cci_pmu, i);
970 pmu_set_event(cci_pmu, i, event->hw.config_base);
971 }
972
973 __cci_pmu_disable(cci_pmu);
974
975 pmu_restore_counters(cci_pmu, saved_mask);
976}
977
978#endif /* CONFIG_ARM_CCI5xx_PMU */
979
980static u64 pmu_event_update(struct perf_event *event)
981{
982 struct hw_perf_event *hwc = &event->hw;
983 u64 delta, prev_raw_count, new_raw_count;
984
985 do {
986 prev_raw_count = local64_read(&hwc->prev_count);
987 new_raw_count = pmu_read_counter(event);
988 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
989 new_raw_count) != prev_raw_count);
990
991 delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK;
992
993 local64_add(delta, &event->count);
994
995 return new_raw_count;
996}
997
998static void pmu_read(struct perf_event *event)
999{
1000 pmu_event_update(event);
1001}
1002
1003static void pmu_event_set_period(struct perf_event *event)
1004{
1005 struct hw_perf_event *hwc = &event->hw;
1006 /*
1007 * The CCI PMU counters have a period of 2^32. To account for the
1008 * possiblity of extreme interrupt latency we program for a period of
1009 * half that. Hopefully we can handle the interrupt before another 2^31
1010 * events occur and the counter overtakes its previous value.
1011 */
1012 u64 val = 1ULL << 31;
1013 local64_set(&hwc->prev_count, val);
1014
1015 /*
1016 * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose
1017 * values needs to be sync-ed with the s/w state before the PMU is
1018 * enabled.
1019 * Mark this counter for sync.
1020 */
1021 hwc->state |= PERF_HES_ARCH;
1022}
1023
1024static irqreturn_t pmu_handle_irq(int irq_num, void *dev)
1025{
1026 struct cci_pmu *cci_pmu = dev;
1027 struct cci_pmu_hw_events *events = &cci_pmu->hw_events;
1028 int idx, handled = IRQ_NONE;
1029
1030 raw_spin_lock(&events->pmu_lock);
1031
1032 /* Disable the PMU while we walk through the counters */
1033 __cci_pmu_disable(cci_pmu);
1034 /*
1035 * Iterate over counters and update the corresponding perf events.
1036 * This should work regardless of whether we have per-counter overflow
1037 * interrupt or a combined overflow interrupt.
1038 */
1039 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) {
1040 struct perf_event *event = events->events[idx];
1041
1042 if (!event)
1043 continue;
1044
1045 /* Did this counter overflow? */
1046 if (!(pmu_read_register(cci_pmu, idx, CCI_PMU_OVRFLW) &
1047 CCI_PMU_OVRFLW_FLAG))
1048 continue;
1049
1050 pmu_write_register(cci_pmu, CCI_PMU_OVRFLW_FLAG, idx,
1051 CCI_PMU_OVRFLW);
1052
1053 pmu_event_update(event);
1054 pmu_event_set_period(event);
1055 handled = IRQ_HANDLED;
1056 }
1057
1058 /* Enable the PMU and sync possibly overflowed counters */
1059 __cci_pmu_enable_sync(cci_pmu);
1060 raw_spin_unlock(&events->pmu_lock);
1061
1062 return IRQ_RETVAL(handled);
1063}
1064
1065static int cci_pmu_get_hw(struct cci_pmu *cci_pmu)
1066{
1067 int ret = pmu_request_irq(cci_pmu, pmu_handle_irq);
1068 if (ret) {
1069 pmu_free_irq(cci_pmu);
1070 return ret;
1071 }
1072 return 0;
1073}
1074
1075static void cci_pmu_put_hw(struct cci_pmu *cci_pmu)
1076{
1077 pmu_free_irq(cci_pmu);
1078}
1079
1080static void hw_perf_event_destroy(struct perf_event *event)
1081{
1082 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1083 atomic_t *active_events = &cci_pmu->active_events;
1084 struct mutex *reserve_mutex = &cci_pmu->reserve_mutex;
1085
1086 if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) {
1087 cci_pmu_put_hw(cci_pmu);
1088 mutex_unlock(reserve_mutex);
1089 }
1090}
1091
1092static void cci_pmu_enable(struct pmu *pmu)
1093{
1094 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1095 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1096 bool enabled = !bitmap_empty(hw_events->used_mask, cci_pmu->num_cntrs);
1097 unsigned long flags;
1098
1099 if (!enabled)
1100 return;
1101
1102 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1103 __cci_pmu_enable_sync(cci_pmu);
1104 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1105
1106}
1107
1108static void cci_pmu_disable(struct pmu *pmu)
1109{
1110 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1111 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1112 unsigned long flags;
1113
1114 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1115 __cci_pmu_disable(cci_pmu);
1116 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1117}
1118
1119/*
1120 * Check if the idx represents a non-programmable counter.
1121 * All the fixed event counters are mapped before the programmable
1122 * counters.
1123 */
1124static bool pmu_fixed_hw_idx(struct cci_pmu *cci_pmu, int idx)
1125{
1126 return (idx >= 0) && (idx < cci_pmu->model->fixed_hw_cntrs);
1127}
1128
1129static void cci_pmu_start(struct perf_event *event, int pmu_flags)
1130{
1131 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1132 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1133 struct hw_perf_event *hwc = &event->hw;
1134 int idx = hwc->idx;
1135 unsigned long flags;
1136
1137 /*
1138 * To handle interrupt latency, we always reprogram the period
1139 * regardless of PERF_EF_RELOAD.
1140 */
1141 if (pmu_flags & PERF_EF_RELOAD)
1142 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
1143
1144 hwc->state = 0;
1145
1146 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
1147 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
1148 return;
1149 }
1150
1151 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1152
1153 /* Configure the counter unless you are counting a fixed event */
1154 if (!pmu_fixed_hw_idx(cci_pmu, idx))
1155 pmu_set_event(cci_pmu, idx, hwc->config_base);
1156
1157 pmu_event_set_period(event);
1158 pmu_enable_counter(cci_pmu, idx);
1159
1160 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1161}
1162
1163static void cci_pmu_stop(struct perf_event *event, int pmu_flags)
1164{
1165 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1166 struct hw_perf_event *hwc = &event->hw;
1167 int idx = hwc->idx;
1168
1169 if (hwc->state & PERF_HES_STOPPED)
1170 return;
1171
1172 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
1173 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
1174 return;
1175 }
1176
1177 /*
1178 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See
1179 * cci_pmu_start()
1180 */
1181 pmu_disable_counter(cci_pmu, idx);
1182 pmu_event_update(event);
1183 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
1184}
1185
1186static int cci_pmu_add(struct perf_event *event, int flags)
1187{
1188 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1189 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1190 struct hw_perf_event *hwc = &event->hw;
1191 int idx;
1192
1193 /* If we don't have a space for the counter then finish early. */
1194 idx = pmu_get_event_idx(hw_events, event);
1195 if (idx < 0)
1196 return idx;
1197
1198 event->hw.idx = idx;
1199 hw_events->events[idx] = event;
1200
1201 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1202 if (flags & PERF_EF_START)
1203 cci_pmu_start(event, PERF_EF_RELOAD);
1204
1205 /* Propagate our changes to the userspace mapping. */
1206 perf_event_update_userpage(event);
1207
1208 return 0;
1209}
1210
1211static void cci_pmu_del(struct perf_event *event, int flags)
1212{
1213 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1214 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1215 struct hw_perf_event *hwc = &event->hw;
1216 int idx = hwc->idx;
1217
1218 cci_pmu_stop(event, PERF_EF_UPDATE);
1219 hw_events->events[idx] = NULL;
1220 clear_bit(idx, hw_events->used_mask);
1221
1222 perf_event_update_userpage(event);
1223}
1224
1225static int validate_event(struct pmu *cci_pmu,
1226 struct cci_pmu_hw_events *hw_events,
1227 struct perf_event *event)
1228{
1229 if (is_software_event(event))
1230 return 1;
1231
1232 /*
1233 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
1234 * core perf code won't check that the pmu->ctx == leader->ctx
1235 * until after pmu->event_init(event).
1236 */
1237 if (event->pmu != cci_pmu)
1238 return 0;
1239
1240 if (event->state < PERF_EVENT_STATE_OFF)
1241 return 1;
1242
1243 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
1244 return 1;
1245
1246 return pmu_get_event_idx(hw_events, event) >= 0;
1247}
1248
1249static int validate_group(struct perf_event *event)
1250{
1251 struct perf_event *sibling, *leader = event->group_leader;
1252 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1253 unsigned long mask[BITS_TO_LONGS(HW_CNTRS_MAX)];
1254 struct cci_pmu_hw_events fake_pmu = {
1255 /*
1256 * Initialise the fake PMU. We only need to populate the
1257 * used_mask for the purposes of validation.
1258 */
1259 .used_mask = mask,
1260 };
1261 bitmap_zero(mask, cci_pmu->num_cntrs);
1262
1263 if (!validate_event(event->pmu, &fake_pmu, leader))
1264 return -EINVAL;
1265
1266 for_each_sibling_event(sibling, leader) {
1267 if (!validate_event(event->pmu, &fake_pmu, sibling))
1268 return -EINVAL;
1269 }
1270
1271 if (!validate_event(event->pmu, &fake_pmu, event))
1272 return -EINVAL;
1273
1274 return 0;
1275}
1276
1277static int __hw_perf_event_init(struct perf_event *event)
1278{
1279 struct hw_perf_event *hwc = &event->hw;
1280 int mapping;
1281
1282 mapping = pmu_map_event(event);
1283
1284 if (mapping < 0) {
1285 pr_debug("event %x:%llx not supported\n", event->attr.type,
1286 event->attr.config);
1287 return mapping;
1288 }
1289
1290 /*
1291 * We don't assign an index until we actually place the event onto
1292 * hardware. Use -1 to signify that we haven't decided where to put it
1293 * yet.
1294 */
1295 hwc->idx = -1;
1296 hwc->config_base = 0;
1297 hwc->config = 0;
1298 hwc->event_base = 0;
1299
1300 /*
1301 * Store the event encoding into the config_base field.
1302 */
1303 hwc->config_base |= (unsigned long)mapping;
1304
1305 if (event->group_leader != event) {
1306 if (validate_group(event) != 0)
1307 return -EINVAL;
1308 }
1309
1310 return 0;
1311}
1312
1313static int cci_pmu_event_init(struct perf_event *event)
1314{
1315 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1316 atomic_t *active_events = &cci_pmu->active_events;
1317 int err = 0;
1318
1319 if (event->attr.type != event->pmu->type)
1320 return -ENOENT;
1321
1322 /* Shared by all CPUs, no meaningful state to sample */
1323 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
1324 return -EOPNOTSUPP;
1325
1326 /*
1327 * Following the example set by other "uncore" PMUs, we accept any CPU
1328 * and rewrite its affinity dynamically rather than having perf core
1329 * handle cpu == -1 and pid == -1 for this case.
1330 *
1331 * The perf core will pin online CPUs for the duration of this call and
1332 * the event being installed into its context, so the PMU's CPU can't
1333 * change under our feet.
1334 */
1335 if (event->cpu < 0)
1336 return -EINVAL;
1337 event->cpu = cci_pmu->cpu;
1338
1339 event->destroy = hw_perf_event_destroy;
1340 if (!atomic_inc_not_zero(active_events)) {
1341 mutex_lock(&cci_pmu->reserve_mutex);
1342 if (atomic_read(active_events) == 0)
1343 err = cci_pmu_get_hw(cci_pmu);
1344 if (!err)
1345 atomic_inc(active_events);
1346 mutex_unlock(&cci_pmu->reserve_mutex);
1347 }
1348 if (err)
1349 return err;
1350
1351 err = __hw_perf_event_init(event);
1352 if (err)
1353 hw_perf_event_destroy(event);
1354
1355 return err;
1356}
1357
1358static ssize_t pmu_cpumask_attr_show(struct device *dev,
1359 struct device_attribute *attr, char *buf)
1360{
1361 struct pmu *pmu = dev_get_drvdata(dev);
1362 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1363
1364 return cpumap_print_to_pagebuf(true, buf, cpumask_of(cci_pmu->cpu));
1365}
1366
1367static struct device_attribute pmu_cpumask_attr =
1368 __ATTR(cpumask, S_IRUGO, pmu_cpumask_attr_show, NULL);
1369
1370static struct attribute *pmu_attrs[] = {
1371 &pmu_cpumask_attr.attr,
1372 NULL,
1373};
1374
1375static const struct attribute_group pmu_attr_group = {
1376 .attrs = pmu_attrs,
1377};
1378
1379static struct attribute_group pmu_format_attr_group = {
1380 .name = "format",
1381 .attrs = NULL, /* Filled in cci_pmu_init_attrs */
1382};
1383
1384static struct attribute_group pmu_event_attr_group = {
1385 .name = "events",
1386 .attrs = NULL, /* Filled in cci_pmu_init_attrs */
1387};
1388
1389static const struct attribute_group *pmu_attr_groups[] = {
1390 &pmu_attr_group,
1391 &pmu_format_attr_group,
1392 &pmu_event_attr_group,
1393 NULL
1394};
1395
1396static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev)
1397{
1398 const struct cci_pmu_model *model = cci_pmu->model;
1399 char *name = model->name;
1400 u32 num_cntrs;
1401
1402 if (WARN_ON(model->num_hw_cntrs > NUM_HW_CNTRS_MAX))
1403 return -EINVAL;
1404 if (WARN_ON(model->fixed_hw_cntrs > FIXED_HW_CNTRS_MAX))
1405 return -EINVAL;
1406
1407 pmu_event_attr_group.attrs = model->event_attrs;
1408 pmu_format_attr_group.attrs = model->format_attrs;
1409
1410 cci_pmu->pmu = (struct pmu) {
1411 .module = THIS_MODULE,
1412 .name = cci_pmu->model->name,
1413 .task_ctx_nr = perf_invalid_context,
1414 .pmu_enable = cci_pmu_enable,
1415 .pmu_disable = cci_pmu_disable,
1416 .event_init = cci_pmu_event_init,
1417 .add = cci_pmu_add,
1418 .del = cci_pmu_del,
1419 .start = cci_pmu_start,
1420 .stop = cci_pmu_stop,
1421 .read = pmu_read,
1422 .attr_groups = pmu_attr_groups,
1423 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1424 };
1425
1426 cci_pmu->plat_device = pdev;
1427 num_cntrs = pmu_get_max_counters(cci_pmu);
1428 if (num_cntrs > cci_pmu->model->num_hw_cntrs) {
1429 dev_warn(&pdev->dev,
1430 "PMU implements more counters(%d) than supported by"
1431 " the model(%d), truncated.",
1432 num_cntrs, cci_pmu->model->num_hw_cntrs);
1433 num_cntrs = cci_pmu->model->num_hw_cntrs;
1434 }
1435 cci_pmu->num_cntrs = num_cntrs + cci_pmu->model->fixed_hw_cntrs;
1436
1437 return perf_pmu_register(&cci_pmu->pmu, name, -1);
1438}
1439
1440static int cci_pmu_offline_cpu(unsigned int cpu)
1441{
1442 int target;
1443
1444 if (!g_cci_pmu || cpu != g_cci_pmu->cpu)
1445 return 0;
1446
1447 target = cpumask_any_but(cpu_online_mask, cpu);
1448 if (target >= nr_cpu_ids)
1449 return 0;
1450
1451 perf_pmu_migrate_context(&g_cci_pmu->pmu, cpu, target);
1452 g_cci_pmu->cpu = target;
1453 return 0;
1454}
1455
1456static __maybe_unused struct cci_pmu_model cci_pmu_models[] = {
1457#ifdef CONFIG_ARM_CCI400_PMU
1458 [CCI400_R0] = {
1459 .name = "CCI_400",
1460 .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_4XX, /* Cycle counter */
1461 .num_hw_cntrs = NUM_HW_CNTRS_CII_4XX,
1462 .cntr_size = SZ_4K,
1463 .format_attrs = cci400_pmu_format_attrs,
1464 .event_attrs = cci400_r0_pmu_event_attrs,
1465 .event_ranges = {
1466 [CCI_IF_SLAVE] = {
1467 CCI400_R0_SLAVE_PORT_MIN_EV,
1468 CCI400_R0_SLAVE_PORT_MAX_EV,
1469 },
1470 [CCI_IF_MASTER] = {
1471 CCI400_R0_MASTER_PORT_MIN_EV,
1472 CCI400_R0_MASTER_PORT_MAX_EV,
1473 },
1474 },
1475 .validate_hw_event = cci400_validate_hw_event,
1476 .get_event_idx = cci400_get_event_idx,
1477 },
1478 [CCI400_R1] = {
1479 .name = "CCI_400_r1",
1480 .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_4XX, /* Cycle counter */
1481 .num_hw_cntrs = NUM_HW_CNTRS_CII_4XX,
1482 .cntr_size = SZ_4K,
1483 .format_attrs = cci400_pmu_format_attrs,
1484 .event_attrs = cci400_r1_pmu_event_attrs,
1485 .event_ranges = {
1486 [CCI_IF_SLAVE] = {
1487 CCI400_R1_SLAVE_PORT_MIN_EV,
1488 CCI400_R1_SLAVE_PORT_MAX_EV,
1489 },
1490 [CCI_IF_MASTER] = {
1491 CCI400_R1_MASTER_PORT_MIN_EV,
1492 CCI400_R1_MASTER_PORT_MAX_EV,
1493 },
1494 },
1495 .validate_hw_event = cci400_validate_hw_event,
1496 .get_event_idx = cci400_get_event_idx,
1497 },
1498#endif
1499#ifdef CONFIG_ARM_CCI5xx_PMU
1500 [CCI500_R0] = {
1501 .name = "CCI_500",
1502 .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_5XX,
1503 .num_hw_cntrs = NUM_HW_CNTRS_CII_5XX,
1504 .cntr_size = SZ_64K,
1505 .format_attrs = cci5xx_pmu_format_attrs,
1506 .event_attrs = cci5xx_pmu_event_attrs,
1507 .event_ranges = {
1508 [CCI_IF_SLAVE] = {
1509 CCI5xx_SLAVE_PORT_MIN_EV,
1510 CCI5xx_SLAVE_PORT_MAX_EV,
1511 },
1512 [CCI_IF_MASTER] = {
1513 CCI5xx_MASTER_PORT_MIN_EV,
1514 CCI5xx_MASTER_PORT_MAX_EV,
1515 },
1516 [CCI_IF_GLOBAL] = {
1517 CCI5xx_GLOBAL_PORT_MIN_EV,
1518 CCI5xx_GLOBAL_PORT_MAX_EV,
1519 },
1520 },
1521 .validate_hw_event = cci500_validate_hw_event,
1522 .write_counters = cci5xx_pmu_write_counters,
1523 },
1524 [CCI550_R0] = {
1525 .name = "CCI_550",
1526 .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_5XX,
1527 .num_hw_cntrs = NUM_HW_CNTRS_CII_5XX,
1528 .cntr_size = SZ_64K,
1529 .format_attrs = cci5xx_pmu_format_attrs,
1530 .event_attrs = cci5xx_pmu_event_attrs,
1531 .event_ranges = {
1532 [CCI_IF_SLAVE] = {
1533 CCI5xx_SLAVE_PORT_MIN_EV,
1534 CCI5xx_SLAVE_PORT_MAX_EV,
1535 },
1536 [CCI_IF_MASTER] = {
1537 CCI5xx_MASTER_PORT_MIN_EV,
1538 CCI5xx_MASTER_PORT_MAX_EV,
1539 },
1540 [CCI_IF_GLOBAL] = {
1541 CCI5xx_GLOBAL_PORT_MIN_EV,
1542 CCI5xx_GLOBAL_PORT_MAX_EV,
1543 },
1544 },
1545 .validate_hw_event = cci550_validate_hw_event,
1546 .write_counters = cci5xx_pmu_write_counters,
1547 },
1548#endif
1549};
1550
1551static const struct of_device_id arm_cci_pmu_matches[] = {
1552#ifdef CONFIG_ARM_CCI400_PMU
1553 {
1554 .compatible = "arm,cci-400-pmu",
1555 .data = NULL,
1556 },
1557 {
1558 .compatible = "arm,cci-400-pmu,r0",
1559 .data = &cci_pmu_models[CCI400_R0],
1560 },
1561 {
1562 .compatible = "arm,cci-400-pmu,r1",
1563 .data = &cci_pmu_models[CCI400_R1],
1564 },
1565#endif
1566#ifdef CONFIG_ARM_CCI5xx_PMU
1567 {
1568 .compatible = "arm,cci-500-pmu,r0",
1569 .data = &cci_pmu_models[CCI500_R0],
1570 },
1571 {
1572 .compatible = "arm,cci-550-pmu,r0",
1573 .data = &cci_pmu_models[CCI550_R0],
1574 },
1575#endif
1576 {},
1577};
1578MODULE_DEVICE_TABLE(of, arm_cci_pmu_matches);
1579
1580static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs)
1581{
1582 int i;
1583
1584 for (i = 0; i < nr_irqs; i++)
1585 if (irq == irqs[i])
1586 return true;
1587
1588 return false;
1589}
1590
1591static struct cci_pmu *cci_pmu_alloc(struct device *dev)
1592{
1593 struct cci_pmu *cci_pmu;
1594 const struct cci_pmu_model *model;
1595
1596 /*
1597 * All allocations are devm_* hence we don't have to free
1598 * them explicitly on an error, as it would end up in driver
1599 * detach.
1600 */
1601 cci_pmu = devm_kzalloc(dev, sizeof(*cci_pmu), GFP_KERNEL);
1602 if (!cci_pmu)
1603 return ERR_PTR(-ENOMEM);
1604
1605 cci_pmu->ctrl_base = *(void __iomem **)dev->platform_data;
1606
1607 model = of_device_get_match_data(dev);
1608 if (!model) {
1609 dev_warn(dev,
1610 "DEPRECATED compatible property, requires secure access to CCI registers");
1611 model = probe_cci_model(cci_pmu);
1612 }
1613 if (!model) {
1614 dev_warn(dev, "CCI PMU version not supported\n");
1615 return ERR_PTR(-ENODEV);
1616 }
1617
1618 cci_pmu->model = model;
1619 cci_pmu->irqs = devm_kcalloc(dev, CCI_PMU_MAX_HW_CNTRS(model),
1620 sizeof(*cci_pmu->irqs), GFP_KERNEL);
1621 if (!cci_pmu->irqs)
1622 return ERR_PTR(-ENOMEM);
1623 cci_pmu->hw_events.events = devm_kcalloc(dev,
1624 CCI_PMU_MAX_HW_CNTRS(model),
1625 sizeof(*cci_pmu->hw_events.events),
1626 GFP_KERNEL);
1627 if (!cci_pmu->hw_events.events)
1628 return ERR_PTR(-ENOMEM);
1629 cci_pmu->hw_events.used_mask = devm_bitmap_zalloc(dev,
1630 CCI_PMU_MAX_HW_CNTRS(model),
1631 GFP_KERNEL);
1632 if (!cci_pmu->hw_events.used_mask)
1633 return ERR_PTR(-ENOMEM);
1634
1635 return cci_pmu;
1636}
1637
1638static int cci_pmu_probe(struct platform_device *pdev)
1639{
1640 struct cci_pmu *cci_pmu;
1641 int i, ret, irq;
1642
1643 cci_pmu = cci_pmu_alloc(&pdev->dev);
1644 if (IS_ERR(cci_pmu))
1645 return PTR_ERR(cci_pmu);
1646
1647 cci_pmu->base = devm_platform_ioremap_resource(pdev, 0);
1648 if (IS_ERR(cci_pmu->base))
1649 return -ENOMEM;
1650
1651 /*
1652 * CCI PMU has one overflow interrupt per counter; but some may be tied
1653 * together to a common interrupt.
1654 */
1655 cci_pmu->nr_irqs = 0;
1656 for (i = 0; i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model); i++) {
1657 irq = platform_get_irq(pdev, i);
1658 if (irq < 0)
1659 break;
1660
1661 if (is_duplicate_irq(irq, cci_pmu->irqs, cci_pmu->nr_irqs))
1662 continue;
1663
1664 cci_pmu->irqs[cci_pmu->nr_irqs++] = irq;
1665 }
1666
1667 /*
1668 * Ensure that the device tree has as many interrupts as the number
1669 * of counters.
1670 */
1671 if (i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)) {
1672 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n",
1673 i, CCI_PMU_MAX_HW_CNTRS(cci_pmu->model));
1674 return -EINVAL;
1675 }
1676
1677 raw_spin_lock_init(&cci_pmu->hw_events.pmu_lock);
1678 mutex_init(&cci_pmu->reserve_mutex);
1679 atomic_set(&cci_pmu->active_events, 0);
1680
1681 cci_pmu->cpu = raw_smp_processor_id();
1682 g_cci_pmu = cci_pmu;
1683 cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,
1684 "perf/arm/cci:online", NULL,
1685 cci_pmu_offline_cpu);
1686
1687 ret = cci_pmu_init(cci_pmu, pdev);
1688 if (ret)
1689 goto error_pmu_init;
1690
1691 pr_info("ARM %s PMU driver probed", cci_pmu->model->name);
1692 return 0;
1693
1694error_pmu_init:
1695 cpuhp_remove_state(CPUHP_AP_PERF_ARM_CCI_ONLINE);
1696 g_cci_pmu = NULL;
1697 return ret;
1698}
1699
1700static int cci_pmu_remove(struct platform_device *pdev)
1701{
1702 if (!g_cci_pmu)
1703 return 0;
1704
1705 cpuhp_remove_state(CPUHP_AP_PERF_ARM_CCI_ONLINE);
1706 perf_pmu_unregister(&g_cci_pmu->pmu);
1707 g_cci_pmu = NULL;
1708
1709 return 0;
1710}
1711
1712static struct platform_driver cci_pmu_driver = {
1713 .driver = {
1714 .name = DRIVER_NAME,
1715 .of_match_table = arm_cci_pmu_matches,
1716 .suppress_bind_attrs = true,
1717 },
1718 .probe = cci_pmu_probe,
1719 .remove = cci_pmu_remove,
1720};
1721
1722module_platform_driver(cci_pmu_driver);
1723MODULE_LICENSE("GPL v2");
1724MODULE_DESCRIPTION("ARM CCI PMU support");