Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved.
3 */
4#include <linux/acpi.h>
5#include <linux/bitops.h>
6#include <linux/bug.h>
7#include <linux/cpuhotplug.h>
8#include <linux/cpumask.h>
9#include <linux/device.h>
10#include <linux/errno.h>
11#include <linux/interrupt.h>
12#include <linux/irq.h>
13#include <linux/kernel.h>
14#include <linux/list.h>
15#include <linux/percpu.h>
16#include <linux/perf_event.h>
17#include <linux/platform_device.h>
18#include <linux/smp.h>
19#include <linux/spinlock.h>
20#include <linux/sysfs.h>
21#include <linux/types.h>
22
23#include <asm/barrier.h>
24#include <asm/local64.h>
25#include <asm/sysreg.h>
26#include <soc/qcom/kryo-l2-accessors.h>
27
28#define MAX_L2_CTRS 9
29
30#define L2PMCR_NUM_EV_SHIFT 11
31#define L2PMCR_NUM_EV_MASK 0x1F
32
33#define L2PMCR 0x400
34#define L2PMCNTENCLR 0x403
35#define L2PMCNTENSET 0x404
36#define L2PMINTENCLR 0x405
37#define L2PMINTENSET 0x406
38#define L2PMOVSCLR 0x407
39#define L2PMOVSSET 0x408
40#define L2PMCCNTCR 0x409
41#define L2PMCCNTR 0x40A
42#define L2PMCCNTSR 0x40C
43#define L2PMRESR 0x410
44#define IA_L2PMXEVCNTCR_BASE 0x420
45#define IA_L2PMXEVCNTR_BASE 0x421
46#define IA_L2PMXEVFILTER_BASE 0x423
47#define IA_L2PMXEVTYPER_BASE 0x424
48
49#define IA_L2_REG_OFFSET 0x10
50
51#define L2PMXEVFILTER_SUFILTER_ALL 0x000E0000
52#define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004
53#define L2PMXEVFILTER_ORGFILTER_ALL 0x00000003
54
55#define L2EVTYPER_REG_SHIFT 3
56
57#define L2PMRESR_GROUP_BITS 8
58#define L2PMRESR_GROUP_MASK GENMASK(7, 0)
59
60#define L2CYCLE_CTR_BIT 31
61#define L2CYCLE_CTR_RAW_CODE 0xFE
62
63#define L2PMCR_RESET_ALL 0x6
64#define L2PMCR_COUNTERS_ENABLE 0x1
65#define L2PMCR_COUNTERS_DISABLE 0x0
66
67#define L2PMRESR_EN BIT_ULL(63)
68
69#define L2_EVT_MASK 0x00000FFF
70#define L2_EVT_CODE_MASK 0x00000FF0
71#define L2_EVT_GRP_MASK 0x0000000F
72#define L2_EVT_CODE_SHIFT 4
73#define L2_EVT_GRP_SHIFT 0
74
75#define L2_EVT_CODE(event) (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT)
76#define L2_EVT_GROUP(event) (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT)
77
78#define L2_EVT_GROUP_MAX 7
79
80#define L2_COUNTER_RELOAD BIT_ULL(31)
81#define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63)
82
83
84#define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE)
85
86/*
87 * Events
88 */
89#define L2_EVENT_CYCLES 0xfe
90#define L2_EVENT_DCACHE_OPS 0x400
91#define L2_EVENT_ICACHE_OPS 0x401
92#define L2_EVENT_TLBI 0x402
93#define L2_EVENT_BARRIERS 0x403
94#define L2_EVENT_TOTAL_READS 0x405
95#define L2_EVENT_TOTAL_WRITES 0x406
96#define L2_EVENT_TOTAL_REQUESTS 0x407
97#define L2_EVENT_LDREX 0x420
98#define L2_EVENT_STREX 0x421
99#define L2_EVENT_CLREX 0x422
100
101
102
103struct cluster_pmu;
104
105/*
106 * Aggregate PMU. Implements the core pmu functions and manages
107 * the hardware PMUs.
108 */
109struct l2cache_pmu {
110 struct hlist_node node;
111 u32 num_pmus;
112 struct pmu pmu;
113 int num_counters;
114 cpumask_t cpumask;
115 struct platform_device *pdev;
116 struct cluster_pmu * __percpu *pmu_cluster;
117 struct list_head clusters;
118};
119
120/*
121 * The cache is made up of one or more clusters, each cluster has its own PMU.
122 * Each cluster is associated with one or more CPUs.
123 * This structure represents one of the hardware PMUs.
124 *
125 * Events can be envisioned as a 2-dimensional array. Each column represents
126 * a group of events. There are 8 groups. Only one entry from each
127 * group can be in use at a time.
128 *
129 * Events are specified as 0xCCG, where CC is 2 hex digits specifying
130 * the code (array row) and G specifies the group (column).
131 *
132 * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE
133 * which is outside the above scheme.
134 */
135struct cluster_pmu {
136 struct list_head next;
137 struct perf_event *events[MAX_L2_CTRS];
138 struct l2cache_pmu *l2cache_pmu;
139 DECLARE_BITMAP(used_counters, MAX_L2_CTRS);
140 DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1);
141 int irq;
142 int cluster_id;
143 /* The CPU that is used for collecting events on this cluster */
144 int on_cpu;
145 /* All the CPUs associated with this cluster */
146 cpumask_t cluster_cpus;
147 spinlock_t pmu_lock;
148};
149
150#define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu))
151
152static u32 l2_cycle_ctr_idx;
153static u32 l2_counter_present_mask;
154
155static inline u32 idx_to_reg_bit(u32 idx)
156{
157 if (idx == l2_cycle_ctr_idx)
158 return BIT(L2CYCLE_CTR_BIT);
159
160 return BIT(idx);
161}
162
163static inline struct cluster_pmu *get_cluster_pmu(
164 struct l2cache_pmu *l2cache_pmu, int cpu)
165{
166 return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu);
167}
168
169static void cluster_pmu_reset(void)
170{
171 /* Reset all counters */
172 kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_RESET_ALL);
173 kryo_l2_set_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask);
174 kryo_l2_set_indirect_reg(L2PMINTENCLR, l2_counter_present_mask);
175 kryo_l2_set_indirect_reg(L2PMOVSCLR, l2_counter_present_mask);
176}
177
178static inline void cluster_pmu_enable(void)
179{
180 kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE);
181}
182
183static inline void cluster_pmu_disable(void)
184{
185 kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE);
186}
187
188static inline void cluster_pmu_counter_set_value(u32 idx, u64 value)
189{
190 if (idx == l2_cycle_ctr_idx)
191 kryo_l2_set_indirect_reg(L2PMCCNTR, value);
192 else
193 kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value);
194}
195
196static inline u64 cluster_pmu_counter_get_value(u32 idx)
197{
198 u64 value;
199
200 if (idx == l2_cycle_ctr_idx)
201 value = kryo_l2_get_indirect_reg(L2PMCCNTR);
202 else
203 value = kryo_l2_get_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx));
204
205 return value;
206}
207
208static inline void cluster_pmu_counter_enable(u32 idx)
209{
210 kryo_l2_set_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx));
211}
212
213static inline void cluster_pmu_counter_disable(u32 idx)
214{
215 kryo_l2_set_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx));
216}
217
218static inline void cluster_pmu_counter_enable_interrupt(u32 idx)
219{
220 kryo_l2_set_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx));
221}
222
223static inline void cluster_pmu_counter_disable_interrupt(u32 idx)
224{
225 kryo_l2_set_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx));
226}
227
228static inline void cluster_pmu_set_evccntcr(u32 val)
229{
230 kryo_l2_set_indirect_reg(L2PMCCNTCR, val);
231}
232
233static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val)
234{
235 kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val);
236}
237
238static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val)
239{
240 kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val);
241}
242
243static void cluster_pmu_set_resr(struct cluster_pmu *cluster,
244 u32 event_group, u32 event_cc)
245{
246 u64 field;
247 u64 resr_val;
248 u32 shift;
249 unsigned long flags;
250
251 shift = L2PMRESR_GROUP_BITS * event_group;
252 field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift);
253
254 spin_lock_irqsave(&cluster->pmu_lock, flags);
255
256 resr_val = kryo_l2_get_indirect_reg(L2PMRESR);
257 resr_val &= ~(L2PMRESR_GROUP_MASK << shift);
258 resr_val |= field;
259 resr_val |= L2PMRESR_EN;
260 kryo_l2_set_indirect_reg(L2PMRESR, resr_val);
261
262 spin_unlock_irqrestore(&cluster->pmu_lock, flags);
263}
264
265/*
266 * Hardware allows filtering of events based on the originating
267 * CPU. Turn this off by setting filter bits to allow events from
268 * all CPUS, subunits and ID independent events in this cluster.
269 */
270static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr)
271{
272 u32 val = L2PMXEVFILTER_SUFILTER_ALL |
273 L2PMXEVFILTER_ORGFILTER_IDINDEP |
274 L2PMXEVFILTER_ORGFILTER_ALL;
275
276 kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val);
277}
278
279static inline u32 cluster_pmu_getreset_ovsr(void)
280{
281 u32 result = kryo_l2_get_indirect_reg(L2PMOVSSET);
282
283 kryo_l2_set_indirect_reg(L2PMOVSCLR, result);
284 return result;
285}
286
287static inline bool cluster_pmu_has_overflowed(u32 ovsr)
288{
289 return !!(ovsr & l2_counter_present_mask);
290}
291
292static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx)
293{
294 return !!(ovsr & idx_to_reg_bit(idx));
295}
296
297static void l2_cache_event_update(struct perf_event *event)
298{
299 struct hw_perf_event *hwc = &event->hw;
300 u64 delta, prev, now;
301 u32 idx = hwc->idx;
302
303 do {
304 prev = local64_read(&hwc->prev_count);
305 now = cluster_pmu_counter_get_value(idx);
306 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
307
308 /*
309 * The cycle counter is 64-bit, but all other counters are
310 * 32-bit, and we must handle 32-bit overflow explicitly.
311 */
312 delta = now - prev;
313 if (idx != l2_cycle_ctr_idx)
314 delta &= 0xffffffff;
315
316 local64_add(delta, &event->count);
317}
318
319static void l2_cache_cluster_set_period(struct cluster_pmu *cluster,
320 struct hw_perf_event *hwc)
321{
322 u32 idx = hwc->idx;
323 u64 new;
324
325 /*
326 * We limit the max period to half the max counter value so
327 * that even in the case of extreme interrupt latency the
328 * counter will (hopefully) not wrap past its initial value.
329 */
330 if (idx == l2_cycle_ctr_idx)
331 new = L2_CYCLE_COUNTER_RELOAD;
332 else
333 new = L2_COUNTER_RELOAD;
334
335 local64_set(&hwc->prev_count, new);
336 cluster_pmu_counter_set_value(idx, new);
337}
338
339static int l2_cache_get_event_idx(struct cluster_pmu *cluster,
340 struct perf_event *event)
341{
342 struct hw_perf_event *hwc = &event->hw;
343 int idx;
344 int num_ctrs = cluster->l2cache_pmu->num_counters - 1;
345 unsigned int group;
346
347 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
348 if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters))
349 return -EAGAIN;
350
351 return l2_cycle_ctr_idx;
352 }
353
354 idx = find_first_zero_bit(cluster->used_counters, num_ctrs);
355 if (idx == num_ctrs)
356 /* The counters are all in use. */
357 return -EAGAIN;
358
359 /*
360 * Check for column exclusion: event column already in use by another
361 * event. This is for events which are not in the same group.
362 * Conflicting events in the same group are detected in event_init.
363 */
364 group = L2_EVT_GROUP(hwc->config_base);
365 if (test_bit(group, cluster->used_groups))
366 return -EAGAIN;
367
368 set_bit(idx, cluster->used_counters);
369 set_bit(group, cluster->used_groups);
370
371 return idx;
372}
373
374static void l2_cache_clear_event_idx(struct cluster_pmu *cluster,
375 struct perf_event *event)
376{
377 struct hw_perf_event *hwc = &event->hw;
378 int idx = hwc->idx;
379
380 clear_bit(idx, cluster->used_counters);
381 if (hwc->config_base != L2CYCLE_CTR_RAW_CODE)
382 clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups);
383}
384
385static irqreturn_t l2_cache_handle_irq(int irq_num, void *data)
386{
387 struct cluster_pmu *cluster = data;
388 int num_counters = cluster->l2cache_pmu->num_counters;
389 u32 ovsr;
390 int idx;
391
392 ovsr = cluster_pmu_getreset_ovsr();
393 if (!cluster_pmu_has_overflowed(ovsr))
394 return IRQ_NONE;
395
396 for_each_set_bit(idx, cluster->used_counters, num_counters) {
397 struct perf_event *event = cluster->events[idx];
398 struct hw_perf_event *hwc;
399
400 if (WARN_ON_ONCE(!event))
401 continue;
402
403 if (!cluster_pmu_counter_has_overflowed(ovsr, idx))
404 continue;
405
406 l2_cache_event_update(event);
407 hwc = &event->hw;
408
409 l2_cache_cluster_set_period(cluster, hwc);
410 }
411
412 return IRQ_HANDLED;
413}
414
415/*
416 * Implementation of abstract pmu functionality required by
417 * the core perf events code.
418 */
419
420static void l2_cache_pmu_enable(struct pmu *pmu)
421{
422 /*
423 * Although there is only one PMU (per socket) controlling multiple
424 * physical PMUs (per cluster), because we do not support per-task mode
425 * each event is associated with a CPU. Each event has pmu_enable
426 * called on its CPU, so here it is only necessary to enable the
427 * counters for the current CPU.
428 */
429
430 cluster_pmu_enable();
431}
432
433static void l2_cache_pmu_disable(struct pmu *pmu)
434{
435 cluster_pmu_disable();
436}
437
438static int l2_cache_event_init(struct perf_event *event)
439{
440 struct hw_perf_event *hwc = &event->hw;
441 struct cluster_pmu *cluster;
442 struct perf_event *sibling;
443 struct l2cache_pmu *l2cache_pmu;
444
445 if (event->attr.type != event->pmu->type)
446 return -ENOENT;
447
448 l2cache_pmu = to_l2cache_pmu(event->pmu);
449
450 if (hwc->sample_period) {
451 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
452 "Sampling not supported\n");
453 return -EOPNOTSUPP;
454 }
455
456 if (event->cpu < 0) {
457 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
458 "Per-task mode not supported\n");
459 return -EOPNOTSUPP;
460 }
461
462 if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) ||
463 ((event->attr.config & ~L2_EVT_MASK) != 0)) &&
464 (event->attr.config != L2CYCLE_CTR_RAW_CODE)) {
465 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
466 "Invalid config %llx\n",
467 event->attr.config);
468 return -EINVAL;
469 }
470
471 /* Don't allow groups with mixed PMUs, except for s/w events */
472 if (event->group_leader->pmu != event->pmu &&
473 !is_software_event(event->group_leader)) {
474 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
475 "Can't create mixed PMU group\n");
476 return -EINVAL;
477 }
478
479 for_each_sibling_event(sibling, event->group_leader) {
480 if (sibling->pmu != event->pmu &&
481 !is_software_event(sibling)) {
482 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
483 "Can't create mixed PMU group\n");
484 return -EINVAL;
485 }
486 }
487
488 cluster = get_cluster_pmu(l2cache_pmu, event->cpu);
489 if (!cluster) {
490 /* CPU has not been initialised */
491 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
492 "CPU%d not associated with L2 cluster\n", event->cpu);
493 return -EINVAL;
494 }
495
496 /* Ensure all events in a group are on the same cpu */
497 if ((event->group_leader != event) &&
498 (cluster->on_cpu != event->group_leader->cpu)) {
499 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
500 "Can't create group on CPUs %d and %d",
501 event->cpu, event->group_leader->cpu);
502 return -EINVAL;
503 }
504
505 if ((event != event->group_leader) &&
506 !is_software_event(event->group_leader) &&
507 (L2_EVT_GROUP(event->group_leader->attr.config) ==
508 L2_EVT_GROUP(event->attr.config))) {
509 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
510 "Column exclusion: conflicting events %llx %llx\n",
511 event->group_leader->attr.config,
512 event->attr.config);
513 return -EINVAL;
514 }
515
516 for_each_sibling_event(sibling, event->group_leader) {
517 if ((sibling != event) &&
518 !is_software_event(sibling) &&
519 (L2_EVT_GROUP(sibling->attr.config) ==
520 L2_EVT_GROUP(event->attr.config))) {
521 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
522 "Column exclusion: conflicting events %llx %llx\n",
523 sibling->attr.config,
524 event->attr.config);
525 return -EINVAL;
526 }
527 }
528
529 hwc->idx = -1;
530 hwc->config_base = event->attr.config;
531
532 /*
533 * Ensure all events are on the same cpu so all events are in the
534 * same cpu context, to avoid races on pmu_enable etc.
535 */
536 event->cpu = cluster->on_cpu;
537
538 return 0;
539}
540
541static void l2_cache_event_start(struct perf_event *event, int flags)
542{
543 struct cluster_pmu *cluster;
544 struct hw_perf_event *hwc = &event->hw;
545 int idx = hwc->idx;
546 u32 config;
547 u32 event_cc, event_group;
548
549 hwc->state = 0;
550
551 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
552
553 l2_cache_cluster_set_period(cluster, hwc);
554
555 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
556 cluster_pmu_set_evccntcr(0);
557 } else {
558 config = hwc->config_base;
559 event_cc = L2_EVT_CODE(config);
560 event_group = L2_EVT_GROUP(config);
561
562 cluster_pmu_set_evcntcr(idx, 0);
563 cluster_pmu_set_evtyper(idx, event_group);
564 cluster_pmu_set_resr(cluster, event_group, event_cc);
565 cluster_pmu_set_evfilter_sys_mode(idx);
566 }
567
568 cluster_pmu_counter_enable_interrupt(idx);
569 cluster_pmu_counter_enable(idx);
570}
571
572static void l2_cache_event_stop(struct perf_event *event, int flags)
573{
574 struct hw_perf_event *hwc = &event->hw;
575 int idx = hwc->idx;
576
577 if (hwc->state & PERF_HES_STOPPED)
578 return;
579
580 cluster_pmu_counter_disable_interrupt(idx);
581 cluster_pmu_counter_disable(idx);
582
583 if (flags & PERF_EF_UPDATE)
584 l2_cache_event_update(event);
585 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
586}
587
588static int l2_cache_event_add(struct perf_event *event, int flags)
589{
590 struct hw_perf_event *hwc = &event->hw;
591 int idx;
592 int err = 0;
593 struct cluster_pmu *cluster;
594
595 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
596
597 idx = l2_cache_get_event_idx(cluster, event);
598 if (idx < 0)
599 return idx;
600
601 hwc->idx = idx;
602 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
603 cluster->events[idx] = event;
604 local64_set(&hwc->prev_count, 0);
605
606 if (flags & PERF_EF_START)
607 l2_cache_event_start(event, flags);
608
609 /* Propagate changes to the userspace mapping. */
610 perf_event_update_userpage(event);
611
612 return err;
613}
614
615static void l2_cache_event_del(struct perf_event *event, int flags)
616{
617 struct hw_perf_event *hwc = &event->hw;
618 struct cluster_pmu *cluster;
619 int idx = hwc->idx;
620
621 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
622
623 l2_cache_event_stop(event, flags | PERF_EF_UPDATE);
624 cluster->events[idx] = NULL;
625 l2_cache_clear_event_idx(cluster, event);
626
627 perf_event_update_userpage(event);
628}
629
630static void l2_cache_event_read(struct perf_event *event)
631{
632 l2_cache_event_update(event);
633}
634
635static ssize_t l2_cache_pmu_cpumask_show(struct device *dev,
636 struct device_attribute *attr,
637 char *buf)
638{
639 struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev));
640
641 return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask);
642}
643
644static struct device_attribute l2_cache_pmu_cpumask_attr =
645 __ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL);
646
647static struct attribute *l2_cache_pmu_cpumask_attrs[] = {
648 &l2_cache_pmu_cpumask_attr.attr,
649 NULL,
650};
651
652static struct attribute_group l2_cache_pmu_cpumask_group = {
653 .attrs = l2_cache_pmu_cpumask_attrs,
654};
655
656/* CCG format for perf RAW codes. */
657PMU_FORMAT_ATTR(l2_code, "config:4-11");
658PMU_FORMAT_ATTR(l2_group, "config:0-3");
659PMU_FORMAT_ATTR(event, "config:0-11");
660
661static struct attribute *l2_cache_pmu_formats[] = {
662 &format_attr_l2_code.attr,
663 &format_attr_l2_group.attr,
664 &format_attr_event.attr,
665 NULL,
666};
667
668static struct attribute_group l2_cache_pmu_format_group = {
669 .name = "format",
670 .attrs = l2_cache_pmu_formats,
671};
672
673static ssize_t l2cache_pmu_event_show(struct device *dev,
674 struct device_attribute *attr, char *page)
675{
676 struct perf_pmu_events_attr *pmu_attr;
677
678 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
679 return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
680}
681
682#define L2CACHE_EVENT_ATTR(_name, _id) \
683 (&((struct perf_pmu_events_attr[]) { \
684 { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \
685 .id = _id, } \
686 })[0].attr.attr)
687
688static struct attribute *l2_cache_pmu_events[] = {
689 L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES),
690 L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS),
691 L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS),
692 L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI),
693 L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS),
694 L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS),
695 L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES),
696 L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS),
697 L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX),
698 L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX),
699 L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX),
700 NULL
701};
702
703static struct attribute_group l2_cache_pmu_events_group = {
704 .name = "events",
705 .attrs = l2_cache_pmu_events,
706};
707
708static const struct attribute_group *l2_cache_pmu_attr_grps[] = {
709 &l2_cache_pmu_format_group,
710 &l2_cache_pmu_cpumask_group,
711 &l2_cache_pmu_events_group,
712 NULL,
713};
714
715/*
716 * Generic device handlers
717 */
718
719static const struct acpi_device_id l2_cache_pmu_acpi_match[] = {
720 { "QCOM8130", },
721 { }
722};
723
724static int get_num_counters(void)
725{
726 int val;
727
728 val = kryo_l2_get_indirect_reg(L2PMCR);
729
730 /*
731 * Read number of counters from L2PMCR and add 1
732 * for the cycle counter.
733 */
734 return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1;
735}
736
737static struct cluster_pmu *l2_cache_associate_cpu_with_cluster(
738 struct l2cache_pmu *l2cache_pmu, int cpu)
739{
740 u64 mpidr;
741 int cpu_cluster_id;
742 struct cluster_pmu *cluster = NULL;
743
744 /*
745 * This assumes that the cluster_id is in MPIDR[aff1] for
746 * single-threaded cores, and MPIDR[aff2] for multi-threaded
747 * cores. This logic will have to be updated if this changes.
748 */
749 mpidr = read_cpuid_mpidr();
750 if (mpidr & MPIDR_MT_BITMASK)
751 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
752 else
753 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
754
755 list_for_each_entry(cluster, &l2cache_pmu->clusters, next) {
756 if (cluster->cluster_id != cpu_cluster_id)
757 continue;
758
759 dev_info(&l2cache_pmu->pdev->dev,
760 "CPU%d associated with cluster %d\n", cpu,
761 cluster->cluster_id);
762 cpumask_set_cpu(cpu, &cluster->cluster_cpus);
763 *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster;
764 break;
765 }
766
767 return cluster;
768}
769
770static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
771{
772 struct cluster_pmu *cluster;
773 struct l2cache_pmu *l2cache_pmu;
774
775 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
776 cluster = get_cluster_pmu(l2cache_pmu, cpu);
777 if (!cluster) {
778 /* First time this CPU has come online */
779 cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu);
780 if (!cluster) {
781 /* Only if broken firmware doesn't list every cluster */
782 WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu);
783 return 0;
784 }
785 }
786
787 /* If another CPU is managing this cluster, we're done */
788 if (cluster->on_cpu != -1)
789 return 0;
790
791 /*
792 * All CPUs on this cluster were down, use this one.
793 * Reset to put it into sane state.
794 */
795 cluster->on_cpu = cpu;
796 cpumask_set_cpu(cpu, &l2cache_pmu->cpumask);
797 cluster_pmu_reset();
798
799 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu)));
800 enable_irq(cluster->irq);
801
802 return 0;
803}
804
805static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
806{
807 struct cluster_pmu *cluster;
808 struct l2cache_pmu *l2cache_pmu;
809 cpumask_t cluster_online_cpus;
810 unsigned int target;
811
812 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
813 cluster = get_cluster_pmu(l2cache_pmu, cpu);
814 if (!cluster)
815 return 0;
816
817 /* If this CPU is not managing the cluster, we're done */
818 if (cluster->on_cpu != cpu)
819 return 0;
820
821 /* Give up ownership of cluster */
822 cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask);
823 cluster->on_cpu = -1;
824
825 /* Any other CPU for this cluster which is still online */
826 cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus,
827 cpu_online_mask);
828 target = cpumask_any_but(&cluster_online_cpus, cpu);
829 if (target >= nr_cpu_ids) {
830 disable_irq(cluster->irq);
831 return 0;
832 }
833
834 perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target);
835 cluster->on_cpu = target;
836 cpumask_set_cpu(target, &l2cache_pmu->cpumask);
837 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target)));
838
839 return 0;
840}
841
842static int l2_cache_pmu_probe_cluster(struct device *dev, void *data)
843{
844 struct platform_device *pdev = to_platform_device(dev->parent);
845 struct platform_device *sdev = to_platform_device(dev);
846 struct l2cache_pmu *l2cache_pmu = data;
847 struct cluster_pmu *cluster;
848 struct acpi_device *device;
849 unsigned long fw_cluster_id;
850 int err;
851 int irq;
852
853 if (acpi_bus_get_device(ACPI_HANDLE(dev), &device))
854 return -ENODEV;
855
856 if (kstrtoul(device->pnp.unique_id, 10, &fw_cluster_id) < 0) {
857 dev_err(&pdev->dev, "unable to read ACPI uid\n");
858 return -ENODEV;
859 }
860
861 cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL);
862 if (!cluster)
863 return -ENOMEM;
864
865 INIT_LIST_HEAD(&cluster->next);
866 list_add(&cluster->next, &l2cache_pmu->clusters);
867 cluster->cluster_id = fw_cluster_id;
868
869 irq = platform_get_irq(sdev, 0);
870 if (irq < 0)
871 return irq;
872 irq_set_status_flags(irq, IRQ_NOAUTOEN);
873 cluster->irq = irq;
874
875 cluster->l2cache_pmu = l2cache_pmu;
876 cluster->on_cpu = -1;
877
878 err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq,
879 IRQF_NOBALANCING | IRQF_NO_THREAD,
880 "l2-cache-pmu", cluster);
881 if (err) {
882 dev_err(&pdev->dev,
883 "Unable to request IRQ%d for L2 PMU counters\n", irq);
884 return err;
885 }
886
887 dev_info(&pdev->dev,
888 "Registered L2 cache PMU cluster %ld\n", fw_cluster_id);
889
890 spin_lock_init(&cluster->pmu_lock);
891
892 l2cache_pmu->num_pmus++;
893
894 return 0;
895}
896
897static int l2_cache_pmu_probe(struct platform_device *pdev)
898{
899 int err;
900 struct l2cache_pmu *l2cache_pmu;
901
902 l2cache_pmu =
903 devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL);
904 if (!l2cache_pmu)
905 return -ENOMEM;
906
907 INIT_LIST_HEAD(&l2cache_pmu->clusters);
908
909 platform_set_drvdata(pdev, l2cache_pmu);
910 l2cache_pmu->pmu = (struct pmu) {
911 /* suffix is instance id for future use with multiple sockets */
912 .name = "l2cache_0",
913 .task_ctx_nr = perf_invalid_context,
914 .pmu_enable = l2_cache_pmu_enable,
915 .pmu_disable = l2_cache_pmu_disable,
916 .event_init = l2_cache_event_init,
917 .add = l2_cache_event_add,
918 .del = l2_cache_event_del,
919 .start = l2_cache_event_start,
920 .stop = l2_cache_event_stop,
921 .read = l2_cache_event_read,
922 .attr_groups = l2_cache_pmu_attr_grps,
923 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
924 };
925
926 l2cache_pmu->num_counters = get_num_counters();
927 l2cache_pmu->pdev = pdev;
928 l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev,
929 struct cluster_pmu *);
930 if (!l2cache_pmu->pmu_cluster)
931 return -ENOMEM;
932
933 l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1;
934 l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) |
935 BIT(L2CYCLE_CTR_BIT);
936
937 cpumask_clear(&l2cache_pmu->cpumask);
938
939 /* Read cluster info and initialize each cluster */
940 err = device_for_each_child(&pdev->dev, l2cache_pmu,
941 l2_cache_pmu_probe_cluster);
942 if (err)
943 return err;
944
945 if (l2cache_pmu->num_pmus == 0) {
946 dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n");
947 return -ENODEV;
948 }
949
950 err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
951 &l2cache_pmu->node);
952 if (err) {
953 dev_err(&pdev->dev, "Error %d registering hotplug", err);
954 return err;
955 }
956
957 err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1);
958 if (err) {
959 dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err);
960 goto out_unregister;
961 }
962
963 dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n",
964 l2cache_pmu->num_pmus);
965
966 return err;
967
968out_unregister:
969 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
970 &l2cache_pmu->node);
971 return err;
972}
973
974static int l2_cache_pmu_remove(struct platform_device *pdev)
975{
976 struct l2cache_pmu *l2cache_pmu =
977 to_l2cache_pmu(platform_get_drvdata(pdev));
978
979 perf_pmu_unregister(&l2cache_pmu->pmu);
980 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
981 &l2cache_pmu->node);
982 return 0;
983}
984
985static struct platform_driver l2_cache_pmu_driver = {
986 .driver = {
987 .name = "qcom-l2cache-pmu",
988 .acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match),
989 .suppress_bind_attrs = true,
990 },
991 .probe = l2_cache_pmu_probe,
992 .remove = l2_cache_pmu_remove,
993};
994
995static int __init register_l2_cache_pmu_driver(void)
996{
997 int err;
998
999 err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1000 "AP_PERF_ARM_QCOM_L2_ONLINE",
1001 l2cache_pmu_online_cpu,
1002 l2cache_pmu_offline_cpu);
1003 if (err)
1004 return err;
1005
1006 return platform_driver_register(&l2_cache_pmu_driver);
1007}
1008device_initcall(register_l2_cache_pmu_driver);
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved.
3 */
4#include <linux/acpi.h>
5#include <linux/bitops.h>
6#include <linux/bug.h>
7#include <linux/cpuhotplug.h>
8#include <linux/cpumask.h>
9#include <linux/device.h>
10#include <linux/errno.h>
11#include <linux/interrupt.h>
12#include <linux/irq.h>
13#include <linux/kernel.h>
14#include <linux/list.h>
15#include <linux/percpu.h>
16#include <linux/perf_event.h>
17#include <linux/platform_device.h>
18#include <linux/smp.h>
19#include <linux/spinlock.h>
20#include <linux/sysfs.h>
21#include <linux/types.h>
22
23#include <asm/barrier.h>
24#include <asm/local64.h>
25#include <asm/sysreg.h>
26
27#define MAX_L2_CTRS 9
28
29#define L2PMCR_NUM_EV_SHIFT 11
30#define L2PMCR_NUM_EV_MASK 0x1F
31
32#define L2PMCR 0x400
33#define L2PMCNTENCLR 0x403
34#define L2PMCNTENSET 0x404
35#define L2PMINTENCLR 0x405
36#define L2PMINTENSET 0x406
37#define L2PMOVSCLR 0x407
38#define L2PMOVSSET 0x408
39#define L2PMCCNTCR 0x409
40#define L2PMCCNTR 0x40A
41#define L2PMCCNTSR 0x40C
42#define L2PMRESR 0x410
43#define IA_L2PMXEVCNTCR_BASE 0x420
44#define IA_L2PMXEVCNTR_BASE 0x421
45#define IA_L2PMXEVFILTER_BASE 0x423
46#define IA_L2PMXEVTYPER_BASE 0x424
47
48#define IA_L2_REG_OFFSET 0x10
49
50#define L2PMXEVFILTER_SUFILTER_ALL 0x000E0000
51#define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004
52#define L2PMXEVFILTER_ORGFILTER_ALL 0x00000003
53
54#define L2EVTYPER_REG_SHIFT 3
55
56#define L2PMRESR_GROUP_BITS 8
57#define L2PMRESR_GROUP_MASK GENMASK(7, 0)
58
59#define L2CYCLE_CTR_BIT 31
60#define L2CYCLE_CTR_RAW_CODE 0xFE
61
62#define L2PMCR_RESET_ALL 0x6
63#define L2PMCR_COUNTERS_ENABLE 0x1
64#define L2PMCR_COUNTERS_DISABLE 0x0
65
66#define L2PMRESR_EN BIT_ULL(63)
67
68#define L2_EVT_MASK 0x00000FFF
69#define L2_EVT_CODE_MASK 0x00000FF0
70#define L2_EVT_GRP_MASK 0x0000000F
71#define L2_EVT_CODE_SHIFT 4
72#define L2_EVT_GRP_SHIFT 0
73
74#define L2_EVT_CODE(event) (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT)
75#define L2_EVT_GROUP(event) (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT)
76
77#define L2_EVT_GROUP_MAX 7
78
79#define L2_COUNTER_RELOAD BIT_ULL(31)
80#define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63)
81
82#define L2CPUSRSELR_EL1 sys_reg(3, 3, 15, 0, 6)
83#define L2CPUSRDR_EL1 sys_reg(3, 3, 15, 0, 7)
84
85#define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE)
86
87/*
88 * Events
89 */
90#define L2_EVENT_CYCLES 0xfe
91#define L2_EVENT_DCACHE_OPS 0x400
92#define L2_EVENT_ICACHE_OPS 0x401
93#define L2_EVENT_TLBI 0x402
94#define L2_EVENT_BARRIERS 0x403
95#define L2_EVENT_TOTAL_READS 0x405
96#define L2_EVENT_TOTAL_WRITES 0x406
97#define L2_EVENT_TOTAL_REQUESTS 0x407
98#define L2_EVENT_LDREX 0x420
99#define L2_EVENT_STREX 0x421
100#define L2_EVENT_CLREX 0x422
101
102static DEFINE_RAW_SPINLOCK(l2_access_lock);
103
104/**
105 * set_l2_indirect_reg: write value to an L2 register
106 * @reg: Address of L2 register.
107 * @value: Value to be written to register.
108 *
109 * Use architecturally required barriers for ordering between system register
110 * accesses
111 */
112static void set_l2_indirect_reg(u64 reg, u64 val)
113{
114 unsigned long flags;
115
116 raw_spin_lock_irqsave(&l2_access_lock, flags);
117 write_sysreg_s(reg, L2CPUSRSELR_EL1);
118 isb();
119 write_sysreg_s(val, L2CPUSRDR_EL1);
120 isb();
121 raw_spin_unlock_irqrestore(&l2_access_lock, flags);
122}
123
124/**
125 * get_l2_indirect_reg: read an L2 register value
126 * @reg: Address of L2 register.
127 *
128 * Use architecturally required barriers for ordering between system register
129 * accesses
130 */
131static u64 get_l2_indirect_reg(u64 reg)
132{
133 u64 val;
134 unsigned long flags;
135
136 raw_spin_lock_irqsave(&l2_access_lock, flags);
137 write_sysreg_s(reg, L2CPUSRSELR_EL1);
138 isb();
139 val = read_sysreg_s(L2CPUSRDR_EL1);
140 raw_spin_unlock_irqrestore(&l2_access_lock, flags);
141
142 return val;
143}
144
145struct cluster_pmu;
146
147/*
148 * Aggregate PMU. Implements the core pmu functions and manages
149 * the hardware PMUs.
150 */
151struct l2cache_pmu {
152 struct hlist_node node;
153 u32 num_pmus;
154 struct pmu pmu;
155 int num_counters;
156 cpumask_t cpumask;
157 struct platform_device *pdev;
158 struct cluster_pmu * __percpu *pmu_cluster;
159 struct list_head clusters;
160};
161
162/*
163 * The cache is made up of one or more clusters, each cluster has its own PMU.
164 * Each cluster is associated with one or more CPUs.
165 * This structure represents one of the hardware PMUs.
166 *
167 * Events can be envisioned as a 2-dimensional array. Each column represents
168 * a group of events. There are 8 groups. Only one entry from each
169 * group can be in use at a time.
170 *
171 * Events are specified as 0xCCG, where CC is 2 hex digits specifying
172 * the code (array row) and G specifies the group (column).
173 *
174 * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE
175 * which is outside the above scheme.
176 */
177struct cluster_pmu {
178 struct list_head next;
179 struct perf_event *events[MAX_L2_CTRS];
180 struct l2cache_pmu *l2cache_pmu;
181 DECLARE_BITMAP(used_counters, MAX_L2_CTRS);
182 DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1);
183 int irq;
184 int cluster_id;
185 /* The CPU that is used for collecting events on this cluster */
186 int on_cpu;
187 /* All the CPUs associated with this cluster */
188 cpumask_t cluster_cpus;
189 spinlock_t pmu_lock;
190};
191
192#define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu))
193
194static u32 l2_cycle_ctr_idx;
195static u32 l2_counter_present_mask;
196
197static inline u32 idx_to_reg_bit(u32 idx)
198{
199 if (idx == l2_cycle_ctr_idx)
200 return BIT(L2CYCLE_CTR_BIT);
201
202 return BIT(idx);
203}
204
205static inline struct cluster_pmu *get_cluster_pmu(
206 struct l2cache_pmu *l2cache_pmu, int cpu)
207{
208 return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu);
209}
210
211static void cluster_pmu_reset(void)
212{
213 /* Reset all counters */
214 set_l2_indirect_reg(L2PMCR, L2PMCR_RESET_ALL);
215 set_l2_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask);
216 set_l2_indirect_reg(L2PMINTENCLR, l2_counter_present_mask);
217 set_l2_indirect_reg(L2PMOVSCLR, l2_counter_present_mask);
218}
219
220static inline void cluster_pmu_enable(void)
221{
222 set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE);
223}
224
225static inline void cluster_pmu_disable(void)
226{
227 set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE);
228}
229
230static inline void cluster_pmu_counter_set_value(u32 idx, u64 value)
231{
232 if (idx == l2_cycle_ctr_idx)
233 set_l2_indirect_reg(L2PMCCNTR, value);
234 else
235 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value);
236}
237
238static inline u64 cluster_pmu_counter_get_value(u32 idx)
239{
240 u64 value;
241
242 if (idx == l2_cycle_ctr_idx)
243 value = get_l2_indirect_reg(L2PMCCNTR);
244 else
245 value = get_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx));
246
247 return value;
248}
249
250static inline void cluster_pmu_counter_enable(u32 idx)
251{
252 set_l2_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx));
253}
254
255static inline void cluster_pmu_counter_disable(u32 idx)
256{
257 set_l2_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx));
258}
259
260static inline void cluster_pmu_counter_enable_interrupt(u32 idx)
261{
262 set_l2_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx));
263}
264
265static inline void cluster_pmu_counter_disable_interrupt(u32 idx)
266{
267 set_l2_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx));
268}
269
270static inline void cluster_pmu_set_evccntcr(u32 val)
271{
272 set_l2_indirect_reg(L2PMCCNTCR, val);
273}
274
275static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val)
276{
277 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val);
278}
279
280static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val)
281{
282 set_l2_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val);
283}
284
285static void cluster_pmu_set_resr(struct cluster_pmu *cluster,
286 u32 event_group, u32 event_cc)
287{
288 u64 field;
289 u64 resr_val;
290 u32 shift;
291 unsigned long flags;
292
293 shift = L2PMRESR_GROUP_BITS * event_group;
294 field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift);
295
296 spin_lock_irqsave(&cluster->pmu_lock, flags);
297
298 resr_val = get_l2_indirect_reg(L2PMRESR);
299 resr_val &= ~(L2PMRESR_GROUP_MASK << shift);
300 resr_val |= field;
301 resr_val |= L2PMRESR_EN;
302 set_l2_indirect_reg(L2PMRESR, resr_val);
303
304 spin_unlock_irqrestore(&cluster->pmu_lock, flags);
305}
306
307/*
308 * Hardware allows filtering of events based on the originating
309 * CPU. Turn this off by setting filter bits to allow events from
310 * all CPUS, subunits and ID independent events in this cluster.
311 */
312static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr)
313{
314 u32 val = L2PMXEVFILTER_SUFILTER_ALL |
315 L2PMXEVFILTER_ORGFILTER_IDINDEP |
316 L2PMXEVFILTER_ORGFILTER_ALL;
317
318 set_l2_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val);
319}
320
321static inline u32 cluster_pmu_getreset_ovsr(void)
322{
323 u32 result = get_l2_indirect_reg(L2PMOVSSET);
324
325 set_l2_indirect_reg(L2PMOVSCLR, result);
326 return result;
327}
328
329static inline bool cluster_pmu_has_overflowed(u32 ovsr)
330{
331 return !!(ovsr & l2_counter_present_mask);
332}
333
334static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx)
335{
336 return !!(ovsr & idx_to_reg_bit(idx));
337}
338
339static void l2_cache_event_update(struct perf_event *event)
340{
341 struct hw_perf_event *hwc = &event->hw;
342 u64 delta, prev, now;
343 u32 idx = hwc->idx;
344
345 do {
346 prev = local64_read(&hwc->prev_count);
347 now = cluster_pmu_counter_get_value(idx);
348 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
349
350 /*
351 * The cycle counter is 64-bit, but all other counters are
352 * 32-bit, and we must handle 32-bit overflow explicitly.
353 */
354 delta = now - prev;
355 if (idx != l2_cycle_ctr_idx)
356 delta &= 0xffffffff;
357
358 local64_add(delta, &event->count);
359}
360
361static void l2_cache_cluster_set_period(struct cluster_pmu *cluster,
362 struct hw_perf_event *hwc)
363{
364 u32 idx = hwc->idx;
365 u64 new;
366
367 /*
368 * We limit the max period to half the max counter value so
369 * that even in the case of extreme interrupt latency the
370 * counter will (hopefully) not wrap past its initial value.
371 */
372 if (idx == l2_cycle_ctr_idx)
373 new = L2_CYCLE_COUNTER_RELOAD;
374 else
375 new = L2_COUNTER_RELOAD;
376
377 local64_set(&hwc->prev_count, new);
378 cluster_pmu_counter_set_value(idx, new);
379}
380
381static int l2_cache_get_event_idx(struct cluster_pmu *cluster,
382 struct perf_event *event)
383{
384 struct hw_perf_event *hwc = &event->hw;
385 int idx;
386 int num_ctrs = cluster->l2cache_pmu->num_counters - 1;
387 unsigned int group;
388
389 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
390 if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters))
391 return -EAGAIN;
392
393 return l2_cycle_ctr_idx;
394 }
395
396 idx = find_first_zero_bit(cluster->used_counters, num_ctrs);
397 if (idx == num_ctrs)
398 /* The counters are all in use. */
399 return -EAGAIN;
400
401 /*
402 * Check for column exclusion: event column already in use by another
403 * event. This is for events which are not in the same group.
404 * Conflicting events in the same group are detected in event_init.
405 */
406 group = L2_EVT_GROUP(hwc->config_base);
407 if (test_bit(group, cluster->used_groups))
408 return -EAGAIN;
409
410 set_bit(idx, cluster->used_counters);
411 set_bit(group, cluster->used_groups);
412
413 return idx;
414}
415
416static void l2_cache_clear_event_idx(struct cluster_pmu *cluster,
417 struct perf_event *event)
418{
419 struct hw_perf_event *hwc = &event->hw;
420 int idx = hwc->idx;
421
422 clear_bit(idx, cluster->used_counters);
423 if (hwc->config_base != L2CYCLE_CTR_RAW_CODE)
424 clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups);
425}
426
427static irqreturn_t l2_cache_handle_irq(int irq_num, void *data)
428{
429 struct cluster_pmu *cluster = data;
430 int num_counters = cluster->l2cache_pmu->num_counters;
431 u32 ovsr;
432 int idx;
433
434 ovsr = cluster_pmu_getreset_ovsr();
435 if (!cluster_pmu_has_overflowed(ovsr))
436 return IRQ_NONE;
437
438 for_each_set_bit(idx, cluster->used_counters, num_counters) {
439 struct perf_event *event = cluster->events[idx];
440 struct hw_perf_event *hwc;
441
442 if (WARN_ON_ONCE(!event))
443 continue;
444
445 if (!cluster_pmu_counter_has_overflowed(ovsr, idx))
446 continue;
447
448 l2_cache_event_update(event);
449 hwc = &event->hw;
450
451 l2_cache_cluster_set_period(cluster, hwc);
452 }
453
454 return IRQ_HANDLED;
455}
456
457/*
458 * Implementation of abstract pmu functionality required by
459 * the core perf events code.
460 */
461
462static void l2_cache_pmu_enable(struct pmu *pmu)
463{
464 /*
465 * Although there is only one PMU (per socket) controlling multiple
466 * physical PMUs (per cluster), because we do not support per-task mode
467 * each event is associated with a CPU. Each event has pmu_enable
468 * called on its CPU, so here it is only necessary to enable the
469 * counters for the current CPU.
470 */
471
472 cluster_pmu_enable();
473}
474
475static void l2_cache_pmu_disable(struct pmu *pmu)
476{
477 cluster_pmu_disable();
478}
479
480static int l2_cache_event_init(struct perf_event *event)
481{
482 struct hw_perf_event *hwc = &event->hw;
483 struct cluster_pmu *cluster;
484 struct perf_event *sibling;
485 struct l2cache_pmu *l2cache_pmu;
486
487 if (event->attr.type != event->pmu->type)
488 return -ENOENT;
489
490 l2cache_pmu = to_l2cache_pmu(event->pmu);
491
492 if (hwc->sample_period) {
493 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
494 "Sampling not supported\n");
495 return -EOPNOTSUPP;
496 }
497
498 if (event->cpu < 0) {
499 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
500 "Per-task mode not supported\n");
501 return -EOPNOTSUPP;
502 }
503
504 if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) ||
505 ((event->attr.config & ~L2_EVT_MASK) != 0)) &&
506 (event->attr.config != L2CYCLE_CTR_RAW_CODE)) {
507 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
508 "Invalid config %llx\n",
509 event->attr.config);
510 return -EINVAL;
511 }
512
513 /* Don't allow groups with mixed PMUs, except for s/w events */
514 if (event->group_leader->pmu != event->pmu &&
515 !is_software_event(event->group_leader)) {
516 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
517 "Can't create mixed PMU group\n");
518 return -EINVAL;
519 }
520
521 for_each_sibling_event(sibling, event->group_leader) {
522 if (sibling->pmu != event->pmu &&
523 !is_software_event(sibling)) {
524 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
525 "Can't create mixed PMU group\n");
526 return -EINVAL;
527 }
528 }
529
530 cluster = get_cluster_pmu(l2cache_pmu, event->cpu);
531 if (!cluster) {
532 /* CPU has not been initialised */
533 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
534 "CPU%d not associated with L2 cluster\n", event->cpu);
535 return -EINVAL;
536 }
537
538 /* Ensure all events in a group are on the same cpu */
539 if ((event->group_leader != event) &&
540 (cluster->on_cpu != event->group_leader->cpu)) {
541 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
542 "Can't create group on CPUs %d and %d",
543 event->cpu, event->group_leader->cpu);
544 return -EINVAL;
545 }
546
547 if ((event != event->group_leader) &&
548 !is_software_event(event->group_leader) &&
549 (L2_EVT_GROUP(event->group_leader->attr.config) ==
550 L2_EVT_GROUP(event->attr.config))) {
551 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
552 "Column exclusion: conflicting events %llx %llx\n",
553 event->group_leader->attr.config,
554 event->attr.config);
555 return -EINVAL;
556 }
557
558 for_each_sibling_event(sibling, event->group_leader) {
559 if ((sibling != event) &&
560 !is_software_event(sibling) &&
561 (L2_EVT_GROUP(sibling->attr.config) ==
562 L2_EVT_GROUP(event->attr.config))) {
563 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
564 "Column exclusion: conflicting events %llx %llx\n",
565 sibling->attr.config,
566 event->attr.config);
567 return -EINVAL;
568 }
569 }
570
571 hwc->idx = -1;
572 hwc->config_base = event->attr.config;
573
574 /*
575 * Ensure all events are on the same cpu so all events are in the
576 * same cpu context, to avoid races on pmu_enable etc.
577 */
578 event->cpu = cluster->on_cpu;
579
580 return 0;
581}
582
583static void l2_cache_event_start(struct perf_event *event, int flags)
584{
585 struct cluster_pmu *cluster;
586 struct hw_perf_event *hwc = &event->hw;
587 int idx = hwc->idx;
588 u32 config;
589 u32 event_cc, event_group;
590
591 hwc->state = 0;
592
593 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
594
595 l2_cache_cluster_set_period(cluster, hwc);
596
597 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
598 cluster_pmu_set_evccntcr(0);
599 } else {
600 config = hwc->config_base;
601 event_cc = L2_EVT_CODE(config);
602 event_group = L2_EVT_GROUP(config);
603
604 cluster_pmu_set_evcntcr(idx, 0);
605 cluster_pmu_set_evtyper(idx, event_group);
606 cluster_pmu_set_resr(cluster, event_group, event_cc);
607 cluster_pmu_set_evfilter_sys_mode(idx);
608 }
609
610 cluster_pmu_counter_enable_interrupt(idx);
611 cluster_pmu_counter_enable(idx);
612}
613
614static void l2_cache_event_stop(struct perf_event *event, int flags)
615{
616 struct hw_perf_event *hwc = &event->hw;
617 int idx = hwc->idx;
618
619 if (hwc->state & PERF_HES_STOPPED)
620 return;
621
622 cluster_pmu_counter_disable_interrupt(idx);
623 cluster_pmu_counter_disable(idx);
624
625 if (flags & PERF_EF_UPDATE)
626 l2_cache_event_update(event);
627 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
628}
629
630static int l2_cache_event_add(struct perf_event *event, int flags)
631{
632 struct hw_perf_event *hwc = &event->hw;
633 int idx;
634 int err = 0;
635 struct cluster_pmu *cluster;
636
637 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
638
639 idx = l2_cache_get_event_idx(cluster, event);
640 if (idx < 0)
641 return idx;
642
643 hwc->idx = idx;
644 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
645 cluster->events[idx] = event;
646 local64_set(&hwc->prev_count, 0);
647
648 if (flags & PERF_EF_START)
649 l2_cache_event_start(event, flags);
650
651 /* Propagate changes to the userspace mapping. */
652 perf_event_update_userpage(event);
653
654 return err;
655}
656
657static void l2_cache_event_del(struct perf_event *event, int flags)
658{
659 struct hw_perf_event *hwc = &event->hw;
660 struct cluster_pmu *cluster;
661 int idx = hwc->idx;
662
663 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
664
665 l2_cache_event_stop(event, flags | PERF_EF_UPDATE);
666 cluster->events[idx] = NULL;
667 l2_cache_clear_event_idx(cluster, event);
668
669 perf_event_update_userpage(event);
670}
671
672static void l2_cache_event_read(struct perf_event *event)
673{
674 l2_cache_event_update(event);
675}
676
677static ssize_t l2_cache_pmu_cpumask_show(struct device *dev,
678 struct device_attribute *attr,
679 char *buf)
680{
681 struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev));
682
683 return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask);
684}
685
686static struct device_attribute l2_cache_pmu_cpumask_attr =
687 __ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL);
688
689static struct attribute *l2_cache_pmu_cpumask_attrs[] = {
690 &l2_cache_pmu_cpumask_attr.attr,
691 NULL,
692};
693
694static struct attribute_group l2_cache_pmu_cpumask_group = {
695 .attrs = l2_cache_pmu_cpumask_attrs,
696};
697
698/* CCG format for perf RAW codes. */
699PMU_FORMAT_ATTR(l2_code, "config:4-11");
700PMU_FORMAT_ATTR(l2_group, "config:0-3");
701PMU_FORMAT_ATTR(event, "config:0-11");
702
703static struct attribute *l2_cache_pmu_formats[] = {
704 &format_attr_l2_code.attr,
705 &format_attr_l2_group.attr,
706 &format_attr_event.attr,
707 NULL,
708};
709
710static struct attribute_group l2_cache_pmu_format_group = {
711 .name = "format",
712 .attrs = l2_cache_pmu_formats,
713};
714
715static ssize_t l2cache_pmu_event_show(struct device *dev,
716 struct device_attribute *attr, char *page)
717{
718 struct perf_pmu_events_attr *pmu_attr;
719
720 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
721 return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
722}
723
724#define L2CACHE_EVENT_ATTR(_name, _id) \
725 (&((struct perf_pmu_events_attr[]) { \
726 { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \
727 .id = _id, } \
728 })[0].attr.attr)
729
730static struct attribute *l2_cache_pmu_events[] = {
731 L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES),
732 L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS),
733 L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS),
734 L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI),
735 L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS),
736 L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS),
737 L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES),
738 L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS),
739 L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX),
740 L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX),
741 L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX),
742 NULL
743};
744
745static struct attribute_group l2_cache_pmu_events_group = {
746 .name = "events",
747 .attrs = l2_cache_pmu_events,
748};
749
750static const struct attribute_group *l2_cache_pmu_attr_grps[] = {
751 &l2_cache_pmu_format_group,
752 &l2_cache_pmu_cpumask_group,
753 &l2_cache_pmu_events_group,
754 NULL,
755};
756
757/*
758 * Generic device handlers
759 */
760
761static const struct acpi_device_id l2_cache_pmu_acpi_match[] = {
762 { "QCOM8130", },
763 { }
764};
765
766static int get_num_counters(void)
767{
768 int val;
769
770 val = get_l2_indirect_reg(L2PMCR);
771
772 /*
773 * Read number of counters from L2PMCR and add 1
774 * for the cycle counter.
775 */
776 return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1;
777}
778
779static struct cluster_pmu *l2_cache_associate_cpu_with_cluster(
780 struct l2cache_pmu *l2cache_pmu, int cpu)
781{
782 u64 mpidr;
783 int cpu_cluster_id;
784 struct cluster_pmu *cluster = NULL;
785
786 /*
787 * This assumes that the cluster_id is in MPIDR[aff1] for
788 * single-threaded cores, and MPIDR[aff2] for multi-threaded
789 * cores. This logic will have to be updated if this changes.
790 */
791 mpidr = read_cpuid_mpidr();
792 if (mpidr & MPIDR_MT_BITMASK)
793 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
794 else
795 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
796
797 list_for_each_entry(cluster, &l2cache_pmu->clusters, next) {
798 if (cluster->cluster_id != cpu_cluster_id)
799 continue;
800
801 dev_info(&l2cache_pmu->pdev->dev,
802 "CPU%d associated with cluster %d\n", cpu,
803 cluster->cluster_id);
804 cpumask_set_cpu(cpu, &cluster->cluster_cpus);
805 *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster;
806 break;
807 }
808
809 return cluster;
810}
811
812static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
813{
814 struct cluster_pmu *cluster;
815 struct l2cache_pmu *l2cache_pmu;
816
817 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
818 cluster = get_cluster_pmu(l2cache_pmu, cpu);
819 if (!cluster) {
820 /* First time this CPU has come online */
821 cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu);
822 if (!cluster) {
823 /* Only if broken firmware doesn't list every cluster */
824 WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu);
825 return 0;
826 }
827 }
828
829 /* If another CPU is managing this cluster, we're done */
830 if (cluster->on_cpu != -1)
831 return 0;
832
833 /*
834 * All CPUs on this cluster were down, use this one.
835 * Reset to put it into sane state.
836 */
837 cluster->on_cpu = cpu;
838 cpumask_set_cpu(cpu, &l2cache_pmu->cpumask);
839 cluster_pmu_reset();
840
841 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu)));
842 enable_irq(cluster->irq);
843
844 return 0;
845}
846
847static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
848{
849 struct cluster_pmu *cluster;
850 struct l2cache_pmu *l2cache_pmu;
851 cpumask_t cluster_online_cpus;
852 unsigned int target;
853
854 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
855 cluster = get_cluster_pmu(l2cache_pmu, cpu);
856 if (!cluster)
857 return 0;
858
859 /* If this CPU is not managing the cluster, we're done */
860 if (cluster->on_cpu != cpu)
861 return 0;
862
863 /* Give up ownership of cluster */
864 cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask);
865 cluster->on_cpu = -1;
866
867 /* Any other CPU for this cluster which is still online */
868 cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus,
869 cpu_online_mask);
870 target = cpumask_any_but(&cluster_online_cpus, cpu);
871 if (target >= nr_cpu_ids) {
872 disable_irq(cluster->irq);
873 return 0;
874 }
875
876 perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target);
877 cluster->on_cpu = target;
878 cpumask_set_cpu(target, &l2cache_pmu->cpumask);
879 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target)));
880
881 return 0;
882}
883
884static int l2_cache_pmu_probe_cluster(struct device *dev, void *data)
885{
886 struct platform_device *pdev = to_platform_device(dev->parent);
887 struct platform_device *sdev = to_platform_device(dev);
888 struct l2cache_pmu *l2cache_pmu = data;
889 struct cluster_pmu *cluster;
890 struct acpi_device *device;
891 unsigned long fw_cluster_id;
892 int err;
893 int irq;
894
895 if (acpi_bus_get_device(ACPI_HANDLE(dev), &device))
896 return -ENODEV;
897
898 if (kstrtoul(device->pnp.unique_id, 10, &fw_cluster_id) < 0) {
899 dev_err(&pdev->dev, "unable to read ACPI uid\n");
900 return -ENODEV;
901 }
902
903 cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL);
904 if (!cluster)
905 return -ENOMEM;
906
907 INIT_LIST_HEAD(&cluster->next);
908 list_add(&cluster->next, &l2cache_pmu->clusters);
909 cluster->cluster_id = fw_cluster_id;
910
911 irq = platform_get_irq(sdev, 0);
912 if (irq < 0)
913 return irq;
914 irq_set_status_flags(irq, IRQ_NOAUTOEN);
915 cluster->irq = irq;
916
917 cluster->l2cache_pmu = l2cache_pmu;
918 cluster->on_cpu = -1;
919
920 err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq,
921 IRQF_NOBALANCING | IRQF_NO_THREAD,
922 "l2-cache-pmu", cluster);
923 if (err) {
924 dev_err(&pdev->dev,
925 "Unable to request IRQ%d for L2 PMU counters\n", irq);
926 return err;
927 }
928
929 dev_info(&pdev->dev,
930 "Registered L2 cache PMU cluster %ld\n", fw_cluster_id);
931
932 spin_lock_init(&cluster->pmu_lock);
933
934 l2cache_pmu->num_pmus++;
935
936 return 0;
937}
938
939static int l2_cache_pmu_probe(struct platform_device *pdev)
940{
941 int err;
942 struct l2cache_pmu *l2cache_pmu;
943
944 l2cache_pmu =
945 devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL);
946 if (!l2cache_pmu)
947 return -ENOMEM;
948
949 INIT_LIST_HEAD(&l2cache_pmu->clusters);
950
951 platform_set_drvdata(pdev, l2cache_pmu);
952 l2cache_pmu->pmu = (struct pmu) {
953 /* suffix is instance id for future use with multiple sockets */
954 .name = "l2cache_0",
955 .task_ctx_nr = perf_invalid_context,
956 .pmu_enable = l2_cache_pmu_enable,
957 .pmu_disable = l2_cache_pmu_disable,
958 .event_init = l2_cache_event_init,
959 .add = l2_cache_event_add,
960 .del = l2_cache_event_del,
961 .start = l2_cache_event_start,
962 .stop = l2_cache_event_stop,
963 .read = l2_cache_event_read,
964 .attr_groups = l2_cache_pmu_attr_grps,
965 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
966 };
967
968 l2cache_pmu->num_counters = get_num_counters();
969 l2cache_pmu->pdev = pdev;
970 l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev,
971 struct cluster_pmu *);
972 if (!l2cache_pmu->pmu_cluster)
973 return -ENOMEM;
974
975 l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1;
976 l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) |
977 BIT(L2CYCLE_CTR_BIT);
978
979 cpumask_clear(&l2cache_pmu->cpumask);
980
981 /* Read cluster info and initialize each cluster */
982 err = device_for_each_child(&pdev->dev, l2cache_pmu,
983 l2_cache_pmu_probe_cluster);
984 if (err)
985 return err;
986
987 if (l2cache_pmu->num_pmus == 0) {
988 dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n");
989 return -ENODEV;
990 }
991
992 err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
993 &l2cache_pmu->node);
994 if (err) {
995 dev_err(&pdev->dev, "Error %d registering hotplug", err);
996 return err;
997 }
998
999 err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1);
1000 if (err) {
1001 dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err);
1002 goto out_unregister;
1003 }
1004
1005 dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n",
1006 l2cache_pmu->num_pmus);
1007
1008 return err;
1009
1010out_unregister:
1011 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1012 &l2cache_pmu->node);
1013 return err;
1014}
1015
1016static int l2_cache_pmu_remove(struct platform_device *pdev)
1017{
1018 struct l2cache_pmu *l2cache_pmu =
1019 to_l2cache_pmu(platform_get_drvdata(pdev));
1020
1021 perf_pmu_unregister(&l2cache_pmu->pmu);
1022 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1023 &l2cache_pmu->node);
1024 return 0;
1025}
1026
1027static struct platform_driver l2_cache_pmu_driver = {
1028 .driver = {
1029 .name = "qcom-l2cache-pmu",
1030 .acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match),
1031 },
1032 .probe = l2_cache_pmu_probe,
1033 .remove = l2_cache_pmu_remove,
1034};
1035
1036static int __init register_l2_cache_pmu_driver(void)
1037{
1038 int err;
1039
1040 err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1041 "AP_PERF_ARM_QCOM_L2_ONLINE",
1042 l2cache_pmu_online_cpu,
1043 l2cache_pmu_offline_cpu);
1044 if (err)
1045 return err;
1046
1047 return platform_driver_register(&l2_cache_pmu_driver);
1048}
1049device_initcall(register_l2_cache_pmu_driver);