Loading...
Note: File does not exist in v5.9.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ARM CoreSight Architecture PMU driver.
4 *
5 * This driver adds support for uncore PMU based on ARM CoreSight Performance
6 * Monitoring Unit Architecture. The PMU is accessible via MMIO registers and
7 * like other uncore PMUs, it does not support process specific events and
8 * cannot be used in sampling mode.
9 *
10 * This code is based on other uncore PMUs like ARM DSU PMU. It provides a
11 * generic implementation to operate the PMU according to CoreSight PMU
12 * architecture and ACPI ARM PMU table (APMT) documents below:
13 * - ARM CoreSight PMU architecture document number: ARM IHI 0091 A.a-00bet0.
14 * - APMT document number: ARM DEN0117.
15 *
16 * The user should refer to the vendor technical documentation to get details
17 * about the supported events.
18 *
19 * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
20 *
21 */
22
23#include <linux/acpi.h>
24#include <linux/cacheinfo.h>
25#include <linux/ctype.h>
26#include <linux/interrupt.h>
27#include <linux/io-64-nonatomic-lo-hi.h>
28#include <linux/module.h>
29#include <linux/perf_event.h>
30#include <linux/platform_device.h>
31#include <acpi/processor.h>
32
33#include "arm_cspmu.h"
34#include "nvidia_cspmu.h"
35
36#define PMUNAME "arm_cspmu"
37#define DRVNAME "arm-cs-arch-pmu"
38
39#define ARM_CSPMU_CPUMASK_ATTR(_name, _config) \
40 ARM_CSPMU_EXT_ATTR(_name, arm_cspmu_cpumask_show, \
41 (unsigned long)_config)
42
43/*
44 * CoreSight PMU Arch register offsets.
45 */
46#define PMEVCNTR_LO 0x0
47#define PMEVCNTR_HI 0x4
48#define PMEVTYPER 0x400
49#define PMCCFILTR 0x47C
50#define PMEVFILTR 0xA00
51#define PMCNTENSET 0xC00
52#define PMCNTENCLR 0xC20
53#define PMINTENSET 0xC40
54#define PMINTENCLR 0xC60
55#define PMOVSCLR 0xC80
56#define PMOVSSET 0xCC0
57#define PMCFGR 0xE00
58#define PMCR 0xE04
59#define PMIIDR 0xE08
60
61/* PMCFGR register field */
62#define PMCFGR_NCG GENMASK(31, 28)
63#define PMCFGR_HDBG BIT(24)
64#define PMCFGR_TRO BIT(23)
65#define PMCFGR_SS BIT(22)
66#define PMCFGR_FZO BIT(21)
67#define PMCFGR_MSI BIT(20)
68#define PMCFGR_UEN BIT(19)
69#define PMCFGR_NA BIT(17)
70#define PMCFGR_EX BIT(16)
71#define PMCFGR_CCD BIT(15)
72#define PMCFGR_CC BIT(14)
73#define PMCFGR_SIZE GENMASK(13, 8)
74#define PMCFGR_N GENMASK(7, 0)
75
76/* PMCR register field */
77#define PMCR_TRO BIT(11)
78#define PMCR_HDBG BIT(10)
79#define PMCR_FZO BIT(9)
80#define PMCR_NA BIT(8)
81#define PMCR_DP BIT(5)
82#define PMCR_X BIT(4)
83#define PMCR_D BIT(3)
84#define PMCR_C BIT(2)
85#define PMCR_P BIT(1)
86#define PMCR_E BIT(0)
87
88/* Each SET/CLR register supports up to 32 counters. */
89#define ARM_CSPMU_SET_CLR_COUNTER_SHIFT 5
90#define ARM_CSPMU_SET_CLR_COUNTER_NUM \
91 (1 << ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
92
93/* Convert counter idx into SET/CLR register number. */
94#define COUNTER_TO_SET_CLR_ID(idx) \
95 (idx >> ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
96
97/* Convert counter idx into SET/CLR register bit. */
98#define COUNTER_TO_SET_CLR_BIT(idx) \
99 (idx & (ARM_CSPMU_SET_CLR_COUNTER_NUM - 1))
100
101#define ARM_CSPMU_ACTIVE_CPU_MASK 0x0
102#define ARM_CSPMU_ASSOCIATED_CPU_MASK 0x1
103
104/* Check if field f in flags is set with value v */
105#define CHECK_APMT_FLAG(flags, f, v) \
106 ((flags & (ACPI_APMT_FLAGS_ ## f)) == (ACPI_APMT_FLAGS_ ## f ## _ ## v))
107
108/* Check and use default if implementer doesn't provide attribute callback */
109#define CHECK_DEFAULT_IMPL_OPS(ops, callback) \
110 do { \
111 if (!ops->callback) \
112 ops->callback = arm_cspmu_ ## callback; \
113 } while (0)
114
115/*
116 * Maximum poll count for reading counter value using high-low-high sequence.
117 */
118#define HILOHI_MAX_POLL 1000
119
120/* JEDEC-assigned JEP106 identification code */
121#define ARM_CSPMU_IMPL_ID_NVIDIA 0x36B
122
123static unsigned long arm_cspmu_cpuhp_state;
124
125/*
126 * In CoreSight PMU architecture, all of the MMIO registers are 32-bit except
127 * counter register. The counter register can be implemented as 32-bit or 64-bit
128 * register depending on the value of PMCFGR.SIZE field. For 64-bit access,
129 * single-copy 64-bit atomic support is implementation defined. APMT node flag
130 * is used to identify if the PMU supports 64-bit single copy atomic. If 64-bit
131 * single copy atomic is not supported, the driver treats the register as a pair
132 * of 32-bit register.
133 */
134
135/*
136 * Read 64-bit register as a pair of 32-bit registers using hi-lo-hi sequence.
137 */
138static u64 read_reg64_hilohi(const void __iomem *addr, u32 max_poll_count)
139{
140 u32 val_lo, val_hi;
141 u64 val;
142
143 /* Use high-low-high sequence to avoid tearing */
144 do {
145 if (max_poll_count-- == 0) {
146 pr_err("ARM CSPMU: timeout hi-low-high sequence\n");
147 return 0;
148 }
149
150 val_hi = readl(addr + 4);
151 val_lo = readl(addr);
152 } while (val_hi != readl(addr + 4));
153
154 val = (((u64)val_hi << 32) | val_lo);
155
156 return val;
157}
158
159/* Check if PMU supports 64-bit single copy atomic. */
160static inline bool supports_64bit_atomics(const struct arm_cspmu *cspmu)
161{
162 return CHECK_APMT_FLAG(cspmu->apmt_node->flags, ATOMIC, SUPP);
163}
164
165/* Check if cycle counter is supported. */
166static inline bool supports_cycle_counter(const struct arm_cspmu *cspmu)
167{
168 return (cspmu->pmcfgr & PMCFGR_CC);
169}
170
171/* Get counter size, which is (PMCFGR_SIZE + 1). */
172static inline u32 counter_size(const struct arm_cspmu *cspmu)
173{
174 return FIELD_GET(PMCFGR_SIZE, cspmu->pmcfgr) + 1;
175}
176
177/* Get counter mask. */
178static inline u64 counter_mask(const struct arm_cspmu *cspmu)
179{
180 return GENMASK_ULL(counter_size(cspmu) - 1, 0);
181}
182
183/* Check if counter is implemented as 64-bit register. */
184static inline bool use_64b_counter_reg(const struct arm_cspmu *cspmu)
185{
186 return (counter_size(cspmu) > 32);
187}
188
189ssize_t arm_cspmu_sysfs_event_show(struct device *dev,
190 struct device_attribute *attr, char *buf)
191{
192 struct dev_ext_attribute *eattr =
193 container_of(attr, struct dev_ext_attribute, attr);
194 return sysfs_emit(buf, "event=0x%llx\n",
195 (unsigned long long)eattr->var);
196}
197EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_event_show);
198
199/* Default event list. */
200static struct attribute *arm_cspmu_event_attrs[] = {
201 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
202 NULL,
203};
204
205static struct attribute **
206arm_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
207{
208 struct attribute **attrs;
209
210 attrs = devm_kmemdup(cspmu->dev, arm_cspmu_event_attrs,
211 sizeof(arm_cspmu_event_attrs), GFP_KERNEL);
212
213 return attrs;
214}
215
216static umode_t
217arm_cspmu_event_attr_is_visible(struct kobject *kobj,
218 struct attribute *attr, int unused)
219{
220 struct device *dev = kobj_to_dev(kobj);
221 struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
222 struct perf_pmu_events_attr *eattr;
223
224 eattr = container_of(attr, typeof(*eattr), attr.attr);
225
226 /* Hide cycle event if not supported */
227 if (!supports_cycle_counter(cspmu) &&
228 eattr->id == ARM_CSPMU_EVT_CYCLES_DEFAULT)
229 return 0;
230
231 return attr->mode;
232}
233
234ssize_t arm_cspmu_sysfs_format_show(struct device *dev,
235 struct device_attribute *attr,
236 char *buf)
237{
238 struct dev_ext_attribute *eattr =
239 container_of(attr, struct dev_ext_attribute, attr);
240 return sysfs_emit(buf, "%s\n", (char *)eattr->var);
241}
242EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_format_show);
243
244static struct attribute *arm_cspmu_format_attrs[] = {
245 ARM_CSPMU_FORMAT_EVENT_ATTR,
246 ARM_CSPMU_FORMAT_FILTER_ATTR,
247 NULL,
248};
249
250static struct attribute **
251arm_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
252{
253 struct attribute **attrs;
254
255 attrs = devm_kmemdup(cspmu->dev, arm_cspmu_format_attrs,
256 sizeof(arm_cspmu_format_attrs), GFP_KERNEL);
257
258 return attrs;
259}
260
261static u32 arm_cspmu_event_type(const struct perf_event *event)
262{
263 return event->attr.config & ARM_CSPMU_EVENT_MASK;
264}
265
266static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event)
267{
268 return (event->attr.config == ARM_CSPMU_EVT_CYCLES_DEFAULT);
269}
270
271static u32 arm_cspmu_event_filter(const struct perf_event *event)
272{
273 return event->attr.config1 & ARM_CSPMU_FILTER_MASK;
274}
275
276static ssize_t arm_cspmu_identifier_show(struct device *dev,
277 struct device_attribute *attr,
278 char *page)
279{
280 struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
281
282 return sysfs_emit(page, "%s\n", cspmu->identifier);
283}
284
285static struct device_attribute arm_cspmu_identifier_attr =
286 __ATTR(identifier, 0444, arm_cspmu_identifier_show, NULL);
287
288static struct attribute *arm_cspmu_identifier_attrs[] = {
289 &arm_cspmu_identifier_attr.attr,
290 NULL,
291};
292
293static struct attribute_group arm_cspmu_identifier_attr_group = {
294 .attrs = arm_cspmu_identifier_attrs,
295};
296
297static const char *arm_cspmu_get_identifier(const struct arm_cspmu *cspmu)
298{
299 const char *identifier =
300 devm_kasprintf(cspmu->dev, GFP_KERNEL, "%x",
301 cspmu->impl.pmiidr);
302 return identifier;
303}
304
305static const char *arm_cspmu_type_str[ACPI_APMT_NODE_TYPE_COUNT] = {
306 "mc",
307 "smmu",
308 "pcie",
309 "acpi",
310 "cache",
311};
312
313static const char *arm_cspmu_get_name(const struct arm_cspmu *cspmu)
314{
315 struct device *dev;
316 struct acpi_apmt_node *apmt_node;
317 u8 pmu_type;
318 char *name;
319 char acpi_hid_string[ACPI_ID_LEN] = { 0 };
320 static atomic_t pmu_idx[ACPI_APMT_NODE_TYPE_COUNT] = { 0 };
321
322 dev = cspmu->dev;
323 apmt_node = cspmu->apmt_node;
324 pmu_type = apmt_node->type;
325
326 if (pmu_type >= ACPI_APMT_NODE_TYPE_COUNT) {
327 dev_err(dev, "unsupported PMU type-%u\n", pmu_type);
328 return NULL;
329 }
330
331 if (pmu_type == ACPI_APMT_NODE_TYPE_ACPI) {
332 memcpy(acpi_hid_string,
333 &apmt_node->inst_primary,
334 sizeof(apmt_node->inst_primary));
335 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME,
336 arm_cspmu_type_str[pmu_type],
337 acpi_hid_string,
338 apmt_node->inst_secondary);
339 } else {
340 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%d", PMUNAME,
341 arm_cspmu_type_str[pmu_type],
342 atomic_fetch_inc(&pmu_idx[pmu_type]));
343 }
344
345 return name;
346}
347
348static ssize_t arm_cspmu_cpumask_show(struct device *dev,
349 struct device_attribute *attr,
350 char *buf)
351{
352 struct pmu *pmu = dev_get_drvdata(dev);
353 struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
354 struct dev_ext_attribute *eattr =
355 container_of(attr, struct dev_ext_attribute, attr);
356 unsigned long mask_id = (unsigned long)eattr->var;
357 const cpumask_t *cpumask;
358
359 switch (mask_id) {
360 case ARM_CSPMU_ACTIVE_CPU_MASK:
361 cpumask = &cspmu->active_cpu;
362 break;
363 case ARM_CSPMU_ASSOCIATED_CPU_MASK:
364 cpumask = &cspmu->associated_cpus;
365 break;
366 default:
367 return 0;
368 }
369 return cpumap_print_to_pagebuf(true, buf, cpumask);
370}
371
372static struct attribute *arm_cspmu_cpumask_attrs[] = {
373 ARM_CSPMU_CPUMASK_ATTR(cpumask, ARM_CSPMU_ACTIVE_CPU_MASK),
374 ARM_CSPMU_CPUMASK_ATTR(associated_cpus, ARM_CSPMU_ASSOCIATED_CPU_MASK),
375 NULL,
376};
377
378static struct attribute_group arm_cspmu_cpumask_attr_group = {
379 .attrs = arm_cspmu_cpumask_attrs,
380};
381
382struct impl_match {
383 u32 pmiidr;
384 u32 mask;
385 int (*impl_init_ops)(struct arm_cspmu *cspmu);
386};
387
388static const struct impl_match impl_match[] = {
389 {
390 .pmiidr = ARM_CSPMU_IMPL_ID_NVIDIA,
391 .mask = ARM_CSPMU_PMIIDR_IMPLEMENTER,
392 .impl_init_ops = nv_cspmu_init_ops
393 },
394 {}
395};
396
397static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu)
398{
399 int ret;
400 struct acpi_apmt_node *apmt_node = cspmu->apmt_node;
401 struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
402 const struct impl_match *match = impl_match;
403
404 /*
405 * Get PMU implementer and product id from APMT node.
406 * If APMT node doesn't have implementer/product id, try get it
407 * from PMIIDR.
408 */
409 cspmu->impl.pmiidr =
410 (apmt_node->impl_id) ? apmt_node->impl_id :
411 readl(cspmu->base0 + PMIIDR);
412
413 /* Find implementer specific attribute ops. */
414 for (; match->pmiidr; match++) {
415 const u32 mask = match->mask;
416
417 if ((match->pmiidr & mask) == (cspmu->impl.pmiidr & mask)) {
418 ret = match->impl_init_ops(cspmu);
419 if (ret)
420 return ret;
421
422 break;
423 }
424 }
425
426 /* Use default callbacks if implementer doesn't provide one. */
427 CHECK_DEFAULT_IMPL_OPS(impl_ops, get_event_attrs);
428 CHECK_DEFAULT_IMPL_OPS(impl_ops, get_format_attrs);
429 CHECK_DEFAULT_IMPL_OPS(impl_ops, get_identifier);
430 CHECK_DEFAULT_IMPL_OPS(impl_ops, get_name);
431 CHECK_DEFAULT_IMPL_OPS(impl_ops, is_cycle_counter_event);
432 CHECK_DEFAULT_IMPL_OPS(impl_ops, event_type);
433 CHECK_DEFAULT_IMPL_OPS(impl_ops, event_filter);
434 CHECK_DEFAULT_IMPL_OPS(impl_ops, event_attr_is_visible);
435
436 return 0;
437}
438
439static struct attribute_group *
440arm_cspmu_alloc_event_attr_group(struct arm_cspmu *cspmu)
441{
442 struct attribute_group *event_group;
443 struct device *dev = cspmu->dev;
444 const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
445
446 event_group =
447 devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
448 if (!event_group)
449 return NULL;
450
451 event_group->name = "events";
452 event_group->is_visible = impl_ops->event_attr_is_visible;
453 event_group->attrs = impl_ops->get_event_attrs(cspmu);
454
455 if (!event_group->attrs)
456 return NULL;
457
458 return event_group;
459}
460
461static struct attribute_group *
462arm_cspmu_alloc_format_attr_group(struct arm_cspmu *cspmu)
463{
464 struct attribute_group *format_group;
465 struct device *dev = cspmu->dev;
466
467 format_group =
468 devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
469 if (!format_group)
470 return NULL;
471
472 format_group->name = "format";
473 format_group->attrs = cspmu->impl.ops.get_format_attrs(cspmu);
474
475 if (!format_group->attrs)
476 return NULL;
477
478 return format_group;
479}
480
481static struct attribute_group **
482arm_cspmu_alloc_attr_group(struct arm_cspmu *cspmu)
483{
484 struct attribute_group **attr_groups = NULL;
485 struct device *dev = cspmu->dev;
486 const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
487 int ret;
488
489 ret = arm_cspmu_init_impl_ops(cspmu);
490 if (ret)
491 return NULL;
492
493 cspmu->identifier = impl_ops->get_identifier(cspmu);
494 cspmu->name = impl_ops->get_name(cspmu);
495
496 if (!cspmu->identifier || !cspmu->name)
497 return NULL;
498
499 attr_groups = devm_kcalloc(dev, 5, sizeof(struct attribute_group *),
500 GFP_KERNEL);
501 if (!attr_groups)
502 return NULL;
503
504 attr_groups[0] = arm_cspmu_alloc_event_attr_group(cspmu);
505 attr_groups[1] = arm_cspmu_alloc_format_attr_group(cspmu);
506 attr_groups[2] = &arm_cspmu_identifier_attr_group;
507 attr_groups[3] = &arm_cspmu_cpumask_attr_group;
508
509 if (!attr_groups[0] || !attr_groups[1])
510 return NULL;
511
512 return attr_groups;
513}
514
515static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu)
516{
517 u32 pmcr = 0;
518
519 pmcr |= PMCR_P;
520 pmcr |= PMCR_C;
521 writel(pmcr, cspmu->base0 + PMCR);
522}
523
524static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu)
525{
526 writel(PMCR_E, cspmu->base0 + PMCR);
527}
528
529static inline void arm_cspmu_stop_counters(struct arm_cspmu *cspmu)
530{
531 writel(0, cspmu->base0 + PMCR);
532}
533
534static void arm_cspmu_enable(struct pmu *pmu)
535{
536 bool disabled;
537 struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
538
539 disabled = bitmap_empty(cspmu->hw_events.used_ctrs,
540 cspmu->num_logical_ctrs);
541
542 if (disabled)
543 return;
544
545 arm_cspmu_start_counters(cspmu);
546}
547
548static void arm_cspmu_disable(struct pmu *pmu)
549{
550 struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
551
552 arm_cspmu_stop_counters(cspmu);
553}
554
555static int arm_cspmu_get_event_idx(struct arm_cspmu_hw_events *hw_events,
556 struct perf_event *event)
557{
558 int idx;
559 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
560
561 if (supports_cycle_counter(cspmu)) {
562 if (cspmu->impl.ops.is_cycle_counter_event(event)) {
563 /* Search for available cycle counter. */
564 if (test_and_set_bit(cspmu->cycle_counter_logical_idx,
565 hw_events->used_ctrs))
566 return -EAGAIN;
567
568 return cspmu->cycle_counter_logical_idx;
569 }
570
571 /*
572 * Search a regular counter from the used counter bitmap.
573 * The cycle counter divides the bitmap into two parts. Search
574 * the first then second half to exclude the cycle counter bit.
575 */
576 idx = find_first_zero_bit(hw_events->used_ctrs,
577 cspmu->cycle_counter_logical_idx);
578 if (idx >= cspmu->cycle_counter_logical_idx) {
579 idx = find_next_zero_bit(
580 hw_events->used_ctrs,
581 cspmu->num_logical_ctrs,
582 cspmu->cycle_counter_logical_idx + 1);
583 }
584 } else {
585 idx = find_first_zero_bit(hw_events->used_ctrs,
586 cspmu->num_logical_ctrs);
587 }
588
589 if (idx >= cspmu->num_logical_ctrs)
590 return -EAGAIN;
591
592 set_bit(idx, hw_events->used_ctrs);
593
594 return idx;
595}
596
597static bool arm_cspmu_validate_event(struct pmu *pmu,
598 struct arm_cspmu_hw_events *hw_events,
599 struct perf_event *event)
600{
601 if (is_software_event(event))
602 return true;
603
604 /* Reject groups spanning multiple HW PMUs. */
605 if (event->pmu != pmu)
606 return false;
607
608 return (arm_cspmu_get_event_idx(hw_events, event) >= 0);
609}
610
611/*
612 * Make sure the group of events can be scheduled at once
613 * on the PMU.
614 */
615static bool arm_cspmu_validate_group(struct perf_event *event)
616{
617 struct perf_event *sibling, *leader = event->group_leader;
618 struct arm_cspmu_hw_events fake_hw_events;
619
620 if (event->group_leader == event)
621 return true;
622
623 memset(&fake_hw_events, 0, sizeof(fake_hw_events));
624
625 if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events, leader))
626 return false;
627
628 for_each_sibling_event(sibling, leader) {
629 if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events,
630 sibling))
631 return false;
632 }
633
634 return arm_cspmu_validate_event(event->pmu, &fake_hw_events, event);
635}
636
637static int arm_cspmu_event_init(struct perf_event *event)
638{
639 struct arm_cspmu *cspmu;
640 struct hw_perf_event *hwc = &event->hw;
641
642 cspmu = to_arm_cspmu(event->pmu);
643
644 /*
645 * Following other "uncore" PMUs, we do not support sampling mode or
646 * attach to a task (per-process mode).
647 */
648 if (is_sampling_event(event)) {
649 dev_dbg(cspmu->pmu.dev,
650 "Can't support sampling events\n");
651 return -EOPNOTSUPP;
652 }
653
654 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
655 dev_dbg(cspmu->pmu.dev,
656 "Can't support per-task counters\n");
657 return -EINVAL;
658 }
659
660 /*
661 * Make sure the CPU assignment is on one of the CPUs associated with
662 * this PMU.
663 */
664 if (!cpumask_test_cpu(event->cpu, &cspmu->associated_cpus)) {
665 dev_dbg(cspmu->pmu.dev,
666 "Requested cpu is not associated with the PMU\n");
667 return -EINVAL;
668 }
669
670 /* Enforce the current active CPU to handle the events in this PMU. */
671 event->cpu = cpumask_first(&cspmu->active_cpu);
672 if (event->cpu >= nr_cpu_ids)
673 return -EINVAL;
674
675 if (!arm_cspmu_validate_group(event))
676 return -EINVAL;
677
678 /*
679 * The logical counter id is tracked with hw_perf_event.extra_reg.idx.
680 * The physical counter id is tracked with hw_perf_event.idx.
681 * We don't assign an index until we actually place the event onto
682 * hardware. Use -1 to signify that we haven't decided where to put it
683 * yet.
684 */
685 hwc->idx = -1;
686 hwc->extra_reg.idx = -1;
687 hwc->config = cspmu->impl.ops.event_type(event);
688
689 return 0;
690}
691
692static inline u32 counter_offset(u32 reg_sz, u32 ctr_idx)
693{
694 return (PMEVCNTR_LO + (reg_sz * ctr_idx));
695}
696
697static void arm_cspmu_write_counter(struct perf_event *event, u64 val)
698{
699 u32 offset;
700 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
701
702 if (use_64b_counter_reg(cspmu)) {
703 offset = counter_offset(sizeof(u64), event->hw.idx);
704
705 writeq(val, cspmu->base1 + offset);
706 } else {
707 offset = counter_offset(sizeof(u32), event->hw.idx);
708
709 writel(lower_32_bits(val), cspmu->base1 + offset);
710 }
711}
712
713static u64 arm_cspmu_read_counter(struct perf_event *event)
714{
715 u32 offset;
716 const void __iomem *counter_addr;
717 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
718
719 if (use_64b_counter_reg(cspmu)) {
720 offset = counter_offset(sizeof(u64), event->hw.idx);
721 counter_addr = cspmu->base1 + offset;
722
723 return supports_64bit_atomics(cspmu) ?
724 readq(counter_addr) :
725 read_reg64_hilohi(counter_addr, HILOHI_MAX_POLL);
726 }
727
728 offset = counter_offset(sizeof(u32), event->hw.idx);
729 return readl(cspmu->base1 + offset);
730}
731
732/*
733 * arm_cspmu_set_event_period: Set the period for the counter.
734 *
735 * To handle cases of extreme interrupt latency, we program
736 * the counter with half of the max count for the counters.
737 */
738static void arm_cspmu_set_event_period(struct perf_event *event)
739{
740 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
741 u64 val = counter_mask(cspmu) >> 1ULL;
742
743 local64_set(&event->hw.prev_count, val);
744 arm_cspmu_write_counter(event, val);
745}
746
747static void arm_cspmu_enable_counter(struct arm_cspmu *cspmu, int idx)
748{
749 u32 reg_id, reg_bit, inten_off, cnten_off;
750
751 reg_id = COUNTER_TO_SET_CLR_ID(idx);
752 reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
753
754 inten_off = PMINTENSET + (4 * reg_id);
755 cnten_off = PMCNTENSET + (4 * reg_id);
756
757 writel(BIT(reg_bit), cspmu->base0 + inten_off);
758 writel(BIT(reg_bit), cspmu->base0 + cnten_off);
759}
760
761static void arm_cspmu_disable_counter(struct arm_cspmu *cspmu, int idx)
762{
763 u32 reg_id, reg_bit, inten_off, cnten_off;
764
765 reg_id = COUNTER_TO_SET_CLR_ID(idx);
766 reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
767
768 inten_off = PMINTENCLR + (4 * reg_id);
769 cnten_off = PMCNTENCLR + (4 * reg_id);
770
771 writel(BIT(reg_bit), cspmu->base0 + cnten_off);
772 writel(BIT(reg_bit), cspmu->base0 + inten_off);
773}
774
775static void arm_cspmu_event_update(struct perf_event *event)
776{
777 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
778 struct hw_perf_event *hwc = &event->hw;
779 u64 delta, prev, now;
780
781 do {
782 prev = local64_read(&hwc->prev_count);
783 now = arm_cspmu_read_counter(event);
784 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
785
786 delta = (now - prev) & counter_mask(cspmu);
787 local64_add(delta, &event->count);
788}
789
790static inline void arm_cspmu_set_event(struct arm_cspmu *cspmu,
791 struct hw_perf_event *hwc)
792{
793 u32 offset = PMEVTYPER + (4 * hwc->idx);
794
795 writel(hwc->config, cspmu->base0 + offset);
796}
797
798static inline void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu,
799 struct hw_perf_event *hwc,
800 u32 filter)
801{
802 u32 offset = PMEVFILTR + (4 * hwc->idx);
803
804 writel(filter, cspmu->base0 + offset);
805}
806
807static inline void arm_cspmu_set_cc_filter(struct arm_cspmu *cspmu, u32 filter)
808{
809 u32 offset = PMCCFILTR;
810
811 writel(filter, cspmu->base0 + offset);
812}
813
814static void arm_cspmu_start(struct perf_event *event, int pmu_flags)
815{
816 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
817 struct hw_perf_event *hwc = &event->hw;
818 u32 filter;
819
820 /* We always reprogram the counter */
821 if (pmu_flags & PERF_EF_RELOAD)
822 WARN_ON(!(hwc->state & PERF_HES_UPTODATE));
823
824 arm_cspmu_set_event_period(event);
825
826 filter = cspmu->impl.ops.event_filter(event);
827
828 if (event->hw.extra_reg.idx == cspmu->cycle_counter_logical_idx) {
829 arm_cspmu_set_cc_filter(cspmu, filter);
830 } else {
831 arm_cspmu_set_event(cspmu, hwc);
832 arm_cspmu_set_ev_filter(cspmu, hwc, filter);
833 }
834
835 hwc->state = 0;
836
837 arm_cspmu_enable_counter(cspmu, hwc->idx);
838}
839
840static void arm_cspmu_stop(struct perf_event *event, int pmu_flags)
841{
842 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
843 struct hw_perf_event *hwc = &event->hw;
844
845 if (hwc->state & PERF_HES_STOPPED)
846 return;
847
848 arm_cspmu_disable_counter(cspmu, hwc->idx);
849 arm_cspmu_event_update(event);
850
851 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
852}
853
854static inline u32 to_phys_idx(struct arm_cspmu *cspmu, u32 idx)
855{
856 return (idx == cspmu->cycle_counter_logical_idx) ?
857 ARM_CSPMU_CYCLE_CNTR_IDX : idx;
858}
859
860static int arm_cspmu_add(struct perf_event *event, int flags)
861{
862 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
863 struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
864 struct hw_perf_event *hwc = &event->hw;
865 int idx;
866
867 if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
868 &cspmu->associated_cpus)))
869 return -ENOENT;
870
871 idx = arm_cspmu_get_event_idx(hw_events, event);
872 if (idx < 0)
873 return idx;
874
875 hw_events->events[idx] = event;
876 hwc->idx = to_phys_idx(cspmu, idx);
877 hwc->extra_reg.idx = idx;
878 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
879
880 if (flags & PERF_EF_START)
881 arm_cspmu_start(event, PERF_EF_RELOAD);
882
883 /* Propagate changes to the userspace mapping. */
884 perf_event_update_userpage(event);
885
886 return 0;
887}
888
889static void arm_cspmu_del(struct perf_event *event, int flags)
890{
891 struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
892 struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
893 struct hw_perf_event *hwc = &event->hw;
894 int idx = hwc->extra_reg.idx;
895
896 arm_cspmu_stop(event, PERF_EF_UPDATE);
897
898 hw_events->events[idx] = NULL;
899
900 clear_bit(idx, hw_events->used_ctrs);
901
902 perf_event_update_userpage(event);
903}
904
905static void arm_cspmu_read(struct perf_event *event)
906{
907 arm_cspmu_event_update(event);
908}
909
910static struct arm_cspmu *arm_cspmu_alloc(struct platform_device *pdev)
911{
912 struct acpi_apmt_node *apmt_node;
913 struct arm_cspmu *cspmu;
914 struct device *dev;
915
916 dev = &pdev->dev;
917 apmt_node = *(struct acpi_apmt_node **)dev_get_platdata(dev);
918 if (!apmt_node) {
919 dev_err(dev, "failed to get APMT node\n");
920 return NULL;
921 }
922
923 cspmu = devm_kzalloc(dev, sizeof(*cspmu), GFP_KERNEL);
924 if (!cspmu)
925 return NULL;
926
927 cspmu->dev = dev;
928 cspmu->apmt_node = apmt_node;
929
930 platform_set_drvdata(pdev, cspmu);
931
932 return cspmu;
933}
934
935static int arm_cspmu_init_mmio(struct arm_cspmu *cspmu)
936{
937 struct device *dev;
938 struct platform_device *pdev;
939 struct acpi_apmt_node *apmt_node;
940
941 dev = cspmu->dev;
942 pdev = to_platform_device(dev);
943 apmt_node = cspmu->apmt_node;
944
945 /* Base address for page 0. */
946 cspmu->base0 = devm_platform_ioremap_resource(pdev, 0);
947 if (IS_ERR(cspmu->base0)) {
948 dev_err(dev, "ioremap failed for page-0 resource\n");
949 return PTR_ERR(cspmu->base0);
950 }
951
952 /* Base address for page 1 if supported. Otherwise point to page 0. */
953 cspmu->base1 = cspmu->base0;
954 if (CHECK_APMT_FLAG(apmt_node->flags, DUAL_PAGE, SUPP)) {
955 cspmu->base1 = devm_platform_ioremap_resource(pdev, 1);
956 if (IS_ERR(cspmu->base1)) {
957 dev_err(dev, "ioremap failed for page-1 resource\n");
958 return PTR_ERR(cspmu->base1);
959 }
960 }
961
962 cspmu->pmcfgr = readl(cspmu->base0 + PMCFGR);
963
964 cspmu->num_logical_ctrs = FIELD_GET(PMCFGR_N, cspmu->pmcfgr) + 1;
965
966 cspmu->cycle_counter_logical_idx = ARM_CSPMU_MAX_HW_CNTRS;
967
968 if (supports_cycle_counter(cspmu)) {
969 /*
970 * The last logical counter is mapped to cycle counter if
971 * there is a gap between regular and cycle counter. Otherwise,
972 * logical and physical have 1-to-1 mapping.
973 */
974 cspmu->cycle_counter_logical_idx =
975 (cspmu->num_logical_ctrs <= ARM_CSPMU_CYCLE_CNTR_IDX) ?
976 cspmu->num_logical_ctrs - 1 :
977 ARM_CSPMU_CYCLE_CNTR_IDX;
978 }
979
980 cspmu->num_set_clr_reg =
981 DIV_ROUND_UP(cspmu->num_logical_ctrs,
982 ARM_CSPMU_SET_CLR_COUNTER_NUM);
983
984 cspmu->hw_events.events =
985 devm_kcalloc(dev, cspmu->num_logical_ctrs,
986 sizeof(*cspmu->hw_events.events), GFP_KERNEL);
987
988 if (!cspmu->hw_events.events)
989 return -ENOMEM;
990
991 return 0;
992}
993
994static inline int arm_cspmu_get_reset_overflow(struct arm_cspmu *cspmu,
995 u32 *pmovs)
996{
997 int i;
998 u32 pmovclr_offset = PMOVSCLR;
999 u32 has_overflowed = 0;
1000
1001 for (i = 0; i < cspmu->num_set_clr_reg; ++i) {
1002 pmovs[i] = readl(cspmu->base1 + pmovclr_offset);
1003 has_overflowed |= pmovs[i];
1004 writel(pmovs[i], cspmu->base1 + pmovclr_offset);
1005 pmovclr_offset += sizeof(u32);
1006 }
1007
1008 return has_overflowed != 0;
1009}
1010
1011static irqreturn_t arm_cspmu_handle_irq(int irq_num, void *dev)
1012{
1013 int idx, has_overflowed;
1014 struct perf_event *event;
1015 struct arm_cspmu *cspmu = dev;
1016 DECLARE_BITMAP(pmovs, ARM_CSPMU_MAX_HW_CNTRS);
1017 bool handled = false;
1018
1019 arm_cspmu_stop_counters(cspmu);
1020
1021 has_overflowed = arm_cspmu_get_reset_overflow(cspmu, (u32 *)pmovs);
1022 if (!has_overflowed)
1023 goto done;
1024
1025 for_each_set_bit(idx, cspmu->hw_events.used_ctrs,
1026 cspmu->num_logical_ctrs) {
1027 event = cspmu->hw_events.events[idx];
1028
1029 if (!event)
1030 continue;
1031
1032 if (!test_bit(event->hw.idx, pmovs))
1033 continue;
1034
1035 arm_cspmu_event_update(event);
1036 arm_cspmu_set_event_period(event);
1037
1038 handled = true;
1039 }
1040
1041done:
1042 arm_cspmu_start_counters(cspmu);
1043 return IRQ_RETVAL(handled);
1044}
1045
1046static int arm_cspmu_request_irq(struct arm_cspmu *cspmu)
1047{
1048 int irq, ret;
1049 struct device *dev;
1050 struct platform_device *pdev;
1051 struct acpi_apmt_node *apmt_node;
1052
1053 dev = cspmu->dev;
1054 pdev = to_platform_device(dev);
1055 apmt_node = cspmu->apmt_node;
1056
1057 /* Skip IRQ request if the PMU does not support overflow interrupt. */
1058 if (apmt_node->ovflw_irq == 0)
1059 return 0;
1060
1061 irq = platform_get_irq(pdev, 0);
1062 if (irq < 0)
1063 return irq;
1064
1065 ret = devm_request_irq(dev, irq, arm_cspmu_handle_irq,
1066 IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(dev),
1067 cspmu);
1068 if (ret) {
1069 dev_err(dev, "Could not request IRQ %d\n", irq);
1070 return ret;
1071 }
1072
1073 cspmu->irq = irq;
1074
1075 return 0;
1076}
1077
1078static inline int arm_cspmu_find_cpu_container(int cpu, u32 container_uid)
1079{
1080 u32 acpi_uid;
1081 struct device *cpu_dev = get_cpu_device(cpu);
1082 struct acpi_device *acpi_dev = ACPI_COMPANION(cpu_dev);
1083
1084 if (!cpu_dev)
1085 return -ENODEV;
1086
1087 while (acpi_dev) {
1088 if (!strcmp(acpi_device_hid(acpi_dev),
1089 ACPI_PROCESSOR_CONTAINER_HID) &&
1090 !kstrtouint(acpi_device_uid(acpi_dev), 0, &acpi_uid) &&
1091 acpi_uid == container_uid)
1092 return 0;
1093
1094 acpi_dev = acpi_dev_parent(acpi_dev);
1095 }
1096
1097 return -ENODEV;
1098}
1099
1100static int arm_cspmu_get_cpus(struct arm_cspmu *cspmu)
1101{
1102 struct device *dev;
1103 struct acpi_apmt_node *apmt_node;
1104 int affinity_flag;
1105 int cpu;
1106
1107 dev = cspmu->pmu.dev;
1108 apmt_node = cspmu->apmt_node;
1109 affinity_flag = apmt_node->flags & ACPI_APMT_FLAGS_AFFINITY;
1110
1111 if (affinity_flag == ACPI_APMT_FLAGS_AFFINITY_PROC) {
1112 for_each_possible_cpu(cpu) {
1113 if (apmt_node->proc_affinity ==
1114 get_acpi_id_for_cpu(cpu)) {
1115 cpumask_set_cpu(cpu, &cspmu->associated_cpus);
1116 break;
1117 }
1118 }
1119 } else {
1120 for_each_possible_cpu(cpu) {
1121 if (arm_cspmu_find_cpu_container(
1122 cpu, apmt_node->proc_affinity))
1123 continue;
1124
1125 cpumask_set_cpu(cpu, &cspmu->associated_cpus);
1126 }
1127 }
1128
1129 if (cpumask_empty(&cspmu->associated_cpus)) {
1130 dev_dbg(dev, "No cpu associated with the PMU\n");
1131 return -ENODEV;
1132 }
1133
1134 return 0;
1135}
1136
1137static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu)
1138{
1139 int ret, capabilities;
1140 struct attribute_group **attr_groups;
1141
1142 attr_groups = arm_cspmu_alloc_attr_group(cspmu);
1143 if (!attr_groups)
1144 return -ENOMEM;
1145
1146 ret = cpuhp_state_add_instance(arm_cspmu_cpuhp_state,
1147 &cspmu->cpuhp_node);
1148 if (ret)
1149 return ret;
1150
1151 capabilities = PERF_PMU_CAP_NO_EXCLUDE;
1152 if (cspmu->irq == 0)
1153 capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1154
1155 cspmu->pmu = (struct pmu){
1156 .task_ctx_nr = perf_invalid_context,
1157 .module = THIS_MODULE,
1158 .pmu_enable = arm_cspmu_enable,
1159 .pmu_disable = arm_cspmu_disable,
1160 .event_init = arm_cspmu_event_init,
1161 .add = arm_cspmu_add,
1162 .del = arm_cspmu_del,
1163 .start = arm_cspmu_start,
1164 .stop = arm_cspmu_stop,
1165 .read = arm_cspmu_read,
1166 .attr_groups = (const struct attribute_group **)attr_groups,
1167 .capabilities = capabilities,
1168 };
1169
1170 /* Hardware counter init */
1171 arm_cspmu_stop_counters(cspmu);
1172 arm_cspmu_reset_counters(cspmu);
1173
1174 ret = perf_pmu_register(&cspmu->pmu, cspmu->name, -1);
1175 if (ret) {
1176 cpuhp_state_remove_instance(arm_cspmu_cpuhp_state,
1177 &cspmu->cpuhp_node);
1178 }
1179
1180 return ret;
1181}
1182
1183static int arm_cspmu_device_probe(struct platform_device *pdev)
1184{
1185 int ret;
1186 struct arm_cspmu *cspmu;
1187
1188 cspmu = arm_cspmu_alloc(pdev);
1189 if (!cspmu)
1190 return -ENOMEM;
1191
1192 ret = arm_cspmu_init_mmio(cspmu);
1193 if (ret)
1194 return ret;
1195
1196 ret = arm_cspmu_request_irq(cspmu);
1197 if (ret)
1198 return ret;
1199
1200 ret = arm_cspmu_get_cpus(cspmu);
1201 if (ret)
1202 return ret;
1203
1204 ret = arm_cspmu_register_pmu(cspmu);
1205 if (ret)
1206 return ret;
1207
1208 return 0;
1209}
1210
1211static int arm_cspmu_device_remove(struct platform_device *pdev)
1212{
1213 struct arm_cspmu *cspmu = platform_get_drvdata(pdev);
1214
1215 perf_pmu_unregister(&cspmu->pmu);
1216 cpuhp_state_remove_instance(arm_cspmu_cpuhp_state, &cspmu->cpuhp_node);
1217
1218 return 0;
1219}
1220
1221static struct platform_driver arm_cspmu_driver = {
1222 .driver = {
1223 .name = DRVNAME,
1224 .suppress_bind_attrs = true,
1225 },
1226 .probe = arm_cspmu_device_probe,
1227 .remove = arm_cspmu_device_remove,
1228};
1229
1230static void arm_cspmu_set_active_cpu(int cpu, struct arm_cspmu *cspmu)
1231{
1232 cpumask_set_cpu(cpu, &cspmu->active_cpu);
1233 WARN_ON(irq_set_affinity(cspmu->irq, &cspmu->active_cpu));
1234}
1235
1236static int arm_cspmu_cpu_online(unsigned int cpu, struct hlist_node *node)
1237{
1238 struct arm_cspmu *cspmu =
1239 hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
1240
1241 if (!cpumask_test_cpu(cpu, &cspmu->associated_cpus))
1242 return 0;
1243
1244 /* If the PMU is already managed, there is nothing to do */
1245 if (!cpumask_empty(&cspmu->active_cpu))
1246 return 0;
1247
1248 /* Use this CPU for event counting */
1249 arm_cspmu_set_active_cpu(cpu, cspmu);
1250
1251 return 0;
1252}
1253
1254static int arm_cspmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1255{
1256 int dst;
1257 struct cpumask online_supported;
1258
1259 struct arm_cspmu *cspmu =
1260 hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
1261
1262 /* Nothing to do if this CPU doesn't own the PMU */
1263 if (!cpumask_test_and_clear_cpu(cpu, &cspmu->active_cpu))
1264 return 0;
1265
1266 /* Choose a new CPU to migrate ownership of the PMU to */
1267 cpumask_and(&online_supported, &cspmu->associated_cpus,
1268 cpu_online_mask);
1269 dst = cpumask_any_but(&online_supported, cpu);
1270 if (dst >= nr_cpu_ids)
1271 return 0;
1272
1273 /* Use this CPU for event counting */
1274 perf_pmu_migrate_context(&cspmu->pmu, cpu, dst);
1275 arm_cspmu_set_active_cpu(dst, cspmu);
1276
1277 return 0;
1278}
1279
1280static int __init arm_cspmu_init(void)
1281{
1282 int ret;
1283
1284 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1285 "perf/arm/cspmu:online",
1286 arm_cspmu_cpu_online,
1287 arm_cspmu_cpu_teardown);
1288 if (ret < 0)
1289 return ret;
1290 arm_cspmu_cpuhp_state = ret;
1291 return platform_driver_register(&arm_cspmu_driver);
1292}
1293
1294static void __exit arm_cspmu_exit(void)
1295{
1296 platform_driver_unregister(&arm_cspmu_driver);
1297 cpuhp_remove_multi_state(arm_cspmu_cpuhp_state);
1298}
1299
1300module_init(arm_cspmu_init);
1301module_exit(arm_cspmu_exit);
1302
1303MODULE_LICENSE("GPL v2");