Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * L220/L310 cache controller support
4 *
5 * Copyright (C) 2016 ARM Limited
6 */
7#include <linux/errno.h>
8#include <linux/hrtimer.h>
9#include <linux/io.h>
10#include <linux/list.h>
11#include <linux/perf_event.h>
12#include <linux/printk.h>
13#include <linux/slab.h>
14#include <linux/types.h>
15
16#include <asm/hardware/cache-l2x0.h>
17
18#define PMU_NR_COUNTERS 2
19
20static void __iomem *l2x0_base;
21static struct pmu *l2x0_pmu;
22static cpumask_t pmu_cpu;
23
24static const char *l2x0_name;
25
26static ktime_t l2x0_pmu_poll_period;
27static struct hrtimer l2x0_pmu_hrtimer;
28
29/*
30 * The L220/PL310 PMU has two equivalent counters, Counter1 and Counter0.
31 * Registers controlling these are laid out in pairs, in descending order, i.e.
32 * the register for Counter1 comes first, followed by the register for
33 * Counter0.
34 * We ensure that idx 0 -> Counter0, and idx1 -> Counter1.
35 */
36static struct perf_event *events[PMU_NR_COUNTERS];
37
38/* Find an unused counter */
39static int l2x0_pmu_find_idx(void)
40{
41 int i;
42
43 for (i = 0; i < PMU_NR_COUNTERS; i++) {
44 if (!events[i])
45 return i;
46 }
47
48 return -1;
49}
50
51/* How many counters are allocated? */
52static int l2x0_pmu_num_active_counters(void)
53{
54 int i, cnt = 0;
55
56 for (i = 0; i < PMU_NR_COUNTERS; i++) {
57 if (events[i])
58 cnt++;
59 }
60
61 return cnt;
62}
63
64static void l2x0_pmu_counter_config_write(int idx, u32 val)
65{
66 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_CFG - 4 * idx);
67}
68
69static u32 l2x0_pmu_counter_read(int idx)
70{
71 return readl_relaxed(l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
72}
73
74static void l2x0_pmu_counter_write(int idx, u32 val)
75{
76 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
77}
78
79static void __l2x0_pmu_enable(void)
80{
81 u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
82 val |= L2X0_EVENT_CNT_CTRL_ENABLE;
83 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
84}
85
86static void __l2x0_pmu_disable(void)
87{
88 u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
89 val &= ~L2X0_EVENT_CNT_CTRL_ENABLE;
90 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
91}
92
93static void l2x0_pmu_enable(struct pmu *pmu)
94{
95 if (l2x0_pmu_num_active_counters() == 0)
96 return;
97
98 __l2x0_pmu_enable();
99}
100
101static void l2x0_pmu_disable(struct pmu *pmu)
102{
103 if (l2x0_pmu_num_active_counters() == 0)
104 return;
105
106 __l2x0_pmu_disable();
107}
108
109static void warn_if_saturated(u32 count)
110{
111 if (count != 0xffffffff)
112 return;
113
114 pr_warn_ratelimited("L2X0 counter saturated. Poll period too long\n");
115}
116
117static void l2x0_pmu_event_read(struct perf_event *event)
118{
119 struct hw_perf_event *hw = &event->hw;
120 u64 prev_count, new_count, mask;
121
122 do {
123 prev_count = local64_read(&hw->prev_count);
124 new_count = l2x0_pmu_counter_read(hw->idx);
125 } while (local64_xchg(&hw->prev_count, new_count) != prev_count);
126
127 mask = GENMASK_ULL(31, 0);
128 local64_add((new_count - prev_count) & mask, &event->count);
129
130 warn_if_saturated(new_count);
131}
132
133static void l2x0_pmu_event_configure(struct perf_event *event)
134{
135 struct hw_perf_event *hw = &event->hw;
136
137 /*
138 * The L2X0 counters saturate at 0xffffffff rather than wrapping, so we
139 * will *always* lose some number of events when a counter saturates,
140 * and have no way of detecting how many were lost.
141 *
142 * To minimize the impact of this, we try to maximize the period by
143 * always starting counters at zero. To ensure that group ratios are
144 * representative, we poll periodically to avoid counters saturating.
145 * See l2x0_pmu_poll().
146 */
147 local64_set(&hw->prev_count, 0);
148 l2x0_pmu_counter_write(hw->idx, 0);
149}
150
151static enum hrtimer_restart l2x0_pmu_poll(struct hrtimer *hrtimer)
152{
153 unsigned long flags;
154 int i;
155
156 local_irq_save(flags);
157 __l2x0_pmu_disable();
158
159 for (i = 0; i < PMU_NR_COUNTERS; i++) {
160 struct perf_event *event = events[i];
161
162 if (!event)
163 continue;
164
165 l2x0_pmu_event_read(event);
166 l2x0_pmu_event_configure(event);
167 }
168
169 __l2x0_pmu_enable();
170 local_irq_restore(flags);
171
172 hrtimer_forward_now(hrtimer, l2x0_pmu_poll_period);
173 return HRTIMER_RESTART;
174}
175
176
177static void __l2x0_pmu_event_enable(int idx, u32 event)
178{
179 u32 val;
180
181 val = event << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
182 val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
183 l2x0_pmu_counter_config_write(idx, val);
184}
185
186static void l2x0_pmu_event_start(struct perf_event *event, int flags)
187{
188 struct hw_perf_event *hw = &event->hw;
189
190 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
191 return;
192
193 if (flags & PERF_EF_RELOAD) {
194 WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE));
195 l2x0_pmu_event_configure(event);
196 }
197
198 hw->state = 0;
199
200 __l2x0_pmu_event_enable(hw->idx, hw->config_base);
201}
202
203static void __l2x0_pmu_event_disable(int idx)
204{
205 u32 val;
206
207 val = L2X0_EVENT_CNT_CFG_SRC_DISABLED << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
208 val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
209 l2x0_pmu_counter_config_write(idx, val);
210}
211
212static void l2x0_pmu_event_stop(struct perf_event *event, int flags)
213{
214 struct hw_perf_event *hw = &event->hw;
215
216 if (WARN_ON_ONCE(event->hw.state & PERF_HES_STOPPED))
217 return;
218
219 __l2x0_pmu_event_disable(hw->idx);
220
221 hw->state |= PERF_HES_STOPPED;
222
223 if (flags & PERF_EF_UPDATE) {
224 l2x0_pmu_event_read(event);
225 hw->state |= PERF_HES_UPTODATE;
226 }
227}
228
229static int l2x0_pmu_event_add(struct perf_event *event, int flags)
230{
231 struct hw_perf_event *hw = &event->hw;
232 int idx = l2x0_pmu_find_idx();
233
234 if (idx == -1)
235 return -EAGAIN;
236
237 /*
238 * Pin the timer, so that the overflows are handled by the chosen
239 * event->cpu (this is the same one as presented in "cpumask"
240 * attribute).
241 */
242 if (l2x0_pmu_num_active_counters() == 0)
243 hrtimer_start(&l2x0_pmu_hrtimer, l2x0_pmu_poll_period,
244 HRTIMER_MODE_REL_PINNED);
245
246 events[idx] = event;
247 hw->idx = idx;
248
249 l2x0_pmu_event_configure(event);
250
251 hw->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
252
253 if (flags & PERF_EF_START)
254 l2x0_pmu_event_start(event, 0);
255
256 return 0;
257}
258
259static void l2x0_pmu_event_del(struct perf_event *event, int flags)
260{
261 struct hw_perf_event *hw = &event->hw;
262
263 l2x0_pmu_event_stop(event, PERF_EF_UPDATE);
264
265 events[hw->idx] = NULL;
266 hw->idx = -1;
267
268 if (l2x0_pmu_num_active_counters() == 0)
269 hrtimer_cancel(&l2x0_pmu_hrtimer);
270}
271
272static bool l2x0_pmu_group_is_valid(struct perf_event *event)
273{
274 struct pmu *pmu = event->pmu;
275 struct perf_event *leader = event->group_leader;
276 struct perf_event *sibling;
277 int num_hw = 0;
278
279 if (leader->pmu == pmu)
280 num_hw++;
281 else if (!is_software_event(leader))
282 return false;
283
284 for_each_sibling_event(sibling, leader) {
285 if (sibling->pmu == pmu)
286 num_hw++;
287 else if (!is_software_event(sibling))
288 return false;
289 }
290
291 return num_hw <= PMU_NR_COUNTERS;
292}
293
294static int l2x0_pmu_event_init(struct perf_event *event)
295{
296 struct hw_perf_event *hw = &event->hw;
297
298 if (event->attr.type != l2x0_pmu->type)
299 return -ENOENT;
300
301 if (is_sampling_event(event) ||
302 event->attach_state & PERF_ATTACH_TASK)
303 return -EINVAL;
304
305 if (event->cpu < 0)
306 return -EINVAL;
307
308 if (event->attr.config & ~L2X0_EVENT_CNT_CFG_SRC_MASK)
309 return -EINVAL;
310
311 hw->config_base = event->attr.config;
312
313 if (!l2x0_pmu_group_is_valid(event))
314 return -EINVAL;
315
316 event->cpu = cpumask_first(&pmu_cpu);
317
318 return 0;
319}
320
321struct l2x0_event_attribute {
322 struct device_attribute attr;
323 unsigned int config;
324 bool pl310_only;
325};
326
327#define L2X0_EVENT_ATTR(_name, _config, _pl310_only) \
328 (&((struct l2x0_event_attribute[]) {{ \
329 .attr = __ATTR(_name, S_IRUGO, l2x0_pmu_event_show, NULL), \
330 .config = _config, \
331 .pl310_only = _pl310_only, \
332 }})[0].attr.attr)
333
334#define L220_PLUS_EVENT_ATTR(_name, _config) \
335 L2X0_EVENT_ATTR(_name, _config, false)
336
337#define PL310_EVENT_ATTR(_name, _config) \
338 L2X0_EVENT_ATTR(_name, _config, true)
339
340static ssize_t l2x0_pmu_event_show(struct device *dev,
341 struct device_attribute *attr, char *buf)
342{
343 struct l2x0_event_attribute *lattr;
344
345 lattr = container_of(attr, typeof(*lattr), attr);
346 return snprintf(buf, PAGE_SIZE, "config=0x%x\n", lattr->config);
347}
348
349static umode_t l2x0_pmu_event_attr_is_visible(struct kobject *kobj,
350 struct attribute *attr,
351 int unused)
352{
353 struct device *dev = kobj_to_dev(kobj);
354 struct pmu *pmu = dev_get_drvdata(dev);
355 struct l2x0_event_attribute *lattr;
356
357 lattr = container_of(attr, typeof(*lattr), attr.attr);
358
359 if (!lattr->pl310_only || strcmp("l2c_310", pmu->name) == 0)
360 return attr->mode;
361
362 return 0;
363}
364
365static struct attribute *l2x0_pmu_event_attrs[] = {
366 L220_PLUS_EVENT_ATTR(co, 0x1),
367 L220_PLUS_EVENT_ATTR(drhit, 0x2),
368 L220_PLUS_EVENT_ATTR(drreq, 0x3),
369 L220_PLUS_EVENT_ATTR(dwhit, 0x4),
370 L220_PLUS_EVENT_ATTR(dwreq, 0x5),
371 L220_PLUS_EVENT_ATTR(dwtreq, 0x6),
372 L220_PLUS_EVENT_ATTR(irhit, 0x7),
373 L220_PLUS_EVENT_ATTR(irreq, 0x8),
374 L220_PLUS_EVENT_ATTR(wa, 0x9),
375 PL310_EVENT_ATTR(ipfalloc, 0xa),
376 PL310_EVENT_ATTR(epfhit, 0xb),
377 PL310_EVENT_ATTR(epfalloc, 0xc),
378 PL310_EVENT_ATTR(srrcvd, 0xd),
379 PL310_EVENT_ATTR(srconf, 0xe),
380 PL310_EVENT_ATTR(epfrcvd, 0xf),
381 NULL
382};
383
384static struct attribute_group l2x0_pmu_event_attrs_group = {
385 .name = "events",
386 .attrs = l2x0_pmu_event_attrs,
387 .is_visible = l2x0_pmu_event_attr_is_visible,
388};
389
390static ssize_t l2x0_pmu_cpumask_show(struct device *dev,
391 struct device_attribute *attr, char *buf)
392{
393 return cpumap_print_to_pagebuf(true, buf, &pmu_cpu);
394}
395
396static struct device_attribute l2x0_pmu_cpumask_attr =
397 __ATTR(cpumask, S_IRUGO, l2x0_pmu_cpumask_show, NULL);
398
399static struct attribute *l2x0_pmu_cpumask_attrs[] = {
400 &l2x0_pmu_cpumask_attr.attr,
401 NULL,
402};
403
404static struct attribute_group l2x0_pmu_cpumask_attr_group = {
405 .attrs = l2x0_pmu_cpumask_attrs,
406};
407
408static const struct attribute_group *l2x0_pmu_attr_groups[] = {
409 &l2x0_pmu_event_attrs_group,
410 &l2x0_pmu_cpumask_attr_group,
411 NULL,
412};
413
414static void l2x0_pmu_reset(void)
415{
416 int i;
417
418 __l2x0_pmu_disable();
419
420 for (i = 0; i < PMU_NR_COUNTERS; i++)
421 __l2x0_pmu_event_disable(i);
422}
423
424static int l2x0_pmu_offline_cpu(unsigned int cpu)
425{
426 unsigned int target;
427
428 if (!cpumask_test_and_clear_cpu(cpu, &pmu_cpu))
429 return 0;
430
431 target = cpumask_any_but(cpu_online_mask, cpu);
432 if (target >= nr_cpu_ids)
433 return 0;
434
435 perf_pmu_migrate_context(l2x0_pmu, cpu, target);
436 cpumask_set_cpu(target, &pmu_cpu);
437
438 return 0;
439}
440
441void l2x0_pmu_suspend(void)
442{
443 int i;
444
445 if (!l2x0_pmu)
446 return;
447
448 l2x0_pmu_disable(l2x0_pmu);
449
450 for (i = 0; i < PMU_NR_COUNTERS; i++) {
451 if (events[i])
452 l2x0_pmu_event_stop(events[i], PERF_EF_UPDATE);
453 }
454
455}
456
457void l2x0_pmu_resume(void)
458{
459 int i;
460
461 if (!l2x0_pmu)
462 return;
463
464 l2x0_pmu_reset();
465
466 for (i = 0; i < PMU_NR_COUNTERS; i++) {
467 if (events[i])
468 l2x0_pmu_event_start(events[i], PERF_EF_RELOAD);
469 }
470
471 l2x0_pmu_enable(l2x0_pmu);
472}
473
474void __init l2x0_pmu_register(void __iomem *base, u32 part)
475{
476 /*
477 * Determine whether we support the PMU, and choose the name for sysfs.
478 * This is also used by l2x0_pmu_event_attr_is_visible to determine
479 * which events to display, as the PL310 PMU supports a superset of
480 * L220 events.
481 *
482 * The L210 PMU has a different programmer's interface, and is not
483 * supported by this driver.
484 *
485 * We must defer registering the PMU until the perf subsystem is up and
486 * running, so just stash the name and base, and leave that to another
487 * initcall.
488 */
489 switch (part & L2X0_CACHE_ID_PART_MASK) {
490 case L2X0_CACHE_ID_PART_L220:
491 l2x0_name = "l2c_220";
492 break;
493 case L2X0_CACHE_ID_PART_L310:
494 l2x0_name = "l2c_310";
495 break;
496 default:
497 return;
498 }
499
500 l2x0_base = base;
501}
502
503static __init int l2x0_pmu_init(void)
504{
505 int ret;
506
507 if (!l2x0_base)
508 return 0;
509
510 l2x0_pmu = kzalloc(sizeof(*l2x0_pmu), GFP_KERNEL);
511 if (!l2x0_pmu) {
512 pr_warn("Unable to allocate L2x0 PMU\n");
513 return -ENOMEM;
514 }
515
516 *l2x0_pmu = (struct pmu) {
517 .task_ctx_nr = perf_invalid_context,
518 .pmu_enable = l2x0_pmu_enable,
519 .pmu_disable = l2x0_pmu_disable,
520 .read = l2x0_pmu_event_read,
521 .start = l2x0_pmu_event_start,
522 .stop = l2x0_pmu_event_stop,
523 .add = l2x0_pmu_event_add,
524 .del = l2x0_pmu_event_del,
525 .event_init = l2x0_pmu_event_init,
526 .attr_groups = l2x0_pmu_attr_groups,
527 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
528 };
529
530 l2x0_pmu_reset();
531
532 /*
533 * We always use a hrtimer rather than an interrupt.
534 * See comments in l2x0_pmu_event_configure and l2x0_pmu_poll.
535 *
536 * Polling once a second allows the counters to fill up to 1/128th on a
537 * quad-core test chip with cores clocked at 400MHz. Hopefully this
538 * leaves sufficient headroom to avoid overflow on production silicon
539 * at higher frequencies.
540 */
541 l2x0_pmu_poll_period = ms_to_ktime(1000);
542 hrtimer_init(&l2x0_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
543 l2x0_pmu_hrtimer.function = l2x0_pmu_poll;
544
545 cpumask_set_cpu(0, &pmu_cpu);
546 ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE,
547 "perf/arm/l2x0:online", NULL,
548 l2x0_pmu_offline_cpu);
549 if (ret)
550 goto out_pmu;
551
552 ret = perf_pmu_register(l2x0_pmu, l2x0_name, -1);
553 if (ret)
554 goto out_cpuhp;
555
556 return 0;
557
558out_cpuhp:
559 cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE);
560out_pmu:
561 kfree(l2x0_pmu);
562 l2x0_pmu = NULL;
563 return ret;
564}
565device_initcall(l2x0_pmu_init);
1/*
2 * L220/L310 cache controller support
3 *
4 * Copyright (C) 2016 ARM Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19#include <linux/errno.h>
20#include <linux/hrtimer.h>
21#include <linux/io.h>
22#include <linux/list.h>
23#include <linux/perf_event.h>
24#include <linux/printk.h>
25#include <linux/slab.h>
26#include <linux/types.h>
27
28#include <asm/hardware/cache-l2x0.h>
29
30#define PMU_NR_COUNTERS 2
31
32static void __iomem *l2x0_base;
33static struct pmu *l2x0_pmu;
34static cpumask_t pmu_cpu;
35
36static const char *l2x0_name;
37
38static ktime_t l2x0_pmu_poll_period;
39static struct hrtimer l2x0_pmu_hrtimer;
40
41/*
42 * The L220/PL310 PMU has two equivalent counters, Counter1 and Counter0.
43 * Registers controlling these are laid out in pairs, in descending order, i.e.
44 * the register for Counter1 comes first, followed by the register for
45 * Counter0.
46 * We ensure that idx 0 -> Counter0, and idx1 -> Counter1.
47 */
48static struct perf_event *events[PMU_NR_COUNTERS];
49
50/* Find an unused counter */
51static int l2x0_pmu_find_idx(void)
52{
53 int i;
54
55 for (i = 0; i < PMU_NR_COUNTERS; i++) {
56 if (!events[i])
57 return i;
58 }
59
60 return -1;
61}
62
63/* How many counters are allocated? */
64static int l2x0_pmu_num_active_counters(void)
65{
66 int i, cnt = 0;
67
68 for (i = 0; i < PMU_NR_COUNTERS; i++) {
69 if (events[i])
70 cnt++;
71 }
72
73 return cnt;
74}
75
76static void l2x0_pmu_counter_config_write(int idx, u32 val)
77{
78 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_CFG - 4 * idx);
79}
80
81static u32 l2x0_pmu_counter_read(int idx)
82{
83 return readl_relaxed(l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
84}
85
86static void l2x0_pmu_counter_write(int idx, u32 val)
87{
88 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
89}
90
91static void __l2x0_pmu_enable(void)
92{
93 u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
94 val |= L2X0_EVENT_CNT_CTRL_ENABLE;
95 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
96}
97
98static void __l2x0_pmu_disable(void)
99{
100 u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
101 val &= ~L2X0_EVENT_CNT_CTRL_ENABLE;
102 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
103}
104
105static void l2x0_pmu_enable(struct pmu *pmu)
106{
107 if (l2x0_pmu_num_active_counters() == 0)
108 return;
109
110 __l2x0_pmu_enable();
111}
112
113static void l2x0_pmu_disable(struct pmu *pmu)
114{
115 if (l2x0_pmu_num_active_counters() == 0)
116 return;
117
118 __l2x0_pmu_disable();
119}
120
121static void warn_if_saturated(u32 count)
122{
123 if (count != 0xffffffff)
124 return;
125
126 pr_warn_ratelimited("L2X0 counter saturated. Poll period too long\n");
127}
128
129static void l2x0_pmu_event_read(struct perf_event *event)
130{
131 struct hw_perf_event *hw = &event->hw;
132 u64 prev_count, new_count, mask;
133
134 do {
135 prev_count = local64_read(&hw->prev_count);
136 new_count = l2x0_pmu_counter_read(hw->idx);
137 } while (local64_xchg(&hw->prev_count, new_count) != prev_count);
138
139 mask = GENMASK_ULL(31, 0);
140 local64_add((new_count - prev_count) & mask, &event->count);
141
142 warn_if_saturated(new_count);
143}
144
145static void l2x0_pmu_event_configure(struct perf_event *event)
146{
147 struct hw_perf_event *hw = &event->hw;
148
149 /*
150 * The L2X0 counters saturate at 0xffffffff rather than wrapping, so we
151 * will *always* lose some number of events when a counter saturates,
152 * and have no way of detecting how many were lost.
153 *
154 * To minimize the impact of this, we try to maximize the period by
155 * always starting counters at zero. To ensure that group ratios are
156 * representative, we poll periodically to avoid counters saturating.
157 * See l2x0_pmu_poll().
158 */
159 local64_set(&hw->prev_count, 0);
160 l2x0_pmu_counter_write(hw->idx, 0);
161}
162
163static enum hrtimer_restart l2x0_pmu_poll(struct hrtimer *hrtimer)
164{
165 unsigned long flags;
166 int i;
167
168 local_irq_save(flags);
169 __l2x0_pmu_disable();
170
171 for (i = 0; i < PMU_NR_COUNTERS; i++) {
172 struct perf_event *event = events[i];
173
174 if (!event)
175 continue;
176
177 l2x0_pmu_event_read(event);
178 l2x0_pmu_event_configure(event);
179 }
180
181 __l2x0_pmu_enable();
182 local_irq_restore(flags);
183
184 hrtimer_forward_now(hrtimer, l2x0_pmu_poll_period);
185 return HRTIMER_RESTART;
186}
187
188
189static void __l2x0_pmu_event_enable(int idx, u32 event)
190{
191 u32 val;
192
193 val = event << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
194 val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
195 l2x0_pmu_counter_config_write(idx, val);
196}
197
198static void l2x0_pmu_event_start(struct perf_event *event, int flags)
199{
200 struct hw_perf_event *hw = &event->hw;
201
202 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
203 return;
204
205 if (flags & PERF_EF_RELOAD) {
206 WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE));
207 l2x0_pmu_event_configure(event);
208 }
209
210 hw->state = 0;
211
212 __l2x0_pmu_event_enable(hw->idx, hw->config_base);
213}
214
215static void __l2x0_pmu_event_disable(int idx)
216{
217 u32 val;
218
219 val = L2X0_EVENT_CNT_CFG_SRC_DISABLED << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
220 val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
221 l2x0_pmu_counter_config_write(idx, val);
222}
223
224static void l2x0_pmu_event_stop(struct perf_event *event, int flags)
225{
226 struct hw_perf_event *hw = &event->hw;
227
228 if (WARN_ON_ONCE(event->hw.state & PERF_HES_STOPPED))
229 return;
230
231 __l2x0_pmu_event_disable(hw->idx);
232
233 hw->state |= PERF_HES_STOPPED;
234
235 if (flags & PERF_EF_UPDATE) {
236 l2x0_pmu_event_read(event);
237 hw->state |= PERF_HES_UPTODATE;
238 }
239}
240
241static int l2x0_pmu_event_add(struct perf_event *event, int flags)
242{
243 struct hw_perf_event *hw = &event->hw;
244 int idx = l2x0_pmu_find_idx();
245
246 if (idx == -1)
247 return -EAGAIN;
248
249 /*
250 * Pin the timer, so that the overflows are handled by the chosen
251 * event->cpu (this is the same one as presented in "cpumask"
252 * attribute).
253 */
254 if (l2x0_pmu_num_active_counters() == 0)
255 hrtimer_start(&l2x0_pmu_hrtimer, l2x0_pmu_poll_period,
256 HRTIMER_MODE_REL_PINNED);
257
258 events[idx] = event;
259 hw->idx = idx;
260
261 l2x0_pmu_event_configure(event);
262
263 hw->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
264
265 if (flags & PERF_EF_START)
266 l2x0_pmu_event_start(event, 0);
267
268 return 0;
269}
270
271static void l2x0_pmu_event_del(struct perf_event *event, int flags)
272{
273 struct hw_perf_event *hw = &event->hw;
274
275 l2x0_pmu_event_stop(event, PERF_EF_UPDATE);
276
277 events[hw->idx] = NULL;
278 hw->idx = -1;
279
280 if (l2x0_pmu_num_active_counters() == 0)
281 hrtimer_cancel(&l2x0_pmu_hrtimer);
282}
283
284static bool l2x0_pmu_group_is_valid(struct perf_event *event)
285{
286 struct pmu *pmu = event->pmu;
287 struct perf_event *leader = event->group_leader;
288 struct perf_event *sibling;
289 int num_hw = 0;
290
291 if (leader->pmu == pmu)
292 num_hw++;
293 else if (!is_software_event(leader))
294 return false;
295
296 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
297 if (sibling->pmu == pmu)
298 num_hw++;
299 else if (!is_software_event(sibling))
300 return false;
301 }
302
303 return num_hw <= PMU_NR_COUNTERS;
304}
305
306static int l2x0_pmu_event_init(struct perf_event *event)
307{
308 struct hw_perf_event *hw = &event->hw;
309
310 if (event->attr.type != l2x0_pmu->type)
311 return -ENOENT;
312
313 if (is_sampling_event(event) ||
314 event->attach_state & PERF_ATTACH_TASK)
315 return -EINVAL;
316
317 if (event->attr.exclude_user ||
318 event->attr.exclude_kernel ||
319 event->attr.exclude_hv ||
320 event->attr.exclude_idle ||
321 event->attr.exclude_host ||
322 event->attr.exclude_guest)
323 return -EINVAL;
324
325 if (event->cpu < 0)
326 return -EINVAL;
327
328 if (event->attr.config & ~L2X0_EVENT_CNT_CFG_SRC_MASK)
329 return -EINVAL;
330
331 hw->config_base = event->attr.config;
332
333 if (!l2x0_pmu_group_is_valid(event))
334 return -EINVAL;
335
336 event->cpu = cpumask_first(&pmu_cpu);
337
338 return 0;
339}
340
341struct l2x0_event_attribute {
342 struct device_attribute attr;
343 unsigned int config;
344 bool pl310_only;
345};
346
347#define L2X0_EVENT_ATTR(_name, _config, _pl310_only) \
348 (&((struct l2x0_event_attribute[]) {{ \
349 .attr = __ATTR(_name, S_IRUGO, l2x0_pmu_event_show, NULL), \
350 .config = _config, \
351 .pl310_only = _pl310_only, \
352 }})[0].attr.attr)
353
354#define L220_PLUS_EVENT_ATTR(_name, _config) \
355 L2X0_EVENT_ATTR(_name, _config, false)
356
357#define PL310_EVENT_ATTR(_name, _config) \
358 L2X0_EVENT_ATTR(_name, _config, true)
359
360static ssize_t l2x0_pmu_event_show(struct device *dev,
361 struct device_attribute *attr, char *buf)
362{
363 struct l2x0_event_attribute *lattr;
364
365 lattr = container_of(attr, typeof(*lattr), attr);
366 return snprintf(buf, PAGE_SIZE, "config=0x%x\n", lattr->config);
367}
368
369static umode_t l2x0_pmu_event_attr_is_visible(struct kobject *kobj,
370 struct attribute *attr,
371 int unused)
372{
373 struct device *dev = kobj_to_dev(kobj);
374 struct pmu *pmu = dev_get_drvdata(dev);
375 struct l2x0_event_attribute *lattr;
376
377 lattr = container_of(attr, typeof(*lattr), attr.attr);
378
379 if (!lattr->pl310_only || strcmp("l2c_310", pmu->name) == 0)
380 return attr->mode;
381
382 return 0;
383}
384
385static struct attribute *l2x0_pmu_event_attrs[] = {
386 L220_PLUS_EVENT_ATTR(co, 0x1),
387 L220_PLUS_EVENT_ATTR(drhit, 0x2),
388 L220_PLUS_EVENT_ATTR(drreq, 0x3),
389 L220_PLUS_EVENT_ATTR(dwhit, 0x4),
390 L220_PLUS_EVENT_ATTR(dwreq, 0x5),
391 L220_PLUS_EVENT_ATTR(dwtreq, 0x6),
392 L220_PLUS_EVENT_ATTR(irhit, 0x7),
393 L220_PLUS_EVENT_ATTR(irreq, 0x8),
394 L220_PLUS_EVENT_ATTR(wa, 0x9),
395 PL310_EVENT_ATTR(ipfalloc, 0xa),
396 PL310_EVENT_ATTR(epfhit, 0xb),
397 PL310_EVENT_ATTR(epfalloc, 0xc),
398 PL310_EVENT_ATTR(srrcvd, 0xd),
399 PL310_EVENT_ATTR(srconf, 0xe),
400 PL310_EVENT_ATTR(epfrcvd, 0xf),
401 NULL
402};
403
404static struct attribute_group l2x0_pmu_event_attrs_group = {
405 .name = "events",
406 .attrs = l2x0_pmu_event_attrs,
407 .is_visible = l2x0_pmu_event_attr_is_visible,
408};
409
410static ssize_t l2x0_pmu_cpumask_show(struct device *dev,
411 struct device_attribute *attr, char *buf)
412{
413 return cpumap_print_to_pagebuf(true, buf, &pmu_cpu);
414}
415
416static struct device_attribute l2x0_pmu_cpumask_attr =
417 __ATTR(cpumask, S_IRUGO, l2x0_pmu_cpumask_show, NULL);
418
419static struct attribute *l2x0_pmu_cpumask_attrs[] = {
420 &l2x0_pmu_cpumask_attr.attr,
421 NULL,
422};
423
424static struct attribute_group l2x0_pmu_cpumask_attr_group = {
425 .attrs = l2x0_pmu_cpumask_attrs,
426};
427
428static const struct attribute_group *l2x0_pmu_attr_groups[] = {
429 &l2x0_pmu_event_attrs_group,
430 &l2x0_pmu_cpumask_attr_group,
431 NULL,
432};
433
434static void l2x0_pmu_reset(void)
435{
436 int i;
437
438 __l2x0_pmu_disable();
439
440 for (i = 0; i < PMU_NR_COUNTERS; i++)
441 __l2x0_pmu_event_disable(i);
442}
443
444static int l2x0_pmu_offline_cpu(unsigned int cpu)
445{
446 unsigned int target;
447
448 if (!cpumask_test_and_clear_cpu(cpu, &pmu_cpu))
449 return 0;
450
451 target = cpumask_any_but(cpu_online_mask, cpu);
452 if (target >= nr_cpu_ids)
453 return 0;
454
455 perf_pmu_migrate_context(l2x0_pmu, cpu, target);
456 cpumask_set_cpu(target, &pmu_cpu);
457
458 return 0;
459}
460
461void l2x0_pmu_suspend(void)
462{
463 int i;
464
465 if (!l2x0_pmu)
466 return;
467
468 l2x0_pmu_disable(l2x0_pmu);
469
470 for (i = 0; i < PMU_NR_COUNTERS; i++) {
471 if (events[i])
472 l2x0_pmu_event_stop(events[i], PERF_EF_UPDATE);
473 }
474
475}
476
477void l2x0_pmu_resume(void)
478{
479 int i;
480
481 if (!l2x0_pmu)
482 return;
483
484 l2x0_pmu_reset();
485
486 for (i = 0; i < PMU_NR_COUNTERS; i++) {
487 if (events[i])
488 l2x0_pmu_event_start(events[i], PERF_EF_RELOAD);
489 }
490
491 l2x0_pmu_enable(l2x0_pmu);
492}
493
494void __init l2x0_pmu_register(void __iomem *base, u32 part)
495{
496 /*
497 * Determine whether we support the PMU, and choose the name for sysfs.
498 * This is also used by l2x0_pmu_event_attr_is_visible to determine
499 * which events to display, as the PL310 PMU supports a superset of
500 * L220 events.
501 *
502 * The L210 PMU has a different programmer's interface, and is not
503 * supported by this driver.
504 *
505 * We must defer registering the PMU until the perf subsystem is up and
506 * running, so just stash the name and base, and leave that to another
507 * initcall.
508 */
509 switch (part & L2X0_CACHE_ID_PART_MASK) {
510 case L2X0_CACHE_ID_PART_L220:
511 l2x0_name = "l2c_220";
512 break;
513 case L2X0_CACHE_ID_PART_L310:
514 l2x0_name = "l2c_310";
515 break;
516 default:
517 return;
518 }
519
520 l2x0_base = base;
521}
522
523static __init int l2x0_pmu_init(void)
524{
525 int ret;
526
527 if (!l2x0_base)
528 return 0;
529
530 l2x0_pmu = kzalloc(sizeof(*l2x0_pmu), GFP_KERNEL);
531 if (!l2x0_pmu) {
532 pr_warn("Unable to allocate L2x0 PMU\n");
533 return -ENOMEM;
534 }
535
536 *l2x0_pmu = (struct pmu) {
537 .task_ctx_nr = perf_invalid_context,
538 .pmu_enable = l2x0_pmu_enable,
539 .pmu_disable = l2x0_pmu_disable,
540 .read = l2x0_pmu_event_read,
541 .start = l2x0_pmu_event_start,
542 .stop = l2x0_pmu_event_stop,
543 .add = l2x0_pmu_event_add,
544 .del = l2x0_pmu_event_del,
545 .event_init = l2x0_pmu_event_init,
546 .attr_groups = l2x0_pmu_attr_groups,
547 };
548
549 l2x0_pmu_reset();
550
551 /*
552 * We always use a hrtimer rather than an interrupt.
553 * See comments in l2x0_pmu_event_configure and l2x0_pmu_poll.
554 *
555 * Polling once a second allows the counters to fill up to 1/128th on a
556 * quad-core test chip with cores clocked at 400MHz. Hopefully this
557 * leaves sufficient headroom to avoid overflow on production silicon
558 * at higher frequencies.
559 */
560 l2x0_pmu_poll_period = ms_to_ktime(1000);
561 hrtimer_init(&l2x0_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
562 l2x0_pmu_hrtimer.function = l2x0_pmu_poll;
563
564 cpumask_set_cpu(0, &pmu_cpu);
565 ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE,
566 "perf/arm/l2x0:online", NULL,
567 l2x0_pmu_offline_cpu);
568 if (ret)
569 goto out_pmu;
570
571 ret = perf_pmu_register(l2x0_pmu, l2x0_name, -1);
572 if (ret)
573 goto out_cpuhp;
574
575 return 0;
576
577out_cpuhp:
578 cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE);
579out_pmu:
580 kfree(l2x0_pmu);
581 l2x0_pmu = NULL;
582 return ret;
583}
584device_initcall(l2x0_pmu_init);