Loading...
Note: File does not exist in v6.2.
1/* Copyright (c) 2014, The Linux Foundation. All rights reserved.
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 */
12
13#include <linux/kernel.h>
14#include <linux/moduleparam.h>
15#include <linux/init.h>
16#include <linux/types.h>
17#include <linux/device.h>
18#include <linux/io.h>
19#include <linux/err.h>
20#include <linux/fs.h>
21#include <linux/slab.h>
22#include <linux/delay.h>
23#include <linux/smp.h>
24#include <linux/sysfs.h>
25#include <linux/stat.h>
26#include <linux/clk.h>
27#include <linux/cpu.h>
28#include <linux/coresight.h>
29#include <linux/coresight-pmu.h>
30#include <linux/pm_wakeup.h>
31#include <linux/amba/bus.h>
32#include <linux/seq_file.h>
33#include <linux/uaccess.h>
34#include <linux/perf_event.h>
35#include <linux/pm_runtime.h>
36#include <asm/sections.h>
37#include <asm/local.h>
38
39#include "coresight-etm4x.h"
40#include "coresight-etm-perf.h"
41
42static int boot_enable;
43module_param_named(boot_enable, boot_enable, int, S_IRUGO);
44
45/* The number of ETMv4 currently registered */
46static int etm4_count;
47static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
48static void etm4_set_default_config(struct etmv4_config *config);
49static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
50 struct perf_event *event);
51
52static enum cpuhp_state hp_online;
53
54static void etm4_os_unlock(struct etmv4_drvdata *drvdata)
55{
56 /* Writing any value to ETMOSLAR unlocks the trace registers */
57 writel_relaxed(0x0, drvdata->base + TRCOSLAR);
58 drvdata->os_unlock = true;
59 isb();
60}
61
62static bool etm4_arch_supported(u8 arch)
63{
64 switch (arch) {
65 case ETM_ARCH_V4:
66 break;
67 default:
68 return false;
69 }
70 return true;
71}
72
73static int etm4_cpu_id(struct coresight_device *csdev)
74{
75 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
76
77 return drvdata->cpu;
78}
79
80static int etm4_trace_id(struct coresight_device *csdev)
81{
82 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
83
84 return drvdata->trcid;
85}
86
87static void etm4_enable_hw(void *info)
88{
89 int i;
90 struct etmv4_drvdata *drvdata = info;
91 struct etmv4_config *config = &drvdata->config;
92
93 CS_UNLOCK(drvdata->base);
94
95 etm4_os_unlock(drvdata);
96
97 /* Disable the trace unit before programming trace registers */
98 writel_relaxed(0, drvdata->base + TRCPRGCTLR);
99
100 /* wait for TRCSTATR.IDLE to go up */
101 if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1))
102 dev_err(drvdata->dev,
103 "timeout while waiting for Idle Trace Status\n");
104
105 writel_relaxed(config->pe_sel, drvdata->base + TRCPROCSELR);
106 writel_relaxed(config->cfg, drvdata->base + TRCCONFIGR);
107 /* nothing specific implemented */
108 writel_relaxed(0x0, drvdata->base + TRCAUXCTLR);
109 writel_relaxed(config->eventctrl0, drvdata->base + TRCEVENTCTL0R);
110 writel_relaxed(config->eventctrl1, drvdata->base + TRCEVENTCTL1R);
111 writel_relaxed(config->stall_ctrl, drvdata->base + TRCSTALLCTLR);
112 writel_relaxed(config->ts_ctrl, drvdata->base + TRCTSCTLR);
113 writel_relaxed(config->syncfreq, drvdata->base + TRCSYNCPR);
114 writel_relaxed(config->ccctlr, drvdata->base + TRCCCCTLR);
115 writel_relaxed(config->bb_ctrl, drvdata->base + TRCBBCTLR);
116 writel_relaxed(drvdata->trcid, drvdata->base + TRCTRACEIDR);
117 writel_relaxed(config->vinst_ctrl, drvdata->base + TRCVICTLR);
118 writel_relaxed(config->viiectlr, drvdata->base + TRCVIIECTLR);
119 writel_relaxed(config->vissctlr,
120 drvdata->base + TRCVISSCTLR);
121 writel_relaxed(config->vipcssctlr,
122 drvdata->base + TRCVIPCSSCTLR);
123 for (i = 0; i < drvdata->nrseqstate - 1; i++)
124 writel_relaxed(config->seq_ctrl[i],
125 drvdata->base + TRCSEQEVRn(i));
126 writel_relaxed(config->seq_rst, drvdata->base + TRCSEQRSTEVR);
127 writel_relaxed(config->seq_state, drvdata->base + TRCSEQSTR);
128 writel_relaxed(config->ext_inp, drvdata->base + TRCEXTINSELR);
129 for (i = 0; i < drvdata->nr_cntr; i++) {
130 writel_relaxed(config->cntrldvr[i],
131 drvdata->base + TRCCNTRLDVRn(i));
132 writel_relaxed(config->cntr_ctrl[i],
133 drvdata->base + TRCCNTCTLRn(i));
134 writel_relaxed(config->cntr_val[i],
135 drvdata->base + TRCCNTVRn(i));
136 }
137
138 /* Resource selector pair 0 is always implemented and reserved */
139 for (i = 0; i < drvdata->nr_resource * 2; i++)
140 writel_relaxed(config->res_ctrl[i],
141 drvdata->base + TRCRSCTLRn(i));
142
143 for (i = 0; i < drvdata->nr_ss_cmp; i++) {
144 writel_relaxed(config->ss_ctrl[i],
145 drvdata->base + TRCSSCCRn(i));
146 writel_relaxed(config->ss_status[i],
147 drvdata->base + TRCSSCSRn(i));
148 writel_relaxed(config->ss_pe_cmp[i],
149 drvdata->base + TRCSSPCICRn(i));
150 }
151 for (i = 0; i < drvdata->nr_addr_cmp; i++) {
152 writeq_relaxed(config->addr_val[i],
153 drvdata->base + TRCACVRn(i));
154 writeq_relaxed(config->addr_acc[i],
155 drvdata->base + TRCACATRn(i));
156 }
157 for (i = 0; i < drvdata->numcidc; i++)
158 writeq_relaxed(config->ctxid_pid[i],
159 drvdata->base + TRCCIDCVRn(i));
160 writel_relaxed(config->ctxid_mask0, drvdata->base + TRCCIDCCTLR0);
161 writel_relaxed(config->ctxid_mask1, drvdata->base + TRCCIDCCTLR1);
162
163 for (i = 0; i < drvdata->numvmidc; i++)
164 writeq_relaxed(config->vmid_val[i],
165 drvdata->base + TRCVMIDCVRn(i));
166 writel_relaxed(config->vmid_mask0, drvdata->base + TRCVMIDCCTLR0);
167 writel_relaxed(config->vmid_mask1, drvdata->base + TRCVMIDCCTLR1);
168
169 /*
170 * Request to keep the trace unit powered and also
171 * emulation of powerdown
172 */
173 writel_relaxed(readl_relaxed(drvdata->base + TRCPDCR) | TRCPDCR_PU,
174 drvdata->base + TRCPDCR);
175
176 /* Enable the trace unit */
177 writel_relaxed(1, drvdata->base + TRCPRGCTLR);
178
179 /* wait for TRCSTATR.IDLE to go back down to '0' */
180 if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 0))
181 dev_err(drvdata->dev,
182 "timeout while waiting for Idle Trace Status\n");
183
184 CS_LOCK(drvdata->base);
185
186 dev_dbg(drvdata->dev, "cpu: %d enable smp call done\n", drvdata->cpu);
187}
188
189static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
190 struct perf_event *event)
191{
192 int ret = 0;
193 struct etmv4_config *config = &drvdata->config;
194 struct perf_event_attr *attr = &event->attr;
195
196 if (!attr) {
197 ret = -EINVAL;
198 goto out;
199 }
200
201 /* Clear configuration from previous run */
202 memset(config, 0, sizeof(struct etmv4_config));
203
204 if (attr->exclude_kernel)
205 config->mode = ETM_MODE_EXCL_KERN;
206
207 if (attr->exclude_user)
208 config->mode = ETM_MODE_EXCL_USER;
209
210 /* Always start from the default config */
211 etm4_set_default_config(config);
212
213 /* Configure filters specified on the perf cmd line, if any. */
214 ret = etm4_set_event_filters(drvdata, event);
215 if (ret)
216 goto out;
217
218 /* Go from generic option to ETMv4 specifics */
219 if (attr->config & BIT(ETM_OPT_CYCACC)) {
220 config->cfg |= BIT(4);
221 /* TRM: Must program this for cycacc to work */
222 config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT;
223 }
224 if (attr->config & BIT(ETM_OPT_TS))
225 /* bit[11], Global timestamp tracing bit */
226 config->cfg |= BIT(11);
227 /* return stack - enable if selected and supported */
228 if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)
229 /* bit[12], Return stack enable bit */
230 config->cfg |= BIT(12);
231
232out:
233 return ret;
234}
235
236static int etm4_enable_perf(struct coresight_device *csdev,
237 struct perf_event *event)
238{
239 int ret = 0;
240 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
241
242 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {
243 ret = -EINVAL;
244 goto out;
245 }
246
247 /* Configure the tracer based on the session's specifics */
248 ret = etm4_parse_event_config(drvdata, event);
249 if (ret)
250 goto out;
251 /* And enable it */
252 etm4_enable_hw(drvdata);
253
254out:
255 return ret;
256}
257
258static int etm4_enable_sysfs(struct coresight_device *csdev)
259{
260 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
261 int ret;
262
263 spin_lock(&drvdata->spinlock);
264
265 /*
266 * Executing etm4_enable_hw on the cpu whose ETM is being enabled
267 * ensures that register writes occur when cpu is powered.
268 */
269 ret = smp_call_function_single(drvdata->cpu,
270 etm4_enable_hw, drvdata, 1);
271 if (ret)
272 goto err;
273
274 drvdata->sticky_enable = true;
275 spin_unlock(&drvdata->spinlock);
276
277 dev_info(drvdata->dev, "ETM tracing enabled\n");
278 return 0;
279
280err:
281 spin_unlock(&drvdata->spinlock);
282 return ret;
283}
284
285static int etm4_enable(struct coresight_device *csdev,
286 struct perf_event *event, u32 mode)
287{
288 int ret;
289 u32 val;
290 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
291
292 val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode);
293
294 /* Someone is already using the tracer */
295 if (val)
296 return -EBUSY;
297
298 switch (mode) {
299 case CS_MODE_SYSFS:
300 ret = etm4_enable_sysfs(csdev);
301 break;
302 case CS_MODE_PERF:
303 ret = etm4_enable_perf(csdev, event);
304 break;
305 default:
306 ret = -EINVAL;
307 }
308
309 /* The tracer didn't start */
310 if (ret)
311 local_set(&drvdata->mode, CS_MODE_DISABLED);
312
313 return ret;
314}
315
316static void etm4_disable_hw(void *info)
317{
318 u32 control;
319 struct etmv4_drvdata *drvdata = info;
320
321 CS_UNLOCK(drvdata->base);
322
323 /* power can be removed from the trace unit now */
324 control = readl_relaxed(drvdata->base + TRCPDCR);
325 control &= ~TRCPDCR_PU;
326 writel_relaxed(control, drvdata->base + TRCPDCR);
327
328 control = readl_relaxed(drvdata->base + TRCPRGCTLR);
329
330 /* EN, bit[0] Trace unit enable bit */
331 control &= ~0x1;
332
333 /* make sure everything completes before disabling */
334 mb();
335 isb();
336 writel_relaxed(control, drvdata->base + TRCPRGCTLR);
337
338 CS_LOCK(drvdata->base);
339
340 dev_dbg(drvdata->dev, "cpu: %d disable smp call done\n", drvdata->cpu);
341}
342
343static int etm4_disable_perf(struct coresight_device *csdev,
344 struct perf_event *event)
345{
346 u32 control;
347 struct etm_filters *filters = event->hw.addr_filters;
348 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
349
350 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
351 return -EINVAL;
352
353 etm4_disable_hw(drvdata);
354
355 /*
356 * Check if the start/stop logic was active when the unit was stopped.
357 * That way we can re-enable the start/stop logic when the process is
358 * scheduled again. Configuration of the start/stop logic happens in
359 * function etm4_set_event_filters().
360 */
361 control = readl_relaxed(drvdata->base + TRCVICTLR);
362 /* TRCVICTLR::SSSTATUS, bit[9] */
363 filters->ssstatus = (control & BIT(9));
364
365 return 0;
366}
367
368static void etm4_disable_sysfs(struct coresight_device *csdev)
369{
370 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
371
372 /*
373 * Taking hotplug lock here protects from clocks getting disabled
374 * with tracing being left on (crash scenario) if user disable occurs
375 * after cpu online mask indicates the cpu is offline but before the
376 * DYING hotplug callback is serviced by the ETM driver.
377 */
378 cpus_read_lock();
379 spin_lock(&drvdata->spinlock);
380
381 /*
382 * Executing etm4_disable_hw on the cpu whose ETM is being disabled
383 * ensures that register writes occur when cpu is powered.
384 */
385 smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);
386
387 spin_unlock(&drvdata->spinlock);
388 cpus_read_unlock();
389
390 dev_info(drvdata->dev, "ETM tracing disabled\n");
391}
392
393static void etm4_disable(struct coresight_device *csdev,
394 struct perf_event *event)
395{
396 u32 mode;
397 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
398
399 /*
400 * For as long as the tracer isn't disabled another entity can't
401 * change its status. As such we can read the status here without
402 * fearing it will change under us.
403 */
404 mode = local_read(&drvdata->mode);
405
406 switch (mode) {
407 case CS_MODE_DISABLED:
408 break;
409 case CS_MODE_SYSFS:
410 etm4_disable_sysfs(csdev);
411 break;
412 case CS_MODE_PERF:
413 etm4_disable_perf(csdev, event);
414 break;
415 }
416
417 if (mode)
418 local_set(&drvdata->mode, CS_MODE_DISABLED);
419}
420
421static const struct coresight_ops_source etm4_source_ops = {
422 .cpu_id = etm4_cpu_id,
423 .trace_id = etm4_trace_id,
424 .enable = etm4_enable,
425 .disable = etm4_disable,
426};
427
428static const struct coresight_ops etm4_cs_ops = {
429 .source_ops = &etm4_source_ops,
430};
431
432static void etm4_init_arch_data(void *info)
433{
434 u32 etmidr0;
435 u32 etmidr1;
436 u32 etmidr2;
437 u32 etmidr3;
438 u32 etmidr4;
439 u32 etmidr5;
440 struct etmv4_drvdata *drvdata = info;
441
442 /* Make sure all registers are accessible */
443 etm4_os_unlock(drvdata);
444
445 CS_UNLOCK(drvdata->base);
446
447 /* find all capabilities of the tracing unit */
448 etmidr0 = readl_relaxed(drvdata->base + TRCIDR0);
449
450 /* INSTP0, bits[2:1] P0 tracing support field */
451 if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2))
452 drvdata->instrp0 = true;
453 else
454 drvdata->instrp0 = false;
455
456 /* TRCBB, bit[5] Branch broadcast tracing support bit */
457 if (BMVAL(etmidr0, 5, 5))
458 drvdata->trcbb = true;
459 else
460 drvdata->trcbb = false;
461
462 /* TRCCOND, bit[6] Conditional instruction tracing support bit */
463 if (BMVAL(etmidr0, 6, 6))
464 drvdata->trccond = true;
465 else
466 drvdata->trccond = false;
467
468 /* TRCCCI, bit[7] Cycle counting instruction bit */
469 if (BMVAL(etmidr0, 7, 7))
470 drvdata->trccci = true;
471 else
472 drvdata->trccci = false;
473
474 /* RETSTACK, bit[9] Return stack bit */
475 if (BMVAL(etmidr0, 9, 9))
476 drvdata->retstack = true;
477 else
478 drvdata->retstack = false;
479
480 /* NUMEVENT, bits[11:10] Number of events field */
481 drvdata->nr_event = BMVAL(etmidr0, 10, 11);
482 /* QSUPP, bits[16:15] Q element support field */
483 drvdata->q_support = BMVAL(etmidr0, 15, 16);
484 /* TSSIZE, bits[28:24] Global timestamp size field */
485 drvdata->ts_size = BMVAL(etmidr0, 24, 28);
486
487 /* base architecture of trace unit */
488 etmidr1 = readl_relaxed(drvdata->base + TRCIDR1);
489 /*
490 * TRCARCHMIN, bits[7:4] architecture the minor version number
491 * TRCARCHMAJ, bits[11:8] architecture major versin number
492 */
493 drvdata->arch = BMVAL(etmidr1, 4, 11);
494
495 /* maximum size of resources */
496 etmidr2 = readl_relaxed(drvdata->base + TRCIDR2);
497 /* CIDSIZE, bits[9:5] Indicates the Context ID size */
498 drvdata->ctxid_size = BMVAL(etmidr2, 5, 9);
499 /* VMIDSIZE, bits[14:10] Indicates the VMID size */
500 drvdata->vmid_size = BMVAL(etmidr2, 10, 14);
501 /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */
502 drvdata->ccsize = BMVAL(etmidr2, 25, 28);
503
504 etmidr3 = readl_relaxed(drvdata->base + TRCIDR3);
505 /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */
506 drvdata->ccitmin = BMVAL(etmidr3, 0, 11);
507 /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */
508 drvdata->s_ex_level = BMVAL(etmidr3, 16, 19);
509 /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */
510 drvdata->ns_ex_level = BMVAL(etmidr3, 20, 23);
511
512 /*
513 * TRCERR, bit[24] whether a trace unit can trace a
514 * system error exception.
515 */
516 if (BMVAL(etmidr3, 24, 24))
517 drvdata->trc_error = true;
518 else
519 drvdata->trc_error = false;
520
521 /* SYNCPR, bit[25] implementation has a fixed synchronization period? */
522 if (BMVAL(etmidr3, 25, 25))
523 drvdata->syncpr = true;
524 else
525 drvdata->syncpr = false;
526
527 /* STALLCTL, bit[26] is stall control implemented? */
528 if (BMVAL(etmidr3, 26, 26))
529 drvdata->stallctl = true;
530 else
531 drvdata->stallctl = false;
532
533 /* SYSSTALL, bit[27] implementation can support stall control? */
534 if (BMVAL(etmidr3, 27, 27))
535 drvdata->sysstall = true;
536 else
537 drvdata->sysstall = false;
538
539 /* NUMPROC, bits[30:28] the number of PEs available for tracing */
540 drvdata->nr_pe = BMVAL(etmidr3, 28, 30);
541
542 /* NOOVERFLOW, bit[31] is trace overflow prevention supported */
543 if (BMVAL(etmidr3, 31, 31))
544 drvdata->nooverflow = true;
545 else
546 drvdata->nooverflow = false;
547
548 /* number of resources trace unit supports */
549 etmidr4 = readl_relaxed(drvdata->base + TRCIDR4);
550 /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */
551 drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3);
552 /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */
553 drvdata->nr_pe_cmp = BMVAL(etmidr4, 12, 15);
554 /*
555 * NUMRSPAIR, bits[19:16]
556 * The number of resource pairs conveyed by the HW starts at 0, i.e a
557 * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on.
558 * As such add 1 to the value of NUMRSPAIR for a better representation.
559 */
560 drvdata->nr_resource = BMVAL(etmidr4, 16, 19) + 1;
561 /*
562 * NUMSSCC, bits[23:20] the number of single-shot
563 * comparator control for tracing
564 */
565 drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23);
566 /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */
567 drvdata->numcidc = BMVAL(etmidr4, 24, 27);
568 /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */
569 drvdata->numvmidc = BMVAL(etmidr4, 28, 31);
570
571 etmidr5 = readl_relaxed(drvdata->base + TRCIDR5);
572 /* NUMEXTIN, bits[8:0] number of external inputs implemented */
573 drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8);
574 /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */
575 drvdata->trcid_size = BMVAL(etmidr5, 16, 21);
576 /* ATBTRIG, bit[22] implementation can support ATB triggers? */
577 if (BMVAL(etmidr5, 22, 22))
578 drvdata->atbtrig = true;
579 else
580 drvdata->atbtrig = false;
581 /*
582 * LPOVERRIDE, bit[23] implementation supports
583 * low-power state override
584 */
585 if (BMVAL(etmidr5, 23, 23))
586 drvdata->lpoverride = true;
587 else
588 drvdata->lpoverride = false;
589 /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */
590 drvdata->nrseqstate = BMVAL(etmidr5, 25, 27);
591 /* NUMCNTR, bits[30:28] number of counters available for tracing */
592 drvdata->nr_cntr = BMVAL(etmidr5, 28, 30);
593 CS_LOCK(drvdata->base);
594}
595
596static void etm4_set_default_config(struct etmv4_config *config)
597{
598 /* disable all events tracing */
599 config->eventctrl0 = 0x0;
600 config->eventctrl1 = 0x0;
601
602 /* disable stalling */
603 config->stall_ctrl = 0x0;
604
605 /* enable trace synchronization every 4096 bytes, if available */
606 config->syncfreq = 0xC;
607
608 /* disable timestamp event */
609 config->ts_ctrl = 0x0;
610
611 /* TRCVICTLR::EVENT = 0x01, select the always on logic */
612 config->vinst_ctrl |= BIT(0);
613}
614
615static u64 etm4_get_access_type(struct etmv4_config *config)
616{
617 u64 access_type = 0;
618
619 /*
620 * EXLEVEL_NS, bits[15:12]
621 * The Exception levels are:
622 * Bit[12] Exception level 0 - Application
623 * Bit[13] Exception level 1 - OS
624 * Bit[14] Exception level 2 - Hypervisor
625 * Bit[15] Never implemented
626 *
627 * Always stay away from hypervisor mode.
628 */
629 access_type = ETM_EXLEVEL_NS_HYP;
630
631 if (config->mode & ETM_MODE_EXCL_KERN)
632 access_type |= ETM_EXLEVEL_NS_OS;
633
634 if (config->mode & ETM_MODE_EXCL_USER)
635 access_type |= ETM_EXLEVEL_NS_APP;
636
637 /*
638 * EXLEVEL_S, bits[11:8], don't trace anything happening
639 * in secure state.
640 */
641 access_type |= (ETM_EXLEVEL_S_APP |
642 ETM_EXLEVEL_S_OS |
643 ETM_EXLEVEL_S_HYP);
644
645 return access_type;
646}
647
648static void etm4_set_comparator_filter(struct etmv4_config *config,
649 u64 start, u64 stop, int comparator)
650{
651 u64 access_type = etm4_get_access_type(config);
652
653 /* First half of default address comparator */
654 config->addr_val[comparator] = start;
655 config->addr_acc[comparator] = access_type;
656 config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;
657
658 /* Second half of default address comparator */
659 config->addr_val[comparator + 1] = stop;
660 config->addr_acc[comparator + 1] = access_type;
661 config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;
662
663 /*
664 * Configure the ViewInst function to include this address range
665 * comparator.
666 *
667 * @comparator is divided by two since it is the index in the
668 * etmv4_config::addr_val array but register TRCVIIECTLR deals with
669 * address range comparator _pairs_.
670 *
671 * Therefore:
672 * index 0 -> compatator pair 0
673 * index 2 -> comparator pair 1
674 * index 4 -> comparator pair 2
675 * ...
676 * index 14 -> comparator pair 7
677 */
678 config->viiectlr |= BIT(comparator / 2);
679}
680
681static void etm4_set_start_stop_filter(struct etmv4_config *config,
682 u64 address, int comparator,
683 enum etm_addr_type type)
684{
685 int shift;
686 u64 access_type = etm4_get_access_type(config);
687
688 /* Configure the comparator */
689 config->addr_val[comparator] = address;
690 config->addr_acc[comparator] = access_type;
691 config->addr_type[comparator] = type;
692
693 /*
694 * Configure ViewInst Start-Stop control register.
695 * Addresses configured to start tracing go from bit 0 to n-1,
696 * while those configured to stop tracing from 16 to 16 + n-1.
697 */
698 shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);
699 config->vissctlr |= BIT(shift + comparator);
700}
701
702static void etm4_set_default_filter(struct etmv4_config *config)
703{
704 u64 start, stop;
705
706 /*
707 * Configure address range comparator '0' to encompass all
708 * possible addresses.
709 */
710 start = 0x0;
711 stop = ~0x0;
712
713 etm4_set_comparator_filter(config, start, stop,
714 ETM_DEFAULT_ADDR_COMP);
715
716 /*
717 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
718 * in the started state
719 */
720 config->vinst_ctrl |= BIT(9);
721
722 /* No start-stop filtering for ViewInst */
723 config->vissctlr = 0x0;
724}
725
726static void etm4_set_default(struct etmv4_config *config)
727{
728 if (WARN_ON_ONCE(!config))
729 return;
730
731 /*
732 * Make default initialisation trace everything
733 *
734 * Select the "always true" resource selector on the
735 * "Enablign Event" line and configure address range comparator
736 * '0' to trace all the possible address range. From there
737 * configure the "include/exclude" engine to include address
738 * range comparator '0'.
739 */
740 etm4_set_default_config(config);
741 etm4_set_default_filter(config);
742}
743
744static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)
745{
746 int nr_comparator, index = 0;
747 struct etmv4_config *config = &drvdata->config;
748
749 /*
750 * nr_addr_cmp holds the number of comparator _pair_, so time 2
751 * for the total number of comparators.
752 */
753 nr_comparator = drvdata->nr_addr_cmp * 2;
754
755 /* Go through the tally of comparators looking for a free one. */
756 while (index < nr_comparator) {
757 switch (type) {
758 case ETM_ADDR_TYPE_RANGE:
759 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&
760 config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)
761 return index;
762
763 /* Address range comparators go in pairs */
764 index += 2;
765 break;
766 case ETM_ADDR_TYPE_START:
767 case ETM_ADDR_TYPE_STOP:
768 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)
769 return index;
770
771 /* Start/stop address can have odd indexes */
772 index += 1;
773 break;
774 default:
775 return -EINVAL;
776 }
777 }
778
779 /* If we are here all the comparators have been used. */
780 return -ENOSPC;
781}
782
783static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
784 struct perf_event *event)
785{
786 int i, comparator, ret = 0;
787 u64 address;
788 struct etmv4_config *config = &drvdata->config;
789 struct etm_filters *filters = event->hw.addr_filters;
790
791 if (!filters)
792 goto default_filter;
793
794 /* Sync events with what Perf got */
795 perf_event_addr_filters_sync(event);
796
797 /*
798 * If there are no filters to deal with simply go ahead with
799 * the default filter, i.e the entire address range.
800 */
801 if (!filters->nr_filters)
802 goto default_filter;
803
804 for (i = 0; i < filters->nr_filters; i++) {
805 struct etm_filter *filter = &filters->etm_filter[i];
806 enum etm_addr_type type = filter->type;
807
808 /* See if a comparator is free. */
809 comparator = etm4_get_next_comparator(drvdata, type);
810 if (comparator < 0) {
811 ret = comparator;
812 goto out;
813 }
814
815 switch (type) {
816 case ETM_ADDR_TYPE_RANGE:
817 etm4_set_comparator_filter(config,
818 filter->start_addr,
819 filter->stop_addr,
820 comparator);
821 /*
822 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
823 * in the started state
824 */
825 config->vinst_ctrl |= BIT(9);
826
827 /* No start-stop filtering for ViewInst */
828 config->vissctlr = 0x0;
829 break;
830 case ETM_ADDR_TYPE_START:
831 case ETM_ADDR_TYPE_STOP:
832 /* Get the right start or stop address */
833 address = (type == ETM_ADDR_TYPE_START ?
834 filter->start_addr :
835 filter->stop_addr);
836
837 /* Configure comparator */
838 etm4_set_start_stop_filter(config, address,
839 comparator, type);
840
841 /*
842 * If filters::ssstatus == 1, trace acquisition was
843 * started but the process was yanked away before the
844 * the stop address was hit. As such the start/stop
845 * logic needs to be re-started so that tracing can
846 * resume where it left.
847 *
848 * The start/stop logic status when a process is
849 * scheduled out is checked in function
850 * etm4_disable_perf().
851 */
852 if (filters->ssstatus)
853 config->vinst_ctrl |= BIT(9);
854
855 /* No include/exclude filtering for ViewInst */
856 config->viiectlr = 0x0;
857 break;
858 default:
859 ret = -EINVAL;
860 goto out;
861 }
862 }
863
864 goto out;
865
866
867default_filter:
868 etm4_set_default_filter(config);
869
870out:
871 return ret;
872}
873
874void etm4_config_trace_mode(struct etmv4_config *config)
875{
876 u32 addr_acc, mode;
877
878 mode = config->mode;
879 mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER);
880
881 /* excluding kernel AND user space doesn't make sense */
882 WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER));
883
884 /* nothing to do if neither flags are set */
885 if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER))
886 return;
887
888 addr_acc = config->addr_acc[ETM_DEFAULT_ADDR_COMP];
889 /* clear default config */
890 addr_acc &= ~(ETM_EXLEVEL_NS_APP | ETM_EXLEVEL_NS_OS);
891
892 /*
893 * EXLEVEL_NS, bits[15:12]
894 * The Exception levels are:
895 * Bit[12] Exception level 0 - Application
896 * Bit[13] Exception level 1 - OS
897 * Bit[14] Exception level 2 - Hypervisor
898 * Bit[15] Never implemented
899 */
900 if (mode & ETM_MODE_EXCL_KERN)
901 addr_acc |= ETM_EXLEVEL_NS_OS;
902 else
903 addr_acc |= ETM_EXLEVEL_NS_APP;
904
905 config->addr_acc[ETM_DEFAULT_ADDR_COMP] = addr_acc;
906 config->addr_acc[ETM_DEFAULT_ADDR_COMP + 1] = addr_acc;
907}
908
909static int etm4_online_cpu(unsigned int cpu)
910{
911 if (!etmdrvdata[cpu])
912 return 0;
913
914 if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable)
915 coresight_enable(etmdrvdata[cpu]->csdev);
916 return 0;
917}
918
919static int etm4_starting_cpu(unsigned int cpu)
920{
921 if (!etmdrvdata[cpu])
922 return 0;
923
924 spin_lock(&etmdrvdata[cpu]->spinlock);
925 if (!etmdrvdata[cpu]->os_unlock) {
926 etm4_os_unlock(etmdrvdata[cpu]);
927 etmdrvdata[cpu]->os_unlock = true;
928 }
929
930 if (local_read(&etmdrvdata[cpu]->mode))
931 etm4_enable_hw(etmdrvdata[cpu]);
932 spin_unlock(&etmdrvdata[cpu]->spinlock);
933 return 0;
934}
935
936static int etm4_dying_cpu(unsigned int cpu)
937{
938 if (!etmdrvdata[cpu])
939 return 0;
940
941 spin_lock(&etmdrvdata[cpu]->spinlock);
942 if (local_read(&etmdrvdata[cpu]->mode))
943 etm4_disable_hw(etmdrvdata[cpu]);
944 spin_unlock(&etmdrvdata[cpu]->spinlock);
945 return 0;
946}
947
948static void etm4_init_trace_id(struct etmv4_drvdata *drvdata)
949{
950 drvdata->trcid = coresight_get_trace_id(drvdata->cpu);
951}
952
953static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
954{
955 int ret;
956 void __iomem *base;
957 struct device *dev = &adev->dev;
958 struct coresight_platform_data *pdata = NULL;
959 struct etmv4_drvdata *drvdata;
960 struct resource *res = &adev->res;
961 struct coresight_desc desc = { 0 };
962 struct device_node *np = adev->dev.of_node;
963
964 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
965 if (!drvdata)
966 return -ENOMEM;
967
968 if (np) {
969 pdata = of_get_coresight_platform_data(dev, np);
970 if (IS_ERR(pdata))
971 return PTR_ERR(pdata);
972 adev->dev.platform_data = pdata;
973 }
974
975 drvdata->dev = &adev->dev;
976 dev_set_drvdata(dev, drvdata);
977
978 /* Validity for the resource is already checked by the AMBA core */
979 base = devm_ioremap_resource(dev, res);
980 if (IS_ERR(base))
981 return PTR_ERR(base);
982
983 drvdata->base = base;
984
985 spin_lock_init(&drvdata->spinlock);
986
987 drvdata->cpu = pdata ? pdata->cpu : 0;
988
989 cpus_read_lock();
990 etmdrvdata[drvdata->cpu] = drvdata;
991
992 if (smp_call_function_single(drvdata->cpu,
993 etm4_init_arch_data, drvdata, 1))
994 dev_err(dev, "ETM arch init failed\n");
995
996 if (!etm4_count++) {
997 cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING,
998 "arm/coresight4:starting",
999 etm4_starting_cpu, etm4_dying_cpu);
1000 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
1001 "arm/coresight4:online",
1002 etm4_online_cpu, NULL);
1003 if (ret < 0)
1004 goto err_arch_supported;
1005 hp_online = ret;
1006 }
1007
1008 cpus_read_unlock();
1009
1010 if (etm4_arch_supported(drvdata->arch) == false) {
1011 ret = -EINVAL;
1012 goto err_arch_supported;
1013 }
1014
1015 etm4_init_trace_id(drvdata);
1016 etm4_set_default(&drvdata->config);
1017
1018 desc.type = CORESIGHT_DEV_TYPE_SOURCE;
1019 desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
1020 desc.ops = &etm4_cs_ops;
1021 desc.pdata = pdata;
1022 desc.dev = dev;
1023 desc.groups = coresight_etmv4_groups;
1024 drvdata->csdev = coresight_register(&desc);
1025 if (IS_ERR(drvdata->csdev)) {
1026 ret = PTR_ERR(drvdata->csdev);
1027 goto err_arch_supported;
1028 }
1029
1030 ret = etm_perf_symlink(drvdata->csdev, true);
1031 if (ret) {
1032 coresight_unregister(drvdata->csdev);
1033 goto err_arch_supported;
1034 }
1035
1036 pm_runtime_put(&adev->dev);
1037 dev_info(dev, "%s initialized\n", (char *)id->data);
1038
1039 if (boot_enable) {
1040 coresight_enable(drvdata->csdev);
1041 drvdata->boot_enable = true;
1042 }
1043
1044 return 0;
1045
1046err_arch_supported:
1047 if (--etm4_count == 0) {
1048 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1049 if (hp_online)
1050 cpuhp_remove_state_nocalls(hp_online);
1051 }
1052 return ret;
1053}
1054
1055static const struct amba_id etm4_ids[] = {
1056 { /* ETM 4.0 - Cortex-A53 */
1057 .id = 0x000bb95d,
1058 .mask = 0x000fffff,
1059 .data = "ETM 4.0",
1060 },
1061 { /* ETM 4.0 - Cortex-A57 */
1062 .id = 0x000bb95e,
1063 .mask = 0x000fffff,
1064 .data = "ETM 4.0",
1065 },
1066 { /* ETM 4.0 - A72, Maia, HiSilicon */
1067 .id = 0x000bb95a,
1068 .mask = 0x000fffff,
1069 .data = "ETM 4.0",
1070 },
1071 { 0, 0},
1072};
1073
1074static struct amba_driver etm4x_driver = {
1075 .drv = {
1076 .name = "coresight-etm4x",
1077 .suppress_bind_attrs = true,
1078 },
1079 .probe = etm4_probe,
1080 .id_table = etm4_ids,
1081};
1082builtin_amba_driver(etm4x_driver);