Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5 */
6
7#include <linux/atomic.h>
8#include <linux/circ_buf.h>
9#include <linux/coresight.h>
10#include <linux/perf_event.h>
11#include <linux/slab.h>
12#include "coresight-priv.h"
13#include "coresight-tmc.h"
14#include "coresight-etm-perf.h"
15
16static int tmc_set_etf_buffer(struct coresight_device *csdev,
17 struct perf_output_handle *handle);
18
19static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
20{
21 int rc = 0;
22
23 CS_UNLOCK(drvdata->base);
24
25 /* Wait for TMCSReady bit to be set */
26 rc = tmc_wait_for_tmcready(drvdata);
27 if (rc) {
28 dev_err(&drvdata->csdev->dev,
29 "Failed to enable: TMC not ready\n");
30 CS_LOCK(drvdata->base);
31 return rc;
32 }
33
34 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
35 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
36 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
37 TMC_FFCR_TRIGON_TRIGIN,
38 drvdata->base + TMC_FFCR);
39
40 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
41 tmc_enable_hw(drvdata);
42
43 CS_LOCK(drvdata->base);
44 return rc;
45}
46
47static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
48{
49 int rc = coresight_claim_device(drvdata->csdev);
50
51 if (rc)
52 return rc;
53
54 rc = __tmc_etb_enable_hw(drvdata);
55 if (rc)
56 coresight_disclaim_device(drvdata->csdev);
57 return rc;
58}
59
60static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
61{
62 char *bufp;
63 u32 read_data, lost;
64
65 /* Check if the buffer wrapped around. */
66 lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
67 bufp = drvdata->buf;
68 drvdata->len = 0;
69 while (1) {
70 read_data = readl_relaxed(drvdata->base + TMC_RRD);
71 if (read_data == 0xFFFFFFFF)
72 break;
73 memcpy(bufp, &read_data, 4);
74 bufp += 4;
75 drvdata->len += 4;
76 }
77
78 if (lost)
79 coresight_insert_barrier_packet(drvdata->buf);
80 return;
81}
82
83static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
84{
85 CS_UNLOCK(drvdata->base);
86
87 tmc_flush_and_stop(drvdata);
88 /*
89 * When operating in sysFS mode the content of the buffer needs to be
90 * read before the TMC is disabled.
91 */
92 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
93 tmc_etb_dump_hw(drvdata);
94 tmc_disable_hw(drvdata);
95
96 CS_LOCK(drvdata->base);
97}
98
99static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
100{
101 __tmc_etb_disable_hw(drvdata);
102 coresight_disclaim_device(drvdata->csdev);
103}
104
105static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
106{
107 int rc = 0;
108
109 CS_UNLOCK(drvdata->base);
110
111 /* Wait for TMCSReady bit to be set */
112 rc = tmc_wait_for_tmcready(drvdata);
113 if (rc) {
114 dev_err(&drvdata->csdev->dev,
115 "Failed to enable : TMC is not ready\n");
116 CS_LOCK(drvdata->base);
117 return rc;
118 }
119
120 writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
121 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
122 drvdata->base + TMC_FFCR);
123 writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
124 tmc_enable_hw(drvdata);
125
126 CS_LOCK(drvdata->base);
127 return rc;
128}
129
130static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
131{
132 int rc = coresight_claim_device(drvdata->csdev);
133
134 if (rc)
135 return rc;
136
137 rc = __tmc_etf_enable_hw(drvdata);
138 if (rc)
139 coresight_disclaim_device(drvdata->csdev);
140 return rc;
141}
142
143static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
144{
145 struct coresight_device *csdev = drvdata->csdev;
146
147 CS_UNLOCK(drvdata->base);
148
149 tmc_flush_and_stop(drvdata);
150 tmc_disable_hw(drvdata);
151 coresight_disclaim_device_unlocked(csdev);
152 CS_LOCK(drvdata->base);
153}
154
155/*
156 * Return the available trace data in the buffer from @pos, with
157 * a maximum limit of @len, updating the @bufpp on where to
158 * find it.
159 */
160ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
161 loff_t pos, size_t len, char **bufpp)
162{
163 ssize_t actual = len;
164
165 /* Adjust the len to available size @pos */
166 if (pos + actual > drvdata->len)
167 actual = drvdata->len - pos;
168 if (actual > 0)
169 *bufpp = drvdata->buf + pos;
170 return actual;
171}
172
173static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
174{
175 int ret = 0;
176 bool used = false;
177 char *buf = NULL;
178 unsigned long flags;
179 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
180
181 /*
182 * If we don't have a buffer release the lock and allocate memory.
183 * Otherwise keep the lock and move along.
184 */
185 spin_lock_irqsave(&drvdata->spinlock, flags);
186 if (!drvdata->buf) {
187 spin_unlock_irqrestore(&drvdata->spinlock, flags);
188
189 /* Allocating the memory here while outside of the spinlock */
190 buf = kzalloc(drvdata->size, GFP_KERNEL);
191 if (!buf)
192 return -ENOMEM;
193
194 /* Let's try again */
195 spin_lock_irqsave(&drvdata->spinlock, flags);
196 }
197
198 if (drvdata->reading) {
199 ret = -EBUSY;
200 goto out;
201 }
202
203 /*
204 * In sysFS mode we can have multiple writers per sink. Since this
205 * sink is already enabled no memory is needed and the HW need not be
206 * touched.
207 */
208 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
209 csdev->refcnt++;
210 goto out;
211 }
212
213 /*
214 * If drvdata::buf isn't NULL, memory was allocated for a previous
215 * trace run but wasn't read. If so simply zero-out the memory.
216 * Otherwise use the memory allocated above.
217 *
218 * The memory is freed when users read the buffer using the
219 * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for
220 * details.
221 */
222 if (drvdata->buf) {
223 memset(drvdata->buf, 0, drvdata->size);
224 } else {
225 used = true;
226 drvdata->buf = buf;
227 }
228
229 ret = tmc_etb_enable_hw(drvdata);
230 if (!ret) {
231 coresight_set_mode(csdev, CS_MODE_SYSFS);
232 csdev->refcnt++;
233 } else {
234 /* Free up the buffer if we failed to enable */
235 used = false;
236 }
237out:
238 spin_unlock_irqrestore(&drvdata->spinlock, flags);
239
240 /* Free memory outside the spinlock if need be */
241 if (!used)
242 kfree(buf);
243
244 return ret;
245}
246
247static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)
248{
249 int ret = 0;
250 pid_t pid;
251 unsigned long flags;
252 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
253 struct perf_output_handle *handle = data;
254 struct cs_buffers *buf = etm_perf_sink_config(handle);
255
256 spin_lock_irqsave(&drvdata->spinlock, flags);
257 do {
258 ret = -EINVAL;
259 if (drvdata->reading)
260 break;
261 /*
262 * No need to continue if the ETB/ETF is already operated
263 * from sysFS.
264 */
265 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
266 ret = -EBUSY;
267 break;
268 }
269
270 /* Get a handle on the pid of the process to monitor */
271 pid = buf->pid;
272
273 if (drvdata->pid != -1 && drvdata->pid != pid) {
274 ret = -EBUSY;
275 break;
276 }
277
278 ret = tmc_set_etf_buffer(csdev, handle);
279 if (ret)
280 break;
281
282 /*
283 * No HW configuration is needed if the sink is already in
284 * use for this session.
285 */
286 if (drvdata->pid == pid) {
287 csdev->refcnt++;
288 break;
289 }
290
291 ret = tmc_etb_enable_hw(drvdata);
292 if (!ret) {
293 /* Associate with monitored process. */
294 drvdata->pid = pid;
295 coresight_set_mode(csdev, CS_MODE_PERF);
296 csdev->refcnt++;
297 }
298 } while (0);
299 spin_unlock_irqrestore(&drvdata->spinlock, flags);
300
301 return ret;
302}
303
304static int tmc_enable_etf_sink(struct coresight_device *csdev,
305 enum cs_mode mode, void *data)
306{
307 int ret;
308
309 switch (mode) {
310 case CS_MODE_SYSFS:
311 ret = tmc_enable_etf_sink_sysfs(csdev);
312 break;
313 case CS_MODE_PERF:
314 ret = tmc_enable_etf_sink_perf(csdev, data);
315 break;
316 /* We shouldn't be here */
317 default:
318 ret = -EINVAL;
319 break;
320 }
321
322 if (ret)
323 return ret;
324
325 dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n");
326 return 0;
327}
328
329static int tmc_disable_etf_sink(struct coresight_device *csdev)
330{
331 unsigned long flags;
332 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
333
334 spin_lock_irqsave(&drvdata->spinlock, flags);
335
336 if (drvdata->reading) {
337 spin_unlock_irqrestore(&drvdata->spinlock, flags);
338 return -EBUSY;
339 }
340
341 csdev->refcnt--;
342 if (csdev->refcnt) {
343 spin_unlock_irqrestore(&drvdata->spinlock, flags);
344 return -EBUSY;
345 }
346
347 /* Complain if we (somehow) got out of sync */
348 WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);
349 tmc_etb_disable_hw(drvdata);
350 /* Dissociate from monitored process. */
351 drvdata->pid = -1;
352 coresight_set_mode(csdev, CS_MODE_DISABLED);
353
354 spin_unlock_irqrestore(&drvdata->spinlock, flags);
355
356 dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n");
357 return 0;
358}
359
360static int tmc_enable_etf_link(struct coresight_device *csdev,
361 struct coresight_connection *in,
362 struct coresight_connection *out)
363{
364 int ret = 0;
365 unsigned long flags;
366 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
367 bool first_enable = false;
368
369 spin_lock_irqsave(&drvdata->spinlock, flags);
370 if (drvdata->reading) {
371 spin_unlock_irqrestore(&drvdata->spinlock, flags);
372 return -EBUSY;
373 }
374
375 if (csdev->refcnt == 0) {
376 ret = tmc_etf_enable_hw(drvdata);
377 if (!ret) {
378 coresight_set_mode(csdev, CS_MODE_SYSFS);
379 first_enable = true;
380 }
381 }
382 if (!ret)
383 csdev->refcnt++;
384 spin_unlock_irqrestore(&drvdata->spinlock, flags);
385
386 if (first_enable)
387 dev_dbg(&csdev->dev, "TMC-ETF enabled\n");
388 return ret;
389}
390
391static void tmc_disable_etf_link(struct coresight_device *csdev,
392 struct coresight_connection *in,
393 struct coresight_connection *out)
394{
395 unsigned long flags;
396 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
397 bool last_disable = false;
398
399 spin_lock_irqsave(&drvdata->spinlock, flags);
400 if (drvdata->reading) {
401 spin_unlock_irqrestore(&drvdata->spinlock, flags);
402 return;
403 }
404
405 csdev->refcnt--;
406 if (csdev->refcnt == 0) {
407 tmc_etf_disable_hw(drvdata);
408 coresight_set_mode(csdev, CS_MODE_DISABLED);
409 last_disable = true;
410 }
411 spin_unlock_irqrestore(&drvdata->spinlock, flags);
412
413 if (last_disable)
414 dev_dbg(&csdev->dev, "TMC-ETF disabled\n");
415}
416
417static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,
418 struct perf_event *event, void **pages,
419 int nr_pages, bool overwrite)
420{
421 int node;
422 struct cs_buffers *buf;
423
424 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
425
426 /* Allocate memory structure for interaction with Perf */
427 buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
428 if (!buf)
429 return NULL;
430
431 buf->pid = task_pid_nr(event->owner);
432 buf->snapshot = overwrite;
433 buf->nr_pages = nr_pages;
434 buf->data_pages = pages;
435
436 return buf;
437}
438
439static void tmc_free_etf_buffer(void *config)
440{
441 struct cs_buffers *buf = config;
442
443 kfree(buf);
444}
445
446static int tmc_set_etf_buffer(struct coresight_device *csdev,
447 struct perf_output_handle *handle)
448{
449 int ret = 0;
450 unsigned long head;
451 struct cs_buffers *buf = etm_perf_sink_config(handle);
452
453 if (!buf)
454 return -EINVAL;
455
456 /* wrap head around to the amount of space we have */
457 head = handle->head & (((unsigned long)buf->nr_pages << PAGE_SHIFT) - 1);
458
459 /* find the page to write to */
460 buf->cur = head / PAGE_SIZE;
461
462 /* and offset within that page */
463 buf->offset = head % PAGE_SIZE;
464
465 local_set(&buf->data_size, 0);
466
467 return ret;
468}
469
470static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
471 struct perf_output_handle *handle,
472 void *sink_config)
473{
474 bool lost = false;
475 int i, cur;
476 const u32 *barrier;
477 u32 *buf_ptr;
478 u64 read_ptr, write_ptr;
479 u32 status;
480 unsigned long offset, to_read = 0, flags;
481 struct cs_buffers *buf = sink_config;
482 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
483
484 if (!buf)
485 return 0;
486
487 /* This shouldn't happen */
488 if (WARN_ON_ONCE(coresight_get_mode(csdev) != CS_MODE_PERF))
489 return 0;
490
491 spin_lock_irqsave(&drvdata->spinlock, flags);
492
493 /* Don't do anything if another tracer is using this sink */
494 if (csdev->refcnt != 1)
495 goto out;
496
497 CS_UNLOCK(drvdata->base);
498
499 tmc_flush_and_stop(drvdata);
500
501 read_ptr = tmc_read_rrp(drvdata);
502 write_ptr = tmc_read_rwp(drvdata);
503
504 /*
505 * Get a hold of the status register and see if a wrap around
506 * has occurred. If so adjust things accordingly.
507 */
508 status = readl_relaxed(drvdata->base + TMC_STS);
509 if (status & TMC_STS_FULL) {
510 lost = true;
511 to_read = drvdata->size;
512 } else {
513 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
514 }
515
516 /*
517 * The TMC RAM buffer may be bigger than the space available in the
518 * perf ring buffer (handle->size). If so advance the RRP so that we
519 * get the latest trace data. In snapshot mode none of that matters
520 * since we are expected to clobber stale data in favour of the latest
521 * traces.
522 */
523 if (!buf->snapshot && to_read > handle->size) {
524 u32 mask = tmc_get_memwidth_mask(drvdata);
525
526 /*
527 * Make sure the new size is aligned in accordance with the
528 * requirement explained in function tmc_get_memwidth_mask().
529 */
530 to_read = handle->size & mask;
531 /* Move the RAM read pointer up */
532 read_ptr = (write_ptr + drvdata->size) - to_read;
533 /* Make sure we are still within our limits */
534 if (read_ptr > (drvdata->size - 1))
535 read_ptr -= drvdata->size;
536 /* Tell the HW */
537 tmc_write_rrp(drvdata, read_ptr);
538 lost = true;
539 }
540
541 /*
542 * Don't set the TRUNCATED flag in snapshot mode because 1) the
543 * captured buffer is expected to be truncated and 2) a full buffer
544 * prevents the event from being re-enabled by the perf core,
545 * resulting in stale data being send to user space.
546 */
547 if (!buf->snapshot && lost)
548 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
549
550 cur = buf->cur;
551 offset = buf->offset;
552 barrier = coresight_barrier_pkt;
553
554 /* for every byte to read */
555 for (i = 0; i < to_read; i += 4) {
556 buf_ptr = buf->data_pages[cur] + offset;
557 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
558
559 if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) {
560 *buf_ptr = *barrier;
561 barrier++;
562 }
563
564 offset += 4;
565 if (offset >= PAGE_SIZE) {
566 offset = 0;
567 cur++;
568 /* wrap around at the end of the buffer */
569 cur &= buf->nr_pages - 1;
570 }
571 }
572
573 /*
574 * In snapshot mode we simply increment the head by the number of byte
575 * that were written. User space will figure out how many bytes to get
576 * from the AUX buffer based on the position of the head.
577 */
578 if (buf->snapshot)
579 handle->head += to_read;
580
581 /*
582 * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace
583 * data before the aux_head is updated via perf_aux_output_end(), which
584 * is expected by the perf ring buffer.
585 */
586 CS_LOCK(drvdata->base);
587out:
588 spin_unlock_irqrestore(&drvdata->spinlock, flags);
589
590 return to_read;
591}
592
593static const struct coresight_ops_sink tmc_etf_sink_ops = {
594 .enable = tmc_enable_etf_sink,
595 .disable = tmc_disable_etf_sink,
596 .alloc_buffer = tmc_alloc_etf_buffer,
597 .free_buffer = tmc_free_etf_buffer,
598 .update_buffer = tmc_update_etf_buffer,
599};
600
601static const struct coresight_ops_link tmc_etf_link_ops = {
602 .enable = tmc_enable_etf_link,
603 .disable = tmc_disable_etf_link,
604};
605
606const struct coresight_ops tmc_etb_cs_ops = {
607 .sink_ops = &tmc_etf_sink_ops,
608};
609
610const struct coresight_ops tmc_etf_cs_ops = {
611 .sink_ops = &tmc_etf_sink_ops,
612 .link_ops = &tmc_etf_link_ops,
613};
614
615int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
616{
617 enum tmc_mode mode;
618 int ret = 0;
619 unsigned long flags;
620
621 /* config types are set a boot time and never change */
622 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
623 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
624 return -EINVAL;
625
626 spin_lock_irqsave(&drvdata->spinlock, flags);
627
628 if (drvdata->reading) {
629 ret = -EBUSY;
630 goto out;
631 }
632
633 /* Don't interfere if operated from Perf */
634 if (coresight_get_mode(drvdata->csdev) == CS_MODE_PERF) {
635 ret = -EINVAL;
636 goto out;
637 }
638
639 /* If drvdata::buf is NULL the trace data has been read already */
640 if (drvdata->buf == NULL) {
641 ret = -EINVAL;
642 goto out;
643 }
644
645 /* Disable the TMC if need be */
646 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
647 /* There is no point in reading a TMC in HW FIFO mode */
648 mode = readl_relaxed(drvdata->base + TMC_MODE);
649 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
650 ret = -EINVAL;
651 goto out;
652 }
653 __tmc_etb_disable_hw(drvdata);
654 }
655
656 drvdata->reading = true;
657out:
658 spin_unlock_irqrestore(&drvdata->spinlock, flags);
659
660 return ret;
661}
662
663int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
664{
665 char *buf = NULL;
666 enum tmc_mode mode;
667 unsigned long flags;
668 int rc = 0;
669
670 /* config types are set a boot time and never change */
671 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
672 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
673 return -EINVAL;
674
675 spin_lock_irqsave(&drvdata->spinlock, flags);
676
677 /* Re-enable the TMC if need be */
678 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
679 /* There is no point in reading a TMC in HW FIFO mode */
680 mode = readl_relaxed(drvdata->base + TMC_MODE);
681 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
682 spin_unlock_irqrestore(&drvdata->spinlock, flags);
683 return -EINVAL;
684 }
685 /*
686 * The trace run will continue with the same allocated trace
687 * buffer. As such zero-out the buffer so that we don't end
688 * up with stale data.
689 *
690 * Since the tracer is still enabled drvdata::buf
691 * can't be NULL.
692 */
693 memset(drvdata->buf, 0, drvdata->size);
694 rc = __tmc_etb_enable_hw(drvdata);
695 if (rc) {
696 spin_unlock_irqrestore(&drvdata->spinlock, flags);
697 return rc;
698 }
699 } else {
700 /*
701 * The ETB/ETF is not tracing and the buffer was just read.
702 * As such prepare to free the trace buffer.
703 */
704 buf = drvdata->buf;
705 drvdata->buf = NULL;
706 }
707
708 drvdata->reading = false;
709 spin_unlock_irqrestore(&drvdata->spinlock, flags);
710
711 /*
712 * Free allocated memory outside of the spinlock. There is no need
713 * to assert the validity of 'buf' since calling kfree(NULL) is safe.
714 */
715 kfree(buf);
716
717 return 0;
718}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5 */
6
7#include <linux/atomic.h>
8#include <linux/circ_buf.h>
9#include <linux/coresight.h>
10#include <linux/perf_event.h>
11#include <linux/slab.h>
12#include "coresight-priv.h"
13#include "coresight-tmc.h"
14#include "coresight-etm-perf.h"
15
16static int tmc_set_etf_buffer(struct coresight_device *csdev,
17 struct perf_output_handle *handle);
18
19static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
20{
21 CS_UNLOCK(drvdata->base);
22
23 /* Wait for TMCSReady bit to be set */
24 tmc_wait_for_tmcready(drvdata);
25
26 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
27 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
28 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
29 TMC_FFCR_TRIGON_TRIGIN,
30 drvdata->base + TMC_FFCR);
31
32 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
33 tmc_enable_hw(drvdata);
34
35 CS_LOCK(drvdata->base);
36}
37
38static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
39{
40 int rc = coresight_claim_device(drvdata->csdev);
41
42 if (rc)
43 return rc;
44
45 __tmc_etb_enable_hw(drvdata);
46 return 0;
47}
48
49static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
50{
51 char *bufp;
52 u32 read_data, lost;
53
54 /* Check if the buffer wrapped around. */
55 lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
56 bufp = drvdata->buf;
57 drvdata->len = 0;
58 while (1) {
59 read_data = readl_relaxed(drvdata->base + TMC_RRD);
60 if (read_data == 0xFFFFFFFF)
61 break;
62 memcpy(bufp, &read_data, 4);
63 bufp += 4;
64 drvdata->len += 4;
65 }
66
67 if (lost)
68 coresight_insert_barrier_packet(drvdata->buf);
69 return;
70}
71
72static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
73{
74 CS_UNLOCK(drvdata->base);
75
76 tmc_flush_and_stop(drvdata);
77 /*
78 * When operating in sysFS mode the content of the buffer needs to be
79 * read before the TMC is disabled.
80 */
81 if (drvdata->mode == CS_MODE_SYSFS)
82 tmc_etb_dump_hw(drvdata);
83 tmc_disable_hw(drvdata);
84
85 CS_LOCK(drvdata->base);
86}
87
88static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
89{
90 __tmc_etb_disable_hw(drvdata);
91 coresight_disclaim_device(drvdata->csdev);
92}
93
94static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
95{
96 CS_UNLOCK(drvdata->base);
97
98 /* Wait for TMCSReady bit to be set */
99 tmc_wait_for_tmcready(drvdata);
100
101 writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
102 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
103 drvdata->base + TMC_FFCR);
104 writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
105 tmc_enable_hw(drvdata);
106
107 CS_LOCK(drvdata->base);
108}
109
110static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
111{
112 int rc = coresight_claim_device(drvdata->csdev);
113
114 if (rc)
115 return rc;
116
117 __tmc_etf_enable_hw(drvdata);
118 return 0;
119}
120
121static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
122{
123 struct coresight_device *csdev = drvdata->csdev;
124
125 CS_UNLOCK(drvdata->base);
126
127 tmc_flush_and_stop(drvdata);
128 tmc_disable_hw(drvdata);
129 coresight_disclaim_device_unlocked(csdev);
130 CS_LOCK(drvdata->base);
131}
132
133/*
134 * Return the available trace data in the buffer from @pos, with
135 * a maximum limit of @len, updating the @bufpp on where to
136 * find it.
137 */
138ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
139 loff_t pos, size_t len, char **bufpp)
140{
141 ssize_t actual = len;
142
143 /* Adjust the len to available size @pos */
144 if (pos + actual > drvdata->len)
145 actual = drvdata->len - pos;
146 if (actual > 0)
147 *bufpp = drvdata->buf + pos;
148 return actual;
149}
150
151static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
152{
153 int ret = 0;
154 bool used = false;
155 char *buf = NULL;
156 unsigned long flags;
157 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
158
159 /*
160 * If we don't have a buffer release the lock and allocate memory.
161 * Otherwise keep the lock and move along.
162 */
163 spin_lock_irqsave(&drvdata->spinlock, flags);
164 if (!drvdata->buf) {
165 spin_unlock_irqrestore(&drvdata->spinlock, flags);
166
167 /* Allocating the memory here while outside of the spinlock */
168 buf = kzalloc(drvdata->size, GFP_KERNEL);
169 if (!buf)
170 return -ENOMEM;
171
172 /* Let's try again */
173 spin_lock_irqsave(&drvdata->spinlock, flags);
174 }
175
176 if (drvdata->reading) {
177 ret = -EBUSY;
178 goto out;
179 }
180
181 /*
182 * In sysFS mode we can have multiple writers per sink. Since this
183 * sink is already enabled no memory is needed and the HW need not be
184 * touched.
185 */
186 if (drvdata->mode == CS_MODE_SYSFS) {
187 atomic_inc(csdev->refcnt);
188 goto out;
189 }
190
191 /*
192 * If drvdata::buf isn't NULL, memory was allocated for a previous
193 * trace run but wasn't read. If so simply zero-out the memory.
194 * Otherwise use the memory allocated above.
195 *
196 * The memory is freed when users read the buffer using the
197 * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for
198 * details.
199 */
200 if (drvdata->buf) {
201 memset(drvdata->buf, 0, drvdata->size);
202 } else {
203 used = true;
204 drvdata->buf = buf;
205 }
206
207 ret = tmc_etb_enable_hw(drvdata);
208 if (!ret) {
209 drvdata->mode = CS_MODE_SYSFS;
210 atomic_inc(csdev->refcnt);
211 } else {
212 /* Free up the buffer if we failed to enable */
213 used = false;
214 }
215out:
216 spin_unlock_irqrestore(&drvdata->spinlock, flags);
217
218 /* Free memory outside the spinlock if need be */
219 if (!used)
220 kfree(buf);
221
222 return ret;
223}
224
225static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)
226{
227 int ret = 0;
228 pid_t pid;
229 unsigned long flags;
230 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
231 struct perf_output_handle *handle = data;
232 struct cs_buffers *buf = etm_perf_sink_config(handle);
233
234 spin_lock_irqsave(&drvdata->spinlock, flags);
235 do {
236 ret = -EINVAL;
237 if (drvdata->reading)
238 break;
239 /*
240 * No need to continue if the ETB/ETF is already operated
241 * from sysFS.
242 */
243 if (drvdata->mode == CS_MODE_SYSFS) {
244 ret = -EBUSY;
245 break;
246 }
247
248 /* Get a handle on the pid of the process to monitor */
249 pid = buf->pid;
250
251 if (drvdata->pid != -1 && drvdata->pid != pid) {
252 ret = -EBUSY;
253 break;
254 }
255
256 ret = tmc_set_etf_buffer(csdev, handle);
257 if (ret)
258 break;
259
260 /*
261 * No HW configuration is needed if the sink is already in
262 * use for this session.
263 */
264 if (drvdata->pid == pid) {
265 atomic_inc(csdev->refcnt);
266 break;
267 }
268
269 ret = tmc_etb_enable_hw(drvdata);
270 if (!ret) {
271 /* Associate with monitored process. */
272 drvdata->pid = pid;
273 drvdata->mode = CS_MODE_PERF;
274 atomic_inc(csdev->refcnt);
275 }
276 } while (0);
277 spin_unlock_irqrestore(&drvdata->spinlock, flags);
278
279 return ret;
280}
281
282static int tmc_enable_etf_sink(struct coresight_device *csdev,
283 u32 mode, void *data)
284{
285 int ret;
286
287 switch (mode) {
288 case CS_MODE_SYSFS:
289 ret = tmc_enable_etf_sink_sysfs(csdev);
290 break;
291 case CS_MODE_PERF:
292 ret = tmc_enable_etf_sink_perf(csdev, data);
293 break;
294 /* We shouldn't be here */
295 default:
296 ret = -EINVAL;
297 break;
298 }
299
300 if (ret)
301 return ret;
302
303 dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n");
304 return 0;
305}
306
307static int tmc_disable_etf_sink(struct coresight_device *csdev)
308{
309 unsigned long flags;
310 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
311
312 spin_lock_irqsave(&drvdata->spinlock, flags);
313
314 if (drvdata->reading) {
315 spin_unlock_irqrestore(&drvdata->spinlock, flags);
316 return -EBUSY;
317 }
318
319 if (atomic_dec_return(csdev->refcnt)) {
320 spin_unlock_irqrestore(&drvdata->spinlock, flags);
321 return -EBUSY;
322 }
323
324 /* Complain if we (somehow) got out of sync */
325 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
326 tmc_etb_disable_hw(drvdata);
327 /* Dissociate from monitored process. */
328 drvdata->pid = -1;
329 drvdata->mode = CS_MODE_DISABLED;
330
331 spin_unlock_irqrestore(&drvdata->spinlock, flags);
332
333 dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n");
334 return 0;
335}
336
337static int tmc_enable_etf_link(struct coresight_device *csdev,
338 int inport, int outport)
339{
340 int ret = 0;
341 unsigned long flags;
342 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
343 bool first_enable = false;
344
345 spin_lock_irqsave(&drvdata->spinlock, flags);
346 if (drvdata->reading) {
347 spin_unlock_irqrestore(&drvdata->spinlock, flags);
348 return -EBUSY;
349 }
350
351 if (atomic_read(&csdev->refcnt[0]) == 0) {
352 ret = tmc_etf_enable_hw(drvdata);
353 if (!ret) {
354 drvdata->mode = CS_MODE_SYSFS;
355 first_enable = true;
356 }
357 }
358 if (!ret)
359 atomic_inc(&csdev->refcnt[0]);
360 spin_unlock_irqrestore(&drvdata->spinlock, flags);
361
362 if (first_enable)
363 dev_dbg(&csdev->dev, "TMC-ETF enabled\n");
364 return ret;
365}
366
367static void tmc_disable_etf_link(struct coresight_device *csdev,
368 int inport, int outport)
369{
370 unsigned long flags;
371 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
372 bool last_disable = false;
373
374 spin_lock_irqsave(&drvdata->spinlock, flags);
375 if (drvdata->reading) {
376 spin_unlock_irqrestore(&drvdata->spinlock, flags);
377 return;
378 }
379
380 if (atomic_dec_return(&csdev->refcnt[0]) == 0) {
381 tmc_etf_disable_hw(drvdata);
382 drvdata->mode = CS_MODE_DISABLED;
383 last_disable = true;
384 }
385 spin_unlock_irqrestore(&drvdata->spinlock, flags);
386
387 if (last_disable)
388 dev_dbg(&csdev->dev, "TMC-ETF disabled\n");
389}
390
391static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,
392 struct perf_event *event, void **pages,
393 int nr_pages, bool overwrite)
394{
395 int node;
396 struct cs_buffers *buf;
397
398 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
399
400 /* Allocate memory structure for interaction with Perf */
401 buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
402 if (!buf)
403 return NULL;
404
405 buf->pid = task_pid_nr(event->owner);
406 buf->snapshot = overwrite;
407 buf->nr_pages = nr_pages;
408 buf->data_pages = pages;
409
410 return buf;
411}
412
413static void tmc_free_etf_buffer(void *config)
414{
415 struct cs_buffers *buf = config;
416
417 kfree(buf);
418}
419
420static int tmc_set_etf_buffer(struct coresight_device *csdev,
421 struct perf_output_handle *handle)
422{
423 int ret = 0;
424 unsigned long head;
425 struct cs_buffers *buf = etm_perf_sink_config(handle);
426
427 if (!buf)
428 return -EINVAL;
429
430 /* wrap head around to the amount of space we have */
431 head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
432
433 /* find the page to write to */
434 buf->cur = head / PAGE_SIZE;
435
436 /* and offset within that page */
437 buf->offset = head % PAGE_SIZE;
438
439 local_set(&buf->data_size, 0);
440
441 return ret;
442}
443
444static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
445 struct perf_output_handle *handle,
446 void *sink_config)
447{
448 bool lost = false;
449 int i, cur;
450 const u32 *barrier;
451 u32 *buf_ptr;
452 u64 read_ptr, write_ptr;
453 u32 status;
454 unsigned long offset, to_read = 0, flags;
455 struct cs_buffers *buf = sink_config;
456 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
457
458 if (!buf)
459 return 0;
460
461 /* This shouldn't happen */
462 if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
463 return 0;
464
465 spin_lock_irqsave(&drvdata->spinlock, flags);
466
467 /* Don't do anything if another tracer is using this sink */
468 if (atomic_read(csdev->refcnt) != 1)
469 goto out;
470
471 CS_UNLOCK(drvdata->base);
472
473 tmc_flush_and_stop(drvdata);
474
475 read_ptr = tmc_read_rrp(drvdata);
476 write_ptr = tmc_read_rwp(drvdata);
477
478 /*
479 * Get a hold of the status register and see if a wrap around
480 * has occurred. If so adjust things accordingly.
481 */
482 status = readl_relaxed(drvdata->base + TMC_STS);
483 if (status & TMC_STS_FULL) {
484 lost = true;
485 to_read = drvdata->size;
486 } else {
487 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
488 }
489
490 /*
491 * The TMC RAM buffer may be bigger than the space available in the
492 * perf ring buffer (handle->size). If so advance the RRP so that we
493 * get the latest trace data. In snapshot mode none of that matters
494 * since we are expected to clobber stale data in favour of the latest
495 * traces.
496 */
497 if (!buf->snapshot && to_read > handle->size) {
498 u32 mask = tmc_get_memwidth_mask(drvdata);
499
500 /*
501 * Make sure the new size is aligned in accordance with the
502 * requirement explained in function tmc_get_memwidth_mask().
503 */
504 to_read = handle->size & mask;
505 /* Move the RAM read pointer up */
506 read_ptr = (write_ptr + drvdata->size) - to_read;
507 /* Make sure we are still within our limits */
508 if (read_ptr > (drvdata->size - 1))
509 read_ptr -= drvdata->size;
510 /* Tell the HW */
511 tmc_write_rrp(drvdata, read_ptr);
512 lost = true;
513 }
514
515 /*
516 * Don't set the TRUNCATED flag in snapshot mode because 1) the
517 * captured buffer is expected to be truncated and 2) a full buffer
518 * prevents the event from being re-enabled by the perf core,
519 * resulting in stale data being send to user space.
520 */
521 if (!buf->snapshot && lost)
522 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
523
524 cur = buf->cur;
525 offset = buf->offset;
526 barrier = coresight_barrier_pkt;
527
528 /* for every byte to read */
529 for (i = 0; i < to_read; i += 4) {
530 buf_ptr = buf->data_pages[cur] + offset;
531 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
532
533 if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) {
534 *buf_ptr = *barrier;
535 barrier++;
536 }
537
538 offset += 4;
539 if (offset >= PAGE_SIZE) {
540 offset = 0;
541 cur++;
542 /* wrap around at the end of the buffer */
543 cur &= buf->nr_pages - 1;
544 }
545 }
546
547 /*
548 * In snapshot mode we simply increment the head by the number of byte
549 * that were written. User space function cs_etm_find_snapshot() will
550 * figure out how many bytes to get from the AUX buffer based on the
551 * position of the head.
552 */
553 if (buf->snapshot)
554 handle->head += to_read;
555
556 CS_LOCK(drvdata->base);
557out:
558 spin_unlock_irqrestore(&drvdata->spinlock, flags);
559
560 return to_read;
561}
562
563static const struct coresight_ops_sink tmc_etf_sink_ops = {
564 .enable = tmc_enable_etf_sink,
565 .disable = tmc_disable_etf_sink,
566 .alloc_buffer = tmc_alloc_etf_buffer,
567 .free_buffer = tmc_free_etf_buffer,
568 .update_buffer = tmc_update_etf_buffer,
569};
570
571static const struct coresight_ops_link tmc_etf_link_ops = {
572 .enable = tmc_enable_etf_link,
573 .disable = tmc_disable_etf_link,
574};
575
576const struct coresight_ops tmc_etb_cs_ops = {
577 .sink_ops = &tmc_etf_sink_ops,
578};
579
580const struct coresight_ops tmc_etf_cs_ops = {
581 .sink_ops = &tmc_etf_sink_ops,
582 .link_ops = &tmc_etf_link_ops,
583};
584
585int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
586{
587 enum tmc_mode mode;
588 int ret = 0;
589 unsigned long flags;
590
591 /* config types are set a boot time and never change */
592 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
593 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
594 return -EINVAL;
595
596 spin_lock_irqsave(&drvdata->spinlock, flags);
597
598 if (drvdata->reading) {
599 ret = -EBUSY;
600 goto out;
601 }
602
603 /* Don't interfere if operated from Perf */
604 if (drvdata->mode == CS_MODE_PERF) {
605 ret = -EINVAL;
606 goto out;
607 }
608
609 /* If drvdata::buf is NULL the trace data has been read already */
610 if (drvdata->buf == NULL) {
611 ret = -EINVAL;
612 goto out;
613 }
614
615 /* Disable the TMC if need be */
616 if (drvdata->mode == CS_MODE_SYSFS) {
617 /* There is no point in reading a TMC in HW FIFO mode */
618 mode = readl_relaxed(drvdata->base + TMC_MODE);
619 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
620 ret = -EINVAL;
621 goto out;
622 }
623 __tmc_etb_disable_hw(drvdata);
624 }
625
626 drvdata->reading = true;
627out:
628 spin_unlock_irqrestore(&drvdata->spinlock, flags);
629
630 return ret;
631}
632
633int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
634{
635 char *buf = NULL;
636 enum tmc_mode mode;
637 unsigned long flags;
638
639 /* config types are set a boot time and never change */
640 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
641 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
642 return -EINVAL;
643
644 spin_lock_irqsave(&drvdata->spinlock, flags);
645
646 /* Re-enable the TMC if need be */
647 if (drvdata->mode == CS_MODE_SYSFS) {
648 /* There is no point in reading a TMC in HW FIFO mode */
649 mode = readl_relaxed(drvdata->base + TMC_MODE);
650 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
651 spin_unlock_irqrestore(&drvdata->spinlock, flags);
652 return -EINVAL;
653 }
654 /*
655 * The trace run will continue with the same allocated trace
656 * buffer. As such zero-out the buffer so that we don't end
657 * up with stale data.
658 *
659 * Since the tracer is still enabled drvdata::buf
660 * can't be NULL.
661 */
662 memset(drvdata->buf, 0, drvdata->size);
663 __tmc_etb_enable_hw(drvdata);
664 } else {
665 /*
666 * The ETB/ETF is not tracing and the buffer was just read.
667 * As such prepare to free the trace buffer.
668 */
669 buf = drvdata->buf;
670 drvdata->buf = NULL;
671 }
672
673 drvdata->reading = false;
674 spin_unlock_irqrestore(&drvdata->spinlock, flags);
675
676 /*
677 * Free allocated memory outside of the spinlock. There is no need
678 * to assert the validity of 'buf' since calling kfree(NULL) is safe.
679 */
680 kfree(buf);
681
682 return 0;
683}