Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5 */
6
7#include <linux/atomic.h>
8#include <linux/coresight.h>
9#include <linux/dma-mapping.h>
10#include <linux/iommu.h>
11#include <linux/idr.h>
12#include <linux/mutex.h>
13#include <linux/refcount.h>
14#include <linux/slab.h>
15#include <linux/types.h>
16#include <linux/vmalloc.h>
17#include "coresight-catu.h"
18#include "coresight-etm-perf.h"
19#include "coresight-priv.h"
20#include "coresight-tmc.h"
21
22struct etr_flat_buf {
23 struct device *dev;
24 dma_addr_t daddr;
25 void *vaddr;
26 size_t size;
27};
28
29/*
30 * etr_perf_buffer - Perf buffer used for ETR
31 * @drvdata - The ETR drvdaga this buffer has been allocated for.
32 * @etr_buf - Actual buffer used by the ETR
33 * @pid - The PID this etr_perf_buffer belongs to.
34 * @snaphost - Perf session mode
35 * @nr_pages - Number of pages in the ring buffer.
36 * @pages - Array of Pages in the ring buffer.
37 */
38struct etr_perf_buffer {
39 struct tmc_drvdata *drvdata;
40 struct etr_buf *etr_buf;
41 pid_t pid;
42 bool snapshot;
43 int nr_pages;
44 void **pages;
45};
46
47/* Convert the perf index to an offset within the ETR buffer */
48#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
49
50/* Lower limit for ETR hardware buffer */
51#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
52
53/*
54 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
55 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
56 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
57 * contain more than one SG buffer and tables.
58 *
59 * A table entry has the following format:
60 *
61 * ---Bit31------------Bit4-------Bit1-----Bit0--
62 * | Address[39:12] | SBZ | Entry Type |
63 * ----------------------------------------------
64 *
65 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
66 * always zero.
67 *
68 * Entry type:
69 * b00 - Reserved.
70 * b01 - Last entry in the tables, points to 4K page buffer.
71 * b10 - Normal entry, points to 4K page buffer.
72 * b11 - Link. The address points to the base of next table.
73 */
74
75typedef u32 sgte_t;
76
77#define ETR_SG_PAGE_SHIFT 12
78#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
79#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
80#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
81#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
82
83#define ETR_SG_ET_MASK 0x3
84#define ETR_SG_ET_LAST 0x1
85#define ETR_SG_ET_NORMAL 0x2
86#define ETR_SG_ET_LINK 0x3
87
88#define ETR_SG_ADDR_SHIFT 4
89
90#define ETR_SG_ENTRY(addr, type) \
91 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
92 (type & ETR_SG_ET_MASK))
93
94#define ETR_SG_ADDR(entry) \
95 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
96#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
97
98/*
99 * struct etr_sg_table : ETR SG Table
100 * @sg_table: Generic SG Table holding the data/table pages.
101 * @hwaddr: hwaddress used by the TMC, which is the base
102 * address of the table.
103 */
104struct etr_sg_table {
105 struct tmc_sg_table *sg_table;
106 dma_addr_t hwaddr;
107};
108
109/*
110 * tmc_etr_sg_table_entries: Total number of table entries required to map
111 * @nr_pages system pages.
112 *
113 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
114 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
115 * with the last entry pointing to another page of table entries.
116 * If we spill over to a new page for mapping 1 entry, we could as
117 * well replace the link entry of the previous page with the last entry.
118 */
119static inline unsigned long __attribute_const__
120tmc_etr_sg_table_entries(int nr_pages)
121{
122 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
123 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
124 /*
125 * If we spill over to a new page for 1 entry, we could as well
126 * make it the LAST entry in the previous page, skipping the Link
127 * address.
128 */
129 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
130 nr_sglinks--;
131 return nr_sgpages + nr_sglinks;
132}
133
134/*
135 * tmc_pages_get_offset: Go through all the pages in the tmc_pages
136 * and map the device address @addr to an offset within the virtual
137 * contiguous buffer.
138 */
139static long
140tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
141{
142 int i;
143 dma_addr_t page_start;
144
145 for (i = 0; i < tmc_pages->nr_pages; i++) {
146 page_start = tmc_pages->daddrs[i];
147 if (addr >= page_start && addr < (page_start + PAGE_SIZE))
148 return i * PAGE_SIZE + (addr - page_start);
149 }
150
151 return -EINVAL;
152}
153
154/*
155 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
156 * If the pages were not allocated in tmc_pages_alloc(), we would
157 * simply drop the refcount.
158 */
159static void tmc_pages_free(struct tmc_pages *tmc_pages,
160 struct device *dev, enum dma_data_direction dir)
161{
162 int i;
163 struct device *real_dev = dev->parent;
164
165 for (i = 0; i < tmc_pages->nr_pages; i++) {
166 if (tmc_pages->daddrs && tmc_pages->daddrs[i])
167 dma_unmap_page(real_dev, tmc_pages->daddrs[i],
168 PAGE_SIZE, dir);
169 if (tmc_pages->pages && tmc_pages->pages[i])
170 __free_page(tmc_pages->pages[i]);
171 }
172
173 kfree(tmc_pages->pages);
174 kfree(tmc_pages->daddrs);
175 tmc_pages->pages = NULL;
176 tmc_pages->daddrs = NULL;
177 tmc_pages->nr_pages = 0;
178}
179
180/*
181 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
182 * If @pages is not NULL, the list of page virtual addresses are
183 * used as the data pages. The pages are then dma_map'ed for @dev
184 * with dma_direction @dir.
185 *
186 * Returns 0 upon success, else the error number.
187 */
188static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
189 struct device *dev, int node,
190 enum dma_data_direction dir, void **pages)
191{
192 int i, nr_pages;
193 dma_addr_t paddr;
194 struct page *page;
195 struct device *real_dev = dev->parent;
196
197 nr_pages = tmc_pages->nr_pages;
198 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
199 GFP_KERNEL);
200 if (!tmc_pages->daddrs)
201 return -ENOMEM;
202 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
203 GFP_KERNEL);
204 if (!tmc_pages->pages) {
205 kfree(tmc_pages->daddrs);
206 tmc_pages->daddrs = NULL;
207 return -ENOMEM;
208 }
209
210 for (i = 0; i < nr_pages; i++) {
211 if (pages && pages[i]) {
212 page = virt_to_page(pages[i]);
213 /* Hold a refcount on the page */
214 get_page(page);
215 } else {
216 page = alloc_pages_node(node,
217 GFP_KERNEL | __GFP_ZERO, 0);
218 if (!page)
219 goto err;
220 }
221 paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
222 if (dma_mapping_error(real_dev, paddr))
223 goto err;
224 tmc_pages->daddrs[i] = paddr;
225 tmc_pages->pages[i] = page;
226 }
227 return 0;
228err:
229 tmc_pages_free(tmc_pages, dev, dir);
230 return -ENOMEM;
231}
232
233static inline long
234tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
235{
236 return tmc_pages_get_offset(&sg_table->data_pages, addr);
237}
238
239static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
240{
241 if (sg_table->table_vaddr)
242 vunmap(sg_table->table_vaddr);
243 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
244}
245
246static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
247{
248 if (sg_table->data_vaddr)
249 vunmap(sg_table->data_vaddr);
250 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
251}
252
253void tmc_free_sg_table(struct tmc_sg_table *sg_table)
254{
255 tmc_free_table_pages(sg_table);
256 tmc_free_data_pages(sg_table);
257}
258EXPORT_SYMBOL_GPL(tmc_free_sg_table);
259
260/*
261 * Alloc pages for the table. Since this will be used by the device,
262 * allocate the pages closer to the device (i.e, dev_to_node(dev)
263 * rather than the CPU node).
264 */
265static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
266{
267 int rc;
268 struct tmc_pages *table_pages = &sg_table->table_pages;
269
270 rc = tmc_pages_alloc(table_pages, sg_table->dev,
271 dev_to_node(sg_table->dev),
272 DMA_TO_DEVICE, NULL);
273 if (rc)
274 return rc;
275 sg_table->table_vaddr = vmap(table_pages->pages,
276 table_pages->nr_pages,
277 VM_MAP,
278 PAGE_KERNEL);
279 if (!sg_table->table_vaddr)
280 rc = -ENOMEM;
281 else
282 sg_table->table_daddr = table_pages->daddrs[0];
283 return rc;
284}
285
286static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
287{
288 int rc;
289
290 /* Allocate data pages on the node requested by the caller */
291 rc = tmc_pages_alloc(&sg_table->data_pages,
292 sg_table->dev, sg_table->node,
293 DMA_FROM_DEVICE, pages);
294 if (!rc) {
295 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
296 sg_table->data_pages.nr_pages,
297 VM_MAP,
298 PAGE_KERNEL);
299 if (!sg_table->data_vaddr)
300 rc = -ENOMEM;
301 }
302 return rc;
303}
304
305/*
306 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
307 * and data buffers. TMC writes to the data buffers and reads from the SG
308 * Table pages.
309 *
310 * @dev - Coresight device to which page should be DMA mapped.
311 * @node - Numa node for mem allocations
312 * @nr_tpages - Number of pages for the table entries.
313 * @nr_dpages - Number of pages for Data buffer.
314 * @pages - Optional list of virtual address of pages.
315 */
316struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
317 int node,
318 int nr_tpages,
319 int nr_dpages,
320 void **pages)
321{
322 long rc;
323 struct tmc_sg_table *sg_table;
324
325 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
326 if (!sg_table)
327 return ERR_PTR(-ENOMEM);
328 sg_table->data_pages.nr_pages = nr_dpages;
329 sg_table->table_pages.nr_pages = nr_tpages;
330 sg_table->node = node;
331 sg_table->dev = dev;
332
333 rc = tmc_alloc_data_pages(sg_table, pages);
334 if (!rc)
335 rc = tmc_alloc_table_pages(sg_table);
336 if (rc) {
337 tmc_free_sg_table(sg_table);
338 kfree(sg_table);
339 return ERR_PTR(rc);
340 }
341
342 return sg_table;
343}
344EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
345
346/*
347 * tmc_sg_table_sync_data_range: Sync the data buffer written
348 * by the device from @offset upto a @size bytes.
349 */
350void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
351 u64 offset, u64 size)
352{
353 int i, index, start;
354 int npages = DIV_ROUND_UP(size, PAGE_SIZE);
355 struct device *real_dev = table->dev->parent;
356 struct tmc_pages *data = &table->data_pages;
357
358 start = offset >> PAGE_SHIFT;
359 for (i = start; i < (start + npages); i++) {
360 index = i % data->nr_pages;
361 dma_sync_single_for_cpu(real_dev, data->daddrs[index],
362 PAGE_SIZE, DMA_FROM_DEVICE);
363 }
364}
365EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
366
367/* tmc_sg_sync_table: Sync the page table */
368void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
369{
370 int i;
371 struct device *real_dev = sg_table->dev->parent;
372 struct tmc_pages *table_pages = &sg_table->table_pages;
373
374 for (i = 0; i < table_pages->nr_pages; i++)
375 dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
376 PAGE_SIZE, DMA_TO_DEVICE);
377}
378EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
379
380/*
381 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
382 * in the SG buffer. The @bufpp is updated to point to the buffer.
383 * Returns :
384 * the length of linear data available at @offset.
385 * or
386 * <= 0 if no data is available.
387 */
388ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
389 u64 offset, size_t len, char **bufpp)
390{
391 size_t size;
392 int pg_idx = offset >> PAGE_SHIFT;
393 int pg_offset = offset & (PAGE_SIZE - 1);
394 struct tmc_pages *data_pages = &sg_table->data_pages;
395
396 size = tmc_sg_table_buf_size(sg_table);
397 if (offset >= size)
398 return -EINVAL;
399
400 /* Make sure we don't go beyond the end */
401 len = (len < (size - offset)) ? len : size - offset;
402 /* Respect the page boundaries */
403 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
404 if (len > 0)
405 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
406 return len;
407}
408EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
409
410#ifdef ETR_SG_DEBUG
411/* Map a dma address to virtual address */
412static unsigned long
413tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
414 dma_addr_t addr, bool table)
415{
416 long offset;
417 unsigned long base;
418 struct tmc_pages *tmc_pages;
419
420 if (table) {
421 tmc_pages = &sg_table->table_pages;
422 base = (unsigned long)sg_table->table_vaddr;
423 } else {
424 tmc_pages = &sg_table->data_pages;
425 base = (unsigned long)sg_table->data_vaddr;
426 }
427
428 offset = tmc_pages_get_offset(tmc_pages, addr);
429 if (offset < 0)
430 return 0;
431 return base + offset;
432}
433
434/* Dump the given sg_table */
435static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
436{
437 sgte_t *ptr;
438 int i = 0;
439 dma_addr_t addr;
440 struct tmc_sg_table *sg_table = etr_table->sg_table;
441
442 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
443 etr_table->hwaddr, true);
444 while (ptr) {
445 addr = ETR_SG_ADDR(*ptr);
446 switch (ETR_SG_ET(*ptr)) {
447 case ETR_SG_ET_NORMAL:
448 dev_dbg(sg_table->dev,
449 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
450 ptr++;
451 break;
452 case ETR_SG_ET_LINK:
453 dev_dbg(sg_table->dev,
454 "%05d: *** %p\t:{L} 0x%llx ***\n",
455 i, ptr, addr);
456 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
457 addr, true);
458 break;
459 case ETR_SG_ET_LAST:
460 dev_dbg(sg_table->dev,
461 "%05d: ### %p\t:[L] 0x%llx ###\n",
462 i, ptr, addr);
463 return;
464 default:
465 dev_dbg(sg_table->dev,
466 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
467 i, ptr, addr);
468 return;
469 }
470 i++;
471 }
472 dev_dbg(sg_table->dev, "******* End of Table *****\n");
473}
474#else
475static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
476#endif
477
478/*
479 * Populate the SG Table page table entries from table/data
480 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
481 * So does a Table page. So we keep track of indices of the tables
482 * in each system page and move the pointers accordingly.
483 */
484#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
485static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
486{
487 dma_addr_t paddr;
488 int i, type, nr_entries;
489 int tpidx = 0; /* index to the current system table_page */
490 int sgtidx = 0; /* index to the sg_table within the current syspage */
491 int sgtentry = 0; /* the entry within the sg_table */
492 int dpidx = 0; /* index to the current system data_page */
493 int spidx = 0; /* index to the SG page within the current data page */
494 sgte_t *ptr; /* pointer to the table entry to fill */
495 struct tmc_sg_table *sg_table = etr_table->sg_table;
496 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
497 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
498
499 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
500 /*
501 * Use the contiguous virtual address of the table to update entries.
502 */
503 ptr = sg_table->table_vaddr;
504 /*
505 * Fill all the entries, except the last entry to avoid special
506 * checks within the loop.
507 */
508 for (i = 0; i < nr_entries - 1; i++) {
509 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
510 /*
511 * Last entry in a sg_table page is a link address to
512 * the next table page. If this sg_table is the last
513 * one in the system page, it links to the first
514 * sg_table in the next system page. Otherwise, it
515 * links to the next sg_table page within the system
516 * page.
517 */
518 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
519 paddr = table_daddrs[tpidx + 1];
520 } else {
521 paddr = table_daddrs[tpidx] +
522 (ETR_SG_PAGE_SIZE * (sgtidx + 1));
523 }
524 type = ETR_SG_ET_LINK;
525 } else {
526 /*
527 * Update the indices to the data_pages to point to the
528 * next sg_page in the data buffer.
529 */
530 type = ETR_SG_ET_NORMAL;
531 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
532 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
533 dpidx++;
534 }
535 *ptr++ = ETR_SG_ENTRY(paddr, type);
536 /*
537 * Move to the next table pointer, moving the table page index
538 * if necessary
539 */
540 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
541 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
542 tpidx++;
543 }
544 }
545
546 /* Set up the last entry, which is always a data pointer */
547 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
548 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
549}
550
551/*
552 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
553 * populate the table.
554 *
555 * @dev - Device pointer for the TMC
556 * @node - NUMA node where the memory should be allocated
557 * @size - Total size of the data buffer
558 * @pages - Optional list of page virtual address
559 */
560static struct etr_sg_table *
561tmc_init_etr_sg_table(struct device *dev, int node,
562 unsigned long size, void **pages)
563{
564 int nr_entries, nr_tpages;
565 int nr_dpages = size >> PAGE_SHIFT;
566 struct tmc_sg_table *sg_table;
567 struct etr_sg_table *etr_table;
568
569 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
570 if (!etr_table)
571 return ERR_PTR(-ENOMEM);
572 nr_entries = tmc_etr_sg_table_entries(nr_dpages);
573 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
574
575 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
576 if (IS_ERR(sg_table)) {
577 kfree(etr_table);
578 return ERR_CAST(sg_table);
579 }
580
581 etr_table->sg_table = sg_table;
582 /* TMC should use table base address for DBA */
583 etr_table->hwaddr = sg_table->table_daddr;
584 tmc_etr_sg_table_populate(etr_table);
585 /* Sync the table pages for the HW */
586 tmc_sg_table_sync_table(sg_table);
587 tmc_etr_sg_table_dump(etr_table);
588
589 return etr_table;
590}
591
592/*
593 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
594 */
595static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
596 struct etr_buf *etr_buf, int node,
597 void **pages)
598{
599 struct etr_flat_buf *flat_buf;
600 struct device *real_dev = drvdata->csdev->dev.parent;
601
602 /* We cannot reuse existing pages for flat buf */
603 if (pages)
604 return -EINVAL;
605
606 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
607 if (!flat_buf)
608 return -ENOMEM;
609
610 flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,
611 &flat_buf->daddr,
612 DMA_FROM_DEVICE, GFP_KERNEL);
613 if (!flat_buf->vaddr) {
614 kfree(flat_buf);
615 return -ENOMEM;
616 }
617
618 flat_buf->size = etr_buf->size;
619 flat_buf->dev = &drvdata->csdev->dev;
620 etr_buf->hwaddr = flat_buf->daddr;
621 etr_buf->mode = ETR_MODE_FLAT;
622 etr_buf->private = flat_buf;
623 return 0;
624}
625
626static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
627{
628 struct etr_flat_buf *flat_buf = etr_buf->private;
629
630 if (flat_buf && flat_buf->daddr) {
631 struct device *real_dev = flat_buf->dev->parent;
632
633 dma_free_noncoherent(real_dev, etr_buf->size,
634 flat_buf->vaddr, flat_buf->daddr,
635 DMA_FROM_DEVICE);
636 }
637 kfree(flat_buf);
638}
639
640static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
641{
642 struct etr_flat_buf *flat_buf = etr_buf->private;
643 struct device *real_dev = flat_buf->dev->parent;
644
645 /*
646 * Adjust the buffer to point to the beginning of the trace data
647 * and update the available trace data.
648 */
649 etr_buf->offset = rrp - etr_buf->hwaddr;
650 if (etr_buf->full)
651 etr_buf->len = etr_buf->size;
652 else
653 etr_buf->len = rwp - rrp;
654
655 /*
656 * The driver always starts tracing at the beginning of the buffer,
657 * the only reason why we would get a wrap around is when the buffer
658 * is full. Sync the entire buffer in one go for this case.
659 */
660 if (etr_buf->offset + etr_buf->len > etr_buf->size)
661 dma_sync_single_for_cpu(real_dev, flat_buf->daddr,
662 etr_buf->size, DMA_FROM_DEVICE);
663 else
664 dma_sync_single_for_cpu(real_dev,
665 flat_buf->daddr + etr_buf->offset,
666 etr_buf->len, DMA_FROM_DEVICE);
667}
668
669static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
670 u64 offset, size_t len, char **bufpp)
671{
672 struct etr_flat_buf *flat_buf = etr_buf->private;
673
674 *bufpp = (char *)flat_buf->vaddr + offset;
675 /*
676 * tmc_etr_buf_get_data already adjusts the length to handle
677 * buffer wrapping around.
678 */
679 return len;
680}
681
682static const struct etr_buf_operations etr_flat_buf_ops = {
683 .alloc = tmc_etr_alloc_flat_buf,
684 .free = tmc_etr_free_flat_buf,
685 .sync = tmc_etr_sync_flat_buf,
686 .get_data = tmc_etr_get_data_flat_buf,
687};
688
689/*
690 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
691 * appropriately.
692 */
693static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
694 struct etr_buf *etr_buf, int node,
695 void **pages)
696{
697 struct etr_sg_table *etr_table;
698 struct device *dev = &drvdata->csdev->dev;
699
700 etr_table = tmc_init_etr_sg_table(dev, node,
701 etr_buf->size, pages);
702 if (IS_ERR(etr_table))
703 return -ENOMEM;
704 etr_buf->hwaddr = etr_table->hwaddr;
705 etr_buf->mode = ETR_MODE_ETR_SG;
706 etr_buf->private = etr_table;
707 return 0;
708}
709
710static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
711{
712 struct etr_sg_table *etr_table = etr_buf->private;
713
714 if (etr_table) {
715 tmc_free_sg_table(etr_table->sg_table);
716 kfree(etr_table);
717 }
718}
719
720static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
721 size_t len, char **bufpp)
722{
723 struct etr_sg_table *etr_table = etr_buf->private;
724
725 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
726}
727
728static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
729{
730 long r_offset, w_offset;
731 struct etr_sg_table *etr_table = etr_buf->private;
732 struct tmc_sg_table *table = etr_table->sg_table;
733
734 /* Convert hw address to offset in the buffer */
735 r_offset = tmc_sg_get_data_page_offset(table, rrp);
736 if (r_offset < 0) {
737 dev_warn(table->dev,
738 "Unable to map RRP %llx to offset\n", rrp);
739 etr_buf->len = 0;
740 return;
741 }
742
743 w_offset = tmc_sg_get_data_page_offset(table, rwp);
744 if (w_offset < 0) {
745 dev_warn(table->dev,
746 "Unable to map RWP %llx to offset\n", rwp);
747 etr_buf->len = 0;
748 return;
749 }
750
751 etr_buf->offset = r_offset;
752 if (etr_buf->full)
753 etr_buf->len = etr_buf->size;
754 else
755 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
756 w_offset - r_offset;
757 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
758}
759
760static const struct etr_buf_operations etr_sg_buf_ops = {
761 .alloc = tmc_etr_alloc_sg_buf,
762 .free = tmc_etr_free_sg_buf,
763 .sync = tmc_etr_sync_sg_buf,
764 .get_data = tmc_etr_get_data_sg_buf,
765};
766
767/*
768 * TMC ETR could be connected to a CATU device, which can provide address
769 * translation service. This is represented by the Output port of the TMC
770 * (ETR) connected to the input port of the CATU.
771 *
772 * Returns : coresight_device ptr for the CATU device if a CATU is found.
773 * : NULL otherwise.
774 */
775struct coresight_device *
776tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
777{
778 int i;
779 struct coresight_device *tmp, *etr = drvdata->csdev;
780
781 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
782 return NULL;
783
784 for (i = 0; i < etr->pdata->nr_outport; i++) {
785 tmp = etr->pdata->conns[i].child_dev;
786 if (tmp && coresight_is_catu_device(tmp))
787 return tmp;
788 }
789
790 return NULL;
791}
792EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
793
794static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata,
795 struct etr_buf *etr_buf)
796{
797 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
798
799 if (catu && helper_ops(catu)->enable)
800 return helper_ops(catu)->enable(catu, etr_buf);
801 return 0;
802}
803
804static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata)
805{
806 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
807
808 if (catu && helper_ops(catu)->disable)
809 helper_ops(catu)->disable(catu, drvdata->etr_buf);
810}
811
812static const struct etr_buf_operations *etr_buf_ops[] = {
813 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
814 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
815 [ETR_MODE_CATU] = NULL,
816};
817
818void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
819{
820 etr_buf_ops[ETR_MODE_CATU] = catu;
821}
822EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
823
824void tmc_etr_remove_catu_ops(void)
825{
826 etr_buf_ops[ETR_MODE_CATU] = NULL;
827}
828EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
829
830static inline int tmc_etr_mode_alloc_buf(int mode,
831 struct tmc_drvdata *drvdata,
832 struct etr_buf *etr_buf, int node,
833 void **pages)
834{
835 int rc = -EINVAL;
836
837 switch (mode) {
838 case ETR_MODE_FLAT:
839 case ETR_MODE_ETR_SG:
840 case ETR_MODE_CATU:
841 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
842 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
843 node, pages);
844 if (!rc)
845 etr_buf->ops = etr_buf_ops[mode];
846 return rc;
847 default:
848 return -EINVAL;
849 }
850}
851
852/*
853 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
854 * @drvdata : ETR device details.
855 * @size : size of the requested buffer.
856 * @flags : Required properties for the buffer.
857 * @node : Node for memory allocations.
858 * @pages : An optional list of pages.
859 */
860static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
861 ssize_t size, int flags,
862 int node, void **pages)
863{
864 int rc = -ENOMEM;
865 bool has_etr_sg, has_iommu;
866 bool has_sg, has_catu;
867 struct etr_buf *etr_buf;
868 struct device *dev = &drvdata->csdev->dev;
869
870 has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
871 has_iommu = iommu_get_domain_for_dev(dev->parent);
872 has_catu = !!tmc_etr_get_catu_device(drvdata);
873
874 has_sg = has_catu || has_etr_sg;
875
876 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
877 if (!etr_buf)
878 return ERR_PTR(-ENOMEM);
879
880 etr_buf->size = size;
881
882 /*
883 * If we have to use an existing list of pages, we cannot reliably
884 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
885 * we use the contiguous DMA memory if at least one of the following
886 * conditions is true:
887 * a) The ETR cannot use Scatter-Gather.
888 * b) we have a backing IOMMU
889 * c) The requested memory size is smaller (< 1M).
890 *
891 * Fallback to available mechanisms.
892 *
893 */
894 if (!pages &&
895 (!has_sg || has_iommu || size < SZ_1M))
896 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
897 etr_buf, node, pages);
898 if (rc && has_etr_sg)
899 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
900 etr_buf, node, pages);
901 if (rc && has_catu)
902 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
903 etr_buf, node, pages);
904 if (rc) {
905 kfree(etr_buf);
906 return ERR_PTR(rc);
907 }
908
909 refcount_set(&etr_buf->refcount, 1);
910 dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
911 (unsigned long)size >> 10, etr_buf->mode);
912 return etr_buf;
913}
914
915static void tmc_free_etr_buf(struct etr_buf *etr_buf)
916{
917 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
918 etr_buf->ops->free(etr_buf);
919 kfree(etr_buf);
920}
921
922/*
923 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
924 * with a maximum of @len bytes.
925 * Returns: The size of the linear data available @pos, with *bufpp
926 * updated to point to the buffer.
927 */
928static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
929 u64 offset, size_t len, char **bufpp)
930{
931 /* Adjust the length to limit this transaction to end of buffer */
932 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
933
934 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
935}
936
937static inline s64
938tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
939{
940 ssize_t len;
941 char *bufp;
942
943 len = tmc_etr_buf_get_data(etr_buf, offset,
944 CORESIGHT_BARRIER_PKT_SIZE, &bufp);
945 if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE))
946 return -EINVAL;
947 coresight_insert_barrier_packet(bufp);
948 return offset + CORESIGHT_BARRIER_PKT_SIZE;
949}
950
951/*
952 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
953 * Makes sure the trace data is synced to the memory for consumption.
954 * @etr_buf->offset will hold the offset to the beginning of the trace data
955 * within the buffer, with @etr_buf->len bytes to consume.
956 */
957static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
958{
959 struct etr_buf *etr_buf = drvdata->etr_buf;
960 u64 rrp, rwp;
961 u32 status;
962
963 rrp = tmc_read_rrp(drvdata);
964 rwp = tmc_read_rwp(drvdata);
965 status = readl_relaxed(drvdata->base + TMC_STS);
966
967 /*
968 * If there were memory errors in the session, truncate the
969 * buffer.
970 */
971 if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
972 dev_dbg(&drvdata->csdev->dev,
973 "tmc memory error detected, truncating buffer\n");
974 etr_buf->len = 0;
975 etr_buf->full = false;
976 return;
977 }
978
979 etr_buf->full = !!(status & TMC_STS_FULL);
980
981 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
982
983 etr_buf->ops->sync(etr_buf, rrp, rwp);
984}
985
986static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
987{
988 u32 axictl, sts;
989 struct etr_buf *etr_buf = drvdata->etr_buf;
990
991 CS_UNLOCK(drvdata->base);
992
993 /* Wait for TMCSReady bit to be set */
994 tmc_wait_for_tmcready(drvdata);
995
996 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
997 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
998
999 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
1000 axictl &= ~TMC_AXICTL_CLEAR_MASK;
1001 axictl |= TMC_AXICTL_PROT_CTL_B1;
1002 axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
1003 axictl |= TMC_AXICTL_AXCACHE_OS;
1004
1005 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
1006 axictl &= ~TMC_AXICTL_ARCACHE_MASK;
1007 axictl |= TMC_AXICTL_ARCACHE_OS;
1008 }
1009
1010 if (etr_buf->mode == ETR_MODE_ETR_SG)
1011 axictl |= TMC_AXICTL_SCT_GAT_MODE;
1012
1013 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1014 tmc_write_dba(drvdata, etr_buf->hwaddr);
1015 /*
1016 * If the TMC pointers must be programmed before the session,
1017 * we have to set it properly (i.e, RRP/RWP to base address and
1018 * STS to "not full").
1019 */
1020 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1021 tmc_write_rrp(drvdata, etr_buf->hwaddr);
1022 tmc_write_rwp(drvdata, etr_buf->hwaddr);
1023 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1024 writel_relaxed(sts, drvdata->base + TMC_STS);
1025 }
1026
1027 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1028 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1029 TMC_FFCR_TRIGON_TRIGIN,
1030 drvdata->base + TMC_FFCR);
1031 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1032 tmc_enable_hw(drvdata);
1033
1034 CS_LOCK(drvdata->base);
1035}
1036
1037static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1038 struct etr_buf *etr_buf)
1039{
1040 int rc;
1041
1042 /* Callers should provide an appropriate buffer for use */
1043 if (WARN_ON(!etr_buf))
1044 return -EINVAL;
1045
1046 if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1047 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1048 return -EINVAL;
1049
1050 if (WARN_ON(drvdata->etr_buf))
1051 return -EBUSY;
1052
1053 /*
1054 * If this ETR is connected to a CATU, enable it before we turn
1055 * this on.
1056 */
1057 rc = tmc_etr_enable_catu(drvdata, etr_buf);
1058 if (rc)
1059 return rc;
1060 rc = coresight_claim_device(drvdata->csdev);
1061 if (!rc) {
1062 drvdata->etr_buf = etr_buf;
1063 __tmc_etr_enable_hw(drvdata);
1064 }
1065
1066 return rc;
1067}
1068
1069/*
1070 * Return the available trace data in the buffer (starts at etr_buf->offset,
1071 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1072 * also updating the @bufpp on where to find it. Since the trace data
1073 * starts at anywhere in the buffer, depending on the RRP, we adjust the
1074 * @len returned to handle buffer wrapping around.
1075 *
1076 * We are protected here by drvdata->reading != 0, which ensures the
1077 * sysfs_buf stays alive.
1078 */
1079ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1080 loff_t pos, size_t len, char **bufpp)
1081{
1082 s64 offset;
1083 ssize_t actual = len;
1084 struct etr_buf *etr_buf = drvdata->sysfs_buf;
1085
1086 if (pos + actual > etr_buf->len)
1087 actual = etr_buf->len - pos;
1088 if (actual <= 0)
1089 return actual;
1090
1091 /* Compute the offset from which we read the data */
1092 offset = etr_buf->offset + pos;
1093 if (offset >= etr_buf->size)
1094 offset -= etr_buf->size;
1095 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1096}
1097
1098static struct etr_buf *
1099tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1100{
1101 return tmc_alloc_etr_buf(drvdata, drvdata->size,
1102 0, cpu_to_node(0), NULL);
1103}
1104
1105static void
1106tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1107{
1108 if (buf)
1109 tmc_free_etr_buf(buf);
1110}
1111
1112static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1113{
1114 struct etr_buf *etr_buf = drvdata->etr_buf;
1115
1116 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1117 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1118 drvdata->sysfs_buf = NULL;
1119 } else {
1120 tmc_sync_etr_buf(drvdata);
1121 /*
1122 * Insert barrier packets at the beginning, if there was
1123 * an overflow.
1124 */
1125 if (etr_buf->full)
1126 tmc_etr_buf_insert_barrier_packet(etr_buf,
1127 etr_buf->offset);
1128 }
1129}
1130
1131static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1132{
1133 CS_UNLOCK(drvdata->base);
1134
1135 tmc_flush_and_stop(drvdata);
1136 /*
1137 * When operating in sysFS mode the content of the buffer needs to be
1138 * read before the TMC is disabled.
1139 */
1140 if (drvdata->mode == CS_MODE_SYSFS)
1141 tmc_etr_sync_sysfs_buf(drvdata);
1142
1143 tmc_disable_hw(drvdata);
1144
1145 CS_LOCK(drvdata->base);
1146
1147}
1148
1149void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1150{
1151 __tmc_etr_disable_hw(drvdata);
1152 /* Disable CATU device if this ETR is connected to one */
1153 tmc_etr_disable_catu(drvdata);
1154 coresight_disclaim_device(drvdata->csdev);
1155 /* Reset the ETR buf used by hardware */
1156 drvdata->etr_buf = NULL;
1157}
1158
1159static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1160{
1161 int ret = 0;
1162 unsigned long flags;
1163 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1164 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1165
1166 /*
1167 * If we are enabling the ETR from disabled state, we need to make
1168 * sure we have a buffer with the right size. The etr_buf is not reset
1169 * immediately after we stop the tracing in SYSFS mode as we wait for
1170 * the user to collect the data. We may be able to reuse the existing
1171 * buffer, provided the size matches. Any allocation has to be done
1172 * with the lock released.
1173 */
1174 spin_lock_irqsave(&drvdata->spinlock, flags);
1175 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1176 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1177 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1178
1179 /* Allocate memory with the locks released */
1180 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1181 if (IS_ERR(new_buf))
1182 return PTR_ERR(new_buf);
1183
1184 /* Let's try again */
1185 spin_lock_irqsave(&drvdata->spinlock, flags);
1186 }
1187
1188 if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1189 ret = -EBUSY;
1190 goto out;
1191 }
1192
1193 /*
1194 * In sysFS mode we can have multiple writers per sink. Since this
1195 * sink is already enabled no memory is needed and the HW need not be
1196 * touched, even if the buffer size has changed.
1197 */
1198 if (drvdata->mode == CS_MODE_SYSFS) {
1199 atomic_inc(csdev->refcnt);
1200 goto out;
1201 }
1202
1203 /*
1204 * If we don't have a buffer or it doesn't match the requested size,
1205 * use the buffer allocated above. Otherwise reuse the existing buffer.
1206 */
1207 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1208 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1209 free_buf = sysfs_buf;
1210 drvdata->sysfs_buf = new_buf;
1211 }
1212
1213 ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf);
1214 if (!ret) {
1215 drvdata->mode = CS_MODE_SYSFS;
1216 atomic_inc(csdev->refcnt);
1217 }
1218out:
1219 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1220
1221 /* Free memory outside the spinlock if need be */
1222 if (free_buf)
1223 tmc_etr_free_sysfs_buf(free_buf);
1224
1225 if (!ret)
1226 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1227
1228 return ret;
1229}
1230
1231/*
1232 * alloc_etr_buf: Allocate ETR buffer for use by perf.
1233 * The size of the hardware buffer is dependent on the size configured
1234 * via sysfs and the perf ring buffer size. We prefer to allocate the
1235 * largest possible size, scaling down the size by half until it
1236 * reaches a minimum limit (1M), beyond which we give up.
1237 */
1238static struct etr_buf *
1239alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1240 int nr_pages, void **pages, bool snapshot)
1241{
1242 int node;
1243 struct etr_buf *etr_buf;
1244 unsigned long size;
1245
1246 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1247 /*
1248 * Try to match the perf ring buffer size if it is larger
1249 * than the size requested via sysfs.
1250 */
1251 if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1252 etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT),
1253 0, node, NULL);
1254 if (!IS_ERR(etr_buf))
1255 goto done;
1256 }
1257
1258 /*
1259 * Else switch to configured size for this ETR
1260 * and scale down until we hit the minimum limit.
1261 */
1262 size = drvdata->size;
1263 do {
1264 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1265 if (!IS_ERR(etr_buf))
1266 goto done;
1267 size /= 2;
1268 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1269
1270 return ERR_PTR(-ENOMEM);
1271
1272done:
1273 return etr_buf;
1274}
1275
1276static struct etr_buf *
1277get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1278 struct perf_event *event, int nr_pages,
1279 void **pages, bool snapshot)
1280{
1281 int ret;
1282 pid_t pid = task_pid_nr(event->owner);
1283 struct etr_buf *etr_buf;
1284
1285retry:
1286 /*
1287 * An etr_perf_buffer is associated with an event and holds a reference
1288 * to the AUX ring buffer that was created for that event. In CPU-wide
1289 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1290 * buffer, share a sink. As such an etr_perf_buffer is created for each
1291 * event but a single etr_buf associated with the ETR is shared between
1292 * them. The last event in a trace session will copy the content of the
1293 * etr_buf to its AUX ring buffer. Ring buffer associated to other
1294 * events are simply not used an freed as events are destoyed. We still
1295 * need to allocate a ring buffer for each event since we don't know
1296 * which event will be last.
1297 */
1298
1299 /*
1300 * The first thing to do here is check if an etr_buf has already been
1301 * allocated for this session. If so it is shared with this event,
1302 * otherwise it is created.
1303 */
1304 mutex_lock(&drvdata->idr_mutex);
1305 etr_buf = idr_find(&drvdata->idr, pid);
1306 if (etr_buf) {
1307 refcount_inc(&etr_buf->refcount);
1308 mutex_unlock(&drvdata->idr_mutex);
1309 return etr_buf;
1310 }
1311
1312 /* If we made it here no buffer has been allocated, do so now. */
1313 mutex_unlock(&drvdata->idr_mutex);
1314
1315 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1316 if (IS_ERR(etr_buf))
1317 return etr_buf;
1318
1319 /* Now that we have a buffer, add it to the IDR. */
1320 mutex_lock(&drvdata->idr_mutex);
1321 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1322 mutex_unlock(&drvdata->idr_mutex);
1323
1324 /* Another event with this session ID has allocated this buffer. */
1325 if (ret == -ENOSPC) {
1326 tmc_free_etr_buf(etr_buf);
1327 goto retry;
1328 }
1329
1330 /* The IDR can't allocate room for a new session, abandon ship. */
1331 if (ret == -ENOMEM) {
1332 tmc_free_etr_buf(etr_buf);
1333 return ERR_PTR(ret);
1334 }
1335
1336
1337 return etr_buf;
1338}
1339
1340static struct etr_buf *
1341get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1342 struct perf_event *event, int nr_pages,
1343 void **pages, bool snapshot)
1344{
1345 /*
1346 * In per-thread mode the etr_buf isn't shared, so just go ahead
1347 * with memory allocation.
1348 */
1349 return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1350}
1351
1352static struct etr_buf *
1353get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1354 int nr_pages, void **pages, bool snapshot)
1355{
1356 if (event->cpu == -1)
1357 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1358 pages, snapshot);
1359
1360 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1361 pages, snapshot);
1362}
1363
1364static struct etr_perf_buffer *
1365tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1366 int nr_pages, void **pages, bool snapshot)
1367{
1368 int node;
1369 struct etr_buf *etr_buf;
1370 struct etr_perf_buffer *etr_perf;
1371
1372 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1373
1374 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1375 if (!etr_perf)
1376 return ERR_PTR(-ENOMEM);
1377
1378 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1379 if (!IS_ERR(etr_buf))
1380 goto done;
1381
1382 kfree(etr_perf);
1383 return ERR_PTR(-ENOMEM);
1384
1385done:
1386 /*
1387 * Keep a reference to the ETR this buffer has been allocated for
1388 * in order to have access to the IDR in tmc_free_etr_buffer().
1389 */
1390 etr_perf->drvdata = drvdata;
1391 etr_perf->etr_buf = etr_buf;
1392
1393 return etr_perf;
1394}
1395
1396
1397static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1398 struct perf_event *event, void **pages,
1399 int nr_pages, bool snapshot)
1400{
1401 struct etr_perf_buffer *etr_perf;
1402 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1403
1404 etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1405 nr_pages, pages, snapshot);
1406 if (IS_ERR(etr_perf)) {
1407 dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1408 return NULL;
1409 }
1410
1411 etr_perf->pid = task_pid_nr(event->owner);
1412 etr_perf->snapshot = snapshot;
1413 etr_perf->nr_pages = nr_pages;
1414 etr_perf->pages = pages;
1415
1416 return etr_perf;
1417}
1418
1419static void tmc_free_etr_buffer(void *config)
1420{
1421 struct etr_perf_buffer *etr_perf = config;
1422 struct tmc_drvdata *drvdata = etr_perf->drvdata;
1423 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1424
1425 if (!etr_buf)
1426 goto free_etr_perf_buffer;
1427
1428 mutex_lock(&drvdata->idr_mutex);
1429 /* If we are not the last one to use the buffer, don't touch it. */
1430 if (!refcount_dec_and_test(&etr_buf->refcount)) {
1431 mutex_unlock(&drvdata->idr_mutex);
1432 goto free_etr_perf_buffer;
1433 }
1434
1435 /* We are the last one, remove from the IDR and free the buffer. */
1436 buf = idr_remove(&drvdata->idr, etr_perf->pid);
1437 mutex_unlock(&drvdata->idr_mutex);
1438
1439 /*
1440 * Something went very wrong if the buffer associated with this ID
1441 * is not the same in the IDR. Leak to avoid use after free.
1442 */
1443 if (buf && WARN_ON(buf != etr_buf))
1444 goto free_etr_perf_buffer;
1445
1446 tmc_free_etr_buf(etr_perf->etr_buf);
1447
1448free_etr_perf_buffer:
1449 kfree(etr_perf);
1450}
1451
1452/*
1453 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1454 * buffer to the perf ring buffer.
1455 */
1456static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1457 unsigned long head,
1458 unsigned long src_offset,
1459 unsigned long to_copy)
1460{
1461 long bytes;
1462 long pg_idx, pg_offset;
1463 char **dst_pages, *src_buf;
1464 struct etr_buf *etr_buf = etr_perf->etr_buf;
1465
1466 head = PERF_IDX2OFF(head, etr_perf);
1467 pg_idx = head >> PAGE_SHIFT;
1468 pg_offset = head & (PAGE_SIZE - 1);
1469 dst_pages = (char **)etr_perf->pages;
1470
1471 while (to_copy > 0) {
1472 /*
1473 * In one iteration, we can copy minimum of :
1474 * 1) what is available in the source buffer,
1475 * 2) what is available in the source buffer, before it
1476 * wraps around.
1477 * 3) what is available in the destination page.
1478 * in one iteration.
1479 */
1480 if (src_offset >= etr_buf->size)
1481 src_offset -= etr_buf->size;
1482 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1483 &src_buf);
1484 if (WARN_ON_ONCE(bytes <= 0))
1485 break;
1486 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1487
1488 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1489
1490 to_copy -= bytes;
1491
1492 /* Move destination pointers */
1493 pg_offset += bytes;
1494 if (pg_offset == PAGE_SIZE) {
1495 pg_offset = 0;
1496 if (++pg_idx == etr_perf->nr_pages)
1497 pg_idx = 0;
1498 }
1499
1500 /* Move source pointers */
1501 src_offset += bytes;
1502 }
1503}
1504
1505/*
1506 * tmc_update_etr_buffer : Update the perf ring buffer with the
1507 * available trace data. We use software double buffering at the moment.
1508 *
1509 * TODO: Add support for reusing the perf ring buffer.
1510 */
1511static unsigned long
1512tmc_update_etr_buffer(struct coresight_device *csdev,
1513 struct perf_output_handle *handle,
1514 void *config)
1515{
1516 bool lost = false;
1517 unsigned long flags, offset, size = 0;
1518 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1519 struct etr_perf_buffer *etr_perf = config;
1520 struct etr_buf *etr_buf = etr_perf->etr_buf;
1521
1522 spin_lock_irqsave(&drvdata->spinlock, flags);
1523
1524 /* Don't do anything if another tracer is using this sink */
1525 if (atomic_read(csdev->refcnt) != 1) {
1526 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1527 goto out;
1528 }
1529
1530 if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1531 lost = true;
1532 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1533 goto out;
1534 }
1535
1536 CS_UNLOCK(drvdata->base);
1537
1538 tmc_flush_and_stop(drvdata);
1539 tmc_sync_etr_buf(drvdata);
1540
1541 CS_LOCK(drvdata->base);
1542 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1543
1544 lost = etr_buf->full;
1545 offset = etr_buf->offset;
1546 size = etr_buf->len;
1547
1548 /*
1549 * The ETR buffer may be bigger than the space available in the
1550 * perf ring buffer (handle->size). If so advance the offset so that we
1551 * get the latest trace data. In snapshot mode none of that matters
1552 * since we are expected to clobber stale data in favour of the latest
1553 * traces.
1554 */
1555 if (!etr_perf->snapshot && size > handle->size) {
1556 u32 mask = tmc_get_memwidth_mask(drvdata);
1557
1558 /*
1559 * Make sure the new size is aligned in accordance with the
1560 * requirement explained in function tmc_get_memwidth_mask().
1561 */
1562 size = handle->size & mask;
1563 offset = etr_buf->offset + etr_buf->len - size;
1564
1565 if (offset >= etr_buf->size)
1566 offset -= etr_buf->size;
1567 lost = true;
1568 }
1569
1570 /* Insert barrier packets at the beginning, if there was an overflow */
1571 if (lost)
1572 tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1573 tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);
1574
1575 /*
1576 * In snapshot mode we simply increment the head by the number of byte
1577 * that were written. User space will figure out how many bytes to get
1578 * from the AUX buffer based on the position of the head.
1579 */
1580 if (etr_perf->snapshot)
1581 handle->head += size;
1582
1583 /*
1584 * Ensure that the AUX trace data is visible before the aux_head
1585 * is updated via perf_aux_output_end(), as expected by the
1586 * perf ring buffer.
1587 */
1588 smp_wmb();
1589
1590out:
1591 /*
1592 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1593 * captured buffer is expected to be truncated and 2) a full buffer
1594 * prevents the event from being re-enabled by the perf core,
1595 * resulting in stale data being send to user space.
1596 */
1597 if (!etr_perf->snapshot && lost)
1598 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1599 return size;
1600}
1601
1602static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1603{
1604 int rc = 0;
1605 pid_t pid;
1606 unsigned long flags;
1607 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1608 struct perf_output_handle *handle = data;
1609 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1610
1611 spin_lock_irqsave(&drvdata->spinlock, flags);
1612 /* Don't use this sink if it is already claimed by sysFS */
1613 if (drvdata->mode == CS_MODE_SYSFS) {
1614 rc = -EBUSY;
1615 goto unlock_out;
1616 }
1617
1618 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1619 rc = -EINVAL;
1620 goto unlock_out;
1621 }
1622
1623 /* Get a handle on the pid of the process to monitor */
1624 pid = etr_perf->pid;
1625
1626 /* Do not proceed if this device is associated with another session */
1627 if (drvdata->pid != -1 && drvdata->pid != pid) {
1628 rc = -EBUSY;
1629 goto unlock_out;
1630 }
1631
1632 /*
1633 * No HW configuration is needed if the sink is already in
1634 * use for this session.
1635 */
1636 if (drvdata->pid == pid) {
1637 atomic_inc(csdev->refcnt);
1638 goto unlock_out;
1639 }
1640
1641 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1642 if (!rc) {
1643 /* Associate with monitored process. */
1644 drvdata->pid = pid;
1645 drvdata->mode = CS_MODE_PERF;
1646 drvdata->perf_buf = etr_perf->etr_buf;
1647 atomic_inc(csdev->refcnt);
1648 }
1649
1650unlock_out:
1651 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1652 return rc;
1653}
1654
1655static int tmc_enable_etr_sink(struct coresight_device *csdev,
1656 u32 mode, void *data)
1657{
1658 switch (mode) {
1659 case CS_MODE_SYSFS:
1660 return tmc_enable_etr_sink_sysfs(csdev);
1661 case CS_MODE_PERF:
1662 return tmc_enable_etr_sink_perf(csdev, data);
1663 }
1664
1665 /* We shouldn't be here */
1666 return -EINVAL;
1667}
1668
1669static int tmc_disable_etr_sink(struct coresight_device *csdev)
1670{
1671 unsigned long flags;
1672 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1673
1674 spin_lock_irqsave(&drvdata->spinlock, flags);
1675
1676 if (drvdata->reading) {
1677 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1678 return -EBUSY;
1679 }
1680
1681 if (atomic_dec_return(csdev->refcnt)) {
1682 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1683 return -EBUSY;
1684 }
1685
1686 /* Complain if we (somehow) got out of sync */
1687 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1688 tmc_etr_disable_hw(drvdata);
1689 /* Dissociate from monitored process. */
1690 drvdata->pid = -1;
1691 drvdata->mode = CS_MODE_DISABLED;
1692 /* Reset perf specific data */
1693 drvdata->perf_buf = NULL;
1694
1695 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1696
1697 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1698 return 0;
1699}
1700
1701static const struct coresight_ops_sink tmc_etr_sink_ops = {
1702 .enable = tmc_enable_etr_sink,
1703 .disable = tmc_disable_etr_sink,
1704 .alloc_buffer = tmc_alloc_etr_buffer,
1705 .update_buffer = tmc_update_etr_buffer,
1706 .free_buffer = tmc_free_etr_buffer,
1707};
1708
1709const struct coresight_ops tmc_etr_cs_ops = {
1710 .sink_ops = &tmc_etr_sink_ops,
1711};
1712
1713int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1714{
1715 int ret = 0;
1716 unsigned long flags;
1717
1718 /* config types are set a boot time and never change */
1719 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1720 return -EINVAL;
1721
1722 spin_lock_irqsave(&drvdata->spinlock, flags);
1723 if (drvdata->reading) {
1724 ret = -EBUSY;
1725 goto out;
1726 }
1727
1728 /*
1729 * We can safely allow reads even if the ETR is operating in PERF mode,
1730 * since the sysfs session is captured in mode specific data.
1731 * If drvdata::sysfs_data is NULL the trace data has been read already.
1732 */
1733 if (!drvdata->sysfs_buf) {
1734 ret = -EINVAL;
1735 goto out;
1736 }
1737
1738 /* Disable the TMC if we are trying to read from a running session. */
1739 if (drvdata->mode == CS_MODE_SYSFS)
1740 __tmc_etr_disable_hw(drvdata);
1741
1742 drvdata->reading = true;
1743out:
1744 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1745
1746 return ret;
1747}
1748
1749int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1750{
1751 unsigned long flags;
1752 struct etr_buf *sysfs_buf = NULL;
1753
1754 /* config types are set a boot time and never change */
1755 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1756 return -EINVAL;
1757
1758 spin_lock_irqsave(&drvdata->spinlock, flags);
1759
1760 /* RE-enable the TMC if need be */
1761 if (drvdata->mode == CS_MODE_SYSFS) {
1762 /*
1763 * The trace run will continue with the same allocated trace
1764 * buffer. Since the tracer is still enabled drvdata::buf can't
1765 * be NULL.
1766 */
1767 __tmc_etr_enable_hw(drvdata);
1768 } else {
1769 /*
1770 * The ETR is not tracing and the buffer was just read.
1771 * As such prepare to free the trace buffer.
1772 */
1773 sysfs_buf = drvdata->sysfs_buf;
1774 drvdata->sysfs_buf = NULL;
1775 }
1776
1777 drvdata->reading = false;
1778 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1779
1780 /* Free allocated memory out side of the spinlock */
1781 if (sysfs_buf)
1782 tmc_etr_free_sysfs_buf(sysfs_buf);
1783
1784 return 0;
1785}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5 */
6
7#include <linux/atomic.h>
8#include <linux/coresight.h>
9#include <linux/dma-mapping.h>
10#include <linux/iommu.h>
11#include <linux/idr.h>
12#include <linux/mutex.h>
13#include <linux/refcount.h>
14#include <linux/slab.h>
15#include <linux/types.h>
16#include <linux/vmalloc.h>
17#include "coresight-catu.h"
18#include "coresight-etm-perf.h"
19#include "coresight-priv.h"
20#include "coresight-tmc.h"
21
22struct etr_flat_buf {
23 struct device *dev;
24 dma_addr_t daddr;
25 void *vaddr;
26 size_t size;
27};
28
29/*
30 * etr_perf_buffer - Perf buffer used for ETR
31 * @drvdata - The ETR drvdaga this buffer has been allocated for.
32 * @etr_buf - Actual buffer used by the ETR
33 * @pid - The PID this etr_perf_buffer belongs to.
34 * @snaphost - Perf session mode
35 * @head - handle->head at the beginning of the session.
36 * @nr_pages - Number of pages in the ring buffer.
37 * @pages - Array of Pages in the ring buffer.
38 */
39struct etr_perf_buffer {
40 struct tmc_drvdata *drvdata;
41 struct etr_buf *etr_buf;
42 pid_t pid;
43 bool snapshot;
44 unsigned long head;
45 int nr_pages;
46 void **pages;
47};
48
49/* Convert the perf index to an offset within the ETR buffer */
50#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
51
52/* Lower limit for ETR hardware buffer */
53#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
54
55/*
56 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
57 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
58 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
59 * contain more than one SG buffer and tables.
60 *
61 * A table entry has the following format:
62 *
63 * ---Bit31------------Bit4-------Bit1-----Bit0--
64 * | Address[39:12] | SBZ | Entry Type |
65 * ----------------------------------------------
66 *
67 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
68 * always zero.
69 *
70 * Entry type:
71 * b00 - Reserved.
72 * b01 - Last entry in the tables, points to 4K page buffer.
73 * b10 - Normal entry, points to 4K page buffer.
74 * b11 - Link. The address points to the base of next table.
75 */
76
77typedef u32 sgte_t;
78
79#define ETR_SG_PAGE_SHIFT 12
80#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
81#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
82#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
83#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
84
85#define ETR_SG_ET_MASK 0x3
86#define ETR_SG_ET_LAST 0x1
87#define ETR_SG_ET_NORMAL 0x2
88#define ETR_SG_ET_LINK 0x3
89
90#define ETR_SG_ADDR_SHIFT 4
91
92#define ETR_SG_ENTRY(addr, type) \
93 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
94 (type & ETR_SG_ET_MASK))
95
96#define ETR_SG_ADDR(entry) \
97 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
98#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
99
100/*
101 * struct etr_sg_table : ETR SG Table
102 * @sg_table: Generic SG Table holding the data/table pages.
103 * @hwaddr: hwaddress used by the TMC, which is the base
104 * address of the table.
105 */
106struct etr_sg_table {
107 struct tmc_sg_table *sg_table;
108 dma_addr_t hwaddr;
109};
110
111/*
112 * tmc_etr_sg_table_entries: Total number of table entries required to map
113 * @nr_pages system pages.
114 *
115 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
116 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
117 * with the last entry pointing to another page of table entries.
118 * If we spill over to a new page for mapping 1 entry, we could as
119 * well replace the link entry of the previous page with the last entry.
120 */
121static inline unsigned long __attribute_const__
122tmc_etr_sg_table_entries(int nr_pages)
123{
124 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
125 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
126 /*
127 * If we spill over to a new page for 1 entry, we could as well
128 * make it the LAST entry in the previous page, skipping the Link
129 * address.
130 */
131 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
132 nr_sglinks--;
133 return nr_sgpages + nr_sglinks;
134}
135
136/*
137 * tmc_pages_get_offset: Go through all the pages in the tmc_pages
138 * and map the device address @addr to an offset within the virtual
139 * contiguous buffer.
140 */
141static long
142tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
143{
144 int i;
145 dma_addr_t page_start;
146
147 for (i = 0; i < tmc_pages->nr_pages; i++) {
148 page_start = tmc_pages->daddrs[i];
149 if (addr >= page_start && addr < (page_start + PAGE_SIZE))
150 return i * PAGE_SIZE + (addr - page_start);
151 }
152
153 return -EINVAL;
154}
155
156/*
157 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
158 * If the pages were not allocated in tmc_pages_alloc(), we would
159 * simply drop the refcount.
160 */
161static void tmc_pages_free(struct tmc_pages *tmc_pages,
162 struct device *dev, enum dma_data_direction dir)
163{
164 int i;
165 struct device *real_dev = dev->parent;
166
167 for (i = 0; i < tmc_pages->nr_pages; i++) {
168 if (tmc_pages->daddrs && tmc_pages->daddrs[i])
169 dma_unmap_page(real_dev, tmc_pages->daddrs[i],
170 PAGE_SIZE, dir);
171 if (tmc_pages->pages && tmc_pages->pages[i])
172 __free_page(tmc_pages->pages[i]);
173 }
174
175 kfree(tmc_pages->pages);
176 kfree(tmc_pages->daddrs);
177 tmc_pages->pages = NULL;
178 tmc_pages->daddrs = NULL;
179 tmc_pages->nr_pages = 0;
180}
181
182/*
183 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
184 * If @pages is not NULL, the list of page virtual addresses are
185 * used as the data pages. The pages are then dma_map'ed for @dev
186 * with dma_direction @dir.
187 *
188 * Returns 0 upon success, else the error number.
189 */
190static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
191 struct device *dev, int node,
192 enum dma_data_direction dir, void **pages)
193{
194 int i, nr_pages;
195 dma_addr_t paddr;
196 struct page *page;
197 struct device *real_dev = dev->parent;
198
199 nr_pages = tmc_pages->nr_pages;
200 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
201 GFP_KERNEL);
202 if (!tmc_pages->daddrs)
203 return -ENOMEM;
204 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
205 GFP_KERNEL);
206 if (!tmc_pages->pages) {
207 kfree(tmc_pages->daddrs);
208 tmc_pages->daddrs = NULL;
209 return -ENOMEM;
210 }
211
212 for (i = 0; i < nr_pages; i++) {
213 if (pages && pages[i]) {
214 page = virt_to_page(pages[i]);
215 /* Hold a refcount on the page */
216 get_page(page);
217 } else {
218 page = alloc_pages_node(node,
219 GFP_KERNEL | __GFP_ZERO, 0);
220 }
221 paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
222 if (dma_mapping_error(real_dev, paddr))
223 goto err;
224 tmc_pages->daddrs[i] = paddr;
225 tmc_pages->pages[i] = page;
226 }
227 return 0;
228err:
229 tmc_pages_free(tmc_pages, dev, dir);
230 return -ENOMEM;
231}
232
233static inline long
234tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
235{
236 return tmc_pages_get_offset(&sg_table->data_pages, addr);
237}
238
239static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
240{
241 if (sg_table->table_vaddr)
242 vunmap(sg_table->table_vaddr);
243 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
244}
245
246static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
247{
248 if (sg_table->data_vaddr)
249 vunmap(sg_table->data_vaddr);
250 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
251}
252
253void tmc_free_sg_table(struct tmc_sg_table *sg_table)
254{
255 tmc_free_table_pages(sg_table);
256 tmc_free_data_pages(sg_table);
257}
258
259/*
260 * Alloc pages for the table. Since this will be used by the device,
261 * allocate the pages closer to the device (i.e, dev_to_node(dev)
262 * rather than the CPU node).
263 */
264static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
265{
266 int rc;
267 struct tmc_pages *table_pages = &sg_table->table_pages;
268
269 rc = tmc_pages_alloc(table_pages, sg_table->dev,
270 dev_to_node(sg_table->dev),
271 DMA_TO_DEVICE, NULL);
272 if (rc)
273 return rc;
274 sg_table->table_vaddr = vmap(table_pages->pages,
275 table_pages->nr_pages,
276 VM_MAP,
277 PAGE_KERNEL);
278 if (!sg_table->table_vaddr)
279 rc = -ENOMEM;
280 else
281 sg_table->table_daddr = table_pages->daddrs[0];
282 return rc;
283}
284
285static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
286{
287 int rc;
288
289 /* Allocate data pages on the node requested by the caller */
290 rc = tmc_pages_alloc(&sg_table->data_pages,
291 sg_table->dev, sg_table->node,
292 DMA_FROM_DEVICE, pages);
293 if (!rc) {
294 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
295 sg_table->data_pages.nr_pages,
296 VM_MAP,
297 PAGE_KERNEL);
298 if (!sg_table->data_vaddr)
299 rc = -ENOMEM;
300 }
301 return rc;
302}
303
304/*
305 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
306 * and data buffers. TMC writes to the data buffers and reads from the SG
307 * Table pages.
308 *
309 * @dev - Coresight device to which page should be DMA mapped.
310 * @node - Numa node for mem allocations
311 * @nr_tpages - Number of pages for the table entries.
312 * @nr_dpages - Number of pages for Data buffer.
313 * @pages - Optional list of virtual address of pages.
314 */
315struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
316 int node,
317 int nr_tpages,
318 int nr_dpages,
319 void **pages)
320{
321 long rc;
322 struct tmc_sg_table *sg_table;
323
324 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
325 if (!sg_table)
326 return ERR_PTR(-ENOMEM);
327 sg_table->data_pages.nr_pages = nr_dpages;
328 sg_table->table_pages.nr_pages = nr_tpages;
329 sg_table->node = node;
330 sg_table->dev = dev;
331
332 rc = tmc_alloc_data_pages(sg_table, pages);
333 if (!rc)
334 rc = tmc_alloc_table_pages(sg_table);
335 if (rc) {
336 tmc_free_sg_table(sg_table);
337 kfree(sg_table);
338 return ERR_PTR(rc);
339 }
340
341 return sg_table;
342}
343
344/*
345 * tmc_sg_table_sync_data_range: Sync the data buffer written
346 * by the device from @offset upto a @size bytes.
347 */
348void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
349 u64 offset, u64 size)
350{
351 int i, index, start;
352 int npages = DIV_ROUND_UP(size, PAGE_SIZE);
353 struct device *real_dev = table->dev->parent;
354 struct tmc_pages *data = &table->data_pages;
355
356 start = offset >> PAGE_SHIFT;
357 for (i = start; i < (start + npages); i++) {
358 index = i % data->nr_pages;
359 dma_sync_single_for_cpu(real_dev, data->daddrs[index],
360 PAGE_SIZE, DMA_FROM_DEVICE);
361 }
362}
363
364/* tmc_sg_sync_table: Sync the page table */
365void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
366{
367 int i;
368 struct device *real_dev = sg_table->dev->parent;
369 struct tmc_pages *table_pages = &sg_table->table_pages;
370
371 for (i = 0; i < table_pages->nr_pages; i++)
372 dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
373 PAGE_SIZE, DMA_TO_DEVICE);
374}
375
376/*
377 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
378 * in the SG buffer. The @bufpp is updated to point to the buffer.
379 * Returns :
380 * the length of linear data available at @offset.
381 * or
382 * <= 0 if no data is available.
383 */
384ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
385 u64 offset, size_t len, char **bufpp)
386{
387 size_t size;
388 int pg_idx = offset >> PAGE_SHIFT;
389 int pg_offset = offset & (PAGE_SIZE - 1);
390 struct tmc_pages *data_pages = &sg_table->data_pages;
391
392 size = tmc_sg_table_buf_size(sg_table);
393 if (offset >= size)
394 return -EINVAL;
395
396 /* Make sure we don't go beyond the end */
397 len = (len < (size - offset)) ? len : size - offset;
398 /* Respect the page boundaries */
399 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
400 if (len > 0)
401 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
402 return len;
403}
404
405#ifdef ETR_SG_DEBUG
406/* Map a dma address to virtual address */
407static unsigned long
408tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
409 dma_addr_t addr, bool table)
410{
411 long offset;
412 unsigned long base;
413 struct tmc_pages *tmc_pages;
414
415 if (table) {
416 tmc_pages = &sg_table->table_pages;
417 base = (unsigned long)sg_table->table_vaddr;
418 } else {
419 tmc_pages = &sg_table->data_pages;
420 base = (unsigned long)sg_table->data_vaddr;
421 }
422
423 offset = tmc_pages_get_offset(tmc_pages, addr);
424 if (offset < 0)
425 return 0;
426 return base + offset;
427}
428
429/* Dump the given sg_table */
430static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
431{
432 sgte_t *ptr;
433 int i = 0;
434 dma_addr_t addr;
435 struct tmc_sg_table *sg_table = etr_table->sg_table;
436
437 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
438 etr_table->hwaddr, true);
439 while (ptr) {
440 addr = ETR_SG_ADDR(*ptr);
441 switch (ETR_SG_ET(*ptr)) {
442 case ETR_SG_ET_NORMAL:
443 dev_dbg(sg_table->dev,
444 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
445 ptr++;
446 break;
447 case ETR_SG_ET_LINK:
448 dev_dbg(sg_table->dev,
449 "%05d: *** %p\t:{L} 0x%llx ***\n",
450 i, ptr, addr);
451 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
452 addr, true);
453 break;
454 case ETR_SG_ET_LAST:
455 dev_dbg(sg_table->dev,
456 "%05d: ### %p\t:[L] 0x%llx ###\n",
457 i, ptr, addr);
458 return;
459 default:
460 dev_dbg(sg_table->dev,
461 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
462 i, ptr, addr);
463 return;
464 }
465 i++;
466 }
467 dev_dbg(sg_table->dev, "******* End of Table *****\n");
468}
469#else
470static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
471#endif
472
473/*
474 * Populate the SG Table page table entries from table/data
475 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
476 * So does a Table page. So we keep track of indices of the tables
477 * in each system page and move the pointers accordingly.
478 */
479#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
480static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
481{
482 dma_addr_t paddr;
483 int i, type, nr_entries;
484 int tpidx = 0; /* index to the current system table_page */
485 int sgtidx = 0; /* index to the sg_table within the current syspage */
486 int sgtentry = 0; /* the entry within the sg_table */
487 int dpidx = 0; /* index to the current system data_page */
488 int spidx = 0; /* index to the SG page within the current data page */
489 sgte_t *ptr; /* pointer to the table entry to fill */
490 struct tmc_sg_table *sg_table = etr_table->sg_table;
491 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
492 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
493
494 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
495 /*
496 * Use the contiguous virtual address of the table to update entries.
497 */
498 ptr = sg_table->table_vaddr;
499 /*
500 * Fill all the entries, except the last entry to avoid special
501 * checks within the loop.
502 */
503 for (i = 0; i < nr_entries - 1; i++) {
504 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
505 /*
506 * Last entry in a sg_table page is a link address to
507 * the next table page. If this sg_table is the last
508 * one in the system page, it links to the first
509 * sg_table in the next system page. Otherwise, it
510 * links to the next sg_table page within the system
511 * page.
512 */
513 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
514 paddr = table_daddrs[tpidx + 1];
515 } else {
516 paddr = table_daddrs[tpidx] +
517 (ETR_SG_PAGE_SIZE * (sgtidx + 1));
518 }
519 type = ETR_SG_ET_LINK;
520 } else {
521 /*
522 * Update the indices to the data_pages to point to the
523 * next sg_page in the data buffer.
524 */
525 type = ETR_SG_ET_NORMAL;
526 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
527 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
528 dpidx++;
529 }
530 *ptr++ = ETR_SG_ENTRY(paddr, type);
531 /*
532 * Move to the next table pointer, moving the table page index
533 * if necessary
534 */
535 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
536 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
537 tpidx++;
538 }
539 }
540
541 /* Set up the last entry, which is always a data pointer */
542 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
543 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
544}
545
546/*
547 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
548 * populate the table.
549 *
550 * @dev - Device pointer for the TMC
551 * @node - NUMA node where the memory should be allocated
552 * @size - Total size of the data buffer
553 * @pages - Optional list of page virtual address
554 */
555static struct etr_sg_table *
556tmc_init_etr_sg_table(struct device *dev, int node,
557 unsigned long size, void **pages)
558{
559 int nr_entries, nr_tpages;
560 int nr_dpages = size >> PAGE_SHIFT;
561 struct tmc_sg_table *sg_table;
562 struct etr_sg_table *etr_table;
563
564 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
565 if (!etr_table)
566 return ERR_PTR(-ENOMEM);
567 nr_entries = tmc_etr_sg_table_entries(nr_dpages);
568 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
569
570 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
571 if (IS_ERR(sg_table)) {
572 kfree(etr_table);
573 return ERR_CAST(sg_table);
574 }
575
576 etr_table->sg_table = sg_table;
577 /* TMC should use table base address for DBA */
578 etr_table->hwaddr = sg_table->table_daddr;
579 tmc_etr_sg_table_populate(etr_table);
580 /* Sync the table pages for the HW */
581 tmc_sg_table_sync_table(sg_table);
582 tmc_etr_sg_table_dump(etr_table);
583
584 return etr_table;
585}
586
587/*
588 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
589 */
590static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
591 struct etr_buf *etr_buf, int node,
592 void **pages)
593{
594 struct etr_flat_buf *flat_buf;
595 struct device *real_dev = drvdata->csdev->dev.parent;
596
597 /* We cannot reuse existing pages for flat buf */
598 if (pages)
599 return -EINVAL;
600
601 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
602 if (!flat_buf)
603 return -ENOMEM;
604
605 flat_buf->vaddr = dma_alloc_coherent(real_dev, etr_buf->size,
606 &flat_buf->daddr, GFP_KERNEL);
607 if (!flat_buf->vaddr) {
608 kfree(flat_buf);
609 return -ENOMEM;
610 }
611
612 flat_buf->size = etr_buf->size;
613 flat_buf->dev = &drvdata->csdev->dev;
614 etr_buf->hwaddr = flat_buf->daddr;
615 etr_buf->mode = ETR_MODE_FLAT;
616 etr_buf->private = flat_buf;
617 return 0;
618}
619
620static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
621{
622 struct etr_flat_buf *flat_buf = etr_buf->private;
623
624 if (flat_buf && flat_buf->daddr) {
625 struct device *real_dev = flat_buf->dev->parent;
626
627 dma_free_coherent(real_dev, flat_buf->size,
628 flat_buf->vaddr, flat_buf->daddr);
629 }
630 kfree(flat_buf);
631}
632
633static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
634{
635 /*
636 * Adjust the buffer to point to the beginning of the trace data
637 * and update the available trace data.
638 */
639 etr_buf->offset = rrp - etr_buf->hwaddr;
640 if (etr_buf->full)
641 etr_buf->len = etr_buf->size;
642 else
643 etr_buf->len = rwp - rrp;
644}
645
646static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
647 u64 offset, size_t len, char **bufpp)
648{
649 struct etr_flat_buf *flat_buf = etr_buf->private;
650
651 *bufpp = (char *)flat_buf->vaddr + offset;
652 /*
653 * tmc_etr_buf_get_data already adjusts the length to handle
654 * buffer wrapping around.
655 */
656 return len;
657}
658
659static const struct etr_buf_operations etr_flat_buf_ops = {
660 .alloc = tmc_etr_alloc_flat_buf,
661 .free = tmc_etr_free_flat_buf,
662 .sync = tmc_etr_sync_flat_buf,
663 .get_data = tmc_etr_get_data_flat_buf,
664};
665
666/*
667 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
668 * appropriately.
669 */
670static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
671 struct etr_buf *etr_buf, int node,
672 void **pages)
673{
674 struct etr_sg_table *etr_table;
675 struct device *dev = &drvdata->csdev->dev;
676
677 etr_table = tmc_init_etr_sg_table(dev, node,
678 etr_buf->size, pages);
679 if (IS_ERR(etr_table))
680 return -ENOMEM;
681 etr_buf->hwaddr = etr_table->hwaddr;
682 etr_buf->mode = ETR_MODE_ETR_SG;
683 etr_buf->private = etr_table;
684 return 0;
685}
686
687static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
688{
689 struct etr_sg_table *etr_table = etr_buf->private;
690
691 if (etr_table) {
692 tmc_free_sg_table(etr_table->sg_table);
693 kfree(etr_table);
694 }
695}
696
697static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
698 size_t len, char **bufpp)
699{
700 struct etr_sg_table *etr_table = etr_buf->private;
701
702 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
703}
704
705static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
706{
707 long r_offset, w_offset;
708 struct etr_sg_table *etr_table = etr_buf->private;
709 struct tmc_sg_table *table = etr_table->sg_table;
710
711 /* Convert hw address to offset in the buffer */
712 r_offset = tmc_sg_get_data_page_offset(table, rrp);
713 if (r_offset < 0) {
714 dev_warn(table->dev,
715 "Unable to map RRP %llx to offset\n", rrp);
716 etr_buf->len = 0;
717 return;
718 }
719
720 w_offset = tmc_sg_get_data_page_offset(table, rwp);
721 if (w_offset < 0) {
722 dev_warn(table->dev,
723 "Unable to map RWP %llx to offset\n", rwp);
724 etr_buf->len = 0;
725 return;
726 }
727
728 etr_buf->offset = r_offset;
729 if (etr_buf->full)
730 etr_buf->len = etr_buf->size;
731 else
732 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
733 w_offset - r_offset;
734 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
735}
736
737static const struct etr_buf_operations etr_sg_buf_ops = {
738 .alloc = tmc_etr_alloc_sg_buf,
739 .free = tmc_etr_free_sg_buf,
740 .sync = tmc_etr_sync_sg_buf,
741 .get_data = tmc_etr_get_data_sg_buf,
742};
743
744/*
745 * TMC ETR could be connected to a CATU device, which can provide address
746 * translation service. This is represented by the Output port of the TMC
747 * (ETR) connected to the input port of the CATU.
748 *
749 * Returns : coresight_device ptr for the CATU device if a CATU is found.
750 * : NULL otherwise.
751 */
752struct coresight_device *
753tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
754{
755 int i;
756 struct coresight_device *tmp, *etr = drvdata->csdev;
757
758 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
759 return NULL;
760
761 for (i = 0; i < etr->pdata->nr_outport; i++) {
762 tmp = etr->pdata->conns[i].child_dev;
763 if (tmp && coresight_is_catu_device(tmp))
764 return tmp;
765 }
766
767 return NULL;
768}
769
770static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata,
771 struct etr_buf *etr_buf)
772{
773 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
774
775 if (catu && helper_ops(catu)->enable)
776 return helper_ops(catu)->enable(catu, etr_buf);
777 return 0;
778}
779
780static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata)
781{
782 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
783
784 if (catu && helper_ops(catu)->disable)
785 helper_ops(catu)->disable(catu, drvdata->etr_buf);
786}
787
788static const struct etr_buf_operations *etr_buf_ops[] = {
789 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
790 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
791 [ETR_MODE_CATU] = IS_ENABLED(CONFIG_CORESIGHT_CATU)
792 ? &etr_catu_buf_ops : NULL,
793};
794
795static inline int tmc_etr_mode_alloc_buf(int mode,
796 struct tmc_drvdata *drvdata,
797 struct etr_buf *etr_buf, int node,
798 void **pages)
799{
800 int rc = -EINVAL;
801
802 switch (mode) {
803 case ETR_MODE_FLAT:
804 case ETR_MODE_ETR_SG:
805 case ETR_MODE_CATU:
806 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
807 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
808 node, pages);
809 if (!rc)
810 etr_buf->ops = etr_buf_ops[mode];
811 return rc;
812 default:
813 return -EINVAL;
814 }
815}
816
817/*
818 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
819 * @drvdata : ETR device details.
820 * @size : size of the requested buffer.
821 * @flags : Required properties for the buffer.
822 * @node : Node for memory allocations.
823 * @pages : An optional list of pages.
824 */
825static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
826 ssize_t size, int flags,
827 int node, void **pages)
828{
829 int rc = -ENOMEM;
830 bool has_etr_sg, has_iommu;
831 bool has_sg, has_catu;
832 struct etr_buf *etr_buf;
833 struct device *dev = &drvdata->csdev->dev;
834
835 has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
836 has_iommu = iommu_get_domain_for_dev(dev->parent);
837 has_catu = !!tmc_etr_get_catu_device(drvdata);
838
839 has_sg = has_catu || has_etr_sg;
840
841 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
842 if (!etr_buf)
843 return ERR_PTR(-ENOMEM);
844
845 etr_buf->size = size;
846
847 /*
848 * If we have to use an existing list of pages, we cannot reliably
849 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
850 * we use the contiguous DMA memory if at least one of the following
851 * conditions is true:
852 * a) The ETR cannot use Scatter-Gather.
853 * b) we have a backing IOMMU
854 * c) The requested memory size is smaller (< 1M).
855 *
856 * Fallback to available mechanisms.
857 *
858 */
859 if (!pages &&
860 (!has_sg || has_iommu || size < SZ_1M))
861 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
862 etr_buf, node, pages);
863 if (rc && has_etr_sg)
864 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
865 etr_buf, node, pages);
866 if (rc && has_catu)
867 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
868 etr_buf, node, pages);
869 if (rc) {
870 kfree(etr_buf);
871 return ERR_PTR(rc);
872 }
873
874 refcount_set(&etr_buf->refcount, 1);
875 dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
876 (unsigned long)size >> 10, etr_buf->mode);
877 return etr_buf;
878}
879
880static void tmc_free_etr_buf(struct etr_buf *etr_buf)
881{
882 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
883 etr_buf->ops->free(etr_buf);
884 kfree(etr_buf);
885}
886
887/*
888 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
889 * with a maximum of @len bytes.
890 * Returns: The size of the linear data available @pos, with *bufpp
891 * updated to point to the buffer.
892 */
893static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
894 u64 offset, size_t len, char **bufpp)
895{
896 /* Adjust the length to limit this transaction to end of buffer */
897 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
898
899 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
900}
901
902static inline s64
903tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
904{
905 ssize_t len;
906 char *bufp;
907
908 len = tmc_etr_buf_get_data(etr_buf, offset,
909 CORESIGHT_BARRIER_PKT_SIZE, &bufp);
910 if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE))
911 return -EINVAL;
912 coresight_insert_barrier_packet(bufp);
913 return offset + CORESIGHT_BARRIER_PKT_SIZE;
914}
915
916/*
917 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
918 * Makes sure the trace data is synced to the memory for consumption.
919 * @etr_buf->offset will hold the offset to the beginning of the trace data
920 * within the buffer, with @etr_buf->len bytes to consume.
921 */
922static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
923{
924 struct etr_buf *etr_buf = drvdata->etr_buf;
925 u64 rrp, rwp;
926 u32 status;
927
928 rrp = tmc_read_rrp(drvdata);
929 rwp = tmc_read_rwp(drvdata);
930 status = readl_relaxed(drvdata->base + TMC_STS);
931
932 /*
933 * If there were memory errors in the session, truncate the
934 * buffer.
935 */
936 if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
937 dev_dbg(&drvdata->csdev->dev,
938 "tmc memory error detected, truncating buffer\n");
939 etr_buf->len = 0;
940 etr_buf->full = 0;
941 return;
942 }
943
944 etr_buf->full = status & TMC_STS_FULL;
945
946 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
947
948 etr_buf->ops->sync(etr_buf, rrp, rwp);
949}
950
951static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
952{
953 u32 axictl, sts;
954 struct etr_buf *etr_buf = drvdata->etr_buf;
955
956 CS_UNLOCK(drvdata->base);
957
958 /* Wait for TMCSReady bit to be set */
959 tmc_wait_for_tmcready(drvdata);
960
961 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
962 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
963
964 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
965 axictl &= ~TMC_AXICTL_CLEAR_MASK;
966 axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16);
967 axictl |= TMC_AXICTL_AXCACHE_OS;
968
969 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
970 axictl &= ~TMC_AXICTL_ARCACHE_MASK;
971 axictl |= TMC_AXICTL_ARCACHE_OS;
972 }
973
974 if (etr_buf->mode == ETR_MODE_ETR_SG)
975 axictl |= TMC_AXICTL_SCT_GAT_MODE;
976
977 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
978 tmc_write_dba(drvdata, etr_buf->hwaddr);
979 /*
980 * If the TMC pointers must be programmed before the session,
981 * we have to set it properly (i.e, RRP/RWP to base address and
982 * STS to "not full").
983 */
984 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
985 tmc_write_rrp(drvdata, etr_buf->hwaddr);
986 tmc_write_rwp(drvdata, etr_buf->hwaddr);
987 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
988 writel_relaxed(sts, drvdata->base + TMC_STS);
989 }
990
991 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
992 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
993 TMC_FFCR_TRIGON_TRIGIN,
994 drvdata->base + TMC_FFCR);
995 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
996 tmc_enable_hw(drvdata);
997
998 CS_LOCK(drvdata->base);
999}
1000
1001static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1002 struct etr_buf *etr_buf)
1003{
1004 int rc;
1005
1006 /* Callers should provide an appropriate buffer for use */
1007 if (WARN_ON(!etr_buf))
1008 return -EINVAL;
1009
1010 if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1011 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1012 return -EINVAL;
1013
1014 if (WARN_ON(drvdata->etr_buf))
1015 return -EBUSY;
1016
1017 /*
1018 * If this ETR is connected to a CATU, enable it before we turn
1019 * this on.
1020 */
1021 rc = tmc_etr_enable_catu(drvdata, etr_buf);
1022 if (rc)
1023 return rc;
1024 rc = coresight_claim_device(drvdata->base);
1025 if (!rc) {
1026 drvdata->etr_buf = etr_buf;
1027 __tmc_etr_enable_hw(drvdata);
1028 }
1029
1030 return rc;
1031}
1032
1033/*
1034 * Return the available trace data in the buffer (starts at etr_buf->offset,
1035 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1036 * also updating the @bufpp on where to find it. Since the trace data
1037 * starts at anywhere in the buffer, depending on the RRP, we adjust the
1038 * @len returned to handle buffer wrapping around.
1039 *
1040 * We are protected here by drvdata->reading != 0, which ensures the
1041 * sysfs_buf stays alive.
1042 */
1043ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1044 loff_t pos, size_t len, char **bufpp)
1045{
1046 s64 offset;
1047 ssize_t actual = len;
1048 struct etr_buf *etr_buf = drvdata->sysfs_buf;
1049
1050 if (pos + actual > etr_buf->len)
1051 actual = etr_buf->len - pos;
1052 if (actual <= 0)
1053 return actual;
1054
1055 /* Compute the offset from which we read the data */
1056 offset = etr_buf->offset + pos;
1057 if (offset >= etr_buf->size)
1058 offset -= etr_buf->size;
1059 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1060}
1061
1062static struct etr_buf *
1063tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1064{
1065 return tmc_alloc_etr_buf(drvdata, drvdata->size,
1066 0, cpu_to_node(0), NULL);
1067}
1068
1069static void
1070tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1071{
1072 if (buf)
1073 tmc_free_etr_buf(buf);
1074}
1075
1076static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1077{
1078 struct etr_buf *etr_buf = drvdata->etr_buf;
1079
1080 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1081 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1082 drvdata->sysfs_buf = NULL;
1083 } else {
1084 tmc_sync_etr_buf(drvdata);
1085 /*
1086 * Insert barrier packets at the beginning, if there was
1087 * an overflow.
1088 */
1089 if (etr_buf->full)
1090 tmc_etr_buf_insert_barrier_packet(etr_buf,
1091 etr_buf->offset);
1092 }
1093}
1094
1095static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1096{
1097 CS_UNLOCK(drvdata->base);
1098
1099 tmc_flush_and_stop(drvdata);
1100 /*
1101 * When operating in sysFS mode the content of the buffer needs to be
1102 * read before the TMC is disabled.
1103 */
1104 if (drvdata->mode == CS_MODE_SYSFS)
1105 tmc_etr_sync_sysfs_buf(drvdata);
1106
1107 tmc_disable_hw(drvdata);
1108
1109 CS_LOCK(drvdata->base);
1110
1111}
1112
1113static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1114{
1115 __tmc_etr_disable_hw(drvdata);
1116 /* Disable CATU device if this ETR is connected to one */
1117 tmc_etr_disable_catu(drvdata);
1118 coresight_disclaim_device(drvdata->base);
1119 /* Reset the ETR buf used by hardware */
1120 drvdata->etr_buf = NULL;
1121}
1122
1123static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1124{
1125 int ret = 0;
1126 unsigned long flags;
1127 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1128 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1129
1130 /*
1131 * If we are enabling the ETR from disabled state, we need to make
1132 * sure we have a buffer with the right size. The etr_buf is not reset
1133 * immediately after we stop the tracing in SYSFS mode as we wait for
1134 * the user to collect the data. We may be able to reuse the existing
1135 * buffer, provided the size matches. Any allocation has to be done
1136 * with the lock released.
1137 */
1138 spin_lock_irqsave(&drvdata->spinlock, flags);
1139 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1140 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1141 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1142
1143 /* Allocate memory with the locks released */
1144 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1145 if (IS_ERR(new_buf))
1146 return PTR_ERR(new_buf);
1147
1148 /* Let's try again */
1149 spin_lock_irqsave(&drvdata->spinlock, flags);
1150 }
1151
1152 if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1153 ret = -EBUSY;
1154 goto out;
1155 }
1156
1157 /*
1158 * In sysFS mode we can have multiple writers per sink. Since this
1159 * sink is already enabled no memory is needed and the HW need not be
1160 * touched, even if the buffer size has changed.
1161 */
1162 if (drvdata->mode == CS_MODE_SYSFS) {
1163 atomic_inc(csdev->refcnt);
1164 goto out;
1165 }
1166
1167 /*
1168 * If we don't have a buffer or it doesn't match the requested size,
1169 * use the buffer allocated above. Otherwise reuse the existing buffer.
1170 */
1171 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1172 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1173 free_buf = sysfs_buf;
1174 drvdata->sysfs_buf = new_buf;
1175 }
1176
1177 ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf);
1178 if (!ret) {
1179 drvdata->mode = CS_MODE_SYSFS;
1180 atomic_inc(csdev->refcnt);
1181 }
1182out:
1183 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1184
1185 /* Free memory outside the spinlock if need be */
1186 if (free_buf)
1187 tmc_etr_free_sysfs_buf(free_buf);
1188
1189 if (!ret)
1190 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1191
1192 return ret;
1193}
1194
1195/*
1196 * alloc_etr_buf: Allocate ETR buffer for use by perf.
1197 * The size of the hardware buffer is dependent on the size configured
1198 * via sysfs and the perf ring buffer size. We prefer to allocate the
1199 * largest possible size, scaling down the size by half until it
1200 * reaches a minimum limit (1M), beyond which we give up.
1201 */
1202static struct etr_buf *
1203alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1204 int nr_pages, void **pages, bool snapshot)
1205{
1206 int node;
1207 struct etr_buf *etr_buf;
1208 unsigned long size;
1209
1210 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1211 /*
1212 * Try to match the perf ring buffer size if it is larger
1213 * than the size requested via sysfs.
1214 */
1215 if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1216 etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT),
1217 0, node, NULL);
1218 if (!IS_ERR(etr_buf))
1219 goto done;
1220 }
1221
1222 /*
1223 * Else switch to configured size for this ETR
1224 * and scale down until we hit the minimum limit.
1225 */
1226 size = drvdata->size;
1227 do {
1228 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1229 if (!IS_ERR(etr_buf))
1230 goto done;
1231 size /= 2;
1232 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1233
1234 return ERR_PTR(-ENOMEM);
1235
1236done:
1237 return etr_buf;
1238}
1239
1240static struct etr_buf *
1241get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1242 struct perf_event *event, int nr_pages,
1243 void **pages, bool snapshot)
1244{
1245 int ret;
1246 pid_t pid = task_pid_nr(event->owner);
1247 struct etr_buf *etr_buf;
1248
1249retry:
1250 /*
1251 * An etr_perf_buffer is associated with an event and holds a reference
1252 * to the AUX ring buffer that was created for that event. In CPU-wide
1253 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1254 * buffer, share a sink. As such an etr_perf_buffer is created for each
1255 * event but a single etr_buf associated with the ETR is shared between
1256 * them. The last event in a trace session will copy the content of the
1257 * etr_buf to its AUX ring buffer. Ring buffer associated to other
1258 * events are simply not used an freed as events are destoyed. We still
1259 * need to allocate a ring buffer for each event since we don't know
1260 * which event will be last.
1261 */
1262
1263 /*
1264 * The first thing to do here is check if an etr_buf has already been
1265 * allocated for this session. If so it is shared with this event,
1266 * otherwise it is created.
1267 */
1268 mutex_lock(&drvdata->idr_mutex);
1269 etr_buf = idr_find(&drvdata->idr, pid);
1270 if (etr_buf) {
1271 refcount_inc(&etr_buf->refcount);
1272 mutex_unlock(&drvdata->idr_mutex);
1273 return etr_buf;
1274 }
1275
1276 /* If we made it here no buffer has been allocated, do so now. */
1277 mutex_unlock(&drvdata->idr_mutex);
1278
1279 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1280 if (IS_ERR(etr_buf))
1281 return etr_buf;
1282
1283 /* Now that we have a buffer, add it to the IDR. */
1284 mutex_lock(&drvdata->idr_mutex);
1285 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1286 mutex_unlock(&drvdata->idr_mutex);
1287
1288 /* Another event with this session ID has allocated this buffer. */
1289 if (ret == -ENOSPC) {
1290 tmc_free_etr_buf(etr_buf);
1291 goto retry;
1292 }
1293
1294 /* The IDR can't allocate room for a new session, abandon ship. */
1295 if (ret == -ENOMEM) {
1296 tmc_free_etr_buf(etr_buf);
1297 return ERR_PTR(ret);
1298 }
1299
1300
1301 return etr_buf;
1302}
1303
1304static struct etr_buf *
1305get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1306 struct perf_event *event, int nr_pages,
1307 void **pages, bool snapshot)
1308{
1309 /*
1310 * In per-thread mode the etr_buf isn't shared, so just go ahead
1311 * with memory allocation.
1312 */
1313 return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1314}
1315
1316static struct etr_buf *
1317get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1318 int nr_pages, void **pages, bool snapshot)
1319{
1320 if (event->cpu == -1)
1321 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1322 pages, snapshot);
1323
1324 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1325 pages, snapshot);
1326}
1327
1328static struct etr_perf_buffer *
1329tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1330 int nr_pages, void **pages, bool snapshot)
1331{
1332 int node;
1333 struct etr_buf *etr_buf;
1334 struct etr_perf_buffer *etr_perf;
1335
1336 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1337
1338 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1339 if (!etr_perf)
1340 return ERR_PTR(-ENOMEM);
1341
1342 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1343 if (!IS_ERR(etr_buf))
1344 goto done;
1345
1346 kfree(etr_perf);
1347 return ERR_PTR(-ENOMEM);
1348
1349done:
1350 /*
1351 * Keep a reference to the ETR this buffer has been allocated for
1352 * in order to have access to the IDR in tmc_free_etr_buffer().
1353 */
1354 etr_perf->drvdata = drvdata;
1355 etr_perf->etr_buf = etr_buf;
1356
1357 return etr_perf;
1358}
1359
1360
1361static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1362 struct perf_event *event, void **pages,
1363 int nr_pages, bool snapshot)
1364{
1365 struct etr_perf_buffer *etr_perf;
1366 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1367
1368 etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1369 nr_pages, pages, snapshot);
1370 if (IS_ERR(etr_perf)) {
1371 dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1372 return NULL;
1373 }
1374
1375 etr_perf->pid = task_pid_nr(event->owner);
1376 etr_perf->snapshot = snapshot;
1377 etr_perf->nr_pages = nr_pages;
1378 etr_perf->pages = pages;
1379
1380 return etr_perf;
1381}
1382
1383static void tmc_free_etr_buffer(void *config)
1384{
1385 struct etr_perf_buffer *etr_perf = config;
1386 struct tmc_drvdata *drvdata = etr_perf->drvdata;
1387 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1388
1389 if (!etr_buf)
1390 goto free_etr_perf_buffer;
1391
1392 mutex_lock(&drvdata->idr_mutex);
1393 /* If we are not the last one to use the buffer, don't touch it. */
1394 if (!refcount_dec_and_test(&etr_buf->refcount)) {
1395 mutex_unlock(&drvdata->idr_mutex);
1396 goto free_etr_perf_buffer;
1397 }
1398
1399 /* We are the last one, remove from the IDR and free the buffer. */
1400 buf = idr_remove(&drvdata->idr, etr_perf->pid);
1401 mutex_unlock(&drvdata->idr_mutex);
1402
1403 /*
1404 * Something went very wrong if the buffer associated with this ID
1405 * is not the same in the IDR. Leak to avoid use after free.
1406 */
1407 if (buf && WARN_ON(buf != etr_buf))
1408 goto free_etr_perf_buffer;
1409
1410 tmc_free_etr_buf(etr_perf->etr_buf);
1411
1412free_etr_perf_buffer:
1413 kfree(etr_perf);
1414}
1415
1416/*
1417 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1418 * buffer to the perf ring buffer.
1419 */
1420static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1421 unsigned long src_offset,
1422 unsigned long to_copy)
1423{
1424 long bytes;
1425 long pg_idx, pg_offset;
1426 unsigned long head = etr_perf->head;
1427 char **dst_pages, *src_buf;
1428 struct etr_buf *etr_buf = etr_perf->etr_buf;
1429
1430 head = etr_perf->head;
1431 pg_idx = head >> PAGE_SHIFT;
1432 pg_offset = head & (PAGE_SIZE - 1);
1433 dst_pages = (char **)etr_perf->pages;
1434
1435 while (to_copy > 0) {
1436 /*
1437 * In one iteration, we can copy minimum of :
1438 * 1) what is available in the source buffer,
1439 * 2) what is available in the source buffer, before it
1440 * wraps around.
1441 * 3) what is available in the destination page.
1442 * in one iteration.
1443 */
1444 if (src_offset >= etr_buf->size)
1445 src_offset -= etr_buf->size;
1446 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1447 &src_buf);
1448 if (WARN_ON_ONCE(bytes <= 0))
1449 break;
1450 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1451
1452 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1453
1454 to_copy -= bytes;
1455
1456 /* Move destination pointers */
1457 pg_offset += bytes;
1458 if (pg_offset == PAGE_SIZE) {
1459 pg_offset = 0;
1460 if (++pg_idx == etr_perf->nr_pages)
1461 pg_idx = 0;
1462 }
1463
1464 /* Move source pointers */
1465 src_offset += bytes;
1466 }
1467}
1468
1469/*
1470 * tmc_update_etr_buffer : Update the perf ring buffer with the
1471 * available trace data. We use software double buffering at the moment.
1472 *
1473 * TODO: Add support for reusing the perf ring buffer.
1474 */
1475static unsigned long
1476tmc_update_etr_buffer(struct coresight_device *csdev,
1477 struct perf_output_handle *handle,
1478 void *config)
1479{
1480 bool lost = false;
1481 unsigned long flags, offset, size = 0;
1482 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1483 struct etr_perf_buffer *etr_perf = config;
1484 struct etr_buf *etr_buf = etr_perf->etr_buf;
1485
1486 spin_lock_irqsave(&drvdata->spinlock, flags);
1487
1488 /* Don't do anything if another tracer is using this sink */
1489 if (atomic_read(csdev->refcnt) != 1) {
1490 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1491 goto out;
1492 }
1493
1494 if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1495 lost = true;
1496 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1497 goto out;
1498 }
1499
1500 CS_UNLOCK(drvdata->base);
1501
1502 tmc_flush_and_stop(drvdata);
1503 tmc_sync_etr_buf(drvdata);
1504
1505 CS_LOCK(drvdata->base);
1506 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1507
1508 lost = etr_buf->full;
1509 offset = etr_buf->offset;
1510 size = etr_buf->len;
1511
1512 /*
1513 * The ETR buffer may be bigger than the space available in the
1514 * perf ring buffer (handle->size). If so advance the offset so that we
1515 * get the latest trace data. In snapshot mode none of that matters
1516 * since we are expected to clobber stale data in favour of the latest
1517 * traces.
1518 */
1519 if (!etr_perf->snapshot && size > handle->size) {
1520 u32 mask = tmc_get_memwidth_mask(drvdata);
1521
1522 /*
1523 * Make sure the new size is aligned in accordance with the
1524 * requirement explained in function tmc_get_memwidth_mask().
1525 */
1526 size = handle->size & mask;
1527 offset = etr_buf->offset + etr_buf->len - size;
1528
1529 if (offset >= etr_buf->size)
1530 offset -= etr_buf->size;
1531 lost = true;
1532 }
1533
1534 /* Insert barrier packets at the beginning, if there was an overflow */
1535 if (lost)
1536 tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset);
1537 tmc_etr_sync_perf_buffer(etr_perf, offset, size);
1538
1539 /*
1540 * In snapshot mode we simply increment the head by the number of byte
1541 * that were written. User space function cs_etm_find_snapshot() will
1542 * figure out how many bytes to get from the AUX buffer based on the
1543 * position of the head.
1544 */
1545 if (etr_perf->snapshot)
1546 handle->head += size;
1547out:
1548 /*
1549 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1550 * captured buffer is expected to be truncated and 2) a full buffer
1551 * prevents the event from being re-enabled by the perf core,
1552 * resulting in stale data being send to user space.
1553 */
1554 if (!etr_perf->snapshot && lost)
1555 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1556 return size;
1557}
1558
1559static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1560{
1561 int rc = 0;
1562 pid_t pid;
1563 unsigned long flags;
1564 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1565 struct perf_output_handle *handle = data;
1566 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1567
1568 spin_lock_irqsave(&drvdata->spinlock, flags);
1569 /* Don't use this sink if it is already claimed by sysFS */
1570 if (drvdata->mode == CS_MODE_SYSFS) {
1571 rc = -EBUSY;
1572 goto unlock_out;
1573 }
1574
1575 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1576 rc = -EINVAL;
1577 goto unlock_out;
1578 }
1579
1580 /* Get a handle on the pid of the process to monitor */
1581 pid = etr_perf->pid;
1582
1583 /* Do not proceed if this device is associated with another session */
1584 if (drvdata->pid != -1 && drvdata->pid != pid) {
1585 rc = -EBUSY;
1586 goto unlock_out;
1587 }
1588
1589 etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf);
1590
1591 /*
1592 * No HW configuration is needed if the sink is already in
1593 * use for this session.
1594 */
1595 if (drvdata->pid == pid) {
1596 atomic_inc(csdev->refcnt);
1597 goto unlock_out;
1598 }
1599
1600 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1601 if (!rc) {
1602 /* Associate with monitored process. */
1603 drvdata->pid = pid;
1604 drvdata->mode = CS_MODE_PERF;
1605 drvdata->perf_buf = etr_perf->etr_buf;
1606 atomic_inc(csdev->refcnt);
1607 }
1608
1609unlock_out:
1610 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1611 return rc;
1612}
1613
1614static int tmc_enable_etr_sink(struct coresight_device *csdev,
1615 u32 mode, void *data)
1616{
1617 switch (mode) {
1618 case CS_MODE_SYSFS:
1619 return tmc_enable_etr_sink_sysfs(csdev);
1620 case CS_MODE_PERF:
1621 return tmc_enable_etr_sink_perf(csdev, data);
1622 }
1623
1624 /* We shouldn't be here */
1625 return -EINVAL;
1626}
1627
1628static int tmc_disable_etr_sink(struct coresight_device *csdev)
1629{
1630 unsigned long flags;
1631 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1632
1633 spin_lock_irqsave(&drvdata->spinlock, flags);
1634
1635 if (drvdata->reading) {
1636 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1637 return -EBUSY;
1638 }
1639
1640 if (atomic_dec_return(csdev->refcnt)) {
1641 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1642 return -EBUSY;
1643 }
1644
1645 /* Complain if we (somehow) got out of sync */
1646 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1647 tmc_etr_disable_hw(drvdata);
1648 /* Dissociate from monitored process. */
1649 drvdata->pid = -1;
1650 drvdata->mode = CS_MODE_DISABLED;
1651 /* Reset perf specific data */
1652 drvdata->perf_buf = NULL;
1653
1654 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1655
1656 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1657 return 0;
1658}
1659
1660static const struct coresight_ops_sink tmc_etr_sink_ops = {
1661 .enable = tmc_enable_etr_sink,
1662 .disable = tmc_disable_etr_sink,
1663 .alloc_buffer = tmc_alloc_etr_buffer,
1664 .update_buffer = tmc_update_etr_buffer,
1665 .free_buffer = tmc_free_etr_buffer,
1666};
1667
1668const struct coresight_ops tmc_etr_cs_ops = {
1669 .sink_ops = &tmc_etr_sink_ops,
1670};
1671
1672int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1673{
1674 int ret = 0;
1675 unsigned long flags;
1676
1677 /* config types are set a boot time and never change */
1678 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1679 return -EINVAL;
1680
1681 spin_lock_irqsave(&drvdata->spinlock, flags);
1682 if (drvdata->reading) {
1683 ret = -EBUSY;
1684 goto out;
1685 }
1686
1687 /*
1688 * We can safely allow reads even if the ETR is operating in PERF mode,
1689 * since the sysfs session is captured in mode specific data.
1690 * If drvdata::sysfs_data is NULL the trace data has been read already.
1691 */
1692 if (!drvdata->sysfs_buf) {
1693 ret = -EINVAL;
1694 goto out;
1695 }
1696
1697 /* Disable the TMC if we are trying to read from a running session. */
1698 if (drvdata->mode == CS_MODE_SYSFS)
1699 __tmc_etr_disable_hw(drvdata);
1700
1701 drvdata->reading = true;
1702out:
1703 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1704
1705 return ret;
1706}
1707
1708int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1709{
1710 unsigned long flags;
1711 struct etr_buf *sysfs_buf = NULL;
1712
1713 /* config types are set a boot time and never change */
1714 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1715 return -EINVAL;
1716
1717 spin_lock_irqsave(&drvdata->spinlock, flags);
1718
1719 /* RE-enable the TMC if need be */
1720 if (drvdata->mode == CS_MODE_SYSFS) {
1721 /*
1722 * The trace run will continue with the same allocated trace
1723 * buffer. Since the tracer is still enabled drvdata::buf can't
1724 * be NULL.
1725 */
1726 __tmc_etr_enable_hw(drvdata);
1727 } else {
1728 /*
1729 * The ETR is not tracing and the buffer was just read.
1730 * As such prepare to free the trace buffer.
1731 */
1732 sysfs_buf = drvdata->sysfs_buf;
1733 drvdata->sysfs_buf = NULL;
1734 }
1735
1736 drvdata->reading = false;
1737 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1738
1739 /* Free allocated memory out side of the spinlock */
1740 if (sysfs_buf)
1741 tmc_etr_free_sysfs_buf(sysfs_buf);
1742
1743 return 0;
1744}