Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5 */
6
7#include <linux/atomic.h>
8#include <linux/coresight.h>
9#include <linux/dma-mapping.h>
10#include <linux/iommu.h>
11#include <linux/idr.h>
12#include <linux/mutex.h>
13#include <linux/refcount.h>
14#include <linux/slab.h>
15#include <linux/types.h>
16#include <linux/vmalloc.h>
17#include "coresight-catu.h"
18#include "coresight-etm-perf.h"
19#include "coresight-priv.h"
20#include "coresight-tmc.h"
21
22struct etr_flat_buf {
23 struct device *dev;
24 dma_addr_t daddr;
25 void *vaddr;
26 size_t size;
27};
28
29struct etr_buf_hw {
30 bool has_iommu;
31 bool has_etr_sg;
32 bool has_catu;
33};
34
35/*
36 * etr_perf_buffer - Perf buffer used for ETR
37 * @drvdata - The ETR drvdaga this buffer has been allocated for.
38 * @etr_buf - Actual buffer used by the ETR
39 * @pid - The PID this etr_perf_buffer belongs to.
40 * @snaphost - Perf session mode
41 * @nr_pages - Number of pages in the ring buffer.
42 * @pages - Array of Pages in the ring buffer.
43 */
44struct etr_perf_buffer {
45 struct tmc_drvdata *drvdata;
46 struct etr_buf *etr_buf;
47 pid_t pid;
48 bool snapshot;
49 int nr_pages;
50 void **pages;
51};
52
53/* Convert the perf index to an offset within the ETR buffer */
54#define PERF_IDX2OFF(idx, buf) \
55 ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
56
57/* Lower limit for ETR hardware buffer */
58#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
59
60/*
61 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
62 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
63 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
64 * contain more than one SG buffer and tables.
65 *
66 * A table entry has the following format:
67 *
68 * ---Bit31------------Bit4-------Bit1-----Bit0--
69 * | Address[39:12] | SBZ | Entry Type |
70 * ----------------------------------------------
71 *
72 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
73 * always zero.
74 *
75 * Entry type:
76 * b00 - Reserved.
77 * b01 - Last entry in the tables, points to 4K page buffer.
78 * b10 - Normal entry, points to 4K page buffer.
79 * b11 - Link. The address points to the base of next table.
80 */
81
82typedef u32 sgte_t;
83
84#define ETR_SG_PAGE_SHIFT 12
85#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
86#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
87#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
88#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
89
90#define ETR_SG_ET_MASK 0x3
91#define ETR_SG_ET_LAST 0x1
92#define ETR_SG_ET_NORMAL 0x2
93#define ETR_SG_ET_LINK 0x3
94
95#define ETR_SG_ADDR_SHIFT 4
96
97#define ETR_SG_ENTRY(addr, type) \
98 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
99 (type & ETR_SG_ET_MASK))
100
101#define ETR_SG_ADDR(entry) \
102 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
103#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
104
105/*
106 * struct etr_sg_table : ETR SG Table
107 * @sg_table: Generic SG Table holding the data/table pages.
108 * @hwaddr: hwaddress used by the TMC, which is the base
109 * address of the table.
110 */
111struct etr_sg_table {
112 struct tmc_sg_table *sg_table;
113 dma_addr_t hwaddr;
114};
115
116/*
117 * tmc_etr_sg_table_entries: Total number of table entries required to map
118 * @nr_pages system pages.
119 *
120 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
121 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
122 * with the last entry pointing to another page of table entries.
123 * If we spill over to a new page for mapping 1 entry, we could as
124 * well replace the link entry of the previous page with the last entry.
125 */
126static inline unsigned long __attribute_const__
127tmc_etr_sg_table_entries(int nr_pages)
128{
129 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
130 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
131 /*
132 * If we spill over to a new page for 1 entry, we could as well
133 * make it the LAST entry in the previous page, skipping the Link
134 * address.
135 */
136 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
137 nr_sglinks--;
138 return nr_sgpages + nr_sglinks;
139}
140
141/*
142 * tmc_pages_get_offset: Go through all the pages in the tmc_pages
143 * and map the device address @addr to an offset within the virtual
144 * contiguous buffer.
145 */
146static long
147tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
148{
149 int i;
150 dma_addr_t page_start;
151
152 for (i = 0; i < tmc_pages->nr_pages; i++) {
153 page_start = tmc_pages->daddrs[i];
154 if (addr >= page_start && addr < (page_start + PAGE_SIZE))
155 return i * PAGE_SIZE + (addr - page_start);
156 }
157
158 return -EINVAL;
159}
160
161/*
162 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
163 * If the pages were not allocated in tmc_pages_alloc(), we would
164 * simply drop the refcount.
165 */
166static void tmc_pages_free(struct tmc_pages *tmc_pages,
167 struct device *dev, enum dma_data_direction dir)
168{
169 int i;
170 struct device *real_dev = dev->parent;
171
172 for (i = 0; i < tmc_pages->nr_pages; i++) {
173 if (tmc_pages->daddrs && tmc_pages->daddrs[i])
174 dma_unmap_page(real_dev, tmc_pages->daddrs[i],
175 PAGE_SIZE, dir);
176 if (tmc_pages->pages && tmc_pages->pages[i])
177 __free_page(tmc_pages->pages[i]);
178 }
179
180 kfree(tmc_pages->pages);
181 kfree(tmc_pages->daddrs);
182 tmc_pages->pages = NULL;
183 tmc_pages->daddrs = NULL;
184 tmc_pages->nr_pages = 0;
185}
186
187/*
188 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
189 * If @pages is not NULL, the list of page virtual addresses are
190 * used as the data pages. The pages are then dma_map'ed for @dev
191 * with dma_direction @dir.
192 *
193 * Returns 0 upon success, else the error number.
194 */
195static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
196 struct device *dev, int node,
197 enum dma_data_direction dir, void **pages)
198{
199 int i, nr_pages;
200 dma_addr_t paddr;
201 struct page *page;
202 struct device *real_dev = dev->parent;
203
204 nr_pages = tmc_pages->nr_pages;
205 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
206 GFP_KERNEL);
207 if (!tmc_pages->daddrs)
208 return -ENOMEM;
209 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
210 GFP_KERNEL);
211 if (!tmc_pages->pages) {
212 kfree(tmc_pages->daddrs);
213 tmc_pages->daddrs = NULL;
214 return -ENOMEM;
215 }
216
217 for (i = 0; i < nr_pages; i++) {
218 if (pages && pages[i]) {
219 page = virt_to_page(pages[i]);
220 /* Hold a refcount on the page */
221 get_page(page);
222 } else {
223 page = alloc_pages_node(node,
224 GFP_KERNEL | __GFP_ZERO, 0);
225 if (!page)
226 goto err;
227 }
228 paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
229 if (dma_mapping_error(real_dev, paddr))
230 goto err;
231 tmc_pages->daddrs[i] = paddr;
232 tmc_pages->pages[i] = page;
233 }
234 return 0;
235err:
236 tmc_pages_free(tmc_pages, dev, dir);
237 return -ENOMEM;
238}
239
240static inline long
241tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
242{
243 return tmc_pages_get_offset(&sg_table->data_pages, addr);
244}
245
246static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
247{
248 if (sg_table->table_vaddr)
249 vunmap(sg_table->table_vaddr);
250 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
251}
252
253static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
254{
255 if (sg_table->data_vaddr)
256 vunmap(sg_table->data_vaddr);
257 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
258}
259
260void tmc_free_sg_table(struct tmc_sg_table *sg_table)
261{
262 tmc_free_table_pages(sg_table);
263 tmc_free_data_pages(sg_table);
264}
265EXPORT_SYMBOL_GPL(tmc_free_sg_table);
266
267/*
268 * Alloc pages for the table. Since this will be used by the device,
269 * allocate the pages closer to the device (i.e, dev_to_node(dev)
270 * rather than the CPU node).
271 */
272static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
273{
274 int rc;
275 struct tmc_pages *table_pages = &sg_table->table_pages;
276
277 rc = tmc_pages_alloc(table_pages, sg_table->dev,
278 dev_to_node(sg_table->dev),
279 DMA_TO_DEVICE, NULL);
280 if (rc)
281 return rc;
282 sg_table->table_vaddr = vmap(table_pages->pages,
283 table_pages->nr_pages,
284 VM_MAP,
285 PAGE_KERNEL);
286 if (!sg_table->table_vaddr)
287 rc = -ENOMEM;
288 else
289 sg_table->table_daddr = table_pages->daddrs[0];
290 return rc;
291}
292
293static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
294{
295 int rc;
296
297 /* Allocate data pages on the node requested by the caller */
298 rc = tmc_pages_alloc(&sg_table->data_pages,
299 sg_table->dev, sg_table->node,
300 DMA_FROM_DEVICE, pages);
301 if (!rc) {
302 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
303 sg_table->data_pages.nr_pages,
304 VM_MAP,
305 PAGE_KERNEL);
306 if (!sg_table->data_vaddr)
307 rc = -ENOMEM;
308 }
309 return rc;
310}
311
312/*
313 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
314 * and data buffers. TMC writes to the data buffers and reads from the SG
315 * Table pages.
316 *
317 * @dev - Coresight device to which page should be DMA mapped.
318 * @node - Numa node for mem allocations
319 * @nr_tpages - Number of pages for the table entries.
320 * @nr_dpages - Number of pages for Data buffer.
321 * @pages - Optional list of virtual address of pages.
322 */
323struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
324 int node,
325 int nr_tpages,
326 int nr_dpages,
327 void **pages)
328{
329 long rc;
330 struct tmc_sg_table *sg_table;
331
332 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
333 if (!sg_table)
334 return ERR_PTR(-ENOMEM);
335 sg_table->data_pages.nr_pages = nr_dpages;
336 sg_table->table_pages.nr_pages = nr_tpages;
337 sg_table->node = node;
338 sg_table->dev = dev;
339
340 rc = tmc_alloc_data_pages(sg_table, pages);
341 if (!rc)
342 rc = tmc_alloc_table_pages(sg_table);
343 if (rc) {
344 tmc_free_sg_table(sg_table);
345 kfree(sg_table);
346 return ERR_PTR(rc);
347 }
348
349 return sg_table;
350}
351EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
352
353/*
354 * tmc_sg_table_sync_data_range: Sync the data buffer written
355 * by the device from @offset upto a @size bytes.
356 */
357void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
358 u64 offset, u64 size)
359{
360 int i, index, start;
361 int npages = DIV_ROUND_UP(size, PAGE_SIZE);
362 struct device *real_dev = table->dev->parent;
363 struct tmc_pages *data = &table->data_pages;
364
365 start = offset >> PAGE_SHIFT;
366 for (i = start; i < (start + npages); i++) {
367 index = i % data->nr_pages;
368 dma_sync_single_for_cpu(real_dev, data->daddrs[index],
369 PAGE_SIZE, DMA_FROM_DEVICE);
370 }
371}
372EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
373
374/* tmc_sg_sync_table: Sync the page table */
375void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
376{
377 int i;
378 struct device *real_dev = sg_table->dev->parent;
379 struct tmc_pages *table_pages = &sg_table->table_pages;
380
381 for (i = 0; i < table_pages->nr_pages; i++)
382 dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
383 PAGE_SIZE, DMA_TO_DEVICE);
384}
385EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
386
387/*
388 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
389 * in the SG buffer. The @bufpp is updated to point to the buffer.
390 * Returns :
391 * the length of linear data available at @offset.
392 * or
393 * <= 0 if no data is available.
394 */
395ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
396 u64 offset, size_t len, char **bufpp)
397{
398 size_t size;
399 int pg_idx = offset >> PAGE_SHIFT;
400 int pg_offset = offset & (PAGE_SIZE - 1);
401 struct tmc_pages *data_pages = &sg_table->data_pages;
402
403 size = tmc_sg_table_buf_size(sg_table);
404 if (offset >= size)
405 return -EINVAL;
406
407 /* Make sure we don't go beyond the end */
408 len = (len < (size - offset)) ? len : size - offset;
409 /* Respect the page boundaries */
410 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
411 if (len > 0)
412 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
413 return len;
414}
415EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
416
417#ifdef ETR_SG_DEBUG
418/* Map a dma address to virtual address */
419static unsigned long
420tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
421 dma_addr_t addr, bool table)
422{
423 long offset;
424 unsigned long base;
425 struct tmc_pages *tmc_pages;
426
427 if (table) {
428 tmc_pages = &sg_table->table_pages;
429 base = (unsigned long)sg_table->table_vaddr;
430 } else {
431 tmc_pages = &sg_table->data_pages;
432 base = (unsigned long)sg_table->data_vaddr;
433 }
434
435 offset = tmc_pages_get_offset(tmc_pages, addr);
436 if (offset < 0)
437 return 0;
438 return base + offset;
439}
440
441/* Dump the given sg_table */
442static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
443{
444 sgte_t *ptr;
445 int i = 0;
446 dma_addr_t addr;
447 struct tmc_sg_table *sg_table = etr_table->sg_table;
448
449 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
450 etr_table->hwaddr, true);
451 while (ptr) {
452 addr = ETR_SG_ADDR(*ptr);
453 switch (ETR_SG_ET(*ptr)) {
454 case ETR_SG_ET_NORMAL:
455 dev_dbg(sg_table->dev,
456 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
457 ptr++;
458 break;
459 case ETR_SG_ET_LINK:
460 dev_dbg(sg_table->dev,
461 "%05d: *** %p\t:{L} 0x%llx ***\n",
462 i, ptr, addr);
463 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
464 addr, true);
465 break;
466 case ETR_SG_ET_LAST:
467 dev_dbg(sg_table->dev,
468 "%05d: ### %p\t:[L] 0x%llx ###\n",
469 i, ptr, addr);
470 return;
471 default:
472 dev_dbg(sg_table->dev,
473 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
474 i, ptr, addr);
475 return;
476 }
477 i++;
478 }
479 dev_dbg(sg_table->dev, "******* End of Table *****\n");
480}
481#else
482static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
483#endif
484
485/*
486 * Populate the SG Table page table entries from table/data
487 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
488 * So does a Table page. So we keep track of indices of the tables
489 * in each system page and move the pointers accordingly.
490 */
491#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
492static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
493{
494 dma_addr_t paddr;
495 int i, type, nr_entries;
496 int tpidx = 0; /* index to the current system table_page */
497 int sgtidx = 0; /* index to the sg_table within the current syspage */
498 int sgtentry = 0; /* the entry within the sg_table */
499 int dpidx = 0; /* index to the current system data_page */
500 int spidx = 0; /* index to the SG page within the current data page */
501 sgte_t *ptr; /* pointer to the table entry to fill */
502 struct tmc_sg_table *sg_table = etr_table->sg_table;
503 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
504 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
505
506 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
507 /*
508 * Use the contiguous virtual address of the table to update entries.
509 */
510 ptr = sg_table->table_vaddr;
511 /*
512 * Fill all the entries, except the last entry to avoid special
513 * checks within the loop.
514 */
515 for (i = 0; i < nr_entries - 1; i++) {
516 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
517 /*
518 * Last entry in a sg_table page is a link address to
519 * the next table page. If this sg_table is the last
520 * one in the system page, it links to the first
521 * sg_table in the next system page. Otherwise, it
522 * links to the next sg_table page within the system
523 * page.
524 */
525 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
526 paddr = table_daddrs[tpidx + 1];
527 } else {
528 paddr = table_daddrs[tpidx] +
529 (ETR_SG_PAGE_SIZE * (sgtidx + 1));
530 }
531 type = ETR_SG_ET_LINK;
532 } else {
533 /*
534 * Update the indices to the data_pages to point to the
535 * next sg_page in the data buffer.
536 */
537 type = ETR_SG_ET_NORMAL;
538 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
539 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
540 dpidx++;
541 }
542 *ptr++ = ETR_SG_ENTRY(paddr, type);
543 /*
544 * Move to the next table pointer, moving the table page index
545 * if necessary
546 */
547 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
548 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
549 tpidx++;
550 }
551 }
552
553 /* Set up the last entry, which is always a data pointer */
554 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
555 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
556}
557
558/*
559 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
560 * populate the table.
561 *
562 * @dev - Device pointer for the TMC
563 * @node - NUMA node where the memory should be allocated
564 * @size - Total size of the data buffer
565 * @pages - Optional list of page virtual address
566 */
567static struct etr_sg_table *
568tmc_init_etr_sg_table(struct device *dev, int node,
569 unsigned long size, void **pages)
570{
571 int nr_entries, nr_tpages;
572 int nr_dpages = size >> PAGE_SHIFT;
573 struct tmc_sg_table *sg_table;
574 struct etr_sg_table *etr_table;
575
576 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
577 if (!etr_table)
578 return ERR_PTR(-ENOMEM);
579 nr_entries = tmc_etr_sg_table_entries(nr_dpages);
580 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
581
582 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
583 if (IS_ERR(sg_table)) {
584 kfree(etr_table);
585 return ERR_CAST(sg_table);
586 }
587
588 etr_table->sg_table = sg_table;
589 /* TMC should use table base address for DBA */
590 etr_table->hwaddr = sg_table->table_daddr;
591 tmc_etr_sg_table_populate(etr_table);
592 /* Sync the table pages for the HW */
593 tmc_sg_table_sync_table(sg_table);
594 tmc_etr_sg_table_dump(etr_table);
595
596 return etr_table;
597}
598
599/*
600 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
601 */
602static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
603 struct etr_buf *etr_buf, int node,
604 void **pages)
605{
606 struct etr_flat_buf *flat_buf;
607 struct device *real_dev = drvdata->csdev->dev.parent;
608
609 /* We cannot reuse existing pages for flat buf */
610 if (pages)
611 return -EINVAL;
612
613 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
614 if (!flat_buf)
615 return -ENOMEM;
616
617 flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,
618 &flat_buf->daddr,
619 DMA_FROM_DEVICE,
620 GFP_KERNEL | __GFP_NOWARN);
621 if (!flat_buf->vaddr) {
622 kfree(flat_buf);
623 return -ENOMEM;
624 }
625
626 flat_buf->size = etr_buf->size;
627 flat_buf->dev = &drvdata->csdev->dev;
628 etr_buf->hwaddr = flat_buf->daddr;
629 etr_buf->mode = ETR_MODE_FLAT;
630 etr_buf->private = flat_buf;
631 return 0;
632}
633
634static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
635{
636 struct etr_flat_buf *flat_buf = etr_buf->private;
637
638 if (flat_buf && flat_buf->daddr) {
639 struct device *real_dev = flat_buf->dev->parent;
640
641 dma_free_noncoherent(real_dev, etr_buf->size,
642 flat_buf->vaddr, flat_buf->daddr,
643 DMA_FROM_DEVICE);
644 }
645 kfree(flat_buf);
646}
647
648static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
649{
650 struct etr_flat_buf *flat_buf = etr_buf->private;
651 struct device *real_dev = flat_buf->dev->parent;
652
653 /*
654 * Adjust the buffer to point to the beginning of the trace data
655 * and update the available trace data.
656 */
657 etr_buf->offset = rrp - etr_buf->hwaddr;
658 if (etr_buf->full)
659 etr_buf->len = etr_buf->size;
660 else
661 etr_buf->len = rwp - rrp;
662
663 /*
664 * The driver always starts tracing at the beginning of the buffer,
665 * the only reason why we would get a wrap around is when the buffer
666 * is full. Sync the entire buffer in one go for this case.
667 */
668 if (etr_buf->offset + etr_buf->len > etr_buf->size)
669 dma_sync_single_for_cpu(real_dev, flat_buf->daddr,
670 etr_buf->size, DMA_FROM_DEVICE);
671 else
672 dma_sync_single_for_cpu(real_dev,
673 flat_buf->daddr + etr_buf->offset,
674 etr_buf->len, DMA_FROM_DEVICE);
675}
676
677static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
678 u64 offset, size_t len, char **bufpp)
679{
680 struct etr_flat_buf *flat_buf = etr_buf->private;
681
682 *bufpp = (char *)flat_buf->vaddr + offset;
683 /*
684 * tmc_etr_buf_get_data already adjusts the length to handle
685 * buffer wrapping around.
686 */
687 return len;
688}
689
690static const struct etr_buf_operations etr_flat_buf_ops = {
691 .alloc = tmc_etr_alloc_flat_buf,
692 .free = tmc_etr_free_flat_buf,
693 .sync = tmc_etr_sync_flat_buf,
694 .get_data = tmc_etr_get_data_flat_buf,
695};
696
697/*
698 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
699 * appropriately.
700 */
701static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
702 struct etr_buf *etr_buf, int node,
703 void **pages)
704{
705 struct etr_sg_table *etr_table;
706 struct device *dev = &drvdata->csdev->dev;
707
708 etr_table = tmc_init_etr_sg_table(dev, node,
709 etr_buf->size, pages);
710 if (IS_ERR(etr_table))
711 return -ENOMEM;
712 etr_buf->hwaddr = etr_table->hwaddr;
713 etr_buf->mode = ETR_MODE_ETR_SG;
714 etr_buf->private = etr_table;
715 return 0;
716}
717
718static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
719{
720 struct etr_sg_table *etr_table = etr_buf->private;
721
722 if (etr_table) {
723 tmc_free_sg_table(etr_table->sg_table);
724 kfree(etr_table);
725 }
726}
727
728static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
729 size_t len, char **bufpp)
730{
731 struct etr_sg_table *etr_table = etr_buf->private;
732
733 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
734}
735
736static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
737{
738 long r_offset, w_offset;
739 struct etr_sg_table *etr_table = etr_buf->private;
740 struct tmc_sg_table *table = etr_table->sg_table;
741
742 /* Convert hw address to offset in the buffer */
743 r_offset = tmc_sg_get_data_page_offset(table, rrp);
744 if (r_offset < 0) {
745 dev_warn(table->dev,
746 "Unable to map RRP %llx to offset\n", rrp);
747 etr_buf->len = 0;
748 return;
749 }
750
751 w_offset = tmc_sg_get_data_page_offset(table, rwp);
752 if (w_offset < 0) {
753 dev_warn(table->dev,
754 "Unable to map RWP %llx to offset\n", rwp);
755 etr_buf->len = 0;
756 return;
757 }
758
759 etr_buf->offset = r_offset;
760 if (etr_buf->full)
761 etr_buf->len = etr_buf->size;
762 else
763 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
764 w_offset - r_offset;
765 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
766}
767
768static const struct etr_buf_operations etr_sg_buf_ops = {
769 .alloc = tmc_etr_alloc_sg_buf,
770 .free = tmc_etr_free_sg_buf,
771 .sync = tmc_etr_sync_sg_buf,
772 .get_data = tmc_etr_get_data_sg_buf,
773};
774
775/*
776 * TMC ETR could be connected to a CATU device, which can provide address
777 * translation service. This is represented by the Output port of the TMC
778 * (ETR) connected to the input port of the CATU.
779 *
780 * Returns : coresight_device ptr for the CATU device if a CATU is found.
781 * : NULL otherwise.
782 */
783struct coresight_device *
784tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
785{
786 struct coresight_device *etr = drvdata->csdev;
787 union coresight_dev_subtype catu_subtype = {
788 .helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU
789 };
790
791 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
792 return NULL;
793
794 return coresight_find_output_type(etr->pdata, CORESIGHT_DEV_TYPE_HELPER,
795 catu_subtype);
796}
797EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
798
799static const struct etr_buf_operations *etr_buf_ops[] = {
800 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
801 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
802 [ETR_MODE_CATU] = NULL,
803};
804
805void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
806{
807 etr_buf_ops[ETR_MODE_CATU] = catu;
808}
809EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
810
811void tmc_etr_remove_catu_ops(void)
812{
813 etr_buf_ops[ETR_MODE_CATU] = NULL;
814}
815EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
816
817static inline int tmc_etr_mode_alloc_buf(int mode,
818 struct tmc_drvdata *drvdata,
819 struct etr_buf *etr_buf, int node,
820 void **pages)
821{
822 int rc = -EINVAL;
823
824 switch (mode) {
825 case ETR_MODE_FLAT:
826 case ETR_MODE_ETR_SG:
827 case ETR_MODE_CATU:
828 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
829 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
830 node, pages);
831 if (!rc)
832 etr_buf->ops = etr_buf_ops[mode];
833 return rc;
834 default:
835 return -EINVAL;
836 }
837}
838
839static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)
840{
841 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
842
843 buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent);
844 buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
845 buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);
846}
847
848static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)
849{
850 bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;
851
852 return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;
853}
854
855/*
856 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
857 * @drvdata : ETR device details.
858 * @size : size of the requested buffer.
859 * @flags : Required properties for the buffer.
860 * @node : Node for memory allocations.
861 * @pages : An optional list of pages.
862 */
863static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
864 ssize_t size, int flags,
865 int node, void **pages)
866{
867 int rc = -ENOMEM;
868 struct etr_buf *etr_buf;
869 struct etr_buf_hw buf_hw;
870 struct device *dev = &drvdata->csdev->dev;
871
872 get_etr_buf_hw(dev, &buf_hw);
873 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
874 if (!etr_buf)
875 return ERR_PTR(-ENOMEM);
876
877 etr_buf->size = size;
878
879 /* If there is user directive for buffer mode, try that first */
880 if (drvdata->etr_mode != ETR_MODE_AUTO)
881 rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata,
882 etr_buf, node, pages);
883
884 /*
885 * If we have to use an existing list of pages, we cannot reliably
886 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
887 * we use the contiguous DMA memory if at least one of the following
888 * conditions is true:
889 * a) The ETR cannot use Scatter-Gather.
890 * b) we have a backing IOMMU
891 * c) The requested memory size is smaller (< 1M).
892 *
893 * Fallback to available mechanisms.
894 *
895 */
896 if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size))
897 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
898 etr_buf, node, pages);
899 if (rc && buf_hw.has_etr_sg)
900 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
901 etr_buf, node, pages);
902 if (rc && buf_hw.has_catu)
903 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
904 etr_buf, node, pages);
905 if (rc) {
906 kfree(etr_buf);
907 return ERR_PTR(rc);
908 }
909
910 refcount_set(&etr_buf->refcount, 1);
911 dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
912 (unsigned long)size >> 10, etr_buf->mode);
913 return etr_buf;
914}
915
916static void tmc_free_etr_buf(struct etr_buf *etr_buf)
917{
918 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
919 etr_buf->ops->free(etr_buf);
920 kfree(etr_buf);
921}
922
923/*
924 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
925 * with a maximum of @len bytes.
926 * Returns: The size of the linear data available @pos, with *bufpp
927 * updated to point to the buffer.
928 */
929static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
930 u64 offset, size_t len, char **bufpp)
931{
932 /* Adjust the length to limit this transaction to end of buffer */
933 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
934
935 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
936}
937
938static inline s64
939tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
940{
941 ssize_t len;
942 char *bufp;
943
944 len = tmc_etr_buf_get_data(etr_buf, offset,
945 CORESIGHT_BARRIER_PKT_SIZE, &bufp);
946 if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))
947 return -EINVAL;
948 coresight_insert_barrier_packet(bufp);
949 return offset + CORESIGHT_BARRIER_PKT_SIZE;
950}
951
952/*
953 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
954 * Makes sure the trace data is synced to the memory for consumption.
955 * @etr_buf->offset will hold the offset to the beginning of the trace data
956 * within the buffer, with @etr_buf->len bytes to consume.
957 */
958static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
959{
960 struct etr_buf *etr_buf = drvdata->etr_buf;
961 u64 rrp, rwp;
962 u32 status;
963
964 rrp = tmc_read_rrp(drvdata);
965 rwp = tmc_read_rwp(drvdata);
966 status = readl_relaxed(drvdata->base + TMC_STS);
967
968 /*
969 * If there were memory errors in the session, truncate the
970 * buffer.
971 */
972 if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
973 dev_dbg(&drvdata->csdev->dev,
974 "tmc memory error detected, truncating buffer\n");
975 etr_buf->len = 0;
976 etr_buf->full = false;
977 return;
978 }
979
980 etr_buf->full = !!(status & TMC_STS_FULL);
981
982 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
983
984 etr_buf->ops->sync(etr_buf, rrp, rwp);
985}
986
987static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
988{
989 u32 axictl, sts;
990 struct etr_buf *etr_buf = drvdata->etr_buf;
991 int rc = 0;
992
993 CS_UNLOCK(drvdata->base);
994
995 /* Wait for TMCSReady bit to be set */
996 rc = tmc_wait_for_tmcready(drvdata);
997 if (rc) {
998 dev_err(&drvdata->csdev->dev,
999 "Failed to enable : TMC not ready\n");
1000 CS_LOCK(drvdata->base);
1001 return rc;
1002 }
1003
1004 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
1005 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
1006
1007 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
1008 axictl &= ~TMC_AXICTL_CLEAR_MASK;
1009 axictl |= TMC_AXICTL_PROT_CTL_B1;
1010 axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
1011 axictl |= TMC_AXICTL_AXCACHE_OS;
1012
1013 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
1014 axictl &= ~TMC_AXICTL_ARCACHE_MASK;
1015 axictl |= TMC_AXICTL_ARCACHE_OS;
1016 }
1017
1018 if (etr_buf->mode == ETR_MODE_ETR_SG)
1019 axictl |= TMC_AXICTL_SCT_GAT_MODE;
1020
1021 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1022 tmc_write_dba(drvdata, etr_buf->hwaddr);
1023 /*
1024 * If the TMC pointers must be programmed before the session,
1025 * we have to set it properly (i.e, RRP/RWP to base address and
1026 * STS to "not full").
1027 */
1028 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1029 tmc_write_rrp(drvdata, etr_buf->hwaddr);
1030 tmc_write_rwp(drvdata, etr_buf->hwaddr);
1031 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1032 writel_relaxed(sts, drvdata->base + TMC_STS);
1033 }
1034
1035 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1036 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1037 TMC_FFCR_TRIGON_TRIGIN,
1038 drvdata->base + TMC_FFCR);
1039 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1040 tmc_enable_hw(drvdata);
1041
1042 CS_LOCK(drvdata->base);
1043 return rc;
1044}
1045
1046static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1047 struct etr_buf *etr_buf)
1048{
1049 int rc;
1050
1051 /* Callers should provide an appropriate buffer for use */
1052 if (WARN_ON(!etr_buf))
1053 return -EINVAL;
1054
1055 if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1056 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1057 return -EINVAL;
1058
1059 if (WARN_ON(drvdata->etr_buf))
1060 return -EBUSY;
1061
1062 rc = coresight_claim_device(drvdata->csdev);
1063 if (!rc) {
1064 drvdata->etr_buf = etr_buf;
1065 rc = __tmc_etr_enable_hw(drvdata);
1066 if (rc) {
1067 drvdata->etr_buf = NULL;
1068 coresight_disclaim_device(drvdata->csdev);
1069 }
1070 }
1071
1072 return rc;
1073}
1074
1075/*
1076 * Return the available trace data in the buffer (starts at etr_buf->offset,
1077 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1078 * also updating the @bufpp on where to find it. Since the trace data
1079 * starts at anywhere in the buffer, depending on the RRP, we adjust the
1080 * @len returned to handle buffer wrapping around.
1081 *
1082 * We are protected here by drvdata->reading != 0, which ensures the
1083 * sysfs_buf stays alive.
1084 */
1085ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1086 loff_t pos, size_t len, char **bufpp)
1087{
1088 s64 offset;
1089 ssize_t actual = len;
1090 struct etr_buf *etr_buf = drvdata->sysfs_buf;
1091
1092 if (pos + actual > etr_buf->len)
1093 actual = etr_buf->len - pos;
1094 if (actual <= 0)
1095 return actual;
1096
1097 /* Compute the offset from which we read the data */
1098 offset = etr_buf->offset + pos;
1099 if (offset >= etr_buf->size)
1100 offset -= etr_buf->size;
1101 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1102}
1103
1104static struct etr_buf *
1105tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1106{
1107 return tmc_alloc_etr_buf(drvdata, drvdata->size,
1108 0, cpu_to_node(0), NULL);
1109}
1110
1111static void
1112tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1113{
1114 if (buf)
1115 tmc_free_etr_buf(buf);
1116}
1117
1118static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1119{
1120 struct etr_buf *etr_buf = drvdata->etr_buf;
1121
1122 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1123 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1124 drvdata->sysfs_buf = NULL;
1125 } else {
1126 tmc_sync_etr_buf(drvdata);
1127 /*
1128 * Insert barrier packets at the beginning, if there was
1129 * an overflow.
1130 */
1131 if (etr_buf->full)
1132 tmc_etr_buf_insert_barrier_packet(etr_buf,
1133 etr_buf->offset);
1134 }
1135}
1136
1137static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1138{
1139 CS_UNLOCK(drvdata->base);
1140
1141 tmc_flush_and_stop(drvdata);
1142 /*
1143 * When operating in sysFS mode the content of the buffer needs to be
1144 * read before the TMC is disabled.
1145 */
1146 if (drvdata->mode == CS_MODE_SYSFS)
1147 tmc_etr_sync_sysfs_buf(drvdata);
1148
1149 tmc_disable_hw(drvdata);
1150
1151 CS_LOCK(drvdata->base);
1152
1153}
1154
1155void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1156{
1157 __tmc_etr_disable_hw(drvdata);
1158 coresight_disclaim_device(drvdata->csdev);
1159 /* Reset the ETR buf used by hardware */
1160 drvdata->etr_buf = NULL;
1161}
1162
1163static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1164{
1165 int ret = 0;
1166 unsigned long flags;
1167 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1168 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1169
1170 /*
1171 * If we are enabling the ETR from disabled state, we need to make
1172 * sure we have a buffer with the right size. The etr_buf is not reset
1173 * immediately after we stop the tracing in SYSFS mode as we wait for
1174 * the user to collect the data. We may be able to reuse the existing
1175 * buffer, provided the size matches. Any allocation has to be done
1176 * with the lock released.
1177 */
1178 spin_lock_irqsave(&drvdata->spinlock, flags);
1179 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1180 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1181 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1182
1183 /* Allocate memory with the locks released */
1184 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1185 if (IS_ERR(new_buf))
1186 return new_buf;
1187
1188 /* Let's try again */
1189 spin_lock_irqsave(&drvdata->spinlock, flags);
1190 }
1191
1192 if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1193 ret = -EBUSY;
1194 goto out;
1195 }
1196
1197 /*
1198 * If we don't have a buffer or it doesn't match the requested size,
1199 * use the buffer allocated above. Otherwise reuse the existing buffer.
1200 */
1201 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1202 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1203 free_buf = sysfs_buf;
1204 drvdata->sysfs_buf = new_buf;
1205 }
1206
1207out:
1208 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1209
1210 /* Free memory outside the spinlock if need be */
1211 if (free_buf)
1212 tmc_etr_free_sysfs_buf(free_buf);
1213 return ret ? ERR_PTR(ret) : drvdata->sysfs_buf;
1214}
1215
1216static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1217{
1218 int ret = 0;
1219 unsigned long flags;
1220 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1221 struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);
1222
1223 if (IS_ERR(sysfs_buf))
1224 return PTR_ERR(sysfs_buf);
1225
1226 spin_lock_irqsave(&drvdata->spinlock, flags);
1227
1228 /*
1229 * In sysFS mode we can have multiple writers per sink. Since this
1230 * sink is already enabled no memory is needed and the HW need not be
1231 * touched, even if the buffer size has changed.
1232 */
1233 if (drvdata->mode == CS_MODE_SYSFS) {
1234 atomic_inc(&csdev->refcnt);
1235 goto out;
1236 }
1237
1238 ret = tmc_etr_enable_hw(drvdata, sysfs_buf);
1239 if (!ret) {
1240 drvdata->mode = CS_MODE_SYSFS;
1241 atomic_inc(&csdev->refcnt);
1242 }
1243
1244out:
1245 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1246
1247 if (!ret)
1248 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1249
1250 return ret;
1251}
1252
1253struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
1254 enum cs_mode mode, void *data)
1255{
1256 struct perf_output_handle *handle = data;
1257 struct etr_perf_buffer *etr_perf;
1258
1259 switch (mode) {
1260 case CS_MODE_SYSFS:
1261 return tmc_etr_get_sysfs_buffer(csdev);
1262 case CS_MODE_PERF:
1263 etr_perf = etm_perf_sink_config(handle);
1264 if (WARN_ON(!etr_perf || !etr_perf->etr_buf))
1265 return ERR_PTR(-EINVAL);
1266 return etr_perf->etr_buf;
1267 default:
1268 return ERR_PTR(-EINVAL);
1269 }
1270}
1271EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);
1272
1273/*
1274 * alloc_etr_buf: Allocate ETR buffer for use by perf.
1275 * The size of the hardware buffer is dependent on the size configured
1276 * via sysfs and the perf ring buffer size. We prefer to allocate the
1277 * largest possible size, scaling down the size by half until it
1278 * reaches a minimum limit (1M), beyond which we give up.
1279 */
1280static struct etr_buf *
1281alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1282 int nr_pages, void **pages, bool snapshot)
1283{
1284 int node;
1285 struct etr_buf *etr_buf;
1286 unsigned long size;
1287
1288 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1289 /*
1290 * Try to match the perf ring buffer size if it is larger
1291 * than the size requested via sysfs.
1292 */
1293 if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1294 etr_buf = tmc_alloc_etr_buf(drvdata, ((ssize_t)nr_pages << PAGE_SHIFT),
1295 0, node, NULL);
1296 if (!IS_ERR(etr_buf))
1297 goto done;
1298 }
1299
1300 /*
1301 * Else switch to configured size for this ETR
1302 * and scale down until we hit the minimum limit.
1303 */
1304 size = drvdata->size;
1305 do {
1306 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1307 if (!IS_ERR(etr_buf))
1308 goto done;
1309 size /= 2;
1310 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1311
1312 return ERR_PTR(-ENOMEM);
1313
1314done:
1315 return etr_buf;
1316}
1317
1318static struct etr_buf *
1319get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1320 struct perf_event *event, int nr_pages,
1321 void **pages, bool snapshot)
1322{
1323 int ret;
1324 pid_t pid = task_pid_nr(event->owner);
1325 struct etr_buf *etr_buf;
1326
1327retry:
1328 /*
1329 * An etr_perf_buffer is associated with an event and holds a reference
1330 * to the AUX ring buffer that was created for that event. In CPU-wide
1331 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1332 * buffer, share a sink. As such an etr_perf_buffer is created for each
1333 * event but a single etr_buf associated with the ETR is shared between
1334 * them. The last event in a trace session will copy the content of the
1335 * etr_buf to its AUX ring buffer. Ring buffer associated to other
1336 * events are simply not used an freed as events are destoyed. We still
1337 * need to allocate a ring buffer for each event since we don't know
1338 * which event will be last.
1339 */
1340
1341 /*
1342 * The first thing to do here is check if an etr_buf has already been
1343 * allocated for this session. If so it is shared with this event,
1344 * otherwise it is created.
1345 */
1346 mutex_lock(&drvdata->idr_mutex);
1347 etr_buf = idr_find(&drvdata->idr, pid);
1348 if (etr_buf) {
1349 refcount_inc(&etr_buf->refcount);
1350 mutex_unlock(&drvdata->idr_mutex);
1351 return etr_buf;
1352 }
1353
1354 /* If we made it here no buffer has been allocated, do so now. */
1355 mutex_unlock(&drvdata->idr_mutex);
1356
1357 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1358 if (IS_ERR(etr_buf))
1359 return etr_buf;
1360
1361 /* Now that we have a buffer, add it to the IDR. */
1362 mutex_lock(&drvdata->idr_mutex);
1363 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1364 mutex_unlock(&drvdata->idr_mutex);
1365
1366 /* Another event with this session ID has allocated this buffer. */
1367 if (ret == -ENOSPC) {
1368 tmc_free_etr_buf(etr_buf);
1369 goto retry;
1370 }
1371
1372 /* The IDR can't allocate room for a new session, abandon ship. */
1373 if (ret == -ENOMEM) {
1374 tmc_free_etr_buf(etr_buf);
1375 return ERR_PTR(ret);
1376 }
1377
1378
1379 return etr_buf;
1380}
1381
1382static struct etr_buf *
1383get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1384 struct perf_event *event, int nr_pages,
1385 void **pages, bool snapshot)
1386{
1387 /*
1388 * In per-thread mode the etr_buf isn't shared, so just go ahead
1389 * with memory allocation.
1390 */
1391 return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1392}
1393
1394static struct etr_buf *
1395get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1396 int nr_pages, void **pages, bool snapshot)
1397{
1398 if (event->cpu == -1)
1399 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1400 pages, snapshot);
1401
1402 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1403 pages, snapshot);
1404}
1405
1406static struct etr_perf_buffer *
1407tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1408 int nr_pages, void **pages, bool snapshot)
1409{
1410 int node;
1411 struct etr_buf *etr_buf;
1412 struct etr_perf_buffer *etr_perf;
1413
1414 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1415
1416 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1417 if (!etr_perf)
1418 return ERR_PTR(-ENOMEM);
1419
1420 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1421 if (!IS_ERR(etr_buf))
1422 goto done;
1423
1424 kfree(etr_perf);
1425 return ERR_PTR(-ENOMEM);
1426
1427done:
1428 /*
1429 * Keep a reference to the ETR this buffer has been allocated for
1430 * in order to have access to the IDR in tmc_free_etr_buffer().
1431 */
1432 etr_perf->drvdata = drvdata;
1433 etr_perf->etr_buf = etr_buf;
1434
1435 return etr_perf;
1436}
1437
1438
1439static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1440 struct perf_event *event, void **pages,
1441 int nr_pages, bool snapshot)
1442{
1443 struct etr_perf_buffer *etr_perf;
1444 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1445
1446 etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1447 nr_pages, pages, snapshot);
1448 if (IS_ERR(etr_perf)) {
1449 dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1450 return NULL;
1451 }
1452
1453 etr_perf->pid = task_pid_nr(event->owner);
1454 etr_perf->snapshot = snapshot;
1455 etr_perf->nr_pages = nr_pages;
1456 etr_perf->pages = pages;
1457
1458 return etr_perf;
1459}
1460
1461static void tmc_free_etr_buffer(void *config)
1462{
1463 struct etr_perf_buffer *etr_perf = config;
1464 struct tmc_drvdata *drvdata = etr_perf->drvdata;
1465 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1466
1467 if (!etr_buf)
1468 goto free_etr_perf_buffer;
1469
1470 mutex_lock(&drvdata->idr_mutex);
1471 /* If we are not the last one to use the buffer, don't touch it. */
1472 if (!refcount_dec_and_test(&etr_buf->refcount)) {
1473 mutex_unlock(&drvdata->idr_mutex);
1474 goto free_etr_perf_buffer;
1475 }
1476
1477 /* We are the last one, remove from the IDR and free the buffer. */
1478 buf = idr_remove(&drvdata->idr, etr_perf->pid);
1479 mutex_unlock(&drvdata->idr_mutex);
1480
1481 /*
1482 * Something went very wrong if the buffer associated with this ID
1483 * is not the same in the IDR. Leak to avoid use after free.
1484 */
1485 if (buf && WARN_ON(buf != etr_buf))
1486 goto free_etr_perf_buffer;
1487
1488 tmc_free_etr_buf(etr_perf->etr_buf);
1489
1490free_etr_perf_buffer:
1491 kfree(etr_perf);
1492}
1493
1494/*
1495 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1496 * buffer to the perf ring buffer.
1497 */
1498static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1499 unsigned long head,
1500 unsigned long src_offset,
1501 unsigned long to_copy)
1502{
1503 long bytes;
1504 long pg_idx, pg_offset;
1505 char **dst_pages, *src_buf;
1506 struct etr_buf *etr_buf = etr_perf->etr_buf;
1507
1508 head = PERF_IDX2OFF(head, etr_perf);
1509 pg_idx = head >> PAGE_SHIFT;
1510 pg_offset = head & (PAGE_SIZE - 1);
1511 dst_pages = (char **)etr_perf->pages;
1512
1513 while (to_copy > 0) {
1514 /*
1515 * In one iteration, we can copy minimum of :
1516 * 1) what is available in the source buffer,
1517 * 2) what is available in the source buffer, before it
1518 * wraps around.
1519 * 3) what is available in the destination page.
1520 * in one iteration.
1521 */
1522 if (src_offset >= etr_buf->size)
1523 src_offset -= etr_buf->size;
1524 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1525 &src_buf);
1526 if (WARN_ON_ONCE(bytes <= 0))
1527 break;
1528 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1529
1530 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1531
1532 to_copy -= bytes;
1533
1534 /* Move destination pointers */
1535 pg_offset += bytes;
1536 if (pg_offset == PAGE_SIZE) {
1537 pg_offset = 0;
1538 if (++pg_idx == etr_perf->nr_pages)
1539 pg_idx = 0;
1540 }
1541
1542 /* Move source pointers */
1543 src_offset += bytes;
1544 }
1545}
1546
1547/*
1548 * tmc_update_etr_buffer : Update the perf ring buffer with the
1549 * available trace data. We use software double buffering at the moment.
1550 *
1551 * TODO: Add support for reusing the perf ring buffer.
1552 */
1553static unsigned long
1554tmc_update_etr_buffer(struct coresight_device *csdev,
1555 struct perf_output_handle *handle,
1556 void *config)
1557{
1558 bool lost = false;
1559 unsigned long flags, offset, size = 0;
1560 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1561 struct etr_perf_buffer *etr_perf = config;
1562 struct etr_buf *etr_buf = etr_perf->etr_buf;
1563
1564 spin_lock_irqsave(&drvdata->spinlock, flags);
1565
1566 /* Don't do anything if another tracer is using this sink */
1567 if (atomic_read(&csdev->refcnt) != 1) {
1568 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1569 goto out;
1570 }
1571
1572 if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1573 lost = true;
1574 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1575 goto out;
1576 }
1577
1578 CS_UNLOCK(drvdata->base);
1579
1580 tmc_flush_and_stop(drvdata);
1581 tmc_sync_etr_buf(drvdata);
1582
1583 CS_LOCK(drvdata->base);
1584 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1585
1586 lost = etr_buf->full;
1587 offset = etr_buf->offset;
1588 size = etr_buf->len;
1589
1590 /*
1591 * The ETR buffer may be bigger than the space available in the
1592 * perf ring buffer (handle->size). If so advance the offset so that we
1593 * get the latest trace data. In snapshot mode none of that matters
1594 * since we are expected to clobber stale data in favour of the latest
1595 * traces.
1596 */
1597 if (!etr_perf->snapshot && size > handle->size) {
1598 u32 mask = tmc_get_memwidth_mask(drvdata);
1599
1600 /*
1601 * Make sure the new size is aligned in accordance with the
1602 * requirement explained in function tmc_get_memwidth_mask().
1603 */
1604 size = handle->size & mask;
1605 offset = etr_buf->offset + etr_buf->len - size;
1606
1607 if (offset >= etr_buf->size)
1608 offset -= etr_buf->size;
1609 lost = true;
1610 }
1611
1612 /* Insert barrier packets at the beginning, if there was an overflow */
1613 if (lost)
1614 tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1615 tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);
1616
1617 /*
1618 * In snapshot mode we simply increment the head by the number of byte
1619 * that were written. User space will figure out how many bytes to get
1620 * from the AUX buffer based on the position of the head.
1621 */
1622 if (etr_perf->snapshot)
1623 handle->head += size;
1624
1625 /*
1626 * Ensure that the AUX trace data is visible before the aux_head
1627 * is updated via perf_aux_output_end(), as expected by the
1628 * perf ring buffer.
1629 */
1630 smp_wmb();
1631
1632out:
1633 /*
1634 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1635 * captured buffer is expected to be truncated and 2) a full buffer
1636 * prevents the event from being re-enabled by the perf core,
1637 * resulting in stale data being send to user space.
1638 */
1639 if (!etr_perf->snapshot && lost)
1640 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1641 return size;
1642}
1643
1644static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1645{
1646 int rc = 0;
1647 pid_t pid;
1648 unsigned long flags;
1649 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1650 struct perf_output_handle *handle = data;
1651 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1652
1653 spin_lock_irqsave(&drvdata->spinlock, flags);
1654 /* Don't use this sink if it is already claimed by sysFS */
1655 if (drvdata->mode == CS_MODE_SYSFS) {
1656 rc = -EBUSY;
1657 goto unlock_out;
1658 }
1659
1660 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1661 rc = -EINVAL;
1662 goto unlock_out;
1663 }
1664
1665 /* Get a handle on the pid of the process to monitor */
1666 pid = etr_perf->pid;
1667
1668 /* Do not proceed if this device is associated with another session */
1669 if (drvdata->pid != -1 && drvdata->pid != pid) {
1670 rc = -EBUSY;
1671 goto unlock_out;
1672 }
1673
1674 /*
1675 * No HW configuration is needed if the sink is already in
1676 * use for this session.
1677 */
1678 if (drvdata->pid == pid) {
1679 atomic_inc(&csdev->refcnt);
1680 goto unlock_out;
1681 }
1682
1683 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1684 if (!rc) {
1685 /* Associate with monitored process. */
1686 drvdata->pid = pid;
1687 drvdata->mode = CS_MODE_PERF;
1688 drvdata->perf_buf = etr_perf->etr_buf;
1689 atomic_inc(&csdev->refcnt);
1690 }
1691
1692unlock_out:
1693 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1694 return rc;
1695}
1696
1697static int tmc_enable_etr_sink(struct coresight_device *csdev,
1698 enum cs_mode mode, void *data)
1699{
1700 switch (mode) {
1701 case CS_MODE_SYSFS:
1702 return tmc_enable_etr_sink_sysfs(csdev);
1703 case CS_MODE_PERF:
1704 return tmc_enable_etr_sink_perf(csdev, data);
1705 default:
1706 return -EINVAL;
1707 }
1708}
1709
1710static int tmc_disable_etr_sink(struct coresight_device *csdev)
1711{
1712 unsigned long flags;
1713 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1714
1715 spin_lock_irqsave(&drvdata->spinlock, flags);
1716
1717 if (drvdata->reading) {
1718 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1719 return -EBUSY;
1720 }
1721
1722 if (atomic_dec_return(&csdev->refcnt)) {
1723 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1724 return -EBUSY;
1725 }
1726
1727 /* Complain if we (somehow) got out of sync */
1728 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1729 tmc_etr_disable_hw(drvdata);
1730 /* Dissociate from monitored process. */
1731 drvdata->pid = -1;
1732 drvdata->mode = CS_MODE_DISABLED;
1733 /* Reset perf specific data */
1734 drvdata->perf_buf = NULL;
1735
1736 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1737
1738 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1739 return 0;
1740}
1741
1742static const struct coresight_ops_sink tmc_etr_sink_ops = {
1743 .enable = tmc_enable_etr_sink,
1744 .disable = tmc_disable_etr_sink,
1745 .alloc_buffer = tmc_alloc_etr_buffer,
1746 .update_buffer = tmc_update_etr_buffer,
1747 .free_buffer = tmc_free_etr_buffer,
1748};
1749
1750const struct coresight_ops tmc_etr_cs_ops = {
1751 .sink_ops = &tmc_etr_sink_ops,
1752};
1753
1754int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1755{
1756 int ret = 0;
1757 unsigned long flags;
1758
1759 /* config types are set a boot time and never change */
1760 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1761 return -EINVAL;
1762
1763 spin_lock_irqsave(&drvdata->spinlock, flags);
1764 if (drvdata->reading) {
1765 ret = -EBUSY;
1766 goto out;
1767 }
1768
1769 /*
1770 * We can safely allow reads even if the ETR is operating in PERF mode,
1771 * since the sysfs session is captured in mode specific data.
1772 * If drvdata::sysfs_data is NULL the trace data has been read already.
1773 */
1774 if (!drvdata->sysfs_buf) {
1775 ret = -EINVAL;
1776 goto out;
1777 }
1778
1779 /* Disable the TMC if we are trying to read from a running session. */
1780 if (drvdata->mode == CS_MODE_SYSFS)
1781 __tmc_etr_disable_hw(drvdata);
1782
1783 drvdata->reading = true;
1784out:
1785 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1786
1787 return ret;
1788}
1789
1790int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1791{
1792 unsigned long flags;
1793 struct etr_buf *sysfs_buf = NULL;
1794
1795 /* config types are set a boot time and never change */
1796 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1797 return -EINVAL;
1798
1799 spin_lock_irqsave(&drvdata->spinlock, flags);
1800
1801 /* RE-enable the TMC if need be */
1802 if (drvdata->mode == CS_MODE_SYSFS) {
1803 /*
1804 * The trace run will continue with the same allocated trace
1805 * buffer. Since the tracer is still enabled drvdata::buf can't
1806 * be NULL.
1807 */
1808 __tmc_etr_enable_hw(drvdata);
1809 } else {
1810 /*
1811 * The ETR is not tracing and the buffer was just read.
1812 * As such prepare to free the trace buffer.
1813 */
1814 sysfs_buf = drvdata->sysfs_buf;
1815 drvdata->sysfs_buf = NULL;
1816 }
1817
1818 drvdata->reading = false;
1819 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1820
1821 /* Free allocated memory out side of the spinlock */
1822 if (sysfs_buf)
1823 tmc_etr_free_sysfs_buf(sysfs_buf);
1824
1825 return 0;
1826}
1827
1828static const char *const buf_modes_str[] = {
1829 [ETR_MODE_FLAT] = "flat",
1830 [ETR_MODE_ETR_SG] = "tmc-sg",
1831 [ETR_MODE_CATU] = "catu",
1832 [ETR_MODE_AUTO] = "auto",
1833};
1834
1835static ssize_t buf_modes_available_show(struct device *dev,
1836 struct device_attribute *attr, char *buf)
1837{
1838 struct etr_buf_hw buf_hw;
1839 ssize_t size = 0;
1840
1841 get_etr_buf_hw(dev, &buf_hw);
1842 size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]);
1843 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]);
1844 if (buf_hw.has_etr_sg)
1845 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]);
1846
1847 if (buf_hw.has_catu)
1848 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]);
1849
1850 size += sysfs_emit_at(buf, size, "\n");
1851 return size;
1852}
1853static DEVICE_ATTR_RO(buf_modes_available);
1854
1855static ssize_t buf_mode_preferred_show(struct device *dev,
1856 struct device_attribute *attr, char *buf)
1857{
1858 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
1859
1860 return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]);
1861}
1862
1863static ssize_t buf_mode_preferred_store(struct device *dev,
1864 struct device_attribute *attr,
1865 const char *buf, size_t size)
1866{
1867 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
1868 struct etr_buf_hw buf_hw;
1869
1870 get_etr_buf_hw(dev, &buf_hw);
1871 if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT]))
1872 drvdata->etr_mode = ETR_MODE_FLAT;
1873 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)
1874 drvdata->etr_mode = ETR_MODE_ETR_SG;
1875 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)
1876 drvdata->etr_mode = ETR_MODE_CATU;
1877 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO]))
1878 drvdata->etr_mode = ETR_MODE_AUTO;
1879 else
1880 return -EINVAL;
1881 return size;
1882}
1883static DEVICE_ATTR_RW(buf_mode_preferred);
1884
1885static struct attribute *coresight_etr_attrs[] = {
1886 &dev_attr_buf_modes_available.attr,
1887 &dev_attr_buf_mode_preferred.attr,
1888 NULL,
1889};
1890
1891const struct attribute_group coresight_etr_group = {
1892 .attrs = coresight_etr_attrs,
1893};