1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
   4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   5 */
   6
   7#include <linux/atomic.h>
   8#include <linux/coresight.h>
   9#include <linux/dma-mapping.h>
  10#include <linux/iommu.h>
  11#include <linux/idr.h>
  12#include <linux/mutex.h>
  13#include <linux/refcount.h>
  14#include <linux/slab.h>
  15#include <linux/types.h>
  16#include <linux/vmalloc.h>
  17#include "coresight-catu.h"
  18#include "coresight-etm-perf.h"
  19#include "coresight-priv.h"
  20#include "coresight-tmc.h"
  21
  22struct etr_flat_buf {
  23	struct device	*dev;
  24	dma_addr_t	daddr;
  25	void		*vaddr;
  26	size_t		size;
  27};
  28
  29struct etr_buf_hw {
  30	bool	has_iommu;
  31	bool	has_etr_sg;
  32	bool	has_catu;
  33};
  34
  35/*
  36 * etr_perf_buffer - Perf buffer used for ETR
  37 * @drvdata		- The ETR drvdaga this buffer has been allocated for.
  38 * @etr_buf		- Actual buffer used by the ETR
  39 * @pid			- The PID this etr_perf_buffer belongs to.
  40 * @snaphost		- Perf session mode
 
  41 * @nr_pages		- Number of pages in the ring buffer.
  42 * @pages		- Array of Pages in the ring buffer.
  43 */
  44struct etr_perf_buffer {
  45	struct tmc_drvdata	*drvdata;
  46	struct etr_buf		*etr_buf;
  47	pid_t			pid;
  48	bool			snapshot;
 
  49	int			nr_pages;
  50	void			**pages;
  51};
  52
  53/* Convert the perf index to an offset within the ETR buffer */
  54#define PERF_IDX2OFF(idx, buf)		\
  55		((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
  56
  57/* Lower limit for ETR hardware buffer */
  58#define TMC_ETR_PERF_MIN_BUF_SIZE	SZ_1M
  59
  60/*
  61 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
  62 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
  63 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
  64 * contain more than one SG buffer and tables.
  65 *
  66 * A table entry has the following format:
  67 *
  68 * ---Bit31------------Bit4-------Bit1-----Bit0--
  69 * |     Address[39:12]    | SBZ |  Entry Type  |
  70 * ----------------------------------------------
  71 *
  72 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
  73 *	    always zero.
  74 *
  75 * Entry type:
  76 *	b00 - Reserved.
  77 *	b01 - Last entry in the tables, points to 4K page buffer.
  78 *	b10 - Normal entry, points to 4K page buffer.
  79 *	b11 - Link. The address points to the base of next table.
  80 */
  81
  82typedef u32 sgte_t;
  83
  84#define ETR_SG_PAGE_SHIFT		12
  85#define ETR_SG_PAGE_SIZE		(1UL << ETR_SG_PAGE_SHIFT)
  86#define ETR_SG_PAGES_PER_SYSPAGE	(PAGE_SIZE / ETR_SG_PAGE_SIZE)
  87#define ETR_SG_PTRS_PER_PAGE		(ETR_SG_PAGE_SIZE / sizeof(sgte_t))
  88#define ETR_SG_PTRS_PER_SYSPAGE		(PAGE_SIZE / sizeof(sgte_t))
  89
  90#define ETR_SG_ET_MASK			0x3
  91#define ETR_SG_ET_LAST			0x1
  92#define ETR_SG_ET_NORMAL		0x2
  93#define ETR_SG_ET_LINK			0x3
  94
  95#define ETR_SG_ADDR_SHIFT		4
  96
  97#define ETR_SG_ENTRY(addr, type) \
  98	(sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
  99		 (type & ETR_SG_ET_MASK))
 100
 101#define ETR_SG_ADDR(entry) \
 102	(((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
 103#define ETR_SG_ET(entry)		((entry) & ETR_SG_ET_MASK)
 104
 105/*
 106 * struct etr_sg_table : ETR SG Table
 107 * @sg_table:		Generic SG Table holding the data/table pages.
 108 * @hwaddr:		hwaddress used by the TMC, which is the base
 109 *			address of the table.
 110 */
 111struct etr_sg_table {
 112	struct tmc_sg_table	*sg_table;
 113	dma_addr_t		hwaddr;
 114};
 115
 116/*
 117 * tmc_etr_sg_table_entries: Total number of table entries required to map
 118 * @nr_pages system pages.
 119 *
 120 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
 121 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
 122 * with the last entry pointing to another page of table entries.
 123 * If we spill over to a new page for mapping 1 entry, we could as
 124 * well replace the link entry of the previous page with the last entry.
 125 */
 126static inline unsigned long __attribute_const__
 127tmc_etr_sg_table_entries(int nr_pages)
 128{
 129	unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
 130	unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
 131	/*
 132	 * If we spill over to a new page for 1 entry, we could as well
 133	 * make it the LAST entry in the previous page, skipping the Link
 134	 * address.
 135	 */
 136	if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
 137		nr_sglinks--;
 138	return nr_sgpages + nr_sglinks;
 139}
 140
 141/*
 142 * tmc_pages_get_offset:  Go through all the pages in the tmc_pages
 143 * and map the device address @addr to an offset within the virtual
 144 * contiguous buffer.
 145 */
 146static long
 147tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
 148{
 149	int i;
 150	dma_addr_t page_start;
 151
 152	for (i = 0; i < tmc_pages->nr_pages; i++) {
 153		page_start = tmc_pages->daddrs[i];
 154		if (addr >= page_start && addr < (page_start + PAGE_SIZE))
 155			return i * PAGE_SIZE + (addr - page_start);
 156	}
 157
 158	return -EINVAL;
 159}
 160
 161/*
 162 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
 163 * If the pages were not allocated in tmc_pages_alloc(), we would
 164 * simply drop the refcount.
 165 */
 166static void tmc_pages_free(struct tmc_pages *tmc_pages,
 167			   struct device *dev, enum dma_data_direction dir)
 168{
 169	int i;
 170	struct device *real_dev = dev->parent;
 171
 172	for (i = 0; i < tmc_pages->nr_pages; i++) {
 173		if (tmc_pages->daddrs && tmc_pages->daddrs[i])
 174			dma_unmap_page(real_dev, tmc_pages->daddrs[i],
 175					 PAGE_SIZE, dir);
 176		if (tmc_pages->pages && tmc_pages->pages[i])
 177			__free_page(tmc_pages->pages[i]);
 178	}
 179
 180	kfree(tmc_pages->pages);
 181	kfree(tmc_pages->daddrs);
 182	tmc_pages->pages = NULL;
 183	tmc_pages->daddrs = NULL;
 184	tmc_pages->nr_pages = 0;
 185}
 186
 187/*
 188 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
 189 * If @pages is not NULL, the list of page virtual addresses are
 190 * used as the data pages. The pages are then dma_map'ed for @dev
 191 * with dma_direction @dir.
 192 *
 193 * Returns 0 upon success, else the error number.
 194 */
 195static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
 196			   struct device *dev, int node,
 197			   enum dma_data_direction dir, void **pages)
 198{
 199	int i, nr_pages;
 200	dma_addr_t paddr;
 201	struct page *page;
 202	struct device *real_dev = dev->parent;
 203
 204	nr_pages = tmc_pages->nr_pages;
 205	tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
 206					 GFP_KERNEL);
 207	if (!tmc_pages->daddrs)
 208		return -ENOMEM;
 209	tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
 210					 GFP_KERNEL);
 211	if (!tmc_pages->pages) {
 212		kfree(tmc_pages->daddrs);
 213		tmc_pages->daddrs = NULL;
 214		return -ENOMEM;
 215	}
 216
 217	for (i = 0; i < nr_pages; i++) {
 218		if (pages && pages[i]) {
 219			page = virt_to_page(pages[i]);
 220			/* Hold a refcount on the page */
 221			get_page(page);
 222		} else {
 223			page = alloc_pages_node(node,
 224						GFP_KERNEL | __GFP_ZERO, 0);
 225			if (!page)
 226				goto err;
 227		}
 228		paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
 229		if (dma_mapping_error(real_dev, paddr))
 230			goto err;
 231		tmc_pages->daddrs[i] = paddr;
 232		tmc_pages->pages[i] = page;
 233	}
 234	return 0;
 235err:
 236	tmc_pages_free(tmc_pages, dev, dir);
 237	return -ENOMEM;
 238}
 239
 240static inline long
 241tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
 242{
 243	return tmc_pages_get_offset(&sg_table->data_pages, addr);
 244}
 245
 246static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
 247{
 248	if (sg_table->table_vaddr)
 249		vunmap(sg_table->table_vaddr);
 250	tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
 251}
 252
 253static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
 254{
 255	if (sg_table->data_vaddr)
 256		vunmap(sg_table->data_vaddr);
 257	tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
 258}
 259
 260void tmc_free_sg_table(struct tmc_sg_table *sg_table)
 261{
 262	tmc_free_table_pages(sg_table);
 263	tmc_free_data_pages(sg_table);
 264}
 265EXPORT_SYMBOL_GPL(tmc_free_sg_table);
 266
 267/*
 268 * Alloc pages for the table. Since this will be used by the device,
 269 * allocate the pages closer to the device (i.e, dev_to_node(dev)
 270 * rather than the CPU node).
 271 */
 272static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
 273{
 274	int rc;
 275	struct tmc_pages *table_pages = &sg_table->table_pages;
 276
 277	rc = tmc_pages_alloc(table_pages, sg_table->dev,
 278			     dev_to_node(sg_table->dev),
 279			     DMA_TO_DEVICE, NULL);
 280	if (rc)
 281		return rc;
 282	sg_table->table_vaddr = vmap(table_pages->pages,
 283				     table_pages->nr_pages,
 284				     VM_MAP,
 285				     PAGE_KERNEL);
 286	if (!sg_table->table_vaddr)
 287		rc = -ENOMEM;
 288	else
 289		sg_table->table_daddr = table_pages->daddrs[0];
 290	return rc;
 291}
 292
 293static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
 294{
 295	int rc;
 296
 297	/* Allocate data pages on the node requested by the caller */
 298	rc = tmc_pages_alloc(&sg_table->data_pages,
 299			     sg_table->dev, sg_table->node,
 300			     DMA_FROM_DEVICE, pages);
 301	if (!rc) {
 302		sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
 303					    sg_table->data_pages.nr_pages,
 304					    VM_MAP,
 305					    PAGE_KERNEL);
 306		if (!sg_table->data_vaddr)
 307			rc = -ENOMEM;
 308	}
 309	return rc;
 310}
 311
 312/*
 313 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
 314 * and data buffers. TMC writes to the data buffers and reads from the SG
 315 * Table pages.
 316 *
 317 * @dev		- Coresight device to which page should be DMA mapped.
 318 * @node	- Numa node for mem allocations
 319 * @nr_tpages	- Number of pages for the table entries.
 320 * @nr_dpages	- Number of pages for Data buffer.
 321 * @pages	- Optional list of virtual address of pages.
 322 */
 323struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
 324					int node,
 325					int nr_tpages,
 326					int nr_dpages,
 327					void **pages)
 328{
 329	long rc;
 330	struct tmc_sg_table *sg_table;
 331
 332	sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
 333	if (!sg_table)
 334		return ERR_PTR(-ENOMEM);
 335	sg_table->data_pages.nr_pages = nr_dpages;
 336	sg_table->table_pages.nr_pages = nr_tpages;
 337	sg_table->node = node;
 338	sg_table->dev = dev;
 339
 340	rc  = tmc_alloc_data_pages(sg_table, pages);
 341	if (!rc)
 342		rc = tmc_alloc_table_pages(sg_table);
 343	if (rc) {
 344		tmc_free_sg_table(sg_table);
 345		kfree(sg_table);
 346		return ERR_PTR(rc);
 347	}
 348
 349	return sg_table;
 350}
 351EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
 352
 353/*
 354 * tmc_sg_table_sync_data_range: Sync the data buffer written
 355 * by the device from @offset upto a @size bytes.
 356 */
 357void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
 358				  u64 offset, u64 size)
 359{
 360	int i, index, start;
 361	int npages = DIV_ROUND_UP(size, PAGE_SIZE);
 362	struct device *real_dev = table->dev->parent;
 363	struct tmc_pages *data = &table->data_pages;
 364
 365	start = offset >> PAGE_SHIFT;
 366	for (i = start; i < (start + npages); i++) {
 367		index = i % data->nr_pages;
 368		dma_sync_single_for_cpu(real_dev, data->daddrs[index],
 369					PAGE_SIZE, DMA_FROM_DEVICE);
 370	}
 371}
 372EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
 373
 374/* tmc_sg_sync_table: Sync the page table */
 375void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
 376{
 377	int i;
 378	struct device *real_dev = sg_table->dev->parent;
 379	struct tmc_pages *table_pages = &sg_table->table_pages;
 380
 381	for (i = 0; i < table_pages->nr_pages; i++)
 382		dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
 383					   PAGE_SIZE, DMA_TO_DEVICE);
 384}
 385EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
 386
 387/*
 388 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
 389 * in the SG buffer. The @bufpp is updated to point to the buffer.
 390 * Returns :
 391 *	the length of linear data available at @offset.
 392 *	or
 393 *	<= 0 if no data is available.
 394 */
 395ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
 396			      u64 offset, size_t len, char **bufpp)
 397{
 398	size_t size;
 399	int pg_idx = offset >> PAGE_SHIFT;
 400	int pg_offset = offset & (PAGE_SIZE - 1);
 401	struct tmc_pages *data_pages = &sg_table->data_pages;
 402
 403	size = tmc_sg_table_buf_size(sg_table);
 404	if (offset >= size)
 405		return -EINVAL;
 406
 407	/* Make sure we don't go beyond the end */
 408	len = (len < (size - offset)) ? len : size - offset;
 409	/* Respect the page boundaries */
 410	len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
 411	if (len > 0)
 412		*bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
 413	return len;
 414}
 415EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
 416
 417#ifdef ETR_SG_DEBUG
 418/* Map a dma address to virtual address */
 419static unsigned long
 420tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
 421		      dma_addr_t addr, bool table)
 422{
 423	long offset;
 424	unsigned long base;
 425	struct tmc_pages *tmc_pages;
 426
 427	if (table) {
 428		tmc_pages = &sg_table->table_pages;
 429		base = (unsigned long)sg_table->table_vaddr;
 430	} else {
 431		tmc_pages = &sg_table->data_pages;
 432		base = (unsigned long)sg_table->data_vaddr;
 433	}
 434
 435	offset = tmc_pages_get_offset(tmc_pages, addr);
 436	if (offset < 0)
 437		return 0;
 438	return base + offset;
 439}
 440
 441/* Dump the given sg_table */
 442static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
 443{
 444	sgte_t *ptr;
 445	int i = 0;
 446	dma_addr_t addr;
 447	struct tmc_sg_table *sg_table = etr_table->sg_table;
 448
 449	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
 450					      etr_table->hwaddr, true);
 451	while (ptr) {
 452		addr = ETR_SG_ADDR(*ptr);
 453		switch (ETR_SG_ET(*ptr)) {
 454		case ETR_SG_ET_NORMAL:
 455			dev_dbg(sg_table->dev,
 456				"%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
 457			ptr++;
 458			break;
 459		case ETR_SG_ET_LINK:
 460			dev_dbg(sg_table->dev,
 461				"%05d: *** %p\t:{L} 0x%llx ***\n",
 462				 i, ptr, addr);
 463			ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
 464							      addr, true);
 465			break;
 466		case ETR_SG_ET_LAST:
 467			dev_dbg(sg_table->dev,
 468				"%05d: ### %p\t:[L] 0x%llx ###\n",
 469				 i, ptr, addr);
 470			return;
 471		default:
 472			dev_dbg(sg_table->dev,
 473				"%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
 474				 i, ptr, addr);
 475			return;
 476		}
 477		i++;
 478	}
 479	dev_dbg(sg_table->dev, "******* End of Table *****\n");
 480}
 481#else
 482static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
 483#endif
 484
 485/*
 486 * Populate the SG Table page table entries from table/data
 487 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
 488 * So does a Table page. So we keep track of indices of the tables
 489 * in each system page and move the pointers accordingly.
 490 */
 491#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
 492static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
 493{
 494	dma_addr_t paddr;
 495	int i, type, nr_entries;
 496	int tpidx = 0; /* index to the current system table_page */
 497	int sgtidx = 0;	/* index to the sg_table within the current syspage */
 498	int sgtentry = 0; /* the entry within the sg_table */
 499	int dpidx = 0; /* index to the current system data_page */
 500	int spidx = 0; /* index to the SG page within the current data page */
 501	sgte_t *ptr; /* pointer to the table entry to fill */
 502	struct tmc_sg_table *sg_table = etr_table->sg_table;
 503	dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
 504	dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
 505
 506	nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
 507	/*
 508	 * Use the contiguous virtual address of the table to update entries.
 509	 */
 510	ptr = sg_table->table_vaddr;
 511	/*
 512	 * Fill all the entries, except the last entry to avoid special
 513	 * checks within the loop.
 514	 */
 515	for (i = 0; i < nr_entries - 1; i++) {
 516		if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
 517			/*
 518			 * Last entry in a sg_table page is a link address to
 519			 * the next table page. If this sg_table is the last
 520			 * one in the system page, it links to the first
 521			 * sg_table in the next system page. Otherwise, it
 522			 * links to the next sg_table page within the system
 523			 * page.
 524			 */
 525			if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
 526				paddr = table_daddrs[tpidx + 1];
 527			} else {
 528				paddr = table_daddrs[tpidx] +
 529					(ETR_SG_PAGE_SIZE * (sgtidx + 1));
 530			}
 531			type = ETR_SG_ET_LINK;
 532		} else {
 533			/*
 534			 * Update the indices to the data_pages to point to the
 535			 * next sg_page in the data buffer.
 536			 */
 537			type = ETR_SG_ET_NORMAL;
 538			paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
 539			if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
 540				dpidx++;
 541		}
 542		*ptr++ = ETR_SG_ENTRY(paddr, type);
 543		/*
 544		 * Move to the next table pointer, moving the table page index
 545		 * if necessary
 546		 */
 547		if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
 548			if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
 549				tpidx++;
 550		}
 551	}
 552
 553	/* Set up the last entry, which is always a data pointer */
 554	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
 555	*ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
 556}
 557
 558/*
 559 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
 560 * populate the table.
 561 *
 562 * @dev		- Device pointer for the TMC
 563 * @node	- NUMA node where the memory should be allocated
 564 * @size	- Total size of the data buffer
 565 * @pages	- Optional list of page virtual address
 566 */
 567static struct etr_sg_table *
 568tmc_init_etr_sg_table(struct device *dev, int node,
 569		      unsigned long size, void **pages)
 570{
 571	int nr_entries, nr_tpages;
 572	int nr_dpages = size >> PAGE_SHIFT;
 573	struct tmc_sg_table *sg_table;
 574	struct etr_sg_table *etr_table;
 575
 576	etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
 577	if (!etr_table)
 578		return ERR_PTR(-ENOMEM);
 579	nr_entries = tmc_etr_sg_table_entries(nr_dpages);
 580	nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
 581
 582	sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
 583	if (IS_ERR(sg_table)) {
 584		kfree(etr_table);
 585		return ERR_CAST(sg_table);
 586	}
 587
 588	etr_table->sg_table = sg_table;
 589	/* TMC should use table base address for DBA */
 590	etr_table->hwaddr = sg_table->table_daddr;
 591	tmc_etr_sg_table_populate(etr_table);
 592	/* Sync the table pages for the HW */
 593	tmc_sg_table_sync_table(sg_table);
 594	tmc_etr_sg_table_dump(etr_table);
 595
 596	return etr_table;
 597}
 598
 599/*
 600 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
 601 */
 602static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
 603				  struct etr_buf *etr_buf, int node,
 604				  void **pages)
 605{
 606	struct etr_flat_buf *flat_buf;
 607	struct device *real_dev = drvdata->csdev->dev.parent;
 608
 609	/* We cannot reuse existing pages for flat buf */
 610	if (pages)
 611		return -EINVAL;
 612
 613	flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
 614	if (!flat_buf)
 615		return -ENOMEM;
 616
 617	flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,
 618						&flat_buf->daddr,
 619						DMA_FROM_DEVICE,
 620						GFP_KERNEL | __GFP_NOWARN);
 621	if (!flat_buf->vaddr) {
 622		kfree(flat_buf);
 623		return -ENOMEM;
 624	}
 625
 626	flat_buf->size = etr_buf->size;
 627	flat_buf->dev = &drvdata->csdev->dev;
 628	etr_buf->hwaddr = flat_buf->daddr;
 629	etr_buf->mode = ETR_MODE_FLAT;
 630	etr_buf->private = flat_buf;
 631	return 0;
 632}
 633
 634static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
 635{
 636	struct etr_flat_buf *flat_buf = etr_buf->private;
 637
 638	if (flat_buf && flat_buf->daddr) {
 639		struct device *real_dev = flat_buf->dev->parent;
 640
 641		dma_free_noncoherent(real_dev, etr_buf->size,
 642				     flat_buf->vaddr, flat_buf->daddr,
 643				     DMA_FROM_DEVICE);
 644	}
 645	kfree(flat_buf);
 646}
 647
 648static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
 649{
 650	struct etr_flat_buf *flat_buf = etr_buf->private;
 651	struct device *real_dev = flat_buf->dev->parent;
 652
 653	/*
 654	 * Adjust the buffer to point to the beginning of the trace data
 655	 * and update the available trace data.
 656	 */
 657	etr_buf->offset = rrp - etr_buf->hwaddr;
 658	if (etr_buf->full)
 659		etr_buf->len = etr_buf->size;
 660	else
 661		etr_buf->len = rwp - rrp;
 662
 663	/*
 664	 * The driver always starts tracing at the beginning of the buffer,
 665	 * the only reason why we would get a wrap around is when the buffer
 666	 * is full.  Sync the entire buffer in one go for this case.
 667	 */
 668	if (etr_buf->offset + etr_buf->len > etr_buf->size)
 669		dma_sync_single_for_cpu(real_dev, flat_buf->daddr,
 670					etr_buf->size, DMA_FROM_DEVICE);
 671	else
 672		dma_sync_single_for_cpu(real_dev,
 673					flat_buf->daddr + etr_buf->offset,
 674					etr_buf->len, DMA_FROM_DEVICE);
 675}
 676
 677static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
 678					 u64 offset, size_t len, char **bufpp)
 679{
 680	struct etr_flat_buf *flat_buf = etr_buf->private;
 681
 682	*bufpp = (char *)flat_buf->vaddr + offset;
 683	/*
 684	 * tmc_etr_buf_get_data already adjusts the length to handle
 685	 * buffer wrapping around.
 686	 */
 687	return len;
 688}
 689
 690static const struct etr_buf_operations etr_flat_buf_ops = {
 691	.alloc = tmc_etr_alloc_flat_buf,
 692	.free = tmc_etr_free_flat_buf,
 693	.sync = tmc_etr_sync_flat_buf,
 694	.get_data = tmc_etr_get_data_flat_buf,
 695};
 696
 697/*
 698 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
 699 * appropriately.
 700 */
 701static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
 702				struct etr_buf *etr_buf, int node,
 703				void **pages)
 704{
 705	struct etr_sg_table *etr_table;
 706	struct device *dev = &drvdata->csdev->dev;
 707
 708	etr_table = tmc_init_etr_sg_table(dev, node,
 709					  etr_buf->size, pages);
 710	if (IS_ERR(etr_table))
 711		return -ENOMEM;
 712	etr_buf->hwaddr = etr_table->hwaddr;
 713	etr_buf->mode = ETR_MODE_ETR_SG;
 714	etr_buf->private = etr_table;
 715	return 0;
 716}
 717
 718static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
 719{
 720	struct etr_sg_table *etr_table = etr_buf->private;
 721
 722	if (etr_table) {
 723		tmc_free_sg_table(etr_table->sg_table);
 724		kfree(etr_table);
 725	}
 726}
 727
 728static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
 729				       size_t len, char **bufpp)
 730{
 731	struct etr_sg_table *etr_table = etr_buf->private;
 732
 733	return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
 734}
 735
 736static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
 737{
 738	long r_offset, w_offset;
 739	struct etr_sg_table *etr_table = etr_buf->private;
 740	struct tmc_sg_table *table = etr_table->sg_table;
 741
 742	/* Convert hw address to offset in the buffer */
 743	r_offset = tmc_sg_get_data_page_offset(table, rrp);
 744	if (r_offset < 0) {
 745		dev_warn(table->dev,
 746			 "Unable to map RRP %llx to offset\n", rrp);
 747		etr_buf->len = 0;
 748		return;
 749	}
 750
 751	w_offset = tmc_sg_get_data_page_offset(table, rwp);
 752	if (w_offset < 0) {
 753		dev_warn(table->dev,
 754			 "Unable to map RWP %llx to offset\n", rwp);
 755		etr_buf->len = 0;
 756		return;
 757	}
 758
 759	etr_buf->offset = r_offset;
 760	if (etr_buf->full)
 761		etr_buf->len = etr_buf->size;
 762	else
 763		etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
 764				w_offset - r_offset;
 765	tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
 766}
 767
 768static const struct etr_buf_operations etr_sg_buf_ops = {
 769	.alloc = tmc_etr_alloc_sg_buf,
 770	.free = tmc_etr_free_sg_buf,
 771	.sync = tmc_etr_sync_sg_buf,
 772	.get_data = tmc_etr_get_data_sg_buf,
 773};
 774
 775/*
 776 * TMC ETR could be connected to a CATU device, which can provide address
 777 * translation service. This is represented by the Output port of the TMC
 778 * (ETR) connected to the input port of the CATU.
 779 *
 780 * Returns	: coresight_device ptr for the CATU device if a CATU is found.
 781 *		: NULL otherwise.
 782 */
 783struct coresight_device *
 784tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
 785{
 786	struct coresight_device *etr = drvdata->csdev;
 787	union coresight_dev_subtype catu_subtype = {
 788		.helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU
 789	};
 790
 791	if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
 792		return NULL;
 793
 794	return coresight_find_output_type(etr->pdata, CORESIGHT_DEV_TYPE_HELPER,
 795					  catu_subtype);
 796}
 797EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
 
 798
 799static const struct etr_buf_operations *etr_buf_ops[] = {
 800	[ETR_MODE_FLAT] = &etr_flat_buf_ops,
 801	[ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
 802	[ETR_MODE_CATU] = NULL,
 803};
 804
 805void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
 
 806{
 807	etr_buf_ops[ETR_MODE_CATU] = catu;
 
 
 
 
 808}
 809EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
 810
 811void tmc_etr_remove_catu_ops(void)
 812{
 813	etr_buf_ops[ETR_MODE_CATU] = NULL;
 
 
 
 814}
 815EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
 
 
 
 
 
 
 816
 817static inline int tmc_etr_mode_alloc_buf(int mode,
 818					 struct tmc_drvdata *drvdata,
 819					 struct etr_buf *etr_buf, int node,
 820					 void **pages)
 821{
 822	int rc = -EINVAL;
 823
 824	switch (mode) {
 825	case ETR_MODE_FLAT:
 826	case ETR_MODE_ETR_SG:
 827	case ETR_MODE_CATU:
 828		if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
 829			rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
 830						      node, pages);
 831		if (!rc)
 832			etr_buf->ops = etr_buf_ops[mode];
 833		return rc;
 834	default:
 835		return -EINVAL;
 836	}
 837}
 838
 839static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)
 840{
 841	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
 842
 843	buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent);
 844	buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
 845	buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);
 846}
 847
 848static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)
 849{
 850	bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;
 851
 852	return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;
 853}
 854
 855/*
 856 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
 857 * @drvdata	: ETR device details.
 858 * @size	: size of the requested buffer.
 859 * @flags	: Required properties for the buffer.
 860 * @node	: Node for memory allocations.
 861 * @pages	: An optional list of pages.
 862 */
 863static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
 864					 ssize_t size, int flags,
 865					 int node, void **pages)
 866{
 867	int rc = -ENOMEM;
 
 
 868	struct etr_buf *etr_buf;
 869	struct etr_buf_hw buf_hw;
 870	struct device *dev = &drvdata->csdev->dev;
 871
 872	get_etr_buf_hw(dev, &buf_hw);
 
 
 
 
 
 873	etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
 874	if (!etr_buf)
 875		return ERR_PTR(-ENOMEM);
 876
 877	etr_buf->size = size;
 878
 879	/* If there is user directive for buffer mode, try that first */
 880	if (drvdata->etr_mode != ETR_MODE_AUTO)
 881		rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata,
 882					    etr_buf, node, pages);
 883
 884	/*
 885	 * If we have to use an existing list of pages, we cannot reliably
 886	 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
 887	 * we use the contiguous DMA memory if at least one of the following
 888	 * conditions is true:
 889	 *  a) The ETR cannot use Scatter-Gather.
 890	 *  b) we have a backing IOMMU
 891	 *  c) The requested memory size is smaller (< 1M).
 892	 *
 893	 * Fallback to available mechanisms.
 894	 *
 895	 */
 896	if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size))
 
 897		rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
 898					    etr_buf, node, pages);
 899	if (rc && buf_hw.has_etr_sg)
 900		rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
 901					    etr_buf, node, pages);
 902	if (rc && buf_hw.has_catu)
 903		rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
 904					    etr_buf, node, pages);
 905	if (rc) {
 906		kfree(etr_buf);
 907		return ERR_PTR(rc);
 908	}
 909
 910	refcount_set(&etr_buf->refcount, 1);
 911	dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
 912		(unsigned long)size >> 10, etr_buf->mode);
 913	return etr_buf;
 914}
 915
 916static void tmc_free_etr_buf(struct etr_buf *etr_buf)
 917{
 918	WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
 919	etr_buf->ops->free(etr_buf);
 920	kfree(etr_buf);
 921}
 922
 923/*
 924 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
 925 * with a maximum of @len bytes.
 926 * Returns: The size of the linear data available @pos, with *bufpp
 927 * updated to point to the buffer.
 928 */
 929static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
 930				    u64 offset, size_t len, char **bufpp)
 931{
 932	/* Adjust the length to limit this transaction to end of buffer */
 933	len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
 934
 935	return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
 936}
 937
 938static inline s64
 939tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
 940{
 941	ssize_t len;
 942	char *bufp;
 943
 944	len = tmc_etr_buf_get_data(etr_buf, offset,
 945				   CORESIGHT_BARRIER_PKT_SIZE, &bufp);
 946	if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))
 947		return -EINVAL;
 948	coresight_insert_barrier_packet(bufp);
 949	return offset + CORESIGHT_BARRIER_PKT_SIZE;
 950}
 951
 952/*
 953 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
 954 * Makes sure the trace data is synced to the memory for consumption.
 955 * @etr_buf->offset will hold the offset to the beginning of the trace data
 956 * within the buffer, with @etr_buf->len bytes to consume.
 957 */
 958static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
 959{
 960	struct etr_buf *etr_buf = drvdata->etr_buf;
 961	u64 rrp, rwp;
 962	u32 status;
 963
 964	rrp = tmc_read_rrp(drvdata);
 965	rwp = tmc_read_rwp(drvdata);
 966	status = readl_relaxed(drvdata->base + TMC_STS);
 967
 968	/*
 969	 * If there were memory errors in the session, truncate the
 970	 * buffer.
 971	 */
 972	if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
 973		dev_dbg(&drvdata->csdev->dev,
 974			"tmc memory error detected, truncating buffer\n");
 975		etr_buf->len = 0;
 976		etr_buf->full = false;
 977		return;
 978	}
 979
 980	etr_buf->full = !!(status & TMC_STS_FULL);
 981
 982	WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
 983
 984	etr_buf->ops->sync(etr_buf, rrp, rwp);
 985}
 986
 987static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
 988{
 989	u32 axictl, sts;
 990	struct etr_buf *etr_buf = drvdata->etr_buf;
 991	int rc = 0;
 992
 993	CS_UNLOCK(drvdata->base);
 994
 995	/* Wait for TMCSReady bit to be set */
 996	rc = tmc_wait_for_tmcready(drvdata);
 997	if (rc) {
 998		dev_err(&drvdata->csdev->dev,
 999			"Failed to enable : TMC not ready\n");
1000		CS_LOCK(drvdata->base);
1001		return rc;
1002	}
1003
1004	writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
1005	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
1006
1007	axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
1008	axictl &= ~TMC_AXICTL_CLEAR_MASK;
1009	axictl |= TMC_AXICTL_PROT_CTL_B1;
1010	axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
1011	axictl |= TMC_AXICTL_AXCACHE_OS;
1012
1013	if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
1014		axictl &= ~TMC_AXICTL_ARCACHE_MASK;
1015		axictl |= TMC_AXICTL_ARCACHE_OS;
1016	}
1017
1018	if (etr_buf->mode == ETR_MODE_ETR_SG)
1019		axictl |= TMC_AXICTL_SCT_GAT_MODE;
1020
1021	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1022	tmc_write_dba(drvdata, etr_buf->hwaddr);
1023	/*
1024	 * If the TMC pointers must be programmed before the session,
1025	 * we have to set it properly (i.e, RRP/RWP to base address and
1026	 * STS to "not full").
1027	 */
1028	if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1029		tmc_write_rrp(drvdata, etr_buf->hwaddr);
1030		tmc_write_rwp(drvdata, etr_buf->hwaddr);
1031		sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1032		writel_relaxed(sts, drvdata->base + TMC_STS);
1033	}
1034
1035	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1036		       TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1037		       TMC_FFCR_TRIGON_TRIGIN,
1038		       drvdata->base + TMC_FFCR);
1039	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1040	tmc_enable_hw(drvdata);
1041
1042	CS_LOCK(drvdata->base);
1043	return rc;
1044}
1045
1046static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1047			     struct etr_buf *etr_buf)
1048{
1049	int rc;
1050
1051	/* Callers should provide an appropriate buffer for use */
1052	if (WARN_ON(!etr_buf))
1053		return -EINVAL;
1054
1055	if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1056	    WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1057		return -EINVAL;
1058
1059	if (WARN_ON(drvdata->etr_buf))
1060		return -EBUSY;
1061
1062	rc = coresight_claim_device(drvdata->csdev);
 
 
 
 
 
 
 
1063	if (!rc) {
1064		drvdata->etr_buf = etr_buf;
1065		rc = __tmc_etr_enable_hw(drvdata);
1066		if (rc) {
1067			drvdata->etr_buf = NULL;
1068			coresight_disclaim_device(drvdata->csdev);
1069		}
1070	}
1071
1072	return rc;
1073}
1074
1075/*
1076 * Return the available trace data in the buffer (starts at etr_buf->offset,
1077 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1078 * also updating the @bufpp on where to find it. Since the trace data
1079 * starts at anywhere in the buffer, depending on the RRP, we adjust the
1080 * @len returned to handle buffer wrapping around.
1081 *
1082 * We are protected here by drvdata->reading != 0, which ensures the
1083 * sysfs_buf stays alive.
1084 */
1085ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1086				loff_t pos, size_t len, char **bufpp)
1087{
1088	s64 offset;
1089	ssize_t actual = len;
1090	struct etr_buf *etr_buf = drvdata->sysfs_buf;
1091
1092	if (pos + actual > etr_buf->len)
1093		actual = etr_buf->len - pos;
1094	if (actual <= 0)
1095		return actual;
1096
1097	/* Compute the offset from which we read the data */
1098	offset = etr_buf->offset + pos;
1099	if (offset >= etr_buf->size)
1100		offset -= etr_buf->size;
1101	return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1102}
1103
1104static struct etr_buf *
1105tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1106{
1107	return tmc_alloc_etr_buf(drvdata, drvdata->size,
1108				 0, cpu_to_node(0), NULL);
1109}
1110
1111static void
1112tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1113{
1114	if (buf)
1115		tmc_free_etr_buf(buf);
1116}
1117
1118static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1119{
1120	struct etr_buf *etr_buf = drvdata->etr_buf;
1121
1122	if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1123		tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1124		drvdata->sysfs_buf = NULL;
1125	} else {
1126		tmc_sync_etr_buf(drvdata);
1127		/*
1128		 * Insert barrier packets at the beginning, if there was
1129		 * an overflow.
1130		 */
1131		if (etr_buf->full)
1132			tmc_etr_buf_insert_barrier_packet(etr_buf,
1133							  etr_buf->offset);
1134	}
1135}
1136
1137static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1138{
1139	CS_UNLOCK(drvdata->base);
1140
1141	tmc_flush_and_stop(drvdata);
1142	/*
1143	 * When operating in sysFS mode the content of the buffer needs to be
1144	 * read before the TMC is disabled.
1145	 */
1146	if (drvdata->mode == CS_MODE_SYSFS)
1147		tmc_etr_sync_sysfs_buf(drvdata);
1148
1149	tmc_disable_hw(drvdata);
1150
1151	CS_LOCK(drvdata->base);
1152
1153}
1154
1155void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1156{
1157	__tmc_etr_disable_hw(drvdata);
1158	coresight_disclaim_device(drvdata->csdev);
 
 
1159	/* Reset the ETR buf used by hardware */
1160	drvdata->etr_buf = NULL;
1161}
1162
1163static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1164{
1165	int ret = 0;
1166	unsigned long flags;
1167	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1168	struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1169
1170	/*
1171	 * If we are enabling the ETR from disabled state, we need to make
1172	 * sure we have a buffer with the right size. The etr_buf is not reset
1173	 * immediately after we stop the tracing in SYSFS mode as we wait for
1174	 * the user to collect the data. We may be able to reuse the existing
1175	 * buffer, provided the size matches. Any allocation has to be done
1176	 * with the lock released.
1177	 */
1178	spin_lock_irqsave(&drvdata->spinlock, flags);
1179	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1180	if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1181		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1182
1183		/* Allocate memory with the locks released */
1184		free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1185		if (IS_ERR(new_buf))
1186			return new_buf;
1187
1188		/* Let's try again */
1189		spin_lock_irqsave(&drvdata->spinlock, flags);
1190	}
1191
1192	if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1193		ret = -EBUSY;
1194		goto out;
1195	}
1196
1197	/*
 
 
 
 
 
 
 
 
 
 
1198	 * If we don't have a buffer or it doesn't match the requested size,
1199	 * use the buffer allocated above. Otherwise reuse the existing buffer.
1200	 */
1201	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1202	if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1203		free_buf = sysfs_buf;
1204		drvdata->sysfs_buf = new_buf;
1205	}
1206
 
 
 
 
 
1207out:
1208	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1209
1210	/* Free memory outside the spinlock if need be */
1211	if (free_buf)
1212		tmc_etr_free_sysfs_buf(free_buf);
1213	return ret ? ERR_PTR(ret) : drvdata->sysfs_buf;
1214}
1215
1216static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1217{
1218	int ret = 0;
1219	unsigned long flags;
1220	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1221	struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);
1222
1223	if (IS_ERR(sysfs_buf))
1224		return PTR_ERR(sysfs_buf);
1225
1226	spin_lock_irqsave(&drvdata->spinlock, flags);
1227
1228	/*
1229	 * In sysFS mode we can have multiple writers per sink.  Since this
1230	 * sink is already enabled no memory is needed and the HW need not be
1231	 * touched, even if the buffer size has changed.
1232	 */
1233	if (drvdata->mode == CS_MODE_SYSFS) {
1234		atomic_inc(&csdev->refcnt);
1235		goto out;
1236	}
1237
1238	ret = tmc_etr_enable_hw(drvdata, sysfs_buf);
1239	if (!ret) {
1240		drvdata->mode = CS_MODE_SYSFS;
1241		atomic_inc(&csdev->refcnt);
1242	}
1243
1244out:
1245	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1246
1247	if (!ret)
1248		dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1249
1250	return ret;
1251}
1252
1253struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
1254				   enum cs_mode mode, void *data)
1255{
1256	struct perf_output_handle *handle = data;
1257	struct etr_perf_buffer *etr_perf;
1258
1259	switch (mode) {
1260	case CS_MODE_SYSFS:
1261		return tmc_etr_get_sysfs_buffer(csdev);
1262	case CS_MODE_PERF:
1263		etr_perf = etm_perf_sink_config(handle);
1264		if (WARN_ON(!etr_perf || !etr_perf->etr_buf))
1265			return ERR_PTR(-EINVAL);
1266		return etr_perf->etr_buf;
1267	default:
1268		return ERR_PTR(-EINVAL);
1269	}
1270}
1271EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);
1272
1273/*
1274 * alloc_etr_buf: Allocate ETR buffer for use by perf.
1275 * The size of the hardware buffer is dependent on the size configured
1276 * via sysfs and the perf ring buffer size. We prefer to allocate the
1277 * largest possible size, scaling down the size by half until it
1278 * reaches a minimum limit (1M), beyond which we give up.
1279 */
1280static struct etr_buf *
1281alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1282	      int nr_pages, void **pages, bool snapshot)
1283{
1284	int node;
1285	struct etr_buf *etr_buf;
1286	unsigned long size;
1287
1288	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1289	/*
1290	 * Try to match the perf ring buffer size if it is larger
1291	 * than the size requested via sysfs.
1292	 */
1293	if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1294		etr_buf = tmc_alloc_etr_buf(drvdata, ((ssize_t)nr_pages << PAGE_SHIFT),
1295					    0, node, NULL);
1296		if (!IS_ERR(etr_buf))
1297			goto done;
1298	}
1299
1300	/*
1301	 * Else switch to configured size for this ETR
1302	 * and scale down until we hit the minimum limit.
1303	 */
1304	size = drvdata->size;
1305	do {
1306		etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1307		if (!IS_ERR(etr_buf))
1308			goto done;
1309		size /= 2;
1310	} while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1311
1312	return ERR_PTR(-ENOMEM);
1313
1314done:
1315	return etr_buf;
1316}
1317
1318static struct etr_buf *
1319get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1320			  struct perf_event *event, int nr_pages,
1321			  void **pages, bool snapshot)
1322{
1323	int ret;
1324	pid_t pid = task_pid_nr(event->owner);
1325	struct etr_buf *etr_buf;
1326
1327retry:
1328	/*
1329	 * An etr_perf_buffer is associated with an event and holds a reference
1330	 * to the AUX ring buffer that was created for that event.  In CPU-wide
1331	 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1332	 * buffer, share a sink.  As such an etr_perf_buffer is created for each
1333	 * event but a single etr_buf associated with the ETR is shared between
1334	 * them.  The last event in a trace session will copy the content of the
1335	 * etr_buf to its AUX ring buffer.  Ring buffer associated to other
1336	 * events are simply not used an freed as events are destoyed.  We still
1337	 * need to allocate a ring buffer for each event since we don't know
1338	 * which event will be last.
1339	 */
1340
1341	/*
1342	 * The first thing to do here is check if an etr_buf has already been
1343	 * allocated for this session.  If so it is shared with this event,
1344	 * otherwise it is created.
1345	 */
1346	mutex_lock(&drvdata->idr_mutex);
1347	etr_buf = idr_find(&drvdata->idr, pid);
1348	if (etr_buf) {
1349		refcount_inc(&etr_buf->refcount);
1350		mutex_unlock(&drvdata->idr_mutex);
1351		return etr_buf;
1352	}
1353
1354	/* If we made it here no buffer has been allocated, do so now. */
1355	mutex_unlock(&drvdata->idr_mutex);
1356
1357	etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1358	if (IS_ERR(etr_buf))
1359		return etr_buf;
1360
1361	/* Now that we have a buffer, add it to the IDR. */
1362	mutex_lock(&drvdata->idr_mutex);
1363	ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1364	mutex_unlock(&drvdata->idr_mutex);
1365
1366	/* Another event with this session ID has allocated this buffer. */
1367	if (ret == -ENOSPC) {
1368		tmc_free_etr_buf(etr_buf);
1369		goto retry;
1370	}
1371
1372	/* The IDR can't allocate room for a new session, abandon ship. */
1373	if (ret == -ENOMEM) {
1374		tmc_free_etr_buf(etr_buf);
1375		return ERR_PTR(ret);
1376	}
1377
1378
1379	return etr_buf;
1380}
1381
1382static struct etr_buf *
1383get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1384			    struct perf_event *event, int nr_pages,
1385			    void **pages, bool snapshot)
1386{
1387	/*
1388	 * In per-thread mode the etr_buf isn't shared, so just go ahead
1389	 * with memory allocation.
1390	 */
1391	return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1392}
1393
1394static struct etr_buf *
1395get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1396		 int nr_pages, void **pages, bool snapshot)
1397{
1398	if (event->cpu == -1)
1399		return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1400						   pages, snapshot);
1401
1402	return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1403					 pages, snapshot);
1404}
1405
1406static struct etr_perf_buffer *
1407tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1408		       int nr_pages, void **pages, bool snapshot)
1409{
1410	int node;
1411	struct etr_buf *etr_buf;
1412	struct etr_perf_buffer *etr_perf;
1413
1414	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1415
1416	etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1417	if (!etr_perf)
1418		return ERR_PTR(-ENOMEM);
1419
1420	etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1421	if (!IS_ERR(etr_buf))
1422		goto done;
1423
1424	kfree(etr_perf);
1425	return ERR_PTR(-ENOMEM);
1426
1427done:
1428	/*
1429	 * Keep a reference to the ETR this buffer has been allocated for
1430	 * in order to have access to the IDR in tmc_free_etr_buffer().
1431	 */
1432	etr_perf->drvdata = drvdata;
1433	etr_perf->etr_buf = etr_buf;
1434
1435	return etr_perf;
1436}
1437
1438
1439static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1440				  struct perf_event *event, void **pages,
1441				  int nr_pages, bool snapshot)
1442{
1443	struct etr_perf_buffer *etr_perf;
1444	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1445
1446	etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1447					  nr_pages, pages, snapshot);
1448	if (IS_ERR(etr_perf)) {
1449		dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1450		return NULL;
1451	}
1452
1453	etr_perf->pid = task_pid_nr(event->owner);
1454	etr_perf->snapshot = snapshot;
1455	etr_perf->nr_pages = nr_pages;
1456	etr_perf->pages = pages;
1457
1458	return etr_perf;
1459}
1460
1461static void tmc_free_etr_buffer(void *config)
1462{
1463	struct etr_perf_buffer *etr_perf = config;
1464	struct tmc_drvdata *drvdata = etr_perf->drvdata;
1465	struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1466
1467	if (!etr_buf)
1468		goto free_etr_perf_buffer;
1469
1470	mutex_lock(&drvdata->idr_mutex);
1471	/* If we are not the last one to use the buffer, don't touch it. */
1472	if (!refcount_dec_and_test(&etr_buf->refcount)) {
1473		mutex_unlock(&drvdata->idr_mutex);
1474		goto free_etr_perf_buffer;
1475	}
1476
1477	/* We are the last one, remove from the IDR and free the buffer. */
1478	buf = idr_remove(&drvdata->idr, etr_perf->pid);
1479	mutex_unlock(&drvdata->idr_mutex);
1480
1481	/*
1482	 * Something went very wrong if the buffer associated with this ID
1483	 * is not the same in the IDR.  Leak to avoid use after free.
1484	 */
1485	if (buf && WARN_ON(buf != etr_buf))
1486		goto free_etr_perf_buffer;
1487
1488	tmc_free_etr_buf(etr_perf->etr_buf);
1489
1490free_etr_perf_buffer:
1491	kfree(etr_perf);
1492}
1493
1494/*
1495 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1496 * buffer to the perf ring buffer.
1497 */
1498static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1499				     unsigned long head,
1500				     unsigned long src_offset,
1501				     unsigned long to_copy)
1502{
1503	long bytes;
1504	long pg_idx, pg_offset;
 
1505	char **dst_pages, *src_buf;
1506	struct etr_buf *etr_buf = etr_perf->etr_buf;
1507
1508	head = PERF_IDX2OFF(head, etr_perf);
1509	pg_idx = head >> PAGE_SHIFT;
1510	pg_offset = head & (PAGE_SIZE - 1);
1511	dst_pages = (char **)etr_perf->pages;
1512
1513	while (to_copy > 0) {
1514		/*
1515		 * In one iteration, we can copy minimum of :
1516		 *  1) what is available in the source buffer,
1517		 *  2) what is available in the source buffer, before it
1518		 *     wraps around.
1519		 *  3) what is available in the destination page.
1520		 * in one iteration.
1521		 */
1522		if (src_offset >= etr_buf->size)
1523			src_offset -= etr_buf->size;
1524		bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1525					     &src_buf);
1526		if (WARN_ON_ONCE(bytes <= 0))
1527			break;
1528		bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1529
1530		memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1531
1532		to_copy -= bytes;
1533
1534		/* Move destination pointers */
1535		pg_offset += bytes;
1536		if (pg_offset == PAGE_SIZE) {
1537			pg_offset = 0;
1538			if (++pg_idx == etr_perf->nr_pages)
1539				pg_idx = 0;
1540		}
1541
1542		/* Move source pointers */
1543		src_offset += bytes;
1544	}
1545}
1546
1547/*
1548 * tmc_update_etr_buffer : Update the perf ring buffer with the
1549 * available trace data. We use software double buffering at the moment.
1550 *
1551 * TODO: Add support for reusing the perf ring buffer.
1552 */
1553static unsigned long
1554tmc_update_etr_buffer(struct coresight_device *csdev,
1555		      struct perf_output_handle *handle,
1556		      void *config)
1557{
1558	bool lost = false;
1559	unsigned long flags, offset, size = 0;
1560	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1561	struct etr_perf_buffer *etr_perf = config;
1562	struct etr_buf *etr_buf = etr_perf->etr_buf;
1563
1564	spin_lock_irqsave(&drvdata->spinlock, flags);
1565
1566	/* Don't do anything if another tracer is using this sink */
1567	if (atomic_read(&csdev->refcnt) != 1) {
1568		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1569		goto out;
1570	}
1571
1572	if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1573		lost = true;
1574		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1575		goto out;
1576	}
1577
1578	CS_UNLOCK(drvdata->base);
1579
1580	tmc_flush_and_stop(drvdata);
1581	tmc_sync_etr_buf(drvdata);
1582
1583	CS_LOCK(drvdata->base);
1584	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1585
1586	lost = etr_buf->full;
1587	offset = etr_buf->offset;
1588	size = etr_buf->len;
1589
1590	/*
1591	 * The ETR buffer may be bigger than the space available in the
1592	 * perf ring buffer (handle->size).  If so advance the offset so that we
1593	 * get the latest trace data.  In snapshot mode none of that matters
1594	 * since we are expected to clobber stale data in favour of the latest
1595	 * traces.
1596	 */
1597	if (!etr_perf->snapshot && size > handle->size) {
1598		u32 mask = tmc_get_memwidth_mask(drvdata);
1599
1600		/*
1601		 * Make sure the new size is aligned in accordance with the
1602		 * requirement explained in function tmc_get_memwidth_mask().
1603		 */
1604		size = handle->size & mask;
1605		offset = etr_buf->offset + etr_buf->len - size;
1606
1607		if (offset >= etr_buf->size)
1608			offset -= etr_buf->size;
1609		lost = true;
1610	}
1611
1612	/* Insert barrier packets at the beginning, if there was an overflow */
1613	if (lost)
1614		tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1615	tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);
1616
1617	/*
1618	 * In snapshot mode we simply increment the head by the number of byte
1619	 * that were written.  User space will figure out how many bytes to get
1620	 * from the AUX buffer based on the position of the head.
 
1621	 */
1622	if (etr_perf->snapshot)
1623		handle->head += size;
1624
1625	/*
1626	 * Ensure that the AUX trace data is visible before the aux_head
1627	 * is updated via perf_aux_output_end(), as expected by the
1628	 * perf ring buffer.
1629	 */
1630	smp_wmb();
1631
1632out:
1633	/*
1634	 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1635	 * captured buffer is expected to be truncated and 2) a full buffer
1636	 * prevents the event from being re-enabled by the perf core,
1637	 * resulting in stale data being send to user space.
1638	 */
1639	if (!etr_perf->snapshot && lost)
1640		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1641	return size;
1642}
1643
1644static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1645{
1646	int rc = 0;
1647	pid_t pid;
1648	unsigned long flags;
1649	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1650	struct perf_output_handle *handle = data;
1651	struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1652
1653	spin_lock_irqsave(&drvdata->spinlock, flags);
1654	 /* Don't use this sink if it is already claimed by sysFS */
1655	if (drvdata->mode == CS_MODE_SYSFS) {
1656		rc = -EBUSY;
1657		goto unlock_out;
1658	}
1659
1660	if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1661		rc = -EINVAL;
1662		goto unlock_out;
1663	}
1664
1665	/* Get a handle on the pid of the process to monitor */
1666	pid = etr_perf->pid;
1667
1668	/* Do not proceed if this device is associated with another session */
1669	if (drvdata->pid != -1 && drvdata->pid != pid) {
1670		rc = -EBUSY;
1671		goto unlock_out;
1672	}
1673
 
 
1674	/*
1675	 * No HW configuration is needed if the sink is already in
1676	 * use for this session.
1677	 */
1678	if (drvdata->pid == pid) {
1679		atomic_inc(&csdev->refcnt);
1680		goto unlock_out;
1681	}
1682
1683	rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1684	if (!rc) {
1685		/* Associate with monitored process. */
1686		drvdata->pid = pid;
1687		drvdata->mode = CS_MODE_PERF;
1688		drvdata->perf_buf = etr_perf->etr_buf;
1689		atomic_inc(&csdev->refcnt);
1690	}
1691
1692unlock_out:
1693	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1694	return rc;
1695}
1696
1697static int tmc_enable_etr_sink(struct coresight_device *csdev,
1698			       enum cs_mode mode, void *data)
1699{
1700	switch (mode) {
1701	case CS_MODE_SYSFS:
1702		return tmc_enable_etr_sink_sysfs(csdev);
1703	case CS_MODE_PERF:
1704		return tmc_enable_etr_sink_perf(csdev, data);
1705	default:
1706		return -EINVAL;
1707	}
 
 
 
1708}
1709
1710static int tmc_disable_etr_sink(struct coresight_device *csdev)
1711{
1712	unsigned long flags;
1713	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1714
1715	spin_lock_irqsave(&drvdata->spinlock, flags);
1716
1717	if (drvdata->reading) {
1718		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1719		return -EBUSY;
1720	}
1721
1722	if (atomic_dec_return(&csdev->refcnt)) {
1723		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1724		return -EBUSY;
1725	}
1726
1727	/* Complain if we (somehow) got out of sync */
1728	WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1729	tmc_etr_disable_hw(drvdata);
1730	/* Dissociate from monitored process. */
1731	drvdata->pid = -1;
1732	drvdata->mode = CS_MODE_DISABLED;
1733	/* Reset perf specific data */
1734	drvdata->perf_buf = NULL;
1735
1736	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1737
1738	dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1739	return 0;
1740}
1741
1742static const struct coresight_ops_sink tmc_etr_sink_ops = {
1743	.enable		= tmc_enable_etr_sink,
1744	.disable	= tmc_disable_etr_sink,
1745	.alloc_buffer	= tmc_alloc_etr_buffer,
1746	.update_buffer	= tmc_update_etr_buffer,
1747	.free_buffer	= tmc_free_etr_buffer,
1748};
1749
1750const struct coresight_ops tmc_etr_cs_ops = {
1751	.sink_ops	= &tmc_etr_sink_ops,
1752};
1753
1754int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1755{
1756	int ret = 0;
1757	unsigned long flags;
1758
1759	/* config types are set a boot time and never change */
1760	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1761		return -EINVAL;
1762
1763	spin_lock_irqsave(&drvdata->spinlock, flags);
1764	if (drvdata->reading) {
1765		ret = -EBUSY;
1766		goto out;
1767	}
1768
1769	/*
1770	 * We can safely allow reads even if the ETR is operating in PERF mode,
1771	 * since the sysfs session is captured in mode specific data.
1772	 * If drvdata::sysfs_data is NULL the trace data has been read already.
1773	 */
1774	if (!drvdata->sysfs_buf) {
1775		ret = -EINVAL;
1776		goto out;
1777	}
1778
1779	/* Disable the TMC if we are trying to read from a running session. */
1780	if (drvdata->mode == CS_MODE_SYSFS)
1781		__tmc_etr_disable_hw(drvdata);
1782
1783	drvdata->reading = true;
1784out:
1785	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1786
1787	return ret;
1788}
1789
1790int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1791{
1792	unsigned long flags;
1793	struct etr_buf *sysfs_buf = NULL;
1794
1795	/* config types are set a boot time and never change */
1796	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1797		return -EINVAL;
1798
1799	spin_lock_irqsave(&drvdata->spinlock, flags);
1800
1801	/* RE-enable the TMC if need be */
1802	if (drvdata->mode == CS_MODE_SYSFS) {
1803		/*
1804		 * The trace run will continue with the same allocated trace
1805		 * buffer. Since the tracer is still enabled drvdata::buf can't
1806		 * be NULL.
1807		 */
1808		__tmc_etr_enable_hw(drvdata);
1809	} else {
1810		/*
1811		 * The ETR is not tracing and the buffer was just read.
1812		 * As such prepare to free the trace buffer.
1813		 */
1814		sysfs_buf = drvdata->sysfs_buf;
1815		drvdata->sysfs_buf = NULL;
1816	}
1817
1818	drvdata->reading = false;
1819	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1820
1821	/* Free allocated memory out side of the spinlock */
1822	if (sysfs_buf)
1823		tmc_etr_free_sysfs_buf(sysfs_buf);
1824
1825	return 0;
1826}
1827
1828static const char *const buf_modes_str[] = {
1829	[ETR_MODE_FLAT]		= "flat",
1830	[ETR_MODE_ETR_SG]	= "tmc-sg",
1831	[ETR_MODE_CATU]		= "catu",
1832	[ETR_MODE_AUTO]		= "auto",
1833};
1834
1835static ssize_t buf_modes_available_show(struct device *dev,
1836					    struct device_attribute *attr, char *buf)
1837{
1838	struct etr_buf_hw buf_hw;
1839	ssize_t size = 0;
1840
1841	get_etr_buf_hw(dev, &buf_hw);
1842	size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]);
1843	size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]);
1844	if (buf_hw.has_etr_sg)
1845		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]);
1846
1847	if (buf_hw.has_catu)
1848		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]);
1849
1850	size += sysfs_emit_at(buf, size, "\n");
1851	return size;
1852}
1853static DEVICE_ATTR_RO(buf_modes_available);
1854
1855static ssize_t buf_mode_preferred_show(struct device *dev,
1856					 struct device_attribute *attr, char *buf)
1857{
1858	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
1859
1860	return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]);
1861}
1862
1863static ssize_t buf_mode_preferred_store(struct device *dev,
1864					  struct device_attribute *attr,
1865					  const char *buf, size_t size)
1866{
1867	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
1868	struct etr_buf_hw buf_hw;
1869
1870	get_etr_buf_hw(dev, &buf_hw);
1871	if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT]))
1872		drvdata->etr_mode = ETR_MODE_FLAT;
1873	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)
1874		drvdata->etr_mode = ETR_MODE_ETR_SG;
1875	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)
1876		drvdata->etr_mode = ETR_MODE_CATU;
1877	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO]))
1878		drvdata->etr_mode = ETR_MODE_AUTO;
1879	else
1880		return -EINVAL;
1881	return size;
1882}
1883static DEVICE_ATTR_RW(buf_mode_preferred);
1884
1885static struct attribute *coresight_etr_attrs[] = {
1886	&dev_attr_buf_modes_available.attr,
1887	&dev_attr_buf_mode_preferred.attr,
1888	NULL,
1889};
1890
1891const struct attribute_group coresight_etr_group = {
1892	.attrs = coresight_etr_attrs,
1893};