1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Dynamic DMA mapping support.
   4 *
   5 * This implementation is a fallback for platforms that do not support
   6 * I/O TLBs (aka DMA address translation hardware).
   7 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
   8 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
   9 * Copyright (C) 2000, 2003 Hewlett-Packard Co
  10 *	David Mosberger-Tang <davidm@hpl.hp.com>
  11 *
  12 * 03/05/07 davidm	Switch from PCI-DMA to generic device DMA API.
  13 * 00/12/13 davidm	Rename to swiotlb.c and add mark_clean() to avoid
  14 *			unnecessary i-cache flushing.
  15 * 04/07/.. ak		Better overflow handling. Assorted fixes.
  16 * 05/09/10 linville	Add support for syncing ranges, support syncing for
  17 *			DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
  18 * 08/12/11 beckyb	Add highmem support
  19 */
  20
  21#define pr_fmt(fmt) "software IO TLB: " fmt
  22
  23#include <linux/cache.h>
  24#include <linux/cc_platform.h>
  25#include <linux/ctype.h>
  26#include <linux/debugfs.h>
  27#include <linux/dma-direct.h>
  28#include <linux/dma-map-ops.h>
  29#include <linux/export.h>
  30#include <linux/gfp.h>
  31#include <linux/highmem.h>
  32#include <linux/io.h>
  33#include <linux/iommu-helper.h>
  34#include <linux/init.h>
  35#include <linux/memblock.h>
  36#include <linux/mm.h>
  37#include <linux/pfn.h>
  38#include <linux/rculist.h>
  39#include <linux/scatterlist.h>
  40#include <linux/set_memory.h>
  41#include <linux/spinlock.h>
  42#include <linux/string.h>
  43#include <linux/swiotlb.h>
  44#include <linux/types.h>
  45#ifdef CONFIG_DMA_RESTRICTED_POOL
  46#include <linux/of.h>
  47#include <linux/of_fdt.h>
  48#include <linux/of_reserved_mem.h>
  49#include <linux/slab.h>
  50#endif
  51
  52#define CREATE_TRACE_POINTS
  53#include <trace/events/swiotlb.h>
  54
  55#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
  56
  57/*
  58 * Minimum IO TLB size to bother booting with.  Systems with mainly
  59 * 64bit capable cards will only lightly use the swiotlb.  If we can't
  60 * allocate a contiguous 1MB, we're probably in trouble anyway.
  61 */
  62#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
  63
  64#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
  65
  66/**
  67 * struct io_tlb_slot - IO TLB slot descriptor
  68 * @orig_addr:	The original address corresponding to a mapped entry.
  69 * @alloc_size:	Size of the allocated buffer.
  70 * @list:	The free list describing the number of free entries available
  71 *		from each index.
  72 * @pad_slots:	Number of preceding padding slots. Valid only in the first
  73 *		allocated non-padding slot.
  74 */
  75struct io_tlb_slot {
  76	phys_addr_t orig_addr;
  77	size_t alloc_size;
  78	unsigned short list;
  79	unsigned short pad_slots;
  80};
  81
  82static bool swiotlb_force_bounce;
  83static bool swiotlb_force_disable;
  84
  85#ifdef CONFIG_SWIOTLB_DYNAMIC
  86
  87static void swiotlb_dyn_alloc(struct work_struct *work);
  88
  89static struct io_tlb_mem io_tlb_default_mem = {
  90	.lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
  91	.pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
  92	.dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
  93					swiotlb_dyn_alloc),
  94};
  95
  96#else  /* !CONFIG_SWIOTLB_DYNAMIC */
  97
  98static struct io_tlb_mem io_tlb_default_mem;
  99
 100#endif	/* CONFIG_SWIOTLB_DYNAMIC */
 101
 102static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
 103static unsigned long default_nareas;
 104
 105/**
 106 * struct io_tlb_area - IO TLB memory area descriptor
 107 *
 108 * This is a single area with a single lock.
 109 *
 110 * @used:	The number of used IO TLB block.
 111 * @index:	The slot index to start searching in this area for next round.
 112 * @lock:	The lock to protect the above data structures in the map and
 113 *		unmap calls.
 114 */
 115struct io_tlb_area {
 116	unsigned long used;
 117	unsigned int index;
 118	spinlock_t lock;
 119};
 120
 121/*
 122 * Round up number of slabs to the next power of 2. The last area is going
 123 * be smaller than the rest if default_nslabs is not power of two.
 124 * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
 125 * otherwise a segment may span two or more areas. It conflicts with free
 126 * contiguous slots tracking: free slots are treated contiguous no matter
 127 * whether they cross an area boundary.
 128 *
 129 * Return true if default_nslabs is rounded up.
 130 */
 131static bool round_up_default_nslabs(void)
 132{
 133	if (!default_nareas)
 134		return false;
 135
 136	if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
 137		default_nslabs = IO_TLB_SEGSIZE * default_nareas;
 138	else if (is_power_of_2(default_nslabs))
 139		return false;
 140	default_nslabs = roundup_pow_of_two(default_nslabs);
 141	return true;
 142}
 143
 144/**
 145 * swiotlb_adjust_nareas() - adjust the number of areas and slots
 146 * @nareas:	Desired number of areas. Zero is treated as 1.
 147 *
 148 * Adjust the default number of areas in a memory pool.
 149 * The default size of the memory pool may also change to meet minimum area
 150 * size requirements.
 151 */
 152static void swiotlb_adjust_nareas(unsigned int nareas)
 153{
 154	if (!nareas)
 155		nareas = 1;
 156	else if (!is_power_of_2(nareas))
 157		nareas = roundup_pow_of_two(nareas);
 158
 159	default_nareas = nareas;
 160
 161	pr_info("area num %d.\n", nareas);
 162	if (round_up_default_nslabs())
 163		pr_info("SWIOTLB bounce buffer size roundup to %luMB",
 164			(default_nslabs << IO_TLB_SHIFT) >> 20);
 165}
 166
 167/**
 168 * limit_nareas() - get the maximum number of areas for a given memory pool size
 169 * @nareas:	Desired number of areas.
 170 * @nslots:	Total number of slots in the memory pool.
 171 *
 172 * Limit the number of areas to the maximum possible number of areas in
 173 * a memory pool of the given size.
 174 *
 175 * Return: Maximum possible number of areas.
 176 */
 177static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
 178{
 179	if (nslots < nareas * IO_TLB_SEGSIZE)
 180		return nslots / IO_TLB_SEGSIZE;
 181	return nareas;
 182}
 183
 184static int __init
 185setup_io_tlb_npages(char *str)
 186{
 187	if (isdigit(*str)) {
 188		/* avoid tail segment of size < IO_TLB_SEGSIZE */
 189		default_nslabs =
 190			ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
 191	}
 192	if (*str == ',')
 193		++str;
 194	if (isdigit(*str))
 195		swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));
 196	if (*str == ',')
 197		++str;
 198	if (!strcmp(str, "force"))
 199		swiotlb_force_bounce = true;
 200	else if (!strcmp(str, "noforce"))
 201		swiotlb_force_disable = true;
 202
 203	return 0;
 204}
 205early_param("swiotlb", setup_io_tlb_npages);
 206
 207unsigned long swiotlb_size_or_default(void)
 208{
 209	return default_nslabs << IO_TLB_SHIFT;
 210}
 211
 212void __init swiotlb_adjust_size(unsigned long size)
 213{
 214	/*
 215	 * If swiotlb parameter has not been specified, give a chance to
 216	 * architectures such as those supporting memory encryption to
 217	 * adjust/expand SWIOTLB size for their use.
 218	 */
 219	if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
 220		return;
 221
 222	size = ALIGN(size, IO_TLB_SIZE);
 223	default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
 224	if (round_up_default_nslabs())
 225		size = default_nslabs << IO_TLB_SHIFT;
 226	pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
 227}
 228
 229void swiotlb_print_info(void)
 230{
 231	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 232
 233	if (!mem->nslabs) {
 234		pr_warn("No low mem\n");
 235		return;
 236	}
 237
 238	pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
 239	       (mem->nslabs << IO_TLB_SHIFT) >> 20);
 240}
 241
 242static inline unsigned long io_tlb_offset(unsigned long val)
 243{
 244	return val & (IO_TLB_SEGSIZE - 1);
 245}
 246
 247static inline unsigned long nr_slots(u64 val)
 248{
 249	return DIV_ROUND_UP(val, IO_TLB_SIZE);
 250}
 251
 252/*
 253 * Early SWIOTLB allocation may be too early to allow an architecture to
 254 * perform the desired operations.  This function allows the architecture to
 255 * call SWIOTLB when the operations are possible.  It needs to be called
 256 * before the SWIOTLB memory is used.
 257 */
 258void __init swiotlb_update_mem_attributes(void)
 259{
 260	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 261	unsigned long bytes;
 262
 263	if (!mem->nslabs || mem->late_alloc)
 264		return;
 265	bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
 266	set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
 267}
 268
 269static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
 270		unsigned long nslabs, bool late_alloc, unsigned int nareas)
 271{
 272	void *vaddr = phys_to_virt(start);
 273	unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
 274
 275	mem->nslabs = nslabs;
 276	mem->start = start;
 277	mem->end = mem->start + bytes;
 278	mem->late_alloc = late_alloc;
 279	mem->nareas = nareas;
 280	mem->area_nslabs = nslabs / mem->nareas;
 281
 282	for (i = 0; i < mem->nareas; i++) {
 283		spin_lock_init(&mem->areas[i].lock);
 284		mem->areas[i].index = 0;
 285		mem->areas[i].used = 0;
 286	}
 287
 288	for (i = 0; i < mem->nslabs; i++) {
 289		mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),
 290					 mem->nslabs - i);
 291		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
 292		mem->slots[i].alloc_size = 0;
 293		mem->slots[i].pad_slots = 0;
 294	}
 295
 296	memset(vaddr, 0, bytes);
 297	mem->vaddr = vaddr;
 298	return;
 299}
 300
 301/**
 302 * add_mem_pool() - add a memory pool to the allocator
 303 * @mem:	Software IO TLB allocator.
 304 * @pool:	Memory pool to be added.
 305 */
 306static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
 307{
 308#ifdef CONFIG_SWIOTLB_DYNAMIC
 309	spin_lock(&mem->lock);
 310	list_add_rcu(&pool->node, &mem->pools);
 311	mem->nslabs += pool->nslabs;
 312	spin_unlock(&mem->lock);
 313#else
 314	mem->nslabs = pool->nslabs;
 315#endif
 316}
 317
 318static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
 319		unsigned int flags,
 320		int (*remap)(void *tlb, unsigned long nslabs))
 321{
 322	size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
 323	void *tlb;
 324
 325	/*
 326	 * By default allocate the bounce buffer memory from low memory, but
 327	 * allow to pick a location everywhere for hypervisors with guest
 328	 * memory encryption.
 329	 */
 330	if (flags & SWIOTLB_ANY)
 331		tlb = memblock_alloc(bytes, PAGE_SIZE);
 332	else
 333		tlb = memblock_alloc_low(bytes, PAGE_SIZE);
 334
 335	if (!tlb) {
 336		pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
 337			__func__, bytes);
 338		return NULL;
 339	}
 340
 341	if (remap && remap(tlb, nslabs) < 0) {
 342		memblock_free(tlb, PAGE_ALIGN(bytes));
 343		pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
 344		return NULL;
 345	}
 346
 347	return tlb;
 348}
 349
 350/*
 351 * Statically reserve bounce buffer space and initialize bounce buffer data
 352 * structures for the software IO TLB used to implement the DMA API.
 353 */
 354void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
 355		int (*remap)(void *tlb, unsigned long nslabs))
 356{
 357	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 358	unsigned long nslabs;
 359	unsigned int nareas;
 360	size_t alloc_size;
 361	void *tlb;
 362
 363	if (!addressing_limit && !swiotlb_force_bounce)
 364		return;
 365	if (swiotlb_force_disable)
 366		return;
 367
 368	io_tlb_default_mem.force_bounce =
 369		swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
 370
 371#ifdef CONFIG_SWIOTLB_DYNAMIC
 372	if (!remap)
 373		io_tlb_default_mem.can_grow = true;
 374	if (flags & SWIOTLB_ANY)
 375		io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
 376	else
 377		io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
 378#endif
 379
 380	if (!default_nareas)
 381		swiotlb_adjust_nareas(num_possible_cpus());
 382
 383	nslabs = default_nslabs;
 384	nareas = limit_nareas(default_nareas, nslabs);
 385	while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
 386		if (nslabs <= IO_TLB_MIN_SLABS)
 387			return;
 388		nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
 389		nareas = limit_nareas(nareas, nslabs);
 390	}
 391
 392	if (default_nslabs != nslabs) {
 393		pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
 394			default_nslabs, nslabs);
 395		default_nslabs = nslabs;
 396	}
 397
 398	alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
 399	mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
 400	if (!mem->slots) {
 401		pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
 402			__func__, alloc_size, PAGE_SIZE);
 403		return;
 404	}
 405
 406	mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
 407		nareas), SMP_CACHE_BYTES);
 408	if (!mem->areas) {
 409		pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
 410		return;
 411	}
 412
 413	swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas);
 414	add_mem_pool(&io_tlb_default_mem, mem);
 415
 416	if (flags & SWIOTLB_VERBOSE)
 417		swiotlb_print_info();
 418}
 419
 420void __init swiotlb_init(bool addressing_limit, unsigned int flags)
 421{
 422	swiotlb_init_remap(addressing_limit, flags, NULL);
 423}
 424
 425/*
 426 * Systems with larger DMA zones (those that don't support ISA) can
 427 * initialize the swiotlb later using the slab allocator if needed.
 428 * This should be just like above, but with some error catching.
 429 */
 430int swiotlb_init_late(size_t size, gfp_t gfp_mask,
 431		int (*remap)(void *tlb, unsigned long nslabs))
 432{
 433	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 434	unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
 435	unsigned int nareas;
 436	unsigned char *vstart = NULL;
 437	unsigned int order, area_order;
 438	bool retried = false;
 439	int rc = 0;
 440
 441	if (io_tlb_default_mem.nslabs)
 442		return 0;
 443
 444	if (swiotlb_force_disable)
 445		return 0;
 446
 447	io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
 448
 449#ifdef CONFIG_SWIOTLB_DYNAMIC
 450	if (!remap)
 451		io_tlb_default_mem.can_grow = true;
 452	if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
 453		io_tlb_default_mem.phys_limit = DMA_BIT_MASK(zone_dma_bits);
 454	else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
 455		io_tlb_default_mem.phys_limit = DMA_BIT_MASK(32);
 456	else
 457		io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
 458#endif
 459
 460	if (!default_nareas)
 461		swiotlb_adjust_nareas(num_possible_cpus());
 462
 463retry:
 464	order = get_order(nslabs << IO_TLB_SHIFT);
 465	nslabs = SLABS_PER_PAGE << order;
 466
 467	while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
 468		vstart = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
 469						  order);
 470		if (vstart)
 471			break;
 472		order--;
 473		nslabs = SLABS_PER_PAGE << order;
 474		retried = true;
 475	}
 476
 477	if (!vstart)
 478		return -ENOMEM;
 479
 480	if (remap)
 481		rc = remap(vstart, nslabs);
 482	if (rc) {
 483		free_pages((unsigned long)vstart, order);
 484
 485		nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
 486		if (nslabs < IO_TLB_MIN_SLABS)
 487			return rc;
 488		retried = true;
 489		goto retry;
 490	}
 491
 492	if (retried) {
 493		pr_warn("only able to allocate %ld MB\n",
 494			(PAGE_SIZE << order) >> 20);
 495	}
 496
 497	nareas = limit_nareas(default_nareas, nslabs);
 498	area_order = get_order(array_size(sizeof(*mem->areas), nareas));
 499	mem->areas = (struct io_tlb_area *)
 500		__get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
 501	if (!mem->areas)
 502		goto error_area;
 503
 504	mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
 505		get_order(array_size(sizeof(*mem->slots), nslabs)));
 506	if (!mem->slots)
 507		goto error_slots;
 508
 509	set_memory_decrypted((unsigned long)vstart,
 510			     (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
 511	swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
 512				 nareas);
 513	add_mem_pool(&io_tlb_default_mem, mem);
 514
 515	swiotlb_print_info();
 516	return 0;
 517
 518error_slots:
 519	free_pages((unsigned long)mem->areas, area_order);
 520error_area:
 521	free_pages((unsigned long)vstart, order);
 522	return -ENOMEM;
 523}
 524
 525void __init swiotlb_exit(void)
 526{
 527	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 528	unsigned long tbl_vaddr;
 529	size_t tbl_size, slots_size;
 530	unsigned int area_order;
 531
 532	if (swiotlb_force_bounce)
 533		return;
 534
 535	if (!mem->nslabs)
 536		return;
 537
 538	pr_info("tearing down default memory pool\n");
 539	tbl_vaddr = (unsigned long)phys_to_virt(mem->start);
 540	tbl_size = PAGE_ALIGN(mem->end - mem->start);
 541	slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
 542
 543	set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
 544	if (mem->late_alloc) {
 545		area_order = get_order(array_size(sizeof(*mem->areas),
 546			mem->nareas));
 547		free_pages((unsigned long)mem->areas, area_order);
 548		free_pages(tbl_vaddr, get_order(tbl_size));
 549		free_pages((unsigned long)mem->slots, get_order(slots_size));
 550	} else {
 551		memblock_free_late(__pa(mem->areas),
 552			array_size(sizeof(*mem->areas), mem->nareas));
 553		memblock_free_late(mem->start, tbl_size);
 554		memblock_free_late(__pa(mem->slots), slots_size);
 555	}
 556
 557	memset(mem, 0, sizeof(*mem));
 558}
 559
 560#ifdef CONFIG_SWIOTLB_DYNAMIC
 561
 562/**
 563 * alloc_dma_pages() - allocate pages to be used for DMA
 564 * @gfp:	GFP flags for the allocation.
 565 * @bytes:	Size of the buffer.
 566 * @phys_limit:	Maximum allowed physical address of the buffer.
 567 *
 568 * Allocate pages from the buddy allocator. If successful, make the allocated
 569 * pages decrypted that they can be used for DMA.
 570 *
 571 * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
 572 * if the allocated physical address was above @phys_limit.
 573 */
 574static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
 575{
 576	unsigned int order = get_order(bytes);
 577	struct page *page;
 578	phys_addr_t paddr;
 579	void *vaddr;
 580
 581	page = alloc_pages(gfp, order);
 582	if (!page)
 583		return NULL;
 584
 585	paddr = page_to_phys(page);
 586	if (paddr + bytes - 1 > phys_limit) {
 587		__free_pages(page, order);
 588		return ERR_PTR(-EAGAIN);
 589	}
 590
 591	vaddr = phys_to_virt(paddr);
 592	if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
 593		goto error;
 594	return page;
 595
 596error:
 597	/* Intentional leak if pages cannot be encrypted again. */
 598	if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
 599		__free_pages(page, order);
 600	return NULL;
 601}
 602
 603/**
 604 * swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
 605 * @dev:	Device for which a memory pool is allocated.
 606 * @bytes:	Size of the buffer.
 607 * @phys_limit:	Maximum allowed physical address of the buffer.
 608 * @gfp:	GFP flags for the allocation.
 609 *
 610 * Return: Allocated pages, or %NULL on allocation failure.
 611 */
 612static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
 613		u64 phys_limit, gfp_t gfp)
 614{
 615	struct page *page;
 616
 617	/*
 618	 * Allocate from the atomic pools if memory is encrypted and
 619	 * the allocation is atomic, because decrypting may block.
 620	 */
 621	if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
 622		void *vaddr;
 623
 624		if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
 625			return NULL;
 626
 627		return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
 628					   dma_coherent_ok);
 629	}
 630
 631	gfp &= ~GFP_ZONEMASK;
 632	if (phys_limit <= DMA_BIT_MASK(zone_dma_bits))
 633		gfp |= __GFP_DMA;
 634	else if (phys_limit <= DMA_BIT_MASK(32))
 635		gfp |= __GFP_DMA32;
 636
 637	while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
 638		if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
 639		    phys_limit < DMA_BIT_MASK(64) &&
 640		    !(gfp & (__GFP_DMA32 | __GFP_DMA)))
 641			gfp |= __GFP_DMA32;
 642		else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
 643			 !(gfp & __GFP_DMA))
 644			gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
 645		else
 646			return NULL;
 647	}
 648
 649	return page;
 650}
 651
 652/**
 653 * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
 654 * @vaddr:	Virtual address of the buffer.
 655 * @bytes:	Size of the buffer.
 656 */
 657static void swiotlb_free_tlb(void *vaddr, size_t bytes)
 658{
 659	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
 660	    dma_free_from_pool(NULL, vaddr, bytes))
 661		return;
 662
 663	/* Intentional leak if pages cannot be encrypted again. */
 664	if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
 665		__free_pages(virt_to_page(vaddr), get_order(bytes));
 666}
 667
 668/**
 669 * swiotlb_alloc_pool() - allocate a new IO TLB memory pool
 670 * @dev:	Device for which a memory pool is allocated.
 671 * @minslabs:	Minimum number of slabs.
 672 * @nslabs:	Desired (maximum) number of slabs.
 673 * @nareas:	Number of areas.
 674 * @phys_limit:	Maximum DMA buffer physical address.
 675 * @gfp:	GFP flags for the allocations.
 676 *
 677 * Allocate and initialize a new IO TLB memory pool. The actual number of
 678 * slabs may be reduced if allocation of @nslabs fails. If even
 679 * @minslabs cannot be allocated, this function fails.
 680 *
 681 * Return: New memory pool, or %NULL on allocation failure.
 682 */
 683static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
 684		unsigned long minslabs, unsigned long nslabs,
 685		unsigned int nareas, u64 phys_limit, gfp_t gfp)
 686{
 687	struct io_tlb_pool *pool;
 688	unsigned int slot_order;
 689	struct page *tlb;
 690	size_t pool_size;
 691	size_t tlb_size;
 692
 693	if (nslabs > SLABS_PER_PAGE << MAX_PAGE_ORDER) {
 694		nslabs = SLABS_PER_PAGE << MAX_PAGE_ORDER;
 695		nareas = limit_nareas(nareas, nslabs);
 696	}
 697
 698	pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
 699	pool = kzalloc(pool_size, gfp);
 700	if (!pool)
 701		goto error;
 702	pool->areas = (void *)pool + sizeof(*pool);
 703
 704	tlb_size = nslabs << IO_TLB_SHIFT;
 705	while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
 706		if (nslabs <= minslabs)
 707			goto error_tlb;
 708		nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
 709		nareas = limit_nareas(nareas, nslabs);
 710		tlb_size = nslabs << IO_TLB_SHIFT;
 711	}
 712
 713	slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
 714	pool->slots = (struct io_tlb_slot *)
 715		__get_free_pages(gfp, slot_order);
 716	if (!pool->slots)
 717		goto error_slots;
 718
 719	swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas);
 720	return pool;
 721
 722error_slots:
 723	swiotlb_free_tlb(page_address(tlb), tlb_size);
 724error_tlb:
 725	kfree(pool);
 726error:
 727	return NULL;
 728}
 729
 730/**
 731 * swiotlb_dyn_alloc() - dynamic memory pool allocation worker
 732 * @work:	Pointer to dyn_alloc in struct io_tlb_mem.
 733 */
 734static void swiotlb_dyn_alloc(struct work_struct *work)
 735{
 736	struct io_tlb_mem *mem =
 737		container_of(work, struct io_tlb_mem, dyn_alloc);
 738	struct io_tlb_pool *pool;
 739
 740	pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
 741				  default_nareas, mem->phys_limit, GFP_KERNEL);
 742	if (!pool) {
 743		pr_warn_ratelimited("Failed to allocate new pool");
 744		return;
 745	}
 746
 747	add_mem_pool(mem, pool);
 748}
 749
 750/**
 751 * swiotlb_dyn_free() - RCU callback to free a memory pool
 752 * @rcu:	RCU head in the corresponding struct io_tlb_pool.
 753 */
 754static void swiotlb_dyn_free(struct rcu_head *rcu)
 755{
 756	struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
 757	size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
 758	size_t tlb_size = pool->end - pool->start;
 759
 760	free_pages((unsigned long)pool->slots, get_order(slots_size));
 761	swiotlb_free_tlb(pool->vaddr, tlb_size);
 762	kfree(pool);
 763}
 764
 765/**
 766 * swiotlb_find_pool() - find the IO TLB pool for a physical address
 767 * @dev:        Device which has mapped the DMA buffer.
 768 * @paddr:      Physical address within the DMA buffer.
 769 *
 770 * Find the IO TLB memory pool descriptor which contains the given physical
 771 * address, if any.
 
 
 772 *
 773 * Return: Memory pool which contains @paddr, or %NULL if none.
 774 */
 775struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
 776{
 777	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
 778	struct io_tlb_pool *pool;
 779
 780	rcu_read_lock();
 781	list_for_each_entry_rcu(pool, &mem->pools, node) {
 782		if (paddr >= pool->start && paddr < pool->end)
 783			goto out;
 784	}
 785
 786	list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
 787		if (paddr >= pool->start && paddr < pool->end)
 788			goto out;
 789	}
 790	pool = NULL;
 791out:
 792	rcu_read_unlock();
 793	return pool;
 794}
 795
 796/**
 797 * swiotlb_del_pool() - remove an IO TLB pool from a device
 798 * @dev:	Owning device.
 799 * @pool:	Memory pool to be removed.
 800 */
 801static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
 802{
 803	unsigned long flags;
 804
 805	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
 806	list_del_rcu(&pool->node);
 807	spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
 808
 809	call_rcu(&pool->rcu, swiotlb_dyn_free);
 810}
 811
 812#endif	/* CONFIG_SWIOTLB_DYNAMIC */
 813
 814/**
 815 * swiotlb_dev_init() - initialize swiotlb fields in &struct device
 816 * @dev:	Device to be initialized.
 817 */
 818void swiotlb_dev_init(struct device *dev)
 819{
 820	dev->dma_io_tlb_mem = &io_tlb_default_mem;
 821#ifdef CONFIG_SWIOTLB_DYNAMIC
 822	INIT_LIST_HEAD(&dev->dma_io_tlb_pools);
 823	spin_lock_init(&dev->dma_io_tlb_lock);
 824	dev->dma_uses_io_tlb = false;
 825#endif
 826}
 827
 828/**
 829 * swiotlb_align_offset() - Get required offset into an IO TLB allocation.
 830 * @dev:         Owning device.
 831 * @align_mask:  Allocation alignment mask.
 832 * @addr:        DMA address.
 833 *
 834 * Return the minimum offset from the start of an IO TLB allocation which is
 835 * required for a given buffer address and allocation alignment to keep the
 836 * device happy.
 837 *
 838 * First, the address bits covered by min_align_mask must be identical in the
 839 * original address and the bounce buffer address. High bits are preserved by
 840 * choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
 841 * padding bytes before the bounce buffer.
 842 *
 843 * Second, @align_mask specifies which bits of the first allocated slot must
 844 * be zero. This may require allocating additional padding slots, and then the
 845 * offset (in bytes) from the first such padding slot is returned.
 846 */
 847static unsigned int swiotlb_align_offset(struct device *dev,
 848					 unsigned int align_mask, u64 addr)
 849{
 850	return addr & dma_get_min_align_mask(dev) &
 851		(align_mask | (IO_TLB_SIZE - 1));
 852}
 853
 854/*
 855 * Bounce: copy the swiotlb buffer from or back to the original dma location
 856 */
 857static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
 858			   enum dma_data_direction dir)
 859{
 860	struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
 861	int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
 862	phys_addr_t orig_addr = mem->slots[index].orig_addr;
 863	size_t alloc_size = mem->slots[index].alloc_size;
 864	unsigned long pfn = PFN_DOWN(orig_addr);
 865	unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
 866	int tlb_offset;
 867
 868	if (orig_addr == INVALID_PHYS_ADDR)
 869		return;
 870
 871	/*
 872	 * It's valid for tlb_offset to be negative. This can happen when the
 873	 * "offset" returned by swiotlb_align_offset() is non-zero, and the
 874	 * tlb_addr is pointing within the first "offset" bytes of the second
 875	 * or subsequent slots of the allocated swiotlb area. While it's not
 876	 * valid for tlb_addr to be pointing within the first "offset" bytes
 877	 * of the first slot, there's no way to check for such an error since
 878	 * this function can't distinguish the first slot from the second and
 879	 * subsequent slots.
 880	 */
 881	tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
 882		     swiotlb_align_offset(dev, 0, orig_addr);
 883
 884	orig_addr += tlb_offset;
 885	alloc_size -= tlb_offset;
 886
 887	if (size > alloc_size) {
 888		dev_WARN_ONCE(dev, 1,
 889			"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
 890			alloc_size, size);
 891		size = alloc_size;
 892	}
 893
 894	if (PageHighMem(pfn_to_page(pfn))) {
 895		unsigned int offset = orig_addr & ~PAGE_MASK;
 896		struct page *page;
 897		unsigned int sz = 0;
 898		unsigned long flags;
 899
 900		while (size) {
 901			sz = min_t(size_t, PAGE_SIZE - offset, size);
 902
 903			local_irq_save(flags);
 904			page = pfn_to_page(pfn);
 905			if (dir == DMA_TO_DEVICE)
 906				memcpy_from_page(vaddr, page, offset, sz);
 907			else
 908				memcpy_to_page(page, offset, vaddr, sz);
 909			local_irq_restore(flags);
 910
 911			size -= sz;
 912			pfn++;
 913			vaddr += sz;
 914			offset = 0;
 915		}
 916	} else if (dir == DMA_TO_DEVICE) {
 917		memcpy(vaddr, phys_to_virt(orig_addr), size);
 918	} else {
 919		memcpy(phys_to_virt(orig_addr), vaddr, size);
 920	}
 921}
 922
 923static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
 924{
 925	return start + (idx << IO_TLB_SHIFT);
 926}
 927
 928/*
 929 * Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
 930 */
 931static inline unsigned long get_max_slots(unsigned long boundary_mask)
 932{
 933	return (boundary_mask >> IO_TLB_SHIFT) + 1;
 934}
 935
 936static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
 937{
 938	if (index >= mem->area_nslabs)
 939		return 0;
 940	return index;
 941}
 942
 943/*
 944 * Track the total used slots with a global atomic value in order to have
 945 * correct information to determine the high water mark. The mem_used()
 946 * function gives imprecise results because there's no locking across
 947 * multiple areas.
 948 */
 949#ifdef CONFIG_DEBUG_FS
 950static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
 951{
 952	unsigned long old_hiwater, new_used;
 953
 954	new_used = atomic_long_add_return(nslots, &mem->total_used);
 955	old_hiwater = atomic_long_read(&mem->used_hiwater);
 956	do {
 957		if (new_used <= old_hiwater)
 958			break;
 959	} while (!atomic_long_try_cmpxchg(&mem->used_hiwater,
 960					  &old_hiwater, new_used));
 961}
 962
 963static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
 964{
 965	atomic_long_sub(nslots, &mem->total_used);
 966}
 967
 968#else /* !CONFIG_DEBUG_FS */
 969static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
 970{
 971}
 972static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
 973{
 974}
 975#endif /* CONFIG_DEBUG_FS */
 976
 977#ifdef CONFIG_SWIOTLB_DYNAMIC
 978#ifdef CONFIG_DEBUG_FS
 979static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
 980{
 981	atomic_long_add(nslots, &mem->transient_nslabs);
 982}
 983
 984static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
 985{
 986	atomic_long_sub(nslots, &mem->transient_nslabs);
 987}
 988
 989#else /* !CONFIG_DEBUG_FS */
 990static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
 991{
 992}
 993static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
 994{
 995}
 996#endif /* CONFIG_DEBUG_FS */
 997#endif /* CONFIG_SWIOTLB_DYNAMIC */
 998
 999/**
1000 * swiotlb_search_pool_area() - search one memory area in one pool
1001 * @dev:	Device which maps the buffer.
1002 * @pool:	Memory pool to be searched.
1003 * @area_index:	Index of the IO TLB memory area to be searched.
1004 * @orig_addr:	Original (non-bounced) IO buffer address.
1005 * @alloc_size: Total requested size of the bounce buffer,
1006 *		including initial alignment padding.
1007 * @alloc_align_mask:	Required alignment of the allocated buffer.
1008 *
1009 * Find a suitable sequence of IO TLB entries for the request and allocate
1010 * a buffer from the given IO TLB memory area.
1011 * This function takes care of locking.
1012 *
1013 * Return: Index of the first allocated slot, or -1 on error.
1014 */
1015static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
1016		int area_index, phys_addr_t orig_addr, size_t alloc_size,
1017		unsigned int alloc_align_mask)
1018{
1019	struct io_tlb_area *area = pool->areas + area_index;
1020	unsigned long boundary_mask = dma_get_seg_boundary(dev);
1021	dma_addr_t tbl_dma_addr =
1022		phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
1023	unsigned long max_slots = get_max_slots(boundary_mask);
1024	unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
1025	unsigned int nslots = nr_slots(alloc_size), stride;
1026	unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);
1027	unsigned int index, slots_checked, count = 0, i;
1028	unsigned long flags;
1029	unsigned int slot_base;
1030	unsigned int slot_index;
1031
1032	BUG_ON(!nslots);
1033	BUG_ON(area_index >= pool->nareas);
1034
1035	/*
1036	 * Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
1037	 * page-aligned in the absence of any other alignment requirements.
1038	 * 'alloc_align_mask' was later introduced to specify the alignment
1039	 * explicitly, however this is passed as zero for streaming mappings
1040	 * and so we preserve the old behaviour there in case any drivers are
1041	 * relying on it.
1042	 */
1043	if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
1044		alloc_align_mask = PAGE_SIZE - 1;
1045
1046	/*
1047	 * Ensure that the allocation is at least slot-aligned and update
1048	 * 'iotlb_align_mask' to ignore bits that will be preserved when
1049	 * offsetting into the allocation.
1050	 */
1051	alloc_align_mask |= (IO_TLB_SIZE - 1);
1052	iotlb_align_mask &= ~alloc_align_mask;
1053
1054	/*
1055	 * For mappings with an alignment requirement don't bother looping to
1056	 * unaligned slots once we found an aligned one.
1057	 */
1058	stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
1059
1060	spin_lock_irqsave(&area->lock, flags);
1061	if (unlikely(nslots > pool->area_nslabs - area->used))
1062		goto not_found;
1063
1064	slot_base = area_index * pool->area_nslabs;
1065	index = area->index;
1066
1067	for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1068		phys_addr_t tlb_addr;
1069
1070		slot_index = slot_base + index;
1071		tlb_addr = slot_addr(tbl_dma_addr, slot_index);
1072
1073		if ((tlb_addr & alloc_align_mask) ||
1074		    (orig_addr && (tlb_addr & iotlb_align_mask) !=
1075				  (orig_addr & iotlb_align_mask))) {
1076			index = wrap_area_index(pool, index + 1);
1077			slots_checked++;
1078			continue;
1079		}
1080
1081		if (!iommu_is_span_boundary(slot_index, nslots,
1082					    nr_slots(tbl_dma_addr),
1083					    max_slots)) {
1084			if (pool->slots[slot_index].list >= nslots)
1085				goto found;
1086		}
1087		index = wrap_area_index(pool, index + stride);
1088		slots_checked += stride;
1089	}
1090
1091not_found:
1092	spin_unlock_irqrestore(&area->lock, flags);
1093	return -1;
1094
1095found:
1096	/*
1097	 * If we find a slot that indicates we have 'nslots' number of
1098	 * contiguous buffers, we allocate the buffers from that slot onwards
1099	 * and set the list of free entries to '0' indicating unavailable.
1100	 */
1101	for (i = slot_index; i < slot_index + nslots; i++) {
1102		pool->slots[i].list = 0;
1103		pool->slots[i].alloc_size = alloc_size - (offset +
1104				((i - slot_index) << IO_TLB_SHIFT));
1105	}
1106	for (i = slot_index - 1;
1107	     io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
1108	     pool->slots[i].list; i--)
1109		pool->slots[i].list = ++count;
1110
1111	/*
1112	 * Update the indices to avoid searching in the next round.
1113	 */
1114	area->index = wrap_area_index(pool, index + nslots);
1115	area->used += nslots;
1116	spin_unlock_irqrestore(&area->lock, flags);
1117
1118	inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots);
1119	return slot_index;
1120}
1121
1122#ifdef CONFIG_SWIOTLB_DYNAMIC
1123
1124/**
1125 * swiotlb_search_area() - search one memory area in all pools
1126 * @dev:	Device which maps the buffer.
1127 * @start_cpu:	Start CPU number.
1128 * @cpu_offset:	Offset from @start_cpu.
1129 * @orig_addr:	Original (non-bounced) IO buffer address.
1130 * @alloc_size: Total requested size of the bounce buffer,
1131 *		including initial alignment padding.
1132 * @alloc_align_mask:	Required alignment of the allocated buffer.
1133 * @retpool:	Used memory pool, updated on return.
1134 *
1135 * Search one memory area in all pools for a sequence of slots that match the
1136 * allocation constraints.
1137 *
1138 * Return: Index of the first allocated slot, or -1 on error.
1139 */
1140static int swiotlb_search_area(struct device *dev, int start_cpu,
1141		int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1142		unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1143{
1144	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1145	struct io_tlb_pool *pool;
1146	int area_index;
1147	int index = -1;
1148
1149	rcu_read_lock();
1150	list_for_each_entry_rcu(pool, &mem->pools, node) {
1151		if (cpu_offset >= pool->nareas)
1152			continue;
1153		area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1154		index = swiotlb_search_pool_area(dev, pool, area_index,
1155						 orig_addr, alloc_size,
1156						 alloc_align_mask);
1157		if (index >= 0) {
1158			*retpool = pool;
1159			break;
1160		}
1161	}
1162	rcu_read_unlock();
1163	return index;
1164}
1165
1166/**
1167 * swiotlb_find_slots() - search for slots in the whole swiotlb
1168 * @dev:	Device which maps the buffer.
1169 * @orig_addr:	Original (non-bounced) IO buffer address.
1170 * @alloc_size: Total requested size of the bounce buffer,
1171 *		including initial alignment padding.
1172 * @alloc_align_mask:	Required alignment of the allocated buffer.
1173 * @retpool:	Used memory pool, updated on return.
1174 *
1175 * Search through the whole software IO TLB to find a sequence of slots that
1176 * match the allocation constraints.
1177 *
1178 * Return: Index of the first allocated slot, or -1 on error.
1179 */
1180static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1181		size_t alloc_size, unsigned int alloc_align_mask,
1182		struct io_tlb_pool **retpool)
1183{
1184	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1185	struct io_tlb_pool *pool;
1186	unsigned long nslabs;
1187	unsigned long flags;
1188	u64 phys_limit;
1189	int cpu, i;
1190	int index;
1191
1192	if (alloc_size > IO_TLB_SEGSIZE * IO_TLB_SIZE)
1193		return -1;
1194
1195	cpu = raw_smp_processor_id();
1196	for (i = 0; i < default_nareas; ++i) {
1197		index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,
1198					    alloc_align_mask, &pool);
1199		if (index >= 0)
1200			goto found;
1201	}
1202
1203	if (!mem->can_grow)
1204		return -1;
1205
1206	schedule_work(&mem->dyn_alloc);
1207
1208	nslabs = nr_slots(alloc_size);
1209	phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
1210	pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
1211				  GFP_NOWAIT | __GFP_NOWARN);
1212	if (!pool)
1213		return -1;
1214
1215	index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
1216					 alloc_size, alloc_align_mask);
1217	if (index < 0) {
1218		swiotlb_dyn_free(&pool->rcu);
1219		return -1;
1220	}
1221
1222	pool->transient = true;
1223	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
1224	list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);
1225	spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
1226	inc_transient_used(mem, pool->nslabs);
1227
1228found:
1229	WRITE_ONCE(dev->dma_uses_io_tlb, true);
1230
1231	/*
1232	 * The general barrier orders reads and writes against a presumed store
1233	 * of the SWIOTLB buffer address by a device driver (to a driver private
1234	 * data structure). It serves two purposes.
1235	 *
1236	 * First, the store to dev->dma_uses_io_tlb must be ordered before the
1237	 * presumed store. This guarantees that the returned buffer address
1238	 * cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
1239	 *
1240	 * Second, the load from mem->pools must be ordered before the same
1241	 * presumed store. This guarantees that the returned buffer address
1242	 * cannot be observed by another CPU before an update of the RCU list
1243	 * that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1244	 * atomicity).
1245	 *
1246	 * See also the comment in is_swiotlb_buffer().
1247	 */
1248	smp_mb();
1249
1250	*retpool = pool;
1251	return index;
1252}
1253
1254#else  /* !CONFIG_SWIOTLB_DYNAMIC */
1255
1256static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1257		size_t alloc_size, unsigned int alloc_align_mask,
1258		struct io_tlb_pool **retpool)
1259{
1260	struct io_tlb_pool *pool;
1261	int start, i;
1262	int index;
1263
1264	*retpool = pool = &dev->dma_io_tlb_mem->defpool;
1265	i = start = raw_smp_processor_id() & (pool->nareas - 1);
1266	do {
1267		index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1268						 alloc_size, alloc_align_mask);
1269		if (index >= 0)
1270			return index;
1271		if (++i >= pool->nareas)
1272			i = 0;
1273	} while (i != start);
1274	return -1;
1275}
1276
1277#endif /* CONFIG_SWIOTLB_DYNAMIC */
1278
1279#ifdef CONFIG_DEBUG_FS
1280
1281/**
1282 * mem_used() - get number of used slots in an allocator
1283 * @mem:	Software IO TLB allocator.
1284 *
1285 * The result is accurate in this version of the function, because an atomic
1286 * counter is available if CONFIG_DEBUG_FS is set.
1287 *
1288 * Return: Number of used slots.
1289 */
1290static unsigned long mem_used(struct io_tlb_mem *mem)
1291{
1292	return atomic_long_read(&mem->total_used);
1293}
1294
1295#else /* !CONFIG_DEBUG_FS */
1296
1297/**
1298 * mem_pool_used() - get number of used slots in a memory pool
1299 * @pool:	Software IO TLB memory pool.
1300 *
1301 * The result is not accurate, see mem_used().
1302 *
1303 * Return: Approximate number of used slots.
1304 */
1305static unsigned long mem_pool_used(struct io_tlb_pool *pool)
1306{
1307	int i;
1308	unsigned long used = 0;
1309
1310	for (i = 0; i < pool->nareas; i++)
1311		used += pool->areas[i].used;
1312	return used;
1313}
1314
1315/**
1316 * mem_used() - get number of used slots in an allocator
1317 * @mem:	Software IO TLB allocator.
1318 *
1319 * The result is not accurate, because there is no locking of individual
1320 * areas.
1321 *
1322 * Return: Approximate number of used slots.
1323 */
1324static unsigned long mem_used(struct io_tlb_mem *mem)
1325{
1326#ifdef CONFIG_SWIOTLB_DYNAMIC
1327	struct io_tlb_pool *pool;
1328	unsigned long used = 0;
1329
1330	rcu_read_lock();
1331	list_for_each_entry_rcu(pool, &mem->pools, node)
1332		used += mem_pool_used(pool);
1333	rcu_read_unlock();
1334
1335	return used;
1336#else
1337	return mem_pool_used(&mem->defpool);
1338#endif
1339}
1340
1341#endif /* CONFIG_DEBUG_FS */
1342
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1343phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
1344		size_t mapping_size, size_t alloc_size,
1345		unsigned int alloc_align_mask, enum dma_data_direction dir,
1346		unsigned long attrs)
1347{
1348	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1349	unsigned int offset;
1350	struct io_tlb_pool *pool;
1351	unsigned int i;
 
1352	int index;
1353	phys_addr_t tlb_addr;
1354	unsigned short pad_slots;
1355
1356	if (!mem || !mem->nslabs) {
1357		dev_warn_ratelimited(dev,
1358			"Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
1359		return (phys_addr_t)DMA_MAPPING_ERROR;
1360	}
1361
1362	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
1363		pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
1364
1365	if (mapping_size > alloc_size) {
1366		dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
1367			      mapping_size, alloc_size);
1368		return (phys_addr_t)DMA_MAPPING_ERROR;
1369	}
 
 
 
 
1370
1371	offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);
1372	index = swiotlb_find_slots(dev, orig_addr,
1373				   alloc_size + offset, alloc_align_mask, &pool);
1374	if (index == -1) {
1375		if (!(attrs & DMA_ATTR_NO_WARN))
1376			dev_warn_ratelimited(dev,
1377	"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
1378				 alloc_size, mem->nslabs, mem_used(mem));
1379		return (phys_addr_t)DMA_MAPPING_ERROR;
1380	}
1381
1382	/*
 
 
 
 
 
 
1383	 * Save away the mapping from the original address to the DMA address.
1384	 * This is needed when we sync the memory.  Then we sync the buffer if
1385	 * needed.
1386	 */
1387	pad_slots = offset >> IO_TLB_SHIFT;
1388	offset &= (IO_TLB_SIZE - 1);
1389	index += pad_slots;
1390	pool->slots[index].pad_slots = pad_slots;
1391	for (i = 0; i < nr_slots(alloc_size + offset); i++)
1392		pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
1393	tlb_addr = slot_addr(pool->start, index) + offset;
1394	/*
1395	 * When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
1396	 * the original buffer to the TLB buffer before initiating DMA in order
1397	 * to preserve the original's data if the device does a partial write,
1398	 * i.e. if the device doesn't overwrite the entire buffer.  Preserving
1399	 * the original data, even if it's garbage, is necessary to match
1400	 * hardware behavior.  Use of swiotlb is supposed to be transparent,
1401	 * i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1402	 */
1403	swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE);
1404	return tlb_addr;
1405}
1406
1407static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
 
1408{
1409	struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
1410	unsigned long flags;
1411	unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
1412	int index, nslots, aindex;
1413	struct io_tlb_area *area;
1414	int count, i;
1415
1416	index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
1417	index -= mem->slots[index].pad_slots;
1418	nslots = nr_slots(mem->slots[index].alloc_size + offset);
1419	aindex = index / mem->area_nslabs;
1420	area = &mem->areas[aindex];
1421
1422	/*
1423	 * Return the buffer to the free list by setting the corresponding
1424	 * entries to indicate the number of contiguous entries available.
1425	 * While returning the entries to the free list, we merge the entries
1426	 * with slots below and above the pool being returned.
1427	 */
1428	BUG_ON(aindex >= mem->nareas);
1429
1430	spin_lock_irqsave(&area->lock, flags);
1431	if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
1432		count = mem->slots[index + nslots].list;
1433	else
1434		count = 0;
1435
1436	/*
1437	 * Step 1: return the slots to the free list, merging the slots with
1438	 * superceeding slots
1439	 */
1440	for (i = index + nslots - 1; i >= index; i--) {
1441		mem->slots[i].list = ++count;
1442		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
1443		mem->slots[i].alloc_size = 0;
1444		mem->slots[i].pad_slots = 0;
1445	}
1446
1447	/*
1448	 * Step 2: merge the returned slots with the preceding slots, if
1449	 * available (non zero)
1450	 */
1451	for (i = index - 1;
1452	     io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
1453	     i--)
1454		mem->slots[i].list = ++count;
1455	area->used -= nslots;
1456	spin_unlock_irqrestore(&area->lock, flags);
1457
1458	dec_used(dev->dma_io_tlb_mem, nslots);
1459}
1460
1461#ifdef CONFIG_SWIOTLB_DYNAMIC
1462
1463/**
1464 * swiotlb_del_transient() - delete a transient memory pool
1465 * @dev:	Device which mapped the buffer.
1466 * @tlb_addr:	Physical address within a bounce buffer.
 
1467 *
1468 * Check whether the address belongs to a transient SWIOTLB memory pool.
1469 * If yes, then delete the pool.
1470 *
1471 * Return: %true if @tlb_addr belonged to a transient pool that was released.
1472 */
1473static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr)
 
1474{
1475	struct io_tlb_pool *pool;
1476
1477	pool = swiotlb_find_pool(dev, tlb_addr);
1478	if (!pool->transient)
1479		return false;
1480
1481	dec_used(dev->dma_io_tlb_mem, pool->nslabs);
1482	swiotlb_del_pool(dev, pool);
1483	dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs);
1484	return true;
1485}
1486
1487#else  /* !CONFIG_SWIOTLB_DYNAMIC */
1488
1489static inline bool swiotlb_del_transient(struct device *dev,
1490					 phys_addr_t tlb_addr)
1491{
1492	return false;
1493}
1494
1495#endif	/* CONFIG_SWIOTLB_DYNAMIC */
1496
1497/*
1498 * tlb_addr is the physical address of the bounce buffer to unmap.
1499 */
1500void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1501			      size_t mapping_size, enum dma_data_direction dir,
1502			      unsigned long attrs)
1503{
1504	/*
1505	 * First, sync the memory before unmapping the entry
1506	 */
1507	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1508	    (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1509		swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_FROM_DEVICE);
 
1510
1511	if (swiotlb_del_transient(dev, tlb_addr))
1512		return;
1513	swiotlb_release_slots(dev, tlb_addr);
1514}
1515
1516void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1517		size_t size, enum dma_data_direction dir)
 
1518{
1519	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1520		swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE);
1521	else
1522		BUG_ON(dir != DMA_FROM_DEVICE);
1523}
1524
1525void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1526		size_t size, enum dma_data_direction dir)
 
1527{
1528	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1529		swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE);
1530	else
1531		BUG_ON(dir != DMA_TO_DEVICE);
1532}
1533
1534/*
1535 * Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
1536 * to the device copy the data into it as well.
1537 */
1538dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
1539		enum dma_data_direction dir, unsigned long attrs)
1540{
1541	phys_addr_t swiotlb_addr;
1542	dma_addr_t dma_addr;
1543
1544	trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
1545
1546	swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, 0, dir,
1547			attrs);
1548	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
1549		return DMA_MAPPING_ERROR;
1550
1551	/* Ensure that the address returned is DMA'ble */
1552	dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
1553	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
1554		swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
1555			attrs | DMA_ATTR_SKIP_CPU_SYNC);
 
1556		dev_WARN_ONCE(dev, 1,
1557			"swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1558			&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
1559		return DMA_MAPPING_ERROR;
1560	}
1561
1562	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1563		arch_sync_dma_for_device(swiotlb_addr, size, dir);
1564	return dma_addr;
1565}
1566
1567size_t swiotlb_max_mapping_size(struct device *dev)
1568{
1569	int min_align_mask = dma_get_min_align_mask(dev);
1570	int min_align = 0;
1571
1572	/*
1573	 * swiotlb_find_slots() skips slots according to
1574	 * min align mask. This affects max mapping size.
1575	 * Take it into acount here.
1576	 */
1577	if (min_align_mask)
1578		min_align = roundup(min_align_mask, IO_TLB_SIZE);
1579
1580	return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
1581}
1582
1583/**
1584 * is_swiotlb_allocated() - check if the default software IO TLB is initialized
1585 */
1586bool is_swiotlb_allocated(void)
1587{
1588	return io_tlb_default_mem.nslabs;
1589}
1590
1591bool is_swiotlb_active(struct device *dev)
1592{
1593	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1594
1595	return mem && mem->nslabs;
1596}
1597
1598/**
1599 * default_swiotlb_base() - get the base address of the default SWIOTLB
1600 *
1601 * Get the lowest physical address used by the default software IO TLB pool.
1602 */
1603phys_addr_t default_swiotlb_base(void)
1604{
1605#ifdef CONFIG_SWIOTLB_DYNAMIC
1606	io_tlb_default_mem.can_grow = false;
1607#endif
1608	return io_tlb_default_mem.defpool.start;
1609}
1610
1611/**
1612 * default_swiotlb_limit() - get the address limit of the default SWIOTLB
1613 *
1614 * Get the highest physical address used by the default software IO TLB pool.
1615 */
1616phys_addr_t default_swiotlb_limit(void)
1617{
1618#ifdef CONFIG_SWIOTLB_DYNAMIC
1619	return io_tlb_default_mem.phys_limit;
1620#else
1621	return io_tlb_default_mem.defpool.end - 1;
1622#endif
1623}
1624
1625#ifdef CONFIG_DEBUG_FS
1626#ifdef CONFIG_SWIOTLB_DYNAMIC
1627static unsigned long mem_transient_used(struct io_tlb_mem *mem)
1628{
1629	return atomic_long_read(&mem->transient_nslabs);
1630}
1631
1632static int io_tlb_transient_used_get(void *data, u64 *val)
1633{
1634	struct io_tlb_mem *mem = data;
1635
1636	*val = mem_transient_used(mem);
1637	return 0;
1638}
1639
1640DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get,
1641			 NULL, "%llu\n");
1642#endif /* CONFIG_SWIOTLB_DYNAMIC */
1643
1644static int io_tlb_used_get(void *data, u64 *val)
1645{
1646	struct io_tlb_mem *mem = data;
1647
1648	*val = mem_used(mem);
1649	return 0;
1650}
1651
1652static int io_tlb_hiwater_get(void *data, u64 *val)
1653{
1654	struct io_tlb_mem *mem = data;
1655
1656	*val = atomic_long_read(&mem->used_hiwater);
1657	return 0;
1658}
1659
1660static int io_tlb_hiwater_set(void *data, u64 val)
1661{
1662	struct io_tlb_mem *mem = data;
1663
1664	/* Only allow setting to zero */
1665	if (val != 0)
1666		return -EINVAL;
1667
1668	atomic_long_set(&mem->used_hiwater, val);
1669	return 0;
1670}
1671
1672DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
1673DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
1674				io_tlb_hiwater_set, "%llu\n");
1675
1676static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1677					 const char *dirname)
1678{
1679	mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
1680	if (!mem->nslabs)
1681		return;
1682
1683	debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
1684	debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
1685			&fops_io_tlb_used);
1686	debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
1687			&fops_io_tlb_hiwater);
1688#ifdef CONFIG_SWIOTLB_DYNAMIC
1689	debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs,
1690			    mem, &fops_io_tlb_transient_used);
1691#endif
1692}
1693
1694static int __init swiotlb_create_default_debugfs(void)
1695{
1696	swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb");
1697	return 0;
1698}
1699
1700late_initcall(swiotlb_create_default_debugfs);
1701
1702#else  /* !CONFIG_DEBUG_FS */
1703
1704static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1705						const char *dirname)
1706{
1707}
1708
1709#endif	/* CONFIG_DEBUG_FS */
1710
1711#ifdef CONFIG_DMA_RESTRICTED_POOL
1712
1713struct page *swiotlb_alloc(struct device *dev, size_t size)
1714{
1715	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1716	struct io_tlb_pool *pool;
1717	phys_addr_t tlb_addr;
1718	unsigned int align;
1719	int index;
1720
1721	if (!mem)
1722		return NULL;
1723
1724	align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
1725	index = swiotlb_find_slots(dev, 0, size, align, &pool);
1726	if (index == -1)
1727		return NULL;
1728
1729	tlb_addr = slot_addr(pool->start, index);
1730	if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1731		dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1732			      &tlb_addr);
1733		swiotlb_release_slots(dev, tlb_addr);
1734		return NULL;
1735	}
1736
1737	return pfn_to_page(PFN_DOWN(tlb_addr));
1738}
1739
1740bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1741{
1742	phys_addr_t tlb_addr = page_to_phys(page);
 
1743
1744	if (!is_swiotlb_buffer(dev, tlb_addr))
 
1745		return false;
1746
1747	swiotlb_release_slots(dev, tlb_addr);
1748
1749	return true;
1750}
1751
1752static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
1753				    struct device *dev)
1754{
1755	struct io_tlb_mem *mem = rmem->priv;
1756	unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
1757
1758	/* Set Per-device io tlb area to one */
1759	unsigned int nareas = 1;
1760
1761	if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
1762		dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
1763		return -EINVAL;
1764	}
1765
1766	/*
1767	 * Since multiple devices can share the same pool, the private data,
1768	 * io_tlb_mem struct, will be initialized by the first device attached
1769	 * to it.
1770	 */
1771	if (!mem) {
1772		struct io_tlb_pool *pool;
1773
1774		mem = kzalloc(sizeof(*mem), GFP_KERNEL);
1775		if (!mem)
1776			return -ENOMEM;
1777		pool = &mem->defpool;
1778
1779		pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL);
1780		if (!pool->slots) {
1781			kfree(mem);
1782			return -ENOMEM;
1783		}
1784
1785		pool->areas = kcalloc(nareas, sizeof(*pool->areas),
1786				GFP_KERNEL);
1787		if (!pool->areas) {
1788			kfree(pool->slots);
1789			kfree(mem);
1790			return -ENOMEM;
1791		}
1792
1793		set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
1794				     rmem->size >> PAGE_SHIFT);
1795		swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
1796					 false, nareas);
1797		mem->force_bounce = true;
1798		mem->for_alloc = true;
1799#ifdef CONFIG_SWIOTLB_DYNAMIC
1800		spin_lock_init(&mem->lock);
1801		INIT_LIST_HEAD_RCU(&mem->pools);
1802#endif
1803		add_mem_pool(mem, pool);
1804
1805		rmem->priv = mem;
1806
1807		swiotlb_create_debugfs_files(mem, rmem->name);
1808	}
1809
1810	dev->dma_io_tlb_mem = mem;
1811
1812	return 0;
1813}
1814
1815static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
1816					struct device *dev)
1817{
1818	dev->dma_io_tlb_mem = &io_tlb_default_mem;
1819}
1820
1821static const struct reserved_mem_ops rmem_swiotlb_ops = {
1822	.device_init = rmem_swiotlb_device_init,
1823	.device_release = rmem_swiotlb_device_release,
1824};
1825
1826static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
1827{
1828	unsigned long node = rmem->fdt_node;
1829
1830	if (of_get_flat_dt_prop(node, "reusable", NULL) ||
1831	    of_get_flat_dt_prop(node, "linux,cma-default", NULL) ||
1832	    of_get_flat_dt_prop(node, "linux,dma-default", NULL) ||
1833	    of_get_flat_dt_prop(node, "no-map", NULL))
1834		return -EINVAL;
1835
1836	rmem->ops = &rmem_swiotlb_ops;
1837	pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
1838		&rmem->base, (unsigned long)rmem->size / SZ_1M);
1839	return 0;
1840}
1841
1842RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
1843#endif /* CONFIG_DMA_RESTRICTED_POOL */