1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/arch/arm/mm/dma-mapping.c
   4 *
   5 *  Copyright (C) 2000-2004 Russell King
   6 *
   7 *  DMA uncached mapping support.
   8 */
   9#include <linux/module.h>
  10#include <linux/mm.h>
  11#include <linux/genalloc.h>
  12#include <linux/gfp.h>
  13#include <linux/errno.h>
  14#include <linux/list.h>
  15#include <linux/init.h>
  16#include <linux/device.h>
  17#include <linux/dma-direct.h>
  18#include <linux/dma-mapping.h>
  19#include <linux/dma-noncoherent.h>
  20#include <linux/dma-contiguous.h>
  21#include <linux/highmem.h>
  22#include <linux/memblock.h>
  23#include <linux/slab.h>
  24#include <linux/iommu.h>
  25#include <linux/io.h>
  26#include <linux/vmalloc.h>
  27#include <linux/sizes.h>
  28#include <linux/cma.h>
  29
  30#include <asm/memory.h>
  31#include <asm/highmem.h>
  32#include <asm/cacheflush.h>
  33#include <asm/tlbflush.h>
  34#include <asm/mach/arch.h>
  35#include <asm/dma-iommu.h>
  36#include <asm/mach/map.h>
  37#include <asm/system_info.h>
  38#include <asm/dma-contiguous.h>
  39#include <xen/swiotlb-xen.h>
  40
  41#include "dma.h"
  42#include "mm.h"
  43
  44struct arm_dma_alloc_args {
  45	struct device *dev;
  46	size_t size;
  47	gfp_t gfp;
  48	pgprot_t prot;
  49	const void *caller;
  50	bool want_vaddr;
  51	int coherent_flag;
  52};
  53
  54struct arm_dma_free_args {
  55	struct device *dev;
  56	size_t size;
  57	void *cpu_addr;
  58	struct page *page;
  59	bool want_vaddr;
  60};
  61
  62#define NORMAL	    0
  63#define COHERENT    1
  64
  65struct arm_dma_allocator {
  66	void *(*alloc)(struct arm_dma_alloc_args *args,
  67		       struct page **ret_page);
  68	void (*free)(struct arm_dma_free_args *args);
  69};
  70
  71struct arm_dma_buffer {
  72	struct list_head list;
  73	void *virt;
  74	struct arm_dma_allocator *allocator;
  75};
  76
  77static LIST_HEAD(arm_dma_bufs);
  78static DEFINE_SPINLOCK(arm_dma_bufs_lock);
  79
  80static struct arm_dma_buffer *arm_dma_buffer_find(void *virt)
  81{
  82	struct arm_dma_buffer *buf, *found = NULL;
  83	unsigned long flags;
  84
  85	spin_lock_irqsave(&arm_dma_bufs_lock, flags);
  86	list_for_each_entry(buf, &arm_dma_bufs, list) {
  87		if (buf->virt == virt) {
  88			list_del(&buf->list);
  89			found = buf;
  90			break;
  91		}
  92	}
  93	spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
  94	return found;
  95}
  96
  97/*
  98 * The DMA API is built upon the notion of "buffer ownership".  A buffer
  99 * is either exclusively owned by the CPU (and therefore may be accessed
 100 * by it) or exclusively owned by the DMA device.  These helper functions
 101 * represent the transitions between these two ownership states.
 102 *
 103 * Note, however, that on later ARMs, this notion does not work due to
 104 * speculative prefetches.  We model our approach on the assumption that
 105 * the CPU does do speculative prefetches, which means we clean caches
 106 * before transfers and delay cache invalidation until transfer completion.
 107 *
 108 */
 109static void __dma_page_cpu_to_dev(struct page *, unsigned long,
 110		size_t, enum dma_data_direction);
 111static void __dma_page_dev_to_cpu(struct page *, unsigned long,
 112		size_t, enum dma_data_direction);
 113
 114/**
 115 * arm_dma_map_page - map a portion of a page for streaming DMA
 116 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 117 * @page: page that buffer resides in
 118 * @offset: offset into page for start of buffer
 119 * @size: size of buffer to map
 120 * @dir: DMA transfer direction
 121 *
 122 * Ensure that any data held in the cache is appropriately discarded
 123 * or written back.
 124 *
 125 * The device owns this memory once this call has completed.  The CPU
 126 * can regain ownership by calling dma_unmap_page().
 127 */
 128static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
 129	     unsigned long offset, size_t size, enum dma_data_direction dir,
 130	     unsigned long attrs)
 131{
 132	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
 133		__dma_page_cpu_to_dev(page, offset, size, dir);
 134	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 135}
 136
 137static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
 138	     unsigned long offset, size_t size, enum dma_data_direction dir,
 139	     unsigned long attrs)
 140{
 141	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 142}
 143
 144/**
 145 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
 146 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 147 * @handle: DMA address of buffer
 148 * @size: size of buffer (same as passed to dma_map_page)
 149 * @dir: DMA transfer direction (same as passed to dma_map_page)
 150 *
 151 * Unmap a page streaming mode DMA translation.  The handle and size
 152 * must match what was provided in the previous dma_map_page() call.
 153 * All other usages are undefined.
 154 *
 155 * After this call, reads by the CPU to the buffer are guaranteed to see
 156 * whatever the device wrote there.
 157 */
 158static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
 159		size_t size, enum dma_data_direction dir, unsigned long attrs)
 160{
 161	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
 162		__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
 163				      handle & ~PAGE_MASK, size, dir);
 164}
 165
 166static void arm_dma_sync_single_for_cpu(struct device *dev,
 167		dma_addr_t handle, size_t size, enum dma_data_direction dir)
 168{
 169	unsigned int offset = handle & (PAGE_SIZE - 1);
 170	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
 171	__dma_page_dev_to_cpu(page, offset, size, dir);
 172}
 173
 174static void arm_dma_sync_single_for_device(struct device *dev,
 175		dma_addr_t handle, size_t size, enum dma_data_direction dir)
 176{
 177	unsigned int offset = handle & (PAGE_SIZE - 1);
 178	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
 179	__dma_page_cpu_to_dev(page, offset, size, dir);
 180}
 181
 182const struct dma_map_ops arm_dma_ops = {
 183	.alloc			= arm_dma_alloc,
 184	.free			= arm_dma_free,
 185	.mmap			= arm_dma_mmap,
 186	.get_sgtable		= arm_dma_get_sgtable,
 187	.map_page		= arm_dma_map_page,
 188	.unmap_page		= arm_dma_unmap_page,
 189	.map_sg			= arm_dma_map_sg,
 190	.unmap_sg		= arm_dma_unmap_sg,
 191	.map_resource		= dma_direct_map_resource,
 192	.sync_single_for_cpu	= arm_dma_sync_single_for_cpu,
 193	.sync_single_for_device	= arm_dma_sync_single_for_device,
 194	.sync_sg_for_cpu	= arm_dma_sync_sg_for_cpu,
 195	.sync_sg_for_device	= arm_dma_sync_sg_for_device,
 196	.dma_supported		= arm_dma_supported,
 197	.get_required_mask	= dma_direct_get_required_mask,
 198};
 199EXPORT_SYMBOL(arm_dma_ops);
 200
 201static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 202	dma_addr_t *handle, gfp_t gfp, unsigned long attrs);
 203static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
 204				  dma_addr_t handle, unsigned long attrs);
 205static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 206		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 207		 unsigned long attrs);
 208
 209const struct dma_map_ops arm_coherent_dma_ops = {
 210	.alloc			= arm_coherent_dma_alloc,
 211	.free			= arm_coherent_dma_free,
 212	.mmap			= arm_coherent_dma_mmap,
 213	.get_sgtable		= arm_dma_get_sgtable,
 214	.map_page		= arm_coherent_dma_map_page,
 215	.map_sg			= arm_dma_map_sg,
 216	.map_resource		= dma_direct_map_resource,
 217	.dma_supported		= arm_dma_supported,
 218	.get_required_mask	= dma_direct_get_required_mask,
 219};
 220EXPORT_SYMBOL(arm_coherent_dma_ops);
 221
 222static int __dma_supported(struct device *dev, u64 mask, bool warn)
 223{
 224	unsigned long max_dma_pfn = min(max_pfn, arm_dma_pfn_limit);
 225
 226	/*
 227	 * Translate the device's DMA mask to a PFN limit.  This
 228	 * PFN number includes the page which we can DMA to.
 229	 */
 230	if (dma_to_pfn(dev, mask) < max_dma_pfn) {
 231		if (warn)
 232			dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",
 233				 mask,
 234				 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,
 235				 max_dma_pfn + 1);
 236		return 0;
 237	}
 238
 239	return 1;
 240}
 241
 242static u64 get_coherent_dma_mask(struct device *dev)
 243{
 244	u64 mask = (u64)DMA_BIT_MASK(32);
 245
 246	if (dev) {
 247		mask = dev->coherent_dma_mask;
 248
 249		/*
 250		 * Sanity check the DMA mask - it must be non-zero, and
 251		 * must be able to be satisfied by a DMA allocation.
 252		 */
 253		if (mask == 0) {
 254			dev_warn(dev, "coherent DMA mask is unset\n");
 255			return 0;
 256		}
 257
 258		if (!__dma_supported(dev, mask, true))
 259			return 0;
 260	}
 261
 262	return mask;
 263}
 264
 265static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag)
 266{
 267	/*
 268	 * Ensure that the allocated pages are zeroed, and that any data
 269	 * lurking in the kernel direct-mapped region is invalidated.
 270	 */
 271	if (PageHighMem(page)) {
 272		phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
 273		phys_addr_t end = base + size;
 274		while (size > 0) {
 275			void *ptr = kmap_atomic(page);
 276			memset(ptr, 0, PAGE_SIZE);
 277			if (coherent_flag != COHERENT)
 278				dmac_flush_range(ptr, ptr + PAGE_SIZE);
 279			kunmap_atomic(ptr);
 280			page++;
 281			size -= PAGE_SIZE;
 282		}
 283		if (coherent_flag != COHERENT)
 284			outer_flush_range(base, end);
 285	} else {
 286		void *ptr = page_address(page);
 287		memset(ptr, 0, size);
 288		if (coherent_flag != COHERENT) {
 289			dmac_flush_range(ptr, ptr + size);
 290			outer_flush_range(__pa(ptr), __pa(ptr) + size);
 291		}
 292	}
 293}
 294
 295/*
 296 * Allocate a DMA buffer for 'dev' of size 'size' using the
 297 * specified gfp mask.  Note that 'size' must be page aligned.
 298 */
 299static struct page *__dma_alloc_buffer(struct device *dev, size_t size,
 300				       gfp_t gfp, int coherent_flag)
 301{
 302	unsigned long order = get_order(size);
 303	struct page *page, *p, *e;
 304
 305	page = alloc_pages(gfp, order);
 306	if (!page)
 307		return NULL;
 308
 309	/*
 310	 * Now split the huge page and free the excess pages
 311	 */
 312	split_page(page, order);
 313	for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
 314		__free_page(p);
 315
 316	__dma_clear_buffer(page, size, coherent_flag);
 317
 318	return page;
 319}
 320
 321/*
 322 * Free a DMA buffer.  'size' must be page aligned.
 323 */
 324static void __dma_free_buffer(struct page *page, size_t size)
 325{
 326	struct page *e = page + (size >> PAGE_SHIFT);
 327
 328	while (page < e) {
 329		__free_page(page);
 330		page++;
 331	}
 332}
 333
 334static void *__alloc_from_contiguous(struct device *dev, size_t size,
 335				     pgprot_t prot, struct page **ret_page,
 336				     const void *caller, bool want_vaddr,
 337				     int coherent_flag, gfp_t gfp);
 338
 339static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 340				 pgprot_t prot, struct page **ret_page,
 341				 const void *caller, bool want_vaddr);
 342
 343#define DEFAULT_DMA_COHERENT_POOL_SIZE	SZ_256K
 344static struct gen_pool *atomic_pool __ro_after_init;
 345
 346static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
 347
 348static int __init early_coherent_pool(char *p)
 349{
 350	atomic_pool_size = memparse(p, &p);
 351	return 0;
 352}
 353early_param("coherent_pool", early_coherent_pool);
 354
 355/*
 356 * Initialise the coherent pool for atomic allocations.
 357 */
 358static int __init atomic_pool_init(void)
 359{
 360	pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
 361	gfp_t gfp = GFP_KERNEL | GFP_DMA;
 362	struct page *page;
 363	void *ptr;
 364
 365	atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
 366	if (!atomic_pool)
 367		goto out;
 368	/*
 369	 * The atomic pool is only used for non-coherent allocations
 370	 * so we must pass NORMAL for coherent_flag.
 371	 */
 372	if (dev_get_cma_area(NULL))
 373		ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
 374				      &page, atomic_pool_init, true, NORMAL,
 375				      GFP_KERNEL);
 376	else
 377		ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
 378					   &page, atomic_pool_init, true);
 379	if (ptr) {
 380		int ret;
 381
 382		ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
 383					page_to_phys(page),
 384					atomic_pool_size, -1);
 385		if (ret)
 386			goto destroy_genpool;
 387
 388		gen_pool_set_algo(atomic_pool,
 389				gen_pool_first_fit_order_align,
 390				NULL);
 391		pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n",
 392		       atomic_pool_size / 1024);
 393		return 0;
 394	}
 395
 396destroy_genpool:
 397	gen_pool_destroy(atomic_pool);
 398	atomic_pool = NULL;
 399out:
 400	pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
 401	       atomic_pool_size / 1024);
 402	return -ENOMEM;
 403}
 404/*
 405 * CMA is activated by core_initcall, so we must be called after it.
 406 */
 407postcore_initcall(atomic_pool_init);
 408
 
 409struct dma_contig_early_reserve {
 410	phys_addr_t base;
 411	unsigned long size;
 412};
 413
 414static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
 415
 416static int dma_mmu_remap_num __initdata;
 417
 
 418void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
 419{
 420	dma_mmu_remap[dma_mmu_remap_num].base = base;
 421	dma_mmu_remap[dma_mmu_remap_num].size = size;
 422	dma_mmu_remap_num++;
 423}
 
 424
 425void __init dma_contiguous_remap(void)
 426{
 427	int i;
 428	for (i = 0; i < dma_mmu_remap_num; i++) {
 429		phys_addr_t start = dma_mmu_remap[i].base;
 430		phys_addr_t end = start + dma_mmu_remap[i].size;
 431		struct map_desc map;
 432		unsigned long addr;
 433
 434		if (end > arm_lowmem_limit)
 435			end = arm_lowmem_limit;
 436		if (start >= end)
 437			continue;
 438
 439		map.pfn = __phys_to_pfn(start);
 440		map.virtual = __phys_to_virt(start);
 441		map.length = end - start;
 442		map.type = MT_MEMORY_DMA_READY;
 443
 444		/*
 445		 * Clear previous low-memory mapping to ensure that the
 446		 * TLB does not see any conflicting entries, then flush
 447		 * the TLB of the old entries before creating new mappings.
 448		 *
 449		 * This ensures that any speculatively loaded TLB entries
 450		 * (even though they may be rare) can not cause any problems,
 451		 * and ensures that this code is architecturally compliant.
 452		 */
 453		for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
 454		     addr += PMD_SIZE)
 455			pmd_clear(pmd_off_k(addr));
 456
 457		flush_tlb_kernel_range(__phys_to_virt(start),
 458				       __phys_to_virt(end));
 459
 460		iotable_init(&map, 1);
 461	}
 462}
 
 463
 464static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data)
 465{
 466	struct page *page = virt_to_page(addr);
 467	pgprot_t prot = *(pgprot_t *)data;
 468
 469	set_pte_ext(pte, mk_pte(page, prot), 0);
 470	return 0;
 471}
 472
 473static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
 474{
 475	unsigned long start = (unsigned long) page_address(page);
 476	unsigned end = start + size;
 477
 478	apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
 479	flush_tlb_kernel_range(start, end);
 480}
 481
 482static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 483				 pgprot_t prot, struct page **ret_page,
 484				 const void *caller, bool want_vaddr)
 485{
 486	struct page *page;
 487	void *ptr = NULL;
 488	/*
 489	 * __alloc_remap_buffer is only called when the device is
 490	 * non-coherent
 491	 */
 492	page = __dma_alloc_buffer(dev, size, gfp, NORMAL);
 493	if (!page)
 494		return NULL;
 495	if (!want_vaddr)
 496		goto out;
 497
 498	ptr = dma_common_contiguous_remap(page, size, prot, caller);
 499	if (!ptr) {
 500		__dma_free_buffer(page, size);
 501		return NULL;
 502	}
 503
 504 out:
 505	*ret_page = page;
 506	return ptr;
 507}
 508
 509static void *__alloc_from_pool(size_t size, struct page **ret_page)
 510{
 511	unsigned long val;
 512	void *ptr = NULL;
 513
 514	if (!atomic_pool) {
 515		WARN(1, "coherent pool not initialised!\n");
 516		return NULL;
 517	}
 518
 519	val = gen_pool_alloc(atomic_pool, size);
 520	if (val) {
 521		phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
 522
 523		*ret_page = phys_to_page(phys);
 524		ptr = (void *)val;
 525	}
 526
 527	return ptr;
 528}
 529
 530static bool __in_atomic_pool(void *start, size_t size)
 531{
 532	return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
 533}
 534
 535static int __free_from_pool(void *start, size_t size)
 536{
 537	if (!__in_atomic_pool(start, size))
 538		return 0;
 539
 540	gen_pool_free(atomic_pool, (unsigned long)start, size);
 541
 542	return 1;
 543}
 544
 545static void *__alloc_from_contiguous(struct device *dev, size_t size,
 546				     pgprot_t prot, struct page **ret_page,
 547				     const void *caller, bool want_vaddr,
 548				     int coherent_flag, gfp_t gfp)
 549{
 550	unsigned long order = get_order(size);
 551	size_t count = size >> PAGE_SHIFT;
 552	struct page *page;
 553	void *ptr = NULL;
 554
 555	page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN);
 556	if (!page)
 557		return NULL;
 558
 559	__dma_clear_buffer(page, size, coherent_flag);
 560
 561	if (!want_vaddr)
 562		goto out;
 563
 564	if (PageHighMem(page)) {
 565		ptr = dma_common_contiguous_remap(page, size, prot, caller);
 566		if (!ptr) {
 567			dma_release_from_contiguous(dev, page, count);
 568			return NULL;
 569		}
 570	} else {
 571		__dma_remap(page, size, prot);
 572		ptr = page_address(page);
 573	}
 574
 575 out:
 576	*ret_page = page;
 577	return ptr;
 578}
 579
 580static void __free_from_contiguous(struct device *dev, struct page *page,
 581				   void *cpu_addr, size_t size, bool want_vaddr)
 582{
 583	if (want_vaddr) {
 584		if (PageHighMem(page))
 585			dma_common_free_remap(cpu_addr, size);
 586		else
 587			__dma_remap(page, size, PAGE_KERNEL);
 588	}
 589	dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
 590}
 591
 592static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot)
 593{
 594	prot = (attrs & DMA_ATTR_WRITE_COMBINE) ?
 595			pgprot_writecombine(prot) :
 596			pgprot_dmacoherent(prot);
 597	return prot;
 598}
 599
 600static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
 601				   struct page **ret_page)
 602{
 603	struct page *page;
 604	/* __alloc_simple_buffer is only called when the device is coherent */
 605	page = __dma_alloc_buffer(dev, size, gfp, COHERENT);
 606	if (!page)
 607		return NULL;
 608
 609	*ret_page = page;
 610	return page_address(page);
 611}
 612
 613static void *simple_allocator_alloc(struct arm_dma_alloc_args *args,
 614				    struct page **ret_page)
 615{
 616	return __alloc_simple_buffer(args->dev, args->size, args->gfp,
 617				     ret_page);
 618}
 619
 620static void simple_allocator_free(struct arm_dma_free_args *args)
 621{
 622	__dma_free_buffer(args->page, args->size);
 623}
 624
 625static struct arm_dma_allocator simple_allocator = {
 626	.alloc = simple_allocator_alloc,
 627	.free = simple_allocator_free,
 628};
 629
 630static void *cma_allocator_alloc(struct arm_dma_alloc_args *args,
 631				 struct page **ret_page)
 632{
 633	return __alloc_from_contiguous(args->dev, args->size, args->prot,
 634				       ret_page, args->caller,
 635				       args->want_vaddr, args->coherent_flag,
 636				       args->gfp);
 637}
 638
 639static void cma_allocator_free(struct arm_dma_free_args *args)
 640{
 641	__free_from_contiguous(args->dev, args->page, args->cpu_addr,
 642			       args->size, args->want_vaddr);
 643}
 644
 645static struct arm_dma_allocator cma_allocator = {
 646	.alloc = cma_allocator_alloc,
 647	.free = cma_allocator_free,
 648};
 649
 650static void *pool_allocator_alloc(struct arm_dma_alloc_args *args,
 651				  struct page **ret_page)
 652{
 653	return __alloc_from_pool(args->size, ret_page);
 654}
 655
 656static void pool_allocator_free(struct arm_dma_free_args *args)
 657{
 658	__free_from_pool(args->cpu_addr, args->size);
 659}
 660
 661static struct arm_dma_allocator pool_allocator = {
 662	.alloc = pool_allocator_alloc,
 663	.free = pool_allocator_free,
 664};
 665
 666static void *remap_allocator_alloc(struct arm_dma_alloc_args *args,
 667				   struct page **ret_page)
 668{
 669	return __alloc_remap_buffer(args->dev, args->size, args->gfp,
 670				    args->prot, ret_page, args->caller,
 671				    args->want_vaddr);
 672}
 673
 674static void remap_allocator_free(struct arm_dma_free_args *args)
 675{
 676	if (args->want_vaddr)
 677		dma_common_free_remap(args->cpu_addr, args->size);
 678
 679	__dma_free_buffer(args->page, args->size);
 680}
 681
 682static struct arm_dma_allocator remap_allocator = {
 683	.alloc = remap_allocator_alloc,
 684	.free = remap_allocator_free,
 685};
 686
 687static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 688			 gfp_t gfp, pgprot_t prot, bool is_coherent,
 689			 unsigned long attrs, const void *caller)
 690{
 691	u64 mask = get_coherent_dma_mask(dev);
 692	struct page *page = NULL;
 693	void *addr;
 694	bool allowblock, cma;
 695	struct arm_dma_buffer *buf;
 696	struct arm_dma_alloc_args args = {
 697		.dev = dev,
 698		.size = PAGE_ALIGN(size),
 699		.gfp = gfp,
 700		.prot = prot,
 701		.caller = caller,
 702		.want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
 703		.coherent_flag = is_coherent ? COHERENT : NORMAL,
 704	};
 705
 706#ifdef CONFIG_DMA_API_DEBUG
 707	u64 limit = (mask + 1) & ~mask;
 708	if (limit && size >= limit) {
 709		dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
 710			size, mask);
 711		return NULL;
 712	}
 713#endif
 714
 715	if (!mask)
 716		return NULL;
 717
 718	buf = kzalloc(sizeof(*buf),
 719		      gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM));
 720	if (!buf)
 721		return NULL;
 722
 723	if (mask < 0xffffffffULL)
 724		gfp |= GFP_DMA;
 725
 726	/*
 727	 * Following is a work-around (a.k.a. hack) to prevent pages
 728	 * with __GFP_COMP being passed to split_page() which cannot
 729	 * handle them.  The real problem is that this flag probably
 730	 * should be 0 on ARM as it is not supported on this
 731	 * platform; see CONFIG_HUGETLBFS.
 732	 */
 733	gfp &= ~(__GFP_COMP);
 734	args.gfp = gfp;
 735
 736	*handle = DMA_MAPPING_ERROR;
 737	allowblock = gfpflags_allow_blocking(gfp);
 738	cma = allowblock ? dev_get_cma_area(dev) : false;
 739
 740	if (cma)
 741		buf->allocator = &cma_allocator;
 742	else if (is_coherent)
 743		buf->allocator = &simple_allocator;
 744	else if (allowblock)
 745		buf->allocator = &remap_allocator;
 746	else
 747		buf->allocator = &pool_allocator;
 748
 749	addr = buf->allocator->alloc(&args, &page);
 750
 751	if (page) {
 752		unsigned long flags;
 753
 754		*handle = pfn_to_dma(dev, page_to_pfn(page));
 755		buf->virt = args.want_vaddr ? addr : page;
 756
 757		spin_lock_irqsave(&arm_dma_bufs_lock, flags);
 758		list_add(&buf->list, &arm_dma_bufs);
 759		spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
 760	} else {
 761		kfree(buf);
 762	}
 763
 764	return args.want_vaddr ? addr : page;
 765}
 766
 767/*
 768 * Allocate DMA-coherent memory space and return both the kernel remapped
 769 * virtual and bus address for that space.
 770 */
 771void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 772		    gfp_t gfp, unsigned long attrs)
 773{
 774	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
 775
 776	return __dma_alloc(dev, size, handle, gfp, prot, false,
 777			   attrs, __builtin_return_address(0));
 778}
 779
 780static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 781	dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
 782{
 783	return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true,
 784			   attrs, __builtin_return_address(0));
 785}
 786
 787static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 788		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 789		 unsigned long attrs)
 790{
 791	int ret = -ENXIO;
 792	unsigned long nr_vma_pages = vma_pages(vma);
 793	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 794	unsigned long pfn = dma_to_pfn(dev, dma_addr);
 795	unsigned long off = vma->vm_pgoff;
 796
 797	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
 798		return ret;
 799
 800	if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
 801		ret = remap_pfn_range(vma, vma->vm_start,
 802				      pfn + off,
 803				      vma->vm_end - vma->vm_start,
 804				      vma->vm_page_prot);
 805	}
 806
 807	return ret;
 808}
 809
 810/*
 811 * Create userspace mapping for the DMA-coherent memory.
 812 */
 813static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 814		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 815		 unsigned long attrs)
 816{
 817	return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 818}
 819
 820int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 821		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 822		 unsigned long attrs)
 823{
 824	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
 825	return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 826}
 827
 828/*
 829 * Free a buffer as defined by the above mapping.
 830 */
 831static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 832			   dma_addr_t handle, unsigned long attrs,
 833			   bool is_coherent)
 834{
 835	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
 836	struct arm_dma_buffer *buf;
 837	struct arm_dma_free_args args = {
 838		.dev = dev,
 839		.size = PAGE_ALIGN(size),
 840		.cpu_addr = cpu_addr,
 841		.page = page,
 842		.want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
 843	};
 844
 845	buf = arm_dma_buffer_find(cpu_addr);
 846	if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr))
 847		return;
 848
 849	buf->allocator->free(&args);
 850	kfree(buf);
 851}
 852
 853void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 854		  dma_addr_t handle, unsigned long attrs)
 855{
 856	__arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
 857}
 858
 859static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
 860				  dma_addr_t handle, unsigned long attrs)
 861{
 862	__arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
 863}
 864
 865int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
 866		 void *cpu_addr, dma_addr_t handle, size_t size,
 867		 unsigned long attrs)
 868{
 869	unsigned long pfn = dma_to_pfn(dev, handle);
 870	struct page *page;
 871	int ret;
 872
 873	/* If the PFN is not valid, we do not have a struct page */
 874	if (!pfn_valid(pfn))
 875		return -ENXIO;
 876
 877	page = pfn_to_page(pfn);
 878
 879	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
 880	if (unlikely(ret))
 881		return ret;
 882
 883	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
 884	return 0;
 885}
 886
 887static void dma_cache_maint_page(struct page *page, unsigned long offset,
 888	size_t size, enum dma_data_direction dir,
 889	void (*op)(const void *, size_t, int))
 890{
 891	unsigned long pfn;
 892	size_t left = size;
 893
 894	pfn = page_to_pfn(page) + offset / PAGE_SIZE;
 895	offset %= PAGE_SIZE;
 896
 897	/*
 898	 * A single sg entry may refer to multiple physically contiguous
 899	 * pages.  But we still need to process highmem pages individually.
 900	 * If highmem is not configured then the bulk of this loop gets
 901	 * optimized out.
 902	 */
 903	do {
 904		size_t len = left;
 905		void *vaddr;
 906
 907		page = pfn_to_page(pfn);
 908
 909		if (PageHighMem(page)) {
 910			if (len + offset > PAGE_SIZE)
 911				len = PAGE_SIZE - offset;
 912
 913			if (cache_is_vipt_nonaliasing()) {
 914				vaddr = kmap_atomic(page);
 915				op(vaddr + offset, len, dir);
 916				kunmap_atomic(vaddr);
 917			} else {
 918				vaddr = kmap_high_get(page);
 919				if (vaddr) {
 920					op(vaddr + offset, len, dir);
 921					kunmap_high(page);
 922				}
 923			}
 924		} else {
 925			vaddr = page_address(page) + offset;
 926			op(vaddr, len, dir);
 927		}
 928		offset = 0;
 929		pfn++;
 930		left -= len;
 931	} while (left);
 932}
 933
 934/*
 935 * Make an area consistent for devices.
 936 * Note: Drivers should NOT use this function directly, as it will break
 937 * platforms with CONFIG_DMABOUNCE.
 938 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
 939 */
 940static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
 941	size_t size, enum dma_data_direction dir)
 942{
 943	phys_addr_t paddr;
 944
 945	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
 946
 947	paddr = page_to_phys(page) + off;
 948	if (dir == DMA_FROM_DEVICE) {
 949		outer_inv_range(paddr, paddr + size);
 950	} else {
 951		outer_clean_range(paddr, paddr + size);
 952	}
 953	/* FIXME: non-speculating: flush on bidirectional mappings? */
 954}
 955
 956static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
 957	size_t size, enum dma_data_direction dir)
 958{
 959	phys_addr_t paddr = page_to_phys(page) + off;
 960
 961	/* FIXME: non-speculating: not required */
 962	/* in any case, don't bother invalidating if DMA to device */
 963	if (dir != DMA_TO_DEVICE) {
 964		outer_inv_range(paddr, paddr + size);
 965
 966		dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
 967	}
 968
 969	/*
 970	 * Mark the D-cache clean for these pages to avoid extra flushing.
 971	 */
 972	if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
 973		unsigned long pfn;
 974		size_t left = size;
 975
 976		pfn = page_to_pfn(page) + off / PAGE_SIZE;
 977		off %= PAGE_SIZE;
 978		if (off) {
 979			pfn++;
 980			left -= PAGE_SIZE - off;
 981		}
 982		while (left >= PAGE_SIZE) {
 983			page = pfn_to_page(pfn++);
 984			set_bit(PG_dcache_clean, &page->flags);
 985			left -= PAGE_SIZE;
 
 
 986		}
 987	}
 988}
 989
 990/**
 991 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
 992 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 993 * @sg: list of buffers
 994 * @nents: number of buffers to map
 995 * @dir: DMA transfer direction
 996 *
 997 * Map a set of buffers described by scatterlist in streaming mode for DMA.
 998 * This is the scatter-gather version of the dma_map_single interface.
 999 * Here the scatter gather list elements are each tagged with the
1000 * appropriate dma address and length.  They are obtained via
1001 * sg_dma_{address,length}.
1002 *
1003 * Device ownership issues as mentioned for dma_map_single are the same
1004 * here.
1005 */
1006int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1007		enum dma_data_direction dir, unsigned long attrs)
1008{
1009	const struct dma_map_ops *ops = get_dma_ops(dev);
1010	struct scatterlist *s;
1011	int i, j;
1012
1013	for_each_sg(sg, s, nents, i) {
1014#ifdef CONFIG_NEED_SG_DMA_LENGTH
1015		s->dma_length = s->length;
1016#endif
1017		s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
1018						s->length, dir, attrs);
1019		if (dma_mapping_error(dev, s->dma_address))
1020			goto bad_mapping;
1021	}
1022	return nents;
1023
1024 bad_mapping:
1025	for_each_sg(sg, s, i, j)
1026		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1027	return 0;
1028}
1029
1030/**
1031 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1032 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1033 * @sg: list of buffers
1034 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1035 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1036 *
1037 * Unmap a set of streaming mode DMA translations.  Again, CPU access
1038 * rules concerning calls here are the same as for dma_unmap_single().
1039 */
1040void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1041		enum dma_data_direction dir, unsigned long attrs)
1042{
1043	const struct dma_map_ops *ops = get_dma_ops(dev);
1044	struct scatterlist *s;
1045
1046	int i;
1047
1048	for_each_sg(sg, s, nents, i)
1049		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1050}
1051
1052/**
1053 * arm_dma_sync_sg_for_cpu
1054 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1055 * @sg: list of buffers
1056 * @nents: number of buffers to map (returned from dma_map_sg)
1057 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1058 */
1059void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1060			int nents, enum dma_data_direction dir)
1061{
1062	const struct dma_map_ops *ops = get_dma_ops(dev);
1063	struct scatterlist *s;
1064	int i;
1065
1066	for_each_sg(sg, s, nents, i)
1067		ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
1068					 dir);
1069}
1070
1071/**
1072 * arm_dma_sync_sg_for_device
1073 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1074 * @sg: list of buffers
1075 * @nents: number of buffers to map (returned from dma_map_sg)
1076 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1077 */
1078void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1079			int nents, enum dma_data_direction dir)
1080{
1081	const struct dma_map_ops *ops = get_dma_ops(dev);
1082	struct scatterlist *s;
1083	int i;
1084
1085	for_each_sg(sg, s, nents, i)
1086		ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
1087					    dir);
1088}
1089
1090/*
1091 * Return whether the given device DMA address mask can be supported
1092 * properly.  For example, if your device can only drive the low 24-bits
1093 * during bus mastering, then you would pass 0x00ffffff as the mask
1094 * to this function.
1095 */
1096int arm_dma_supported(struct device *dev, u64 mask)
1097{
1098	return __dma_supported(dev, mask, false);
1099}
1100
1101static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
1102{
1103	/*
1104	 * When CONFIG_ARM_LPAE is set, physical address can extend above
1105	 * 32-bits, which then can't be addressed by devices that only support
1106	 * 32-bit DMA.
1107	 * Use the generic dma-direct / swiotlb ops code in that case, as that
1108	 * handles bounce buffering for us.
1109	 */
1110	if (IS_ENABLED(CONFIG_ARM_LPAE))
1111		return NULL;
1112	return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;
1113}
1114
1115#ifdef CONFIG_ARM_DMA_USE_IOMMU
1116
1117static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs)
1118{
1119	int prot = 0;
1120
1121	if (attrs & DMA_ATTR_PRIVILEGED)
1122		prot |= IOMMU_PRIV;
1123
1124	switch (dir) {
1125	case DMA_BIDIRECTIONAL:
1126		return prot | IOMMU_READ | IOMMU_WRITE;
1127	case DMA_TO_DEVICE:
1128		return prot | IOMMU_READ;
1129	case DMA_FROM_DEVICE:
1130		return prot | IOMMU_WRITE;
1131	default:
1132		return prot;
1133	}
1134}
1135
1136/* IOMMU */
1137
1138static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
1139
1140static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1141				      size_t size)
1142{
1143	unsigned int order = get_order(size);
1144	unsigned int align = 0;
1145	unsigned int count, start;
1146	size_t mapping_size = mapping->bits << PAGE_SHIFT;
1147	unsigned long flags;
1148	dma_addr_t iova;
1149	int i;
1150
1151	if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
1152		order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
1153
1154	count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1155	align = (1 << order) - 1;
1156
1157	spin_lock_irqsave(&mapping->lock, flags);
1158	for (i = 0; i < mapping->nr_bitmaps; i++) {
1159		start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1160				mapping->bits, 0, count, align);
1161
1162		if (start > mapping->bits)
1163			continue;
1164
1165		bitmap_set(mapping->bitmaps[i], start, count);
1166		break;
1167	}
1168
1169	/*
1170	 * No unused range found. Try to extend the existing mapping
1171	 * and perform a second attempt to reserve an IO virtual
1172	 * address range of size bytes.
1173	 */
1174	if (i == mapping->nr_bitmaps) {
1175		if (extend_iommu_mapping(mapping)) {
1176			spin_unlock_irqrestore(&mapping->lock, flags);
1177			return DMA_MAPPING_ERROR;
1178		}
1179
1180		start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1181				mapping->bits, 0, count, align);
1182
1183		if (start > mapping->bits) {
1184			spin_unlock_irqrestore(&mapping->lock, flags);
1185			return DMA_MAPPING_ERROR;
1186		}
1187
1188		bitmap_set(mapping->bitmaps[i], start, count);
1189	}
1190	spin_unlock_irqrestore(&mapping->lock, flags);
1191
1192	iova = mapping->base + (mapping_size * i);
1193	iova += start << PAGE_SHIFT;
1194
1195	return iova;
1196}
1197
1198static inline void __free_iova(struct dma_iommu_mapping *mapping,
1199			       dma_addr_t addr, size_t size)
1200{
1201	unsigned int start, count;
1202	size_t mapping_size = mapping->bits << PAGE_SHIFT;
1203	unsigned long flags;
1204	dma_addr_t bitmap_base;
1205	u32 bitmap_index;
1206
1207	if (!size)
1208		return;
1209
1210	bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;
1211	BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);
1212
1213	bitmap_base = mapping->base + mapping_size * bitmap_index;
1214
1215	start = (addr - bitmap_base) >>	PAGE_SHIFT;
1216
1217	if (addr + size > bitmap_base + mapping_size) {
1218		/*
1219		 * The address range to be freed reaches into the iova
1220		 * range of the next bitmap. This should not happen as
1221		 * we don't allow this in __alloc_iova (at the
1222		 * moment).
1223		 */
1224		BUG();
1225	} else
1226		count = size >> PAGE_SHIFT;
1227
1228	spin_lock_irqsave(&mapping->lock, flags);
1229	bitmap_clear(mapping->bitmaps[bitmap_index], start, count);
1230	spin_unlock_irqrestore(&mapping->lock, flags);
1231}
1232
1233/* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */
1234static const int iommu_order_array[] = { 9, 8, 4, 0 };
1235
1236static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1237					  gfp_t gfp, unsigned long attrs,
1238					  int coherent_flag)
1239{
1240	struct page **pages;
1241	int count = size >> PAGE_SHIFT;
1242	int array_size = count * sizeof(struct page *);
1243	int i = 0;
1244	int order_idx = 0;
1245
1246	if (array_size <= PAGE_SIZE)
1247		pages = kzalloc(array_size, GFP_KERNEL);
1248	else
1249		pages = vzalloc(array_size);
1250	if (!pages)
1251		return NULL;
1252
1253	if (attrs & DMA_ATTR_FORCE_CONTIGUOUS)
1254	{
1255		unsigned long order = get_order(size);
1256		struct page *page;
1257
1258		page = dma_alloc_from_contiguous(dev, count, order,
1259						 gfp & __GFP_NOWARN);
1260		if (!page)
1261			goto error;
1262
1263		__dma_clear_buffer(page, size, coherent_flag);
1264
1265		for (i = 0; i < count; i++)
1266			pages[i] = page + i;
1267
1268		return pages;
1269	}
1270
1271	/* Go straight to 4K chunks if caller says it's OK. */
1272	if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES)
1273		order_idx = ARRAY_SIZE(iommu_order_array) - 1;
1274
1275	/*
1276	 * IOMMU can map any pages, so himem can also be used here
1277	 */
1278	gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1279
1280	while (count) {
1281		int j, order;
1282
1283		order = iommu_order_array[order_idx];
1284
1285		/* Drop down when we get small */
1286		if (__fls(count) < order) {
1287			order_idx++;
1288			continue;
1289		}
1290
1291		if (order) {
1292			/* See if it's easy to allocate a high-order chunk */
1293			pages[i] = alloc_pages(gfp | __GFP_NORETRY, order);
1294
1295			/* Go down a notch at first sign of pressure */
1296			if (!pages[i]) {
1297				order_idx++;
1298				continue;
1299			}
1300		} else {
1301			pages[i] = alloc_pages(gfp, 0);
1302			if (!pages[i])
1303				goto error;
1304		}
1305
1306		if (order) {
1307			split_page(pages[i], order);
1308			j = 1 << order;
1309			while (--j)
1310				pages[i + j] = pages[i] + j;
1311		}
1312
1313		__dma_clear_buffer(pages[i], PAGE_SIZE << order, coherent_flag);
1314		i += 1 << order;
1315		count -= 1 << order;
1316	}
1317
1318	return pages;
1319error:
1320	while (i--)
1321		if (pages[i])
1322			__free_pages(pages[i], 0);
1323	kvfree(pages);
1324	return NULL;
1325}
1326
1327static int __iommu_free_buffer(struct device *dev, struct page **pages,
1328			       size_t size, unsigned long attrs)
1329{
1330	int count = size >> PAGE_SHIFT;
1331	int i;
1332
1333	if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
1334		dma_release_from_contiguous(dev, pages[0], count);
1335	} else {
1336		for (i = 0; i < count; i++)
1337			if (pages[i])
1338				__free_pages(pages[i], 0);
1339	}
1340
1341	kvfree(pages);
1342	return 0;
1343}
1344
1345/*
1346 * Create a mapping in device IO address space for specified pages
1347 */
1348static dma_addr_t
1349__iommu_create_mapping(struct device *dev, struct page **pages, size_t size,
1350		       unsigned long attrs)
1351{
1352	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1353	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1354	dma_addr_t dma_addr, iova;
1355	int i;
1356
1357	dma_addr = __alloc_iova(mapping, size);
1358	if (dma_addr == DMA_MAPPING_ERROR)
1359		return dma_addr;
1360
1361	iova = dma_addr;
1362	for (i = 0; i < count; ) {
1363		int ret;
1364
1365		unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1366		phys_addr_t phys = page_to_phys(pages[i]);
1367		unsigned int len, j;
1368
1369		for (j = i + 1; j < count; j++, next_pfn++)
1370			if (page_to_pfn(pages[j]) != next_pfn)
1371				break;
1372
1373		len = (j - i) << PAGE_SHIFT;
1374		ret = iommu_map(mapping->domain, iova, phys, len,
1375				__dma_info_to_prot(DMA_BIDIRECTIONAL, attrs));
 
1376		if (ret < 0)
1377			goto fail;
1378		iova += len;
1379		i = j;
1380	}
1381	return dma_addr;
1382fail:
1383	iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1384	__free_iova(mapping, dma_addr, size);
1385	return DMA_MAPPING_ERROR;
1386}
1387
1388static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1389{
1390	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1391
1392	/*
1393	 * add optional in-page offset from iova to size and align
1394	 * result to page size
1395	 */
1396	size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1397	iova &= PAGE_MASK;
1398
1399	iommu_unmap(mapping->domain, iova, size);
1400	__free_iova(mapping, iova, size);
1401	return 0;
1402}
1403
1404static struct page **__atomic_get_pages(void *addr)
1405{
1406	struct page *page;
1407	phys_addr_t phys;
1408
1409	phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
1410	page = phys_to_page(phys);
1411
1412	return (struct page **)page;
1413}
1414
1415static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs)
1416{
1417	if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
1418		return __atomic_get_pages(cpu_addr);
1419
1420	if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
1421		return cpu_addr;
1422
1423	return dma_common_find_pages(cpu_addr);
1424}
1425
1426static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp,
1427				  dma_addr_t *handle, int coherent_flag,
1428				  unsigned long attrs)
1429{
1430	struct page *page;
1431	void *addr;
1432
1433	if (coherent_flag  == COHERENT)
1434		addr = __alloc_simple_buffer(dev, size, gfp, &page);
1435	else
1436		addr = __alloc_from_pool(size, &page);
1437	if (!addr)
1438		return NULL;
1439
1440	*handle = __iommu_create_mapping(dev, &page, size, attrs);
1441	if (*handle == DMA_MAPPING_ERROR)
1442		goto err_mapping;
1443
1444	return addr;
1445
1446err_mapping:
1447	__free_from_pool(addr, size);
1448	return NULL;
1449}
1450
1451static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1452			dma_addr_t handle, size_t size, int coherent_flag)
1453{
1454	__iommu_remove_mapping(dev, handle, size);
1455	if (coherent_flag == COHERENT)
1456		__dma_free_buffer(virt_to_page(cpu_addr), size);
1457	else
1458		__free_from_pool(cpu_addr, size);
1459}
1460
1461static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size,
1462	    dma_addr_t *handle, gfp_t gfp, unsigned long attrs,
1463	    int coherent_flag)
1464{
1465	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
1466	struct page **pages;
1467	void *addr = NULL;
 
1468
1469	*handle = DMA_MAPPING_ERROR;
1470	size = PAGE_ALIGN(size);
1471
1472	if (coherent_flag  == COHERENT || !gfpflags_allow_blocking(gfp))
1473		return __iommu_alloc_simple(dev, size, gfp, handle,
1474					    coherent_flag, attrs);
1475
1476	/*
1477	 * Following is a work-around (a.k.a. hack) to prevent pages
1478	 * with __GFP_COMP being passed to split_page() which cannot
1479	 * handle them.  The real problem is that this flag probably
1480	 * should be 0 on ARM as it is not supported on this
1481	 * platform; see CONFIG_HUGETLBFS.
1482	 */
1483	gfp &= ~(__GFP_COMP);
1484
1485	pages = __iommu_alloc_buffer(dev, size, gfp, attrs, coherent_flag);
1486	if (!pages)
1487		return NULL;
1488
1489	*handle = __iommu_create_mapping(dev, pages, size, attrs);
1490	if (*handle == DMA_MAPPING_ERROR)
1491		goto err_buffer;
1492
1493	if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
1494		return pages;
1495
1496	addr = dma_common_pages_remap(pages, size, prot,
1497				   __builtin_return_address(0));
1498	if (!addr)
1499		goto err_mapping;
1500
1501	return addr;
1502
1503err_mapping:
1504	__iommu_remove_mapping(dev, *handle, size);
1505err_buffer:
1506	__iommu_free_buffer(dev, pages, size, attrs);
1507	return NULL;
1508}
1509
1510static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1511	    dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1512{
1513	return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, NORMAL);
1514}
1515
1516static void *arm_coherent_iommu_alloc_attrs(struct device *dev, size_t size,
1517		    dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1518{
1519	return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, COHERENT);
1520}
1521
1522static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1523		    void *cpu_addr, dma_addr_t dma_addr, size_t size,
1524		    unsigned long attrs)
1525{
1526	struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1527	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1528	int err;
1529
1530	if (!pages)
1531		return -ENXIO;
1532
1533	if (vma->vm_pgoff >= nr_pages)
1534		return -ENXIO;
1535
 
 
 
1536	err = vm_map_pages(vma, pages, nr_pages);
1537	if (err)
1538		pr_err("Remapping memory failed: %d\n", err);
1539
1540	return err;
1541}
1542static int arm_iommu_mmap_attrs(struct device *dev,
1543		struct vm_area_struct *vma, void *cpu_addr,
1544		dma_addr_t dma_addr, size_t size, unsigned long attrs)
1545{
1546	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
1547
1548	return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs);
1549}
1550
1551static int arm_coherent_iommu_mmap_attrs(struct device *dev,
1552		struct vm_area_struct *vma, void *cpu_addr,
1553		dma_addr_t dma_addr, size_t size, unsigned long attrs)
1554{
1555	return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs);
1556}
1557
1558/*
1559 * free a page as defined by the above mapping.
1560 * Must not be called with IRQs disabled.
1561 */
1562void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1563	dma_addr_t handle, unsigned long attrs, int coherent_flag)
1564{
 
1565	struct page **pages;
1566	size = PAGE_ALIGN(size);
1567
1568	if (coherent_flag == COHERENT || __in_atomic_pool(cpu_addr, size)) {
1569		__iommu_free_atomic(dev, cpu_addr, handle, size, coherent_flag);
1570		return;
1571	}
1572
1573	pages = __iommu_get_pages(cpu_addr, attrs);
1574	if (!pages) {
1575		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1576		return;
1577	}
1578
1579	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0)
1580		dma_common_free_remap(cpu_addr, size);
1581
1582	__iommu_remove_mapping(dev, handle, size);
1583	__iommu_free_buffer(dev, pages, size, attrs);
1584}
1585
1586void arm_iommu_free_attrs(struct device *dev, size_t size,
1587		    void *cpu_addr, dma_addr_t handle, unsigned long attrs)
1588{
1589	__arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, NORMAL);
1590}
1591
1592void arm_coherent_iommu_free_attrs(struct device *dev, size_t size,
1593		    void *cpu_addr, dma_addr_t handle, unsigned long attrs)
1594{
1595	__arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, COHERENT);
1596}
1597
1598static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
1599				 void *cpu_addr, dma_addr_t dma_addr,
1600				 size_t size, unsigned long attrs)
1601{
1602	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1603	struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1604
1605	if (!pages)
1606		return -ENXIO;
1607
1608	return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
1609					 GFP_KERNEL);
1610}
1611
1612/*
1613 * Map a part of the scatter-gather list into contiguous io address space
1614 */
1615static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1616			  size_t size, dma_addr_t *handle,
1617			  enum dma_data_direction dir, unsigned long attrs,
1618			  bool is_coherent)
1619{
1620	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1621	dma_addr_t iova, iova_base;
1622	int ret = 0;
1623	unsigned int count;
1624	struct scatterlist *s;
1625	int prot;
1626
1627	size = PAGE_ALIGN(size);
1628	*handle = DMA_MAPPING_ERROR;
1629
1630	iova_base = iova = __alloc_iova(mapping, size);
1631	if (iova == DMA_MAPPING_ERROR)
1632		return -ENOMEM;
1633
1634	for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1635		phys_addr_t phys = page_to_phys(sg_page(s));
1636		unsigned int len = PAGE_ALIGN(s->offset + s->length);
1637
1638		if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1639			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1640
1641		prot = __dma_info_to_prot(dir, attrs);
1642
1643		ret = iommu_map(mapping->domain, iova, phys, len, prot);
 
1644		if (ret < 0)
1645			goto fail;
1646		count += len >> PAGE_SHIFT;
1647		iova += len;
1648	}
1649	*handle = iova_base;
1650
1651	return 0;
1652fail:
1653	iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1654	__free_iova(mapping, iova_base, size);
1655	return ret;
1656}
1657
1658static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1659		     enum dma_data_direction dir, unsigned long attrs,
1660		     bool is_coherent)
 
 
 
 
 
 
 
 
 
 
 
1661{
1662	struct scatterlist *s = sg, *dma = sg, *start = sg;
1663	int i, count = 0;
1664	unsigned int offset = s->offset;
1665	unsigned int size = s->offset + s->length;
1666	unsigned int max = dma_get_max_seg_size(dev);
1667
1668	for (i = 1; i < nents; i++) {
1669		s = sg_next(s);
1670
1671		s->dma_address = DMA_MAPPING_ERROR;
1672		s->dma_length = 0;
1673
1674		if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1675			if (__map_sg_chunk(dev, start, size, &dma->dma_address,
1676			    dir, attrs, is_coherent) < 0)
 
1677				goto bad_mapping;
1678
1679			dma->dma_address += offset;
1680			dma->dma_length = size - offset;
1681
1682			size = offset = s->offset;
1683			start = s;
1684			dma = sg_next(dma);
1685			count += 1;
1686		}
1687		size += s->length;
1688	}
1689	if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
1690		is_coherent) < 0)
1691		goto bad_mapping;
1692
1693	dma->dma_address += offset;
1694	dma->dma_length = size - offset;
1695
1696	return count+1;
1697
1698bad_mapping:
1699	for_each_sg(sg, s, count, i)
1700		__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1701	return 0;
1702}
1703
1704/**
1705 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1706 * @dev: valid struct device pointer
1707 * @sg: list of buffers
1708 * @nents: number of buffers to map
1709 * @dir: DMA transfer direction
1710 *
1711 * Map a set of i/o coherent buffers described by scatterlist in streaming
1712 * mode for DMA. The scatter gather list elements are merged together (if
1713 * possible) and tagged with the appropriate dma address and length. They are
1714 * obtained via sg_dma_{address,length}.
1715 */
1716int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1717		int nents, enum dma_data_direction dir, unsigned long attrs)
1718{
1719	return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
1720}
1721
1722/**
1723 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1724 * @dev: valid struct device pointer
1725 * @sg: list of buffers
1726 * @nents: number of buffers to map
1727 * @dir: DMA transfer direction
1728 *
1729 * Map a set of buffers described by scatterlist in streaming mode for DMA.
1730 * The scatter gather list elements are merged together (if possible) and
1731 * tagged with the appropriate dma address and length. They are obtained via
1732 * sg_dma_{address,length}.
1733 */
1734int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1735		int nents, enum dma_data_direction dir, unsigned long attrs)
1736{
1737	return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
1738}
1739
1740static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1741		int nents, enum dma_data_direction dir,
1742		unsigned long attrs, bool is_coherent)
1743{
1744	struct scatterlist *s;
1745	int i;
1746
1747	for_each_sg(sg, s, nents, i) {
1748		if (sg_dma_len(s))
1749			__iommu_remove_mapping(dev, sg_dma_address(s),
1750					       sg_dma_len(s));
1751		if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1752			__dma_page_dev_to_cpu(sg_page(s), s->offset,
1753					      s->length, dir);
1754	}
1755}
1756
1757/**
1758 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1759 * @dev: valid struct device pointer
1760 * @sg: list of buffers
1761 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1762 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1763 *
1764 * Unmap a set of streaming mode DMA translations.  Again, CPU access
1765 * rules concerning calls here are the same as for dma_unmap_single().
1766 */
1767void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1768		int nents, enum dma_data_direction dir,
1769		unsigned long attrs)
1770{
1771	__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
1772}
1773
1774/**
1775 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1776 * @dev: valid struct device pointer
1777 * @sg: list of buffers
1778 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1779 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1780 *
1781 * Unmap a set of streaming mode DMA translations.  Again, CPU access
1782 * rules concerning calls here are the same as for dma_unmap_single().
1783 */
1784void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1785			enum dma_data_direction dir,
1786			unsigned long attrs)
1787{
1788	__iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
1789}
1790
1791/**
1792 * arm_iommu_sync_sg_for_cpu
1793 * @dev: valid struct device pointer
1794 * @sg: list of buffers
1795 * @nents: number of buffers to map (returned from dma_map_sg)
1796 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1797 */
1798void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 
1799			int nents, enum dma_data_direction dir)
1800{
1801	struct scatterlist *s;
1802	int i;
1803
 
 
 
1804	for_each_sg(sg, s, nents, i)
1805		__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1806
1807}
1808
1809/**
1810 * arm_iommu_sync_sg_for_device
1811 * @dev: valid struct device pointer
1812 * @sg: list of buffers
1813 * @nents: number of buffers to map (returned from dma_map_sg)
1814 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1815 */
1816void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 
1817			int nents, enum dma_data_direction dir)
1818{
1819	struct scatterlist *s;
1820	int i;
1821
 
 
 
1822	for_each_sg(sg, s, nents, i)
1823		__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1824}
1825
1826
1827/**
1828 * arm_coherent_iommu_map_page
1829 * @dev: valid struct device pointer
1830 * @page: page that buffer resides in
1831 * @offset: offset into page for start of buffer
1832 * @size: size of buffer to map
1833 * @dir: DMA transfer direction
1834 *
1835 * Coherent IOMMU aware version of arm_dma_map_page()
1836 */
1837static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1838	     unsigned long offset, size_t size, enum dma_data_direction dir,
1839	     unsigned long attrs)
1840{
1841	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1842	dma_addr_t dma_addr;
1843	int ret, prot, len = PAGE_ALIGN(size + offset);
1844
 
 
 
1845	dma_addr = __alloc_iova(mapping, len);
1846	if (dma_addr == DMA_MAPPING_ERROR)
1847		return dma_addr;
1848
1849	prot = __dma_info_to_prot(dir, attrs);
1850
1851	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
 
1852	if (ret < 0)
1853		goto fail;
1854
1855	return dma_addr + offset;
1856fail:
1857	__free_iova(mapping, dma_addr, len);
1858	return DMA_MAPPING_ERROR;
1859}
1860
1861/**
1862 * arm_iommu_map_page
1863 * @dev: valid struct device pointer
1864 * @page: page that buffer resides in
1865 * @offset: offset into page for start of buffer
1866 * @size: size of buffer to map
1867 * @dir: DMA transfer direction
1868 *
1869 * IOMMU aware version of arm_dma_map_page()
1870 */
1871static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1872	     unsigned long offset, size_t size, enum dma_data_direction dir,
1873	     unsigned long attrs)
1874{
1875	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1876		__dma_page_cpu_to_dev(page, offset, size, dir);
1877
1878	return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
1879}
1880
1881/**
1882 * arm_coherent_iommu_unmap_page
1883 * @dev: valid struct device pointer
1884 * @handle: DMA address of buffer
1885 * @size: size of buffer (same as passed to dma_map_page)
1886 * @dir: DMA transfer direction (same as passed to dma_map_page)
1887 *
1888 * Coherent IOMMU aware version of arm_dma_unmap_page()
1889 */
1890static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1891		size_t size, enum dma_data_direction dir, unsigned long attrs)
1892{
1893	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1894	dma_addr_t iova = handle & PAGE_MASK;
1895	int offset = handle & ~PAGE_MASK;
1896	int len = PAGE_ALIGN(size + offset);
1897
1898	if (!iova)
1899		return;
1900
1901	iommu_unmap(mapping->domain, iova, len);
1902	__free_iova(mapping, iova, len);
1903}
1904
1905/**
1906 * arm_iommu_unmap_page
1907 * @dev: valid struct device pointer
1908 * @handle: DMA address of buffer
1909 * @size: size of buffer (same as passed to dma_map_page)
1910 * @dir: DMA transfer direction (same as passed to dma_map_page)
1911 *
1912 * IOMMU aware version of arm_dma_unmap_page()
1913 */
1914static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1915		size_t size, enum dma_data_direction dir, unsigned long attrs)
1916{
1917	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1918	dma_addr_t iova = handle & PAGE_MASK;
1919	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1920	int offset = handle & ~PAGE_MASK;
1921	int len = PAGE_ALIGN(size + offset);
1922
1923	if (!iova)
1924		return;
1925
1926	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
 
1927		__dma_page_dev_to_cpu(page, offset, size, dir);
 
1928
1929	iommu_unmap(mapping->domain, iova, len);
1930	__free_iova(mapping, iova, len);
1931}
1932
1933/**
1934 * arm_iommu_map_resource - map a device resource for DMA
1935 * @dev: valid struct device pointer
1936 * @phys_addr: physical address of resource
1937 * @size: size of resource to map
1938 * @dir: DMA transfer direction
1939 */
1940static dma_addr_t arm_iommu_map_resource(struct device *dev,
1941		phys_addr_t phys_addr, size_t size,
1942		enum dma_data_direction dir, unsigned long attrs)
1943{
1944	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1945	dma_addr_t dma_addr;
1946	int ret, prot;
1947	phys_addr_t addr = phys_addr & PAGE_MASK;
1948	unsigned int offset = phys_addr & ~PAGE_MASK;
1949	size_t len = PAGE_ALIGN(size + offset);
1950
1951	dma_addr = __alloc_iova(mapping, len);
1952	if (dma_addr == DMA_MAPPING_ERROR)
1953		return dma_addr;
1954
1955	prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO;
1956
1957	ret = iommu_map(mapping->domain, dma_addr, addr, len, prot);
1958	if (ret < 0)
1959		goto fail;
1960
1961	return dma_addr + offset;
1962fail:
1963	__free_iova(mapping, dma_addr, len);
1964	return DMA_MAPPING_ERROR;
1965}
1966
1967/**
1968 * arm_iommu_unmap_resource - unmap a device DMA resource
1969 * @dev: valid struct device pointer
1970 * @dma_handle: DMA address to resource
1971 * @size: size of resource to map
1972 * @dir: DMA transfer direction
1973 */
1974static void arm_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle,
1975		size_t size, enum dma_data_direction dir,
1976		unsigned long attrs)
1977{
1978	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1979	dma_addr_t iova = dma_handle & PAGE_MASK;
1980	unsigned int offset = dma_handle & ~PAGE_MASK;
1981	size_t len = PAGE_ALIGN(size + offset);
1982
1983	if (!iova)
1984		return;
1985
1986	iommu_unmap(mapping->domain, iova, len);
1987	__free_iova(mapping, iova, len);
1988}
1989
1990static void arm_iommu_sync_single_for_cpu(struct device *dev,
1991		dma_addr_t handle, size_t size, enum dma_data_direction dir)
1992{
1993	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1994	dma_addr_t iova = handle & PAGE_MASK;
1995	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1996	unsigned int offset = handle & ~PAGE_MASK;
1997
1998	if (!iova)
1999		return;
2000
 
2001	__dma_page_dev_to_cpu(page, offset, size, dir);
2002}
2003
2004static void arm_iommu_sync_single_for_device(struct device *dev,
2005		dma_addr_t handle, size_t size, enum dma_data_direction dir)
2006{
2007	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
2008	dma_addr_t iova = handle & PAGE_MASK;
2009	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
2010	unsigned int offset = handle & ~PAGE_MASK;
2011
2012	if (!iova)
2013		return;
2014
 
2015	__dma_page_cpu_to_dev(page, offset, size, dir);
2016}
2017
2018const struct dma_map_ops iommu_ops = {
2019	.alloc		= arm_iommu_alloc_attrs,
2020	.free		= arm_iommu_free_attrs,
2021	.mmap		= arm_iommu_mmap_attrs,
2022	.get_sgtable	= arm_iommu_get_sgtable,
2023
2024	.map_page		= arm_iommu_map_page,
2025	.unmap_page		= arm_iommu_unmap_page,
2026	.sync_single_for_cpu	= arm_iommu_sync_single_for_cpu,
2027	.sync_single_for_device	= arm_iommu_sync_single_for_device,
2028
2029	.map_sg			= arm_iommu_map_sg,
2030	.unmap_sg		= arm_iommu_unmap_sg,
2031	.sync_sg_for_cpu	= arm_iommu_sync_sg_for_cpu,
2032	.sync_sg_for_device	= arm_iommu_sync_sg_for_device,
2033
2034	.map_resource		= arm_iommu_map_resource,
2035	.unmap_resource		= arm_iommu_unmap_resource,
2036
2037	.dma_supported		= arm_dma_supported,
2038};
2039
2040const struct dma_map_ops iommu_coherent_ops = {
2041	.alloc		= arm_coherent_iommu_alloc_attrs,
2042	.free		= arm_coherent_iommu_free_attrs,
2043	.mmap		= arm_coherent_iommu_mmap_attrs,
2044	.get_sgtable	= arm_iommu_get_sgtable,
2045
2046	.map_page	= arm_coherent_iommu_map_page,
2047	.unmap_page	= arm_coherent_iommu_unmap_page,
2048
2049	.map_sg		= arm_coherent_iommu_map_sg,
2050	.unmap_sg	= arm_coherent_iommu_unmap_sg,
2051
2052	.map_resource	= arm_iommu_map_resource,
2053	.unmap_resource	= arm_iommu_unmap_resource,
2054
2055	.dma_supported		= arm_dma_supported,
2056};
2057
2058/**
2059 * arm_iommu_create_mapping
2060 * @bus: pointer to the bus holding the client device (for IOMMU calls)
2061 * @base: start address of the valid IO address space
2062 * @size: maximum size of the valid IO address space
2063 *
2064 * Creates a mapping structure which holds information about used/unused
2065 * IO address ranges, which is required to perform memory allocation and
2066 * mapping with IOMMU aware functions.
2067 *
2068 * The client device need to be attached to the mapping with
2069 * arm_iommu_attach_device function.
2070 */
2071struct dma_iommu_mapping *
2072arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size)
2073{
2074	unsigned int bits = size >> PAGE_SHIFT;
2075	unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
2076	struct dma_iommu_mapping *mapping;
2077	int extensions = 1;
2078	int err = -ENOMEM;
2079
2080	/* currently only 32-bit DMA address space is supported */
2081	if (size > DMA_BIT_MASK(32) + 1)
2082		return ERR_PTR(-ERANGE);
2083
2084	if (!bitmap_size)
2085		return ERR_PTR(-EINVAL);
2086
2087	if (bitmap_size > PAGE_SIZE) {
2088		extensions = bitmap_size / PAGE_SIZE;
2089		bitmap_size = PAGE_SIZE;
2090	}
2091
2092	mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
2093	if (!mapping)
2094		goto err;
2095
2096	mapping->bitmap_size = bitmap_size;
2097	mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *),
2098				   GFP_KERNEL);
2099	if (!mapping->bitmaps)
2100		goto err2;
2101
2102	mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);
2103	if (!mapping->bitmaps[0])
2104		goto err3;
2105
2106	mapping->nr_bitmaps = 1;
2107	mapping->extensions = extensions;
2108	mapping->base = base;
2109	mapping->bits = BITS_PER_BYTE * bitmap_size;
2110
2111	spin_lock_init(&mapping->lock);
2112
2113	mapping->domain = iommu_domain_alloc(bus);
2114	if (!mapping->domain)
 
2115		goto err4;
 
2116
2117	kref_init(&mapping->kref);
2118	return mapping;
2119err4:
2120	kfree(mapping->bitmaps[0]);
2121err3:
2122	kfree(mapping->bitmaps);
2123err2:
2124	kfree(mapping);
2125err:
2126	return ERR_PTR(err);
2127}
2128EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
2129
2130static void release_iommu_mapping(struct kref *kref)
2131{
2132	int i;
2133	struct dma_iommu_mapping *mapping =
2134		container_of(kref, struct dma_iommu_mapping, kref);
2135
2136	iommu_domain_free(mapping->domain);
2137	for (i = 0; i < mapping->nr_bitmaps; i++)
2138		kfree(mapping->bitmaps[i]);
2139	kfree(mapping->bitmaps);
2140	kfree(mapping);
2141}
2142
2143static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
2144{
2145	int next_bitmap;
2146
2147	if (mapping->nr_bitmaps >= mapping->extensions)
2148		return -EINVAL;
2149
2150	next_bitmap = mapping->nr_bitmaps;
2151	mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,
2152						GFP_ATOMIC);
2153	if (!mapping->bitmaps[next_bitmap])
2154		return -ENOMEM;
2155
2156	mapping->nr_bitmaps++;
2157
2158	return 0;
2159}
2160
2161void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
2162{
2163	if (mapping)
2164		kref_put(&mapping->kref, release_iommu_mapping);
2165}
2166EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
2167
2168static int __arm_iommu_attach_device(struct device *dev,
2169				     struct dma_iommu_mapping *mapping)
2170{
2171	int err;
2172
2173	err = iommu_attach_device(mapping->domain, dev);
2174	if (err)
2175		return err;
2176
2177	kref_get(&mapping->kref);
2178	to_dma_iommu_mapping(dev) = mapping;
2179
2180	pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
2181	return 0;
2182}
2183
2184/**
2185 * arm_iommu_attach_device
2186 * @dev: valid struct device pointer
2187 * @mapping: io address space mapping structure (returned from
2188 *	arm_iommu_create_mapping)
2189 *
2190 * Attaches specified io address space mapping to the provided device.
2191 * This replaces the dma operations (dma_map_ops pointer) with the
2192 * IOMMU aware version.
2193 *
2194 * More than one client might be attached to the same io address space
2195 * mapping.
2196 */
2197int arm_iommu_attach_device(struct device *dev,
2198			    struct dma_iommu_mapping *mapping)
2199{
2200	int err;
2201
2202	err = __arm_iommu_attach_device(dev, mapping);
2203	if (err)
2204		return err;
2205
2206	set_dma_ops(dev, &iommu_ops);
2207	return 0;
2208}
2209EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
2210
2211/**
2212 * arm_iommu_detach_device
2213 * @dev: valid struct device pointer
2214 *
2215 * Detaches the provided device from a previously attached map.
2216 * This overwrites the dma_ops pointer with appropriate non-IOMMU ops.
2217 */
2218void arm_iommu_detach_device(struct device *dev)
2219{
2220	struct dma_iommu_mapping *mapping;
2221
2222	mapping = to_dma_iommu_mapping(dev);
2223	if (!mapping) {
2224		dev_warn(dev, "Not attached\n");
2225		return;
2226	}
2227
2228	iommu_detach_device(mapping->domain, dev);
2229	kref_put(&mapping->kref, release_iommu_mapping);
2230	to_dma_iommu_mapping(dev) = NULL;
2231	set_dma_ops(dev, arm_get_dma_map_ops(dev->archdata.dma_coherent));
2232
2233	pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
2234}
2235EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
2236
2237static const struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
2238{
2239	return coherent ? &iommu_coherent_ops : &iommu_ops;
2240}
2241
2242static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2243				    const struct iommu_ops *iommu)
2244{
2245	struct dma_iommu_mapping *mapping;
 
2246
2247	if (!iommu)
2248		return false;
2249
2250	mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
 
2251	if (IS_ERR(mapping)) {
2252		pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
2253				size, dev_name(dev));
2254		return false;
2255	}
2256
2257	if (__arm_iommu_attach_device(dev, mapping)) {
2258		pr_warn("Failed to attached device %s to IOMMU_mapping\n",
2259				dev_name(dev));
2260		arm_iommu_release_mapping(mapping);
2261		return false;
2262	}
2263
2264	return true;
2265}
2266
2267static void arm_teardown_iommu_dma_ops(struct device *dev)
2268{
2269	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
2270
2271	if (!mapping)
2272		return;
2273
2274	arm_iommu_detach_device(dev);
2275	arm_iommu_release_mapping(mapping);
2276}
2277
2278#else
2279
2280static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2281				    const struct iommu_ops *iommu)
2282{
2283	return false;
2284}
2285
2286static void arm_teardown_iommu_dma_ops(struct device *dev) { }
2287
2288#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops
2289
2290#endif	/* CONFIG_ARM_DMA_USE_IOMMU */
2291
2292void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
2293			const struct iommu_ops *iommu, bool coherent)
2294{
2295	const struct dma_map_ops *dma_ops;
2296
2297	dev->archdata.dma_coherent = coherent;
2298#ifdef CONFIG_SWIOTLB
2299	dev->dma_coherent = coherent;
2300#endif
 
 
2301
2302	/*
2303	 * Don't override the dma_ops if they have already been set. Ideally
2304	 * this should be the only location where dma_ops are set, remove this
2305	 * check when all other callers of set_dma_ops will have disappeared.
2306	 */
2307	if (dev->dma_ops)
2308		return;
2309
2310	if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))
2311		dma_ops = arm_get_iommu_dma_map_ops(coherent);
2312	else
2313		dma_ops = arm_get_dma_map_ops(coherent);
2314
2315	set_dma_ops(dev, dma_ops);
2316
2317#ifdef CONFIG_XEN
2318	if (xen_initial_domain())
2319		dev->dma_ops = &xen_swiotlb_dma_ops;
2320#endif
2321	dev->archdata.dma_ops_setup = true;
2322}
2323
2324void arch_teardown_dma_ops(struct device *dev)
2325{
2326	if (!dev->archdata.dma_ops_setup)
2327		return;
2328
2329	arm_teardown_iommu_dma_ops(dev);
2330	/* Let arch_setup_dma_ops() start again from scratch upon re-probe */
2331	set_dma_ops(dev, NULL);
2332}
2333
2334#ifdef CONFIG_SWIOTLB
2335void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
2336		size_t size, enum dma_data_direction dir)
2337{
2338	__dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1),
2339			      size, dir);
2340}
2341
2342void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
2343		size_t size, enum dma_data_direction dir)
2344{
2345	__dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1),
2346			      size, dir);
2347}
2348
2349long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
2350		dma_addr_t dma_addr)
2351{
2352	return dma_to_pfn(dev, dma_addr);
2353}
2354
2355void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
2356		gfp_t gfp, unsigned long attrs)
2357{
2358	return __dma_alloc(dev, size, dma_handle, gfp,
2359			   __get_dma_pgprot(attrs, PAGE_KERNEL), false,
2360			   attrs, __builtin_return_address(0));
2361}
2362
2363void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
2364		dma_addr_t dma_handle, unsigned long attrs)
2365{
2366	__arm_dma_free(dev, size, cpu_addr, dma_handle, attrs, false);
2367}
2368#endif /* CONFIG_SWIOTLB */