1/*
   2 *  linux/arch/arm/mm/dma-mapping.c
   3 *
   4 *  Copyright (C) 2000-2004 Russell King
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 *
  10 *  DMA uncached mapping support.
  11 */
  12#include <linux/bootmem.h>
  13#include <linux/module.h>
  14#include <linux/mm.h>
  15#include <linux/genalloc.h>
  16#include <linux/gfp.h>
  17#include <linux/errno.h>
  18#include <linux/list.h>
  19#include <linux/init.h>
  20#include <linux/device.h>
  21#include <linux/dma-mapping.h>
  22#include <linux/dma-contiguous.h>
  23#include <linux/highmem.h>
  24#include <linux/memblock.h>
  25#include <linux/slab.h>
  26#include <linux/iommu.h>
  27#include <linux/io.h>
  28#include <linux/vmalloc.h>
  29#include <linux/sizes.h>
  30#include <linux/cma.h>
  31
  32#include <asm/memory.h>
  33#include <asm/highmem.h>
  34#include <asm/cacheflush.h>
  35#include <asm/tlbflush.h>
  36#include <asm/mach/arch.h>
  37#include <asm/dma-iommu.h>
  38#include <asm/mach/map.h>
  39#include <asm/system_info.h>
  40#include <asm/dma-contiguous.h>
  41
  42#include "dma.h"
  43#include "mm.h"
  44
  45struct arm_dma_alloc_args {
  46	struct device *dev;
  47	size_t size;
  48	gfp_t gfp;
  49	pgprot_t prot;
  50	const void *caller;
  51	bool want_vaddr;
 
  52};
  53
  54struct arm_dma_free_args {
  55	struct device *dev;
  56	size_t size;
  57	void *cpu_addr;
  58	struct page *page;
  59	bool want_vaddr;
  60};
  61
 
 
 
  62struct arm_dma_allocator {
  63	void *(*alloc)(struct arm_dma_alloc_args *args,
  64		       struct page **ret_page);
  65	void (*free)(struct arm_dma_free_args *args);
  66};
  67
  68struct arm_dma_buffer {
  69	struct list_head list;
  70	void *virt;
  71	struct arm_dma_allocator *allocator;
  72};
  73
  74static LIST_HEAD(arm_dma_bufs);
  75static DEFINE_SPINLOCK(arm_dma_bufs_lock);
  76
  77static struct arm_dma_buffer *arm_dma_buffer_find(void *virt)
  78{
  79	struct arm_dma_buffer *buf, *found = NULL;
  80	unsigned long flags;
  81
  82	spin_lock_irqsave(&arm_dma_bufs_lock, flags);
  83	list_for_each_entry(buf, &arm_dma_bufs, list) {
  84		if (buf->virt == virt) {
  85			list_del(&buf->list);
  86			found = buf;
  87			break;
  88		}
  89	}
  90	spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
  91	return found;
  92}
  93
  94/*
  95 * The DMA API is built upon the notion of "buffer ownership".  A buffer
  96 * is either exclusively owned by the CPU (and therefore may be accessed
  97 * by it) or exclusively owned by the DMA device.  These helper functions
  98 * represent the transitions between these two ownership states.
  99 *
 100 * Note, however, that on later ARMs, this notion does not work due to
 101 * speculative prefetches.  We model our approach on the assumption that
 102 * the CPU does do speculative prefetches, which means we clean caches
 103 * before transfers and delay cache invalidation until transfer completion.
 104 *
 105 */
 106static void __dma_page_cpu_to_dev(struct page *, unsigned long,
 107		size_t, enum dma_data_direction);
 108static void __dma_page_dev_to_cpu(struct page *, unsigned long,
 109		size_t, enum dma_data_direction);
 110
 111/**
 112 * arm_dma_map_page - map a portion of a page for streaming DMA
 113 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 114 * @page: page that buffer resides in
 115 * @offset: offset into page for start of buffer
 116 * @size: size of buffer to map
 117 * @dir: DMA transfer direction
 118 *
 119 * Ensure that any data held in the cache is appropriately discarded
 120 * or written back.
 121 *
 122 * The device owns this memory once this call has completed.  The CPU
 123 * can regain ownership by calling dma_unmap_page().
 124 */
 125static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
 126	     unsigned long offset, size_t size, enum dma_data_direction dir,
 127	     struct dma_attrs *attrs)
 128{
 129	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 130		__dma_page_cpu_to_dev(page, offset, size, dir);
 131	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 132}
 133
 134static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
 135	     unsigned long offset, size_t size, enum dma_data_direction dir,
 136	     struct dma_attrs *attrs)
 137{
 138	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 139}
 140
 141/**
 142 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
 143 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 144 * @handle: DMA address of buffer
 145 * @size: size of buffer (same as passed to dma_map_page)
 146 * @dir: DMA transfer direction (same as passed to dma_map_page)
 147 *
 148 * Unmap a page streaming mode DMA translation.  The handle and size
 149 * must match what was provided in the previous dma_map_page() call.
 150 * All other usages are undefined.
 151 *
 152 * After this call, reads by the CPU to the buffer are guaranteed to see
 153 * whatever the device wrote there.
 154 */
 155static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
 156		size_t size, enum dma_data_direction dir,
 157		struct dma_attrs *attrs)
 158{
 159	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 160		__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
 161				      handle & ~PAGE_MASK, size, dir);
 162}
 163
 164static void arm_dma_sync_single_for_cpu(struct device *dev,
 165		dma_addr_t handle, size_t size, enum dma_data_direction dir)
 166{
 167	unsigned int offset = handle & (PAGE_SIZE - 1);
 168	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
 169	__dma_page_dev_to_cpu(page, offset, size, dir);
 170}
 171
 172static void arm_dma_sync_single_for_device(struct device *dev,
 173		dma_addr_t handle, size_t size, enum dma_data_direction dir)
 174{
 175	unsigned int offset = handle & (PAGE_SIZE - 1);
 176	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
 177	__dma_page_cpu_to_dev(page, offset, size, dir);
 178}
 179
 180struct dma_map_ops arm_dma_ops = {
 181	.alloc			= arm_dma_alloc,
 182	.free			= arm_dma_free,
 183	.mmap			= arm_dma_mmap,
 184	.get_sgtable		= arm_dma_get_sgtable,
 185	.map_page		= arm_dma_map_page,
 186	.unmap_page		= arm_dma_unmap_page,
 187	.map_sg			= arm_dma_map_sg,
 188	.unmap_sg		= arm_dma_unmap_sg,
 189	.sync_single_for_cpu	= arm_dma_sync_single_for_cpu,
 190	.sync_single_for_device	= arm_dma_sync_single_for_device,
 191	.sync_sg_for_cpu	= arm_dma_sync_sg_for_cpu,
 192	.sync_sg_for_device	= arm_dma_sync_sg_for_device,
 193	.set_dma_mask		= arm_dma_set_mask,
 194};
 195EXPORT_SYMBOL(arm_dma_ops);
 196
 197static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 198	dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);
 199static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
 200				  dma_addr_t handle, struct dma_attrs *attrs);
 201static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 202		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 203		 struct dma_attrs *attrs);
 204
 205struct dma_map_ops arm_coherent_dma_ops = {
 206	.alloc			= arm_coherent_dma_alloc,
 207	.free			= arm_coherent_dma_free,
 208	.mmap			= arm_coherent_dma_mmap,
 209	.get_sgtable		= arm_dma_get_sgtable,
 210	.map_page		= arm_coherent_dma_map_page,
 211	.map_sg			= arm_dma_map_sg,
 212	.set_dma_mask		= arm_dma_set_mask,
 213};
 214EXPORT_SYMBOL(arm_coherent_dma_ops);
 215
 216static int __dma_supported(struct device *dev, u64 mask, bool warn)
 217{
 218	unsigned long max_dma_pfn;
 219
 220	/*
 221	 * If the mask allows for more memory than we can address,
 222	 * and we actually have that much memory, then we must
 223	 * indicate that DMA to this device is not supported.
 224	 */
 225	if (sizeof(mask) != sizeof(dma_addr_t) &&
 226	    mask > (dma_addr_t)~0 &&
 227	    dma_to_pfn(dev, ~0) < max_pfn - 1) {
 228		if (warn) {
 229			dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n",
 230				 mask);
 231			dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n");
 232		}
 233		return 0;
 234	}
 235
 236	max_dma_pfn = min(max_pfn, arm_dma_pfn_limit);
 237
 238	/*
 239	 * Translate the device's DMA mask to a PFN limit.  This
 240	 * PFN number includes the page which we can DMA to.
 241	 */
 242	if (dma_to_pfn(dev, mask) < max_dma_pfn) {
 243		if (warn)
 244			dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",
 245				 mask,
 246				 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,
 247				 max_dma_pfn + 1);
 248		return 0;
 249	}
 250
 251	return 1;
 252}
 253
 254static u64 get_coherent_dma_mask(struct device *dev)
 255{
 256	u64 mask = (u64)DMA_BIT_MASK(32);
 257
 258	if (dev) {
 259		mask = dev->coherent_dma_mask;
 260
 261		/*
 262		 * Sanity check the DMA mask - it must be non-zero, and
 263		 * must be able to be satisfied by a DMA allocation.
 264		 */
 265		if (mask == 0) {
 266			dev_warn(dev, "coherent DMA mask is unset\n");
 267			return 0;
 268		}
 269
 270		if (!__dma_supported(dev, mask, true))
 271			return 0;
 272	}
 273
 274	return mask;
 275}
 276
 277static void __dma_clear_buffer(struct page *page, size_t size)
 278{
 279	/*
 280	 * Ensure that the allocated pages are zeroed, and that any data
 281	 * lurking in the kernel direct-mapped region is invalidated.
 282	 */
 283	if (PageHighMem(page)) {
 284		phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
 285		phys_addr_t end = base + size;
 286		while (size > 0) {
 287			void *ptr = kmap_atomic(page);
 288			memset(ptr, 0, PAGE_SIZE);
 289			dmac_flush_range(ptr, ptr + PAGE_SIZE);
 
 290			kunmap_atomic(ptr);
 291			page++;
 292			size -= PAGE_SIZE;
 293		}
 294		outer_flush_range(base, end);
 
 295	} else {
 296		void *ptr = page_address(page);
 297		memset(ptr, 0, size);
 298		dmac_flush_range(ptr, ptr + size);
 299		outer_flush_range(__pa(ptr), __pa(ptr) + size);
 
 
 300	}
 301}
 302
 303/*
 304 * Allocate a DMA buffer for 'dev' of size 'size' using the
 305 * specified gfp mask.  Note that 'size' must be page aligned.
 306 */
 307static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
 
 308{
 309	unsigned long order = get_order(size);
 310	struct page *page, *p, *e;
 311
 312	page = alloc_pages(gfp, order);
 313	if (!page)
 314		return NULL;
 315
 316	/*
 317	 * Now split the huge page and free the excess pages
 318	 */
 319	split_page(page, order);
 320	for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
 321		__free_page(p);
 322
 323	__dma_clear_buffer(page, size);
 324
 325	return page;
 326}
 327
 328/*
 329 * Free a DMA buffer.  'size' must be page aligned.
 330 */
 331static void __dma_free_buffer(struct page *page, size_t size)
 332{
 333	struct page *e = page + (size >> PAGE_SHIFT);
 334
 335	while (page < e) {
 336		__free_page(page);
 337		page++;
 338	}
 339}
 340
 341#ifdef CONFIG_MMU
 342
 343static void *__alloc_from_contiguous(struct device *dev, size_t size,
 344				     pgprot_t prot, struct page **ret_page,
 345				     const void *caller, bool want_vaddr);
 
 346
 347static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 348				 pgprot_t prot, struct page **ret_page,
 349				 const void *caller, bool want_vaddr);
 350
 351static void *
 352__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
 353	const void *caller)
 354{
 355	/*
 356	 * DMA allocation can be mapped to user space, so lets
 357	 * set VM_USERMAP flags too.
 358	 */
 359	return dma_common_contiguous_remap(page, size,
 360			VM_ARM_DMA_CONSISTENT | VM_USERMAP,
 361			prot, caller);
 362}
 363
 364static void __dma_free_remap(void *cpu_addr, size_t size)
 365{
 366	dma_common_free_remap(cpu_addr, size,
 367			VM_ARM_DMA_CONSISTENT | VM_USERMAP);
 368}
 369
 370#define DEFAULT_DMA_COHERENT_POOL_SIZE	SZ_256K
 371static struct gen_pool *atomic_pool;
 372
 373static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE;
 374
 375static int __init early_coherent_pool(char *p)
 376{
 377	atomic_pool_size = memparse(p, &p);
 378	return 0;
 379}
 380early_param("coherent_pool", early_coherent_pool);
 381
 382void __init init_dma_coherent_pool_size(unsigned long size)
 383{
 384	/*
 385	 * Catch any attempt to set the pool size too late.
 386	 */
 387	BUG_ON(atomic_pool);
 388
 389	/*
 390	 * Set architecture specific coherent pool size only if
 391	 * it has not been changed by kernel command line parameter.
 392	 */
 393	if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE)
 394		atomic_pool_size = size;
 395}
 396
 397/*
 398 * Initialise the coherent pool for atomic allocations.
 399 */
 400static int __init atomic_pool_init(void)
 401{
 402	pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
 403	gfp_t gfp = GFP_KERNEL | GFP_DMA;
 404	struct page *page;
 405	void *ptr;
 406
 407	atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
 408	if (!atomic_pool)
 409		goto out;
 410
 
 
 
 411	if (dev_get_cma_area(NULL))
 412		ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
 413					      &page, atomic_pool_init, true);
 
 414	else
 415		ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
 416					   &page, atomic_pool_init, true);
 417	if (ptr) {
 418		int ret;
 419
 420		ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
 421					page_to_phys(page),
 422					atomic_pool_size, -1);
 423		if (ret)
 424			goto destroy_genpool;
 425
 426		gen_pool_set_algo(atomic_pool,
 427				gen_pool_first_fit_order_align,
 428				(void *)PAGE_SHIFT);
 429		pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n",
 430		       atomic_pool_size / 1024);
 431		return 0;
 432	}
 433
 434destroy_genpool:
 435	gen_pool_destroy(atomic_pool);
 436	atomic_pool = NULL;
 437out:
 438	pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n",
 439	       atomic_pool_size / 1024);
 440	return -ENOMEM;
 441}
 442/*
 443 * CMA is activated by core_initcall, so we must be called after it.
 444 */
 445postcore_initcall(atomic_pool_init);
 446
 
 447struct dma_contig_early_reserve {
 448	phys_addr_t base;
 449	unsigned long size;
 450};
 451
 452static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
 453
 454static int dma_mmu_remap_num __initdata;
 455
 
 456void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
 457{
 458	dma_mmu_remap[dma_mmu_remap_num].base = base;
 459	dma_mmu_remap[dma_mmu_remap_num].size = size;
 460	dma_mmu_remap_num++;
 461}
 
 462
 463void __init dma_contiguous_remap(void)
 464{
 465	int i;
 466	for (i = 0; i < dma_mmu_remap_num; i++) {
 467		phys_addr_t start = dma_mmu_remap[i].base;
 468		phys_addr_t end = start + dma_mmu_remap[i].size;
 469		struct map_desc map;
 470		unsigned long addr;
 471
 472		if (end > arm_lowmem_limit)
 473			end = arm_lowmem_limit;
 474		if (start >= end)
 475			continue;
 476
 477		map.pfn = __phys_to_pfn(start);
 478		map.virtual = __phys_to_virt(start);
 479		map.length = end - start;
 480		map.type = MT_MEMORY_DMA_READY;
 481
 482		/*
 483		 * Clear previous low-memory mapping to ensure that the
 484		 * TLB does not see any conflicting entries, then flush
 485		 * the TLB of the old entries before creating new mappings.
 486		 *
 487		 * This ensures that any speculatively loaded TLB entries
 488		 * (even though they may be rare) can not cause any problems,
 489		 * and ensures that this code is architecturally compliant.
 490		 */
 491		for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
 492		     addr += PMD_SIZE)
 493			pmd_clear(pmd_off_k(addr));
 494
 495		flush_tlb_kernel_range(__phys_to_virt(start),
 496				       __phys_to_virt(end));
 497
 498		iotable_init(&map, 1);
 499	}
 500}
 
 501
 502static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
 503			    void *data)
 504{
 505	struct page *page = virt_to_page(addr);
 506	pgprot_t prot = *(pgprot_t *)data;
 507
 508	set_pte_ext(pte, mk_pte(page, prot), 0);
 509	return 0;
 510}
 511
 512static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
 513{
 514	unsigned long start = (unsigned long) page_address(page);
 515	unsigned end = start + size;
 516
 517	apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
 518	flush_tlb_kernel_range(start, end);
 519}
 520
 521static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 522				 pgprot_t prot, struct page **ret_page,
 523				 const void *caller, bool want_vaddr)
 524{
 525	struct page *page;
 526	void *ptr = NULL;
 527	page = __dma_alloc_buffer(dev, size, gfp);
 
 
 
 
 528	if (!page)
 529		return NULL;
 530	if (!want_vaddr)
 531		goto out;
 532
 533	ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
 534	if (!ptr) {
 535		__dma_free_buffer(page, size);
 536		return NULL;
 537	}
 538
 539 out:
 540	*ret_page = page;
 541	return ptr;
 542}
 543
 544static void *__alloc_from_pool(size_t size, struct page **ret_page)
 545{
 546	unsigned long val;
 547	void *ptr = NULL;
 548
 549	if (!atomic_pool) {
 550		WARN(1, "coherent pool not initialised!\n");
 551		return NULL;
 552	}
 553
 554	val = gen_pool_alloc(atomic_pool, size);
 555	if (val) {
 556		phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
 557
 558		*ret_page = phys_to_page(phys);
 559		ptr = (void *)val;
 560	}
 561
 562	return ptr;
 563}
 564
 565static bool __in_atomic_pool(void *start, size_t size)
 566{
 567	return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
 568}
 569
 570static int __free_from_pool(void *start, size_t size)
 571{
 572	if (!__in_atomic_pool(start, size))
 573		return 0;
 574
 575	gen_pool_free(atomic_pool, (unsigned long)start, size);
 576
 577	return 1;
 578}
 579
 580static void *__alloc_from_contiguous(struct device *dev, size_t size,
 581				     pgprot_t prot, struct page **ret_page,
 582				     const void *caller, bool want_vaddr)
 
 583{
 584	unsigned long order = get_order(size);
 585	size_t count = size >> PAGE_SHIFT;
 586	struct page *page;
 587	void *ptr = NULL;
 588
 589	page = dma_alloc_from_contiguous(dev, count, order);
 590	if (!page)
 591		return NULL;
 592
 593	__dma_clear_buffer(page, size);
 594
 595	if (!want_vaddr)
 596		goto out;
 597
 598	if (PageHighMem(page)) {
 599		ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
 600		if (!ptr) {
 601			dma_release_from_contiguous(dev, page, count);
 602			return NULL;
 603		}
 604	} else {
 605		__dma_remap(page, size, prot);
 606		ptr = page_address(page);
 607	}
 608
 609 out:
 610	*ret_page = page;
 611	return ptr;
 612}
 613
 614static void __free_from_contiguous(struct device *dev, struct page *page,
 615				   void *cpu_addr, size_t size, bool want_vaddr)
 616{
 617	if (want_vaddr) {
 618		if (PageHighMem(page))
 619			__dma_free_remap(cpu_addr, size);
 620		else
 621			__dma_remap(page, size, PAGE_KERNEL);
 622	}
 623	dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
 624}
 625
 626static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
 627{
 628	prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
 629			    pgprot_writecombine(prot) :
 630			    pgprot_dmacoherent(prot);
 631	return prot;
 632}
 633
 634#define nommu() 0
 635
 636#else	/* !CONFIG_MMU */
 637
 638#define nommu() 1
 639
 640#define __get_dma_pgprot(attrs, prot)				__pgprot(0)
 641#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv)	NULL
 642#define __alloc_from_pool(size, ret_page)			NULL
 643#define __alloc_from_contiguous(dev, size, prot, ret, c, wv)	NULL
 644#define __free_from_pool(cpu_addr, size)			do { } while (0)
 645#define __free_from_contiguous(dev, page, cpu_addr, size, wv)	do { } while (0)
 646#define __dma_free_remap(cpu_addr, size)			do { } while (0)
 647
 648#endif	/* CONFIG_MMU */
 649
 650static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
 651				   struct page **ret_page)
 652{
 653	struct page *page;
 654	page = __dma_alloc_buffer(dev, size, gfp);
 
 655	if (!page)
 656		return NULL;
 657
 658	*ret_page = page;
 659	return page_address(page);
 660}
 661
 662static void *simple_allocator_alloc(struct arm_dma_alloc_args *args,
 663				    struct page **ret_page)
 664{
 665	return __alloc_simple_buffer(args->dev, args->size, args->gfp,
 666				     ret_page);
 667}
 668
 669static void simple_allocator_free(struct arm_dma_free_args *args)
 670{
 671	__dma_free_buffer(args->page, args->size);
 672}
 673
 674static struct arm_dma_allocator simple_allocator = {
 675	.alloc = simple_allocator_alloc,
 676	.free = simple_allocator_free,
 677};
 678
 679static void *cma_allocator_alloc(struct arm_dma_alloc_args *args,
 680				 struct page **ret_page)
 681{
 682	return __alloc_from_contiguous(args->dev, args->size, args->prot,
 683				       ret_page, args->caller,
 684				       args->want_vaddr);
 
 685}
 686
 687static void cma_allocator_free(struct arm_dma_free_args *args)
 688{
 689	__free_from_contiguous(args->dev, args->page, args->cpu_addr,
 690			       args->size, args->want_vaddr);
 691}
 692
 693static struct arm_dma_allocator cma_allocator = {
 694	.alloc = cma_allocator_alloc,
 695	.free = cma_allocator_free,
 696};
 697
 698static void *pool_allocator_alloc(struct arm_dma_alloc_args *args,
 699				  struct page **ret_page)
 700{
 701	return __alloc_from_pool(args->size, ret_page);
 702}
 703
 704static void pool_allocator_free(struct arm_dma_free_args *args)
 705{
 706	__free_from_pool(args->cpu_addr, args->size);
 707}
 708
 709static struct arm_dma_allocator pool_allocator = {
 710	.alloc = pool_allocator_alloc,
 711	.free = pool_allocator_free,
 712};
 713
 714static void *remap_allocator_alloc(struct arm_dma_alloc_args *args,
 715				   struct page **ret_page)
 716{
 717	return __alloc_remap_buffer(args->dev, args->size, args->gfp,
 718				    args->prot, ret_page, args->caller,
 719				    args->want_vaddr);
 720}
 721
 722static void remap_allocator_free(struct arm_dma_free_args *args)
 723{
 724	if (args->want_vaddr)
 725		__dma_free_remap(args->cpu_addr, args->size);
 726
 727	__dma_free_buffer(args->page, args->size);
 728}
 729
 730static struct arm_dma_allocator remap_allocator = {
 731	.alloc = remap_allocator_alloc,
 732	.free = remap_allocator_free,
 733};
 734
 735static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 736			 gfp_t gfp, pgprot_t prot, bool is_coherent,
 737			 struct dma_attrs *attrs, const void *caller)
 738{
 739	u64 mask = get_coherent_dma_mask(dev);
 740	struct page *page = NULL;
 741	void *addr;
 742	bool allowblock, cma;
 743	struct arm_dma_buffer *buf;
 744	struct arm_dma_alloc_args args = {
 745		.dev = dev,
 746		.size = PAGE_ALIGN(size),
 747		.gfp = gfp,
 748		.prot = prot,
 749		.caller = caller,
 750		.want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs),
 
 751	};
 752
 753#ifdef CONFIG_DMA_API_DEBUG
 754	u64 limit = (mask + 1) & ~mask;
 755	if (limit && size >= limit) {
 756		dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
 757			size, mask);
 758		return NULL;
 759	}
 760#endif
 761
 762	if (!mask)
 763		return NULL;
 764
 765	buf = kzalloc(sizeof(*buf),
 766		      gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM));
 767	if (!buf)
 768		return NULL;
 769
 770	if (mask < 0xffffffffULL)
 771		gfp |= GFP_DMA;
 772
 773	/*
 774	 * Following is a work-around (a.k.a. hack) to prevent pages
 775	 * with __GFP_COMP being passed to split_page() which cannot
 776	 * handle them.  The real problem is that this flag probably
 777	 * should be 0 on ARM as it is not supported on this
 778	 * platform; see CONFIG_HUGETLBFS.
 779	 */
 780	gfp &= ~(__GFP_COMP);
 781	args.gfp = gfp;
 782
 783	*handle = DMA_ERROR_CODE;
 784	allowblock = gfpflags_allow_blocking(gfp);
 785	cma = allowblock ? dev_get_cma_area(dev) : false;
 786
 787	if (cma)
 788		buf->allocator = &cma_allocator;
 789	else if (nommu() || is_coherent)
 790		buf->allocator = &simple_allocator;
 791	else if (allowblock)
 792		buf->allocator = &remap_allocator;
 793	else
 794		buf->allocator = &pool_allocator;
 795
 796	addr = buf->allocator->alloc(&args, &page);
 797
 798	if (page) {
 799		unsigned long flags;
 800
 801		*handle = pfn_to_dma(dev, page_to_pfn(page));
 802		buf->virt = args.want_vaddr ? addr : page;
 803
 804		spin_lock_irqsave(&arm_dma_bufs_lock, flags);
 805		list_add(&buf->list, &arm_dma_bufs);
 806		spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
 807	} else {
 808		kfree(buf);
 809	}
 810
 811	return args.want_vaddr ? addr : page;
 812}
 813
 814/*
 815 * Allocate DMA-coherent memory space and return both the kernel remapped
 816 * virtual and bus address for that space.
 817 */
 818void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 819		    gfp_t gfp, struct dma_attrs *attrs)
 820{
 821	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
 822
 823	return __dma_alloc(dev, size, handle, gfp, prot, false,
 824			   attrs, __builtin_return_address(0));
 825}
 826
 827static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 828	dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
 829{
 830	return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true,
 831			   attrs, __builtin_return_address(0));
 832}
 833
 834static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 835		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 836		 struct dma_attrs *attrs)
 837{
 838	int ret = -ENXIO;
 839#ifdef CONFIG_MMU
 840	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 841	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 842	unsigned long pfn = dma_to_pfn(dev, dma_addr);
 843	unsigned long off = vma->vm_pgoff;
 844
 845	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
 846		return ret;
 847
 848	if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
 849		ret = remap_pfn_range(vma, vma->vm_start,
 850				      pfn + off,
 851				      vma->vm_end - vma->vm_start,
 852				      vma->vm_page_prot);
 853	}
 854#endif	/* CONFIG_MMU */
 855
 856	return ret;
 857}
 858
 859/*
 860 * Create userspace mapping for the DMA-coherent memory.
 861 */
 862static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 863		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 864		 struct dma_attrs *attrs)
 865{
 866	return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 867}
 868
 869int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 870		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 871		 struct dma_attrs *attrs)
 872{
 873#ifdef CONFIG_MMU
 874	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
 875#endif	/* CONFIG_MMU */
 876	return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 877}
 878
 879/*
 880 * Free a buffer as defined by the above mapping.
 881 */
 882static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 883			   dma_addr_t handle, struct dma_attrs *attrs,
 884			   bool is_coherent)
 885{
 886	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
 887	struct arm_dma_buffer *buf;
 888	struct arm_dma_free_args args = {
 889		.dev = dev,
 890		.size = PAGE_ALIGN(size),
 891		.cpu_addr = cpu_addr,
 892		.page = page,
 893		.want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs),
 894	};
 895
 896	buf = arm_dma_buffer_find(cpu_addr);
 897	if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr))
 898		return;
 899
 900	buf->allocator->free(&args);
 901	kfree(buf);
 902}
 903
 904void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 905		  dma_addr_t handle, struct dma_attrs *attrs)
 906{
 907	__arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
 908}
 909
 910static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
 911				  dma_addr_t handle, struct dma_attrs *attrs)
 912{
 913	__arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
 914}
 915
 916int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
 917		 void *cpu_addr, dma_addr_t handle, size_t size,
 918		 struct dma_attrs *attrs)
 919{
 920	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
 921	int ret;
 922
 923	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
 924	if (unlikely(ret))
 925		return ret;
 926
 927	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
 928	return 0;
 929}
 930
 931static void dma_cache_maint_page(struct page *page, unsigned long offset,
 932	size_t size, enum dma_data_direction dir,
 933	void (*op)(const void *, size_t, int))
 934{
 935	unsigned long pfn;
 936	size_t left = size;
 937
 938	pfn = page_to_pfn(page) + offset / PAGE_SIZE;
 939	offset %= PAGE_SIZE;
 940
 941	/*
 942	 * A single sg entry may refer to multiple physically contiguous
 943	 * pages.  But we still need to process highmem pages individually.
 944	 * If highmem is not configured then the bulk of this loop gets
 945	 * optimized out.
 946	 */
 947	do {
 948		size_t len = left;
 949		void *vaddr;
 950
 951		page = pfn_to_page(pfn);
 952
 953		if (PageHighMem(page)) {
 954			if (len + offset > PAGE_SIZE)
 955				len = PAGE_SIZE - offset;
 956
 957			if (cache_is_vipt_nonaliasing()) {
 958				vaddr = kmap_atomic(page);
 959				op(vaddr + offset, len, dir);
 960				kunmap_atomic(vaddr);
 961			} else {
 962				vaddr = kmap_high_get(page);
 963				if (vaddr) {
 964					op(vaddr + offset, len, dir);
 965					kunmap_high(page);
 966				}
 967			}
 968		} else {
 969			vaddr = page_address(page) + offset;
 970			op(vaddr, len, dir);
 971		}
 972		offset = 0;
 973		pfn++;
 974		left -= len;
 975	} while (left);
 976}
 977
 978/*
 979 * Make an area consistent for devices.
 980 * Note: Drivers should NOT use this function directly, as it will break
 981 * platforms with CONFIG_DMABOUNCE.
 982 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
 983 */
 984static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
 985	size_t size, enum dma_data_direction dir)
 986{
 987	phys_addr_t paddr;
 988
 989	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
 990
 991	paddr = page_to_phys(page) + off;
 992	if (dir == DMA_FROM_DEVICE) {
 993		outer_inv_range(paddr, paddr + size);
 994	} else {
 995		outer_clean_range(paddr, paddr + size);
 996	}
 997	/* FIXME: non-speculating: flush on bidirectional mappings? */
 998}
 999
1000static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
1001	size_t size, enum dma_data_direction dir)
1002{
1003	phys_addr_t paddr = page_to_phys(page) + off;
1004
1005	/* FIXME: non-speculating: not required */
1006	/* in any case, don't bother invalidating if DMA to device */
1007	if (dir != DMA_TO_DEVICE) {
1008		outer_inv_range(paddr, paddr + size);
1009
1010		dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
1011	}
1012
1013	/*
1014	 * Mark the D-cache clean for these pages to avoid extra flushing.
1015	 */
1016	if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
1017		unsigned long pfn;
1018		size_t left = size;
1019
1020		pfn = page_to_pfn(page) + off / PAGE_SIZE;
1021		off %= PAGE_SIZE;
1022		if (off) {
1023			pfn++;
1024			left -= PAGE_SIZE - off;
1025		}
1026		while (left >= PAGE_SIZE) {
1027			page = pfn_to_page(pfn++);
1028			set_bit(PG_dcache_clean, &page->flags);
1029			left -= PAGE_SIZE;
 
 
1030		}
1031	}
1032}
1033
1034/**
1035 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
1036 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1037 * @sg: list of buffers
1038 * @nents: number of buffers to map
1039 * @dir: DMA transfer direction
1040 *
1041 * Map a set of buffers described by scatterlist in streaming mode for DMA.
1042 * This is the scatter-gather version of the dma_map_single interface.
1043 * Here the scatter gather list elements are each tagged with the
1044 * appropriate dma address and length.  They are obtained via
1045 * sg_dma_{address,length}.
1046 *
1047 * Device ownership issues as mentioned for dma_map_single are the same
1048 * here.
1049 */
1050int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1051		enum dma_data_direction dir, struct dma_attrs *attrs)
1052{
1053	struct dma_map_ops *ops = get_dma_ops(dev);
1054	struct scatterlist *s;
1055	int i, j;
1056
1057	for_each_sg(sg, s, nents, i) {
1058#ifdef CONFIG_NEED_SG_DMA_LENGTH
1059		s->dma_length = s->length;
1060#endif
1061		s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
1062						s->length, dir, attrs);
1063		if (dma_mapping_error(dev, s->dma_address))
1064			goto bad_mapping;
1065	}
1066	return nents;
1067
1068 bad_mapping:
1069	for_each_sg(sg, s, i, j)
1070		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1071	return 0;
1072}
1073
1074/**
1075 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1076 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1077 * @sg: list of buffers
1078 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1079 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1080 *
1081 * Unmap a set of streaming mode DMA translations.  Again, CPU access
1082 * rules concerning calls here are the same as for dma_unmap_single().
1083 */
1084void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1085		enum dma_data_direction dir, struct dma_attrs *attrs)
1086{
1087	struct dma_map_ops *ops = get_dma_ops(dev);
1088	struct scatterlist *s;
1089
1090	int i;
1091
1092	for_each_sg(sg, s, nents, i)
1093		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1094}
1095
1096/**
1097 * arm_dma_sync_sg_for_cpu
1098 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1099 * @sg: list of buffers
1100 * @nents: number of buffers to map (returned from dma_map_sg)
1101 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1102 */
1103void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1104			int nents, enum dma_data_direction dir)
1105{
1106	struct dma_map_ops *ops = get_dma_ops(dev);
1107	struct scatterlist *s;
1108	int i;
1109
1110	for_each_sg(sg, s, nents, i)
1111		ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
1112					 dir);
1113}
1114
1115/**
1116 * arm_dma_sync_sg_for_device
1117 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1118 * @sg: list of buffers
1119 * @nents: number of buffers to map (returned from dma_map_sg)
1120 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1121 */
1122void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1123			int nents, enum dma_data_direction dir)
1124{
1125	struct dma_map_ops *ops = get_dma_ops(dev);
1126	struct scatterlist *s;
1127	int i;
1128
1129	for_each_sg(sg, s, nents, i)
1130		ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
1131					    dir);
1132}
1133
1134/*
1135 * Return whether the given device DMA address mask can be supported
1136 * properly.  For example, if your device can only drive the low 24-bits
1137 * during bus mastering, then you would pass 0x00ffffff as the mask
1138 * to this function.
1139 */
1140int dma_supported(struct device *dev, u64 mask)
1141{
1142	return __dma_supported(dev, mask, false);
1143}
1144EXPORT_SYMBOL(dma_supported);
1145
1146int arm_dma_set_mask(struct device *dev, u64 dma_mask)
1147{
1148	if (!dev->dma_mask || !dma_supported(dev, dma_mask))
1149		return -EIO;
1150
1151	*dev->dma_mask = dma_mask;
1152
1153	return 0;
1154}
1155
1156#define PREALLOC_DMA_DEBUG_ENTRIES	4096
 
1157
1158static int __init dma_debug_do_init(void)
1159{
1160	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
1161	return 0;
 
 
 
 
 
 
1162}
1163fs_initcall(dma_debug_do_init);
1164
1165#ifdef CONFIG_ARM_DMA_USE_IOMMU
1166
1167/* IOMMU */
1168
1169static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
1170
1171static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1172				      size_t size)
1173{
1174	unsigned int order = get_order(size);
1175	unsigned int align = 0;
1176	unsigned int count, start;
1177	size_t mapping_size = mapping->bits << PAGE_SHIFT;
1178	unsigned long flags;
1179	dma_addr_t iova;
1180	int i;
1181
1182	if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
1183		order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
1184
1185	count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1186	align = (1 << order) - 1;
1187
1188	spin_lock_irqsave(&mapping->lock, flags);
1189	for (i = 0; i < mapping->nr_bitmaps; i++) {
1190		start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1191				mapping->bits, 0, count, align);
1192
1193		if (start > mapping->bits)
1194			continue;
1195
1196		bitmap_set(mapping->bitmaps[i], start, count);
1197		break;
1198	}
1199
1200	/*
1201	 * No unused range found. Try to extend the existing mapping
1202	 * and perform a second attempt to reserve an IO virtual
1203	 * address range of size bytes.
1204	 */
1205	if (i == mapping->nr_bitmaps) {
1206		if (extend_iommu_mapping(mapping)) {
1207			spin_unlock_irqrestore(&mapping->lock, flags);
1208			return DMA_ERROR_CODE;
1209		}
1210
1211		start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1212				mapping->bits, 0, count, align);
1213
1214		if (start > mapping->bits) {
1215			spin_unlock_irqrestore(&mapping->lock, flags);
1216			return DMA_ERROR_CODE;
1217		}
1218
1219		bitmap_set(mapping->bitmaps[i], start, count);
1220	}
1221	spin_unlock_irqrestore(&mapping->lock, flags);
1222
1223	iova = mapping->base + (mapping_size * i);
1224	iova += start << PAGE_SHIFT;
1225
1226	return iova;
1227}
1228
1229static inline void __free_iova(struct dma_iommu_mapping *mapping,
1230			       dma_addr_t addr, size_t size)
1231{
1232	unsigned int start, count;
1233	size_t mapping_size = mapping->bits << PAGE_SHIFT;
1234	unsigned long flags;
1235	dma_addr_t bitmap_base;
1236	u32 bitmap_index;
1237
1238	if (!size)
1239		return;
1240
1241	bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;
1242	BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);
1243
1244	bitmap_base = mapping->base + mapping_size * bitmap_index;
1245
1246	start = (addr - bitmap_base) >>	PAGE_SHIFT;
1247
1248	if (addr + size > bitmap_base + mapping_size) {
1249		/*
1250		 * The address range to be freed reaches into the iova
1251		 * range of the next bitmap. This should not happen as
1252		 * we don't allow this in __alloc_iova (at the
1253		 * moment).
1254		 */
1255		BUG();
1256	} else
1257		count = size >> PAGE_SHIFT;
1258
1259	spin_lock_irqsave(&mapping->lock, flags);
1260	bitmap_clear(mapping->bitmaps[bitmap_index], start, count);
1261	spin_unlock_irqrestore(&mapping->lock, flags);
1262}
1263
1264/* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */
1265static const int iommu_order_array[] = { 9, 8, 4, 0 };
1266
1267static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1268					  gfp_t gfp, struct dma_attrs *attrs)
 
1269{
1270	struct page **pages;
1271	int count = size >> PAGE_SHIFT;
1272	int array_size = count * sizeof(struct page *);
1273	int i = 0;
1274	int order_idx = 0;
1275
1276	if (array_size <= PAGE_SIZE)
1277		pages = kzalloc(array_size, GFP_KERNEL);
1278	else
1279		pages = vzalloc(array_size);
1280	if (!pages)
1281		return NULL;
1282
1283	if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))
1284	{
1285		unsigned long order = get_order(size);
1286		struct page *page;
1287
1288		page = dma_alloc_from_contiguous(dev, count, order);
 
1289		if (!page)
1290			goto error;
1291
1292		__dma_clear_buffer(page, size);
1293
1294		for (i = 0; i < count; i++)
1295			pages[i] = page + i;
1296
1297		return pages;
1298	}
1299
1300	/* Go straight to 4K chunks if caller says it's OK. */
1301	if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs))
1302		order_idx = ARRAY_SIZE(iommu_order_array) - 1;
1303
1304	/*
1305	 * IOMMU can map any pages, so himem can also be used here
1306	 */
1307	gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1308
1309	while (count) {
1310		int j, order;
1311
1312		order = iommu_order_array[order_idx];
1313
1314		/* Drop down when we get small */
1315		if (__fls(count) < order) {
1316			order_idx++;
1317			continue;
1318		}
1319
1320		if (order) {
1321			/* See if it's easy to allocate a high-order chunk */
1322			pages[i] = alloc_pages(gfp | __GFP_NORETRY, order);
1323
1324			/* Go down a notch at first sign of pressure */
1325			if (!pages[i]) {
1326				order_idx++;
1327				continue;
1328			}
1329		} else {
1330			pages[i] = alloc_pages(gfp, 0);
1331			if (!pages[i])
1332				goto error;
1333		}
1334
1335		if (order) {
1336			split_page(pages[i], order);
1337			j = 1 << order;
1338			while (--j)
1339				pages[i + j] = pages[i] + j;
1340		}
1341
1342		__dma_clear_buffer(pages[i], PAGE_SIZE << order);
1343		i += 1 << order;
1344		count -= 1 << order;
1345	}
1346
1347	return pages;
1348error:
1349	while (i--)
1350		if (pages[i])
1351			__free_pages(pages[i], 0);
1352	kvfree(pages);
1353	return NULL;
1354}
1355
1356static int __iommu_free_buffer(struct device *dev, struct page **pages,
1357			       size_t size, struct dma_attrs *attrs)
1358{
1359	int count = size >> PAGE_SHIFT;
1360	int i;
1361
1362	if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {
1363		dma_release_from_contiguous(dev, pages[0], count);
1364	} else {
1365		for (i = 0; i < count; i++)
1366			if (pages[i])
1367				__free_pages(pages[i], 0);
1368	}
1369
1370	kvfree(pages);
1371	return 0;
1372}
1373
1374/*
1375 * Create a CPU mapping for a specified pages
1376 */
1377static void *
1378__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
1379		    const void *caller)
1380{
1381	return dma_common_pages_remap(pages, size,
1382			VM_ARM_DMA_CONSISTENT | VM_USERMAP, prot, caller);
1383}
1384
1385/*
1386 * Create a mapping in device IO address space for specified pages
1387 */
1388static dma_addr_t
1389__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
 
1390{
1391	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1392	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1393	dma_addr_t dma_addr, iova;
1394	int i;
1395
1396	dma_addr = __alloc_iova(mapping, size);
1397	if (dma_addr == DMA_ERROR_CODE)
1398		return dma_addr;
1399
1400	iova = dma_addr;
1401	for (i = 0; i < count; ) {
1402		int ret;
1403
1404		unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1405		phys_addr_t phys = page_to_phys(pages[i]);
1406		unsigned int len, j;
1407
1408		for (j = i + 1; j < count; j++, next_pfn++)
1409			if (page_to_pfn(pages[j]) != next_pfn)
1410				break;
1411
1412		len = (j - i) << PAGE_SHIFT;
1413		ret = iommu_map(mapping->domain, iova, phys, len,
1414				IOMMU_READ|IOMMU_WRITE);
 
1415		if (ret < 0)
1416			goto fail;
1417		iova += len;
1418		i = j;
1419	}
1420	return dma_addr;
1421fail:
1422	iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1423	__free_iova(mapping, dma_addr, size);
1424	return DMA_ERROR_CODE;
1425}
1426
1427static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1428{
1429	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1430
1431	/*
1432	 * add optional in-page offset from iova to size and align
1433	 * result to page size
1434	 */
1435	size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1436	iova &= PAGE_MASK;
1437
1438	iommu_unmap(mapping->domain, iova, size);
1439	__free_iova(mapping, iova, size);
1440	return 0;
1441}
1442
1443static struct page **__atomic_get_pages(void *addr)
1444{
1445	struct page *page;
1446	phys_addr_t phys;
1447
1448	phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
1449	page = phys_to_page(phys);
1450
1451	return (struct page **)page;
1452}
1453
1454static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
1455{
1456	struct vm_struct *area;
1457
1458	if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
1459		return __atomic_get_pages(cpu_addr);
1460
1461	if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1462		return cpu_addr;
1463
1464	area = find_vm_area(cpu_addr);
1465	if (area && (area->flags & VM_ARM_DMA_CONSISTENT))
1466		return area->pages;
1467	return NULL;
1468}
1469
1470static void *__iommu_alloc_atomic(struct device *dev, size_t size,
1471				  dma_addr_t *handle)
 
1472{
1473	struct page *page;
1474	void *addr;
1475
1476	addr = __alloc_from_pool(size, &page);
 
 
 
1477	if (!addr)
1478		return NULL;
1479
1480	*handle = __iommu_create_mapping(dev, &page, size);
1481	if (*handle == DMA_ERROR_CODE)
1482		goto err_mapping;
1483
1484	return addr;
1485
1486err_mapping:
1487	__free_from_pool(addr, size);
1488	return NULL;
1489}
1490
1491static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1492				dma_addr_t handle, size_t size)
1493{
1494	__iommu_remove_mapping(dev, handle, size);
1495	__free_from_pool(cpu_addr, size);
 
 
 
1496}
1497
1498static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1499	    dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
1500{
1501	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
1502	struct page **pages;
1503	void *addr = NULL;
 
1504
1505	*handle = DMA_ERROR_CODE;
1506	size = PAGE_ALIGN(size);
1507
1508	if (!gfpflags_allow_blocking(gfp))
1509		return __iommu_alloc_atomic(dev, size, handle);
 
1510
1511	/*
1512	 * Following is a work-around (a.k.a. hack) to prevent pages
1513	 * with __GFP_COMP being passed to split_page() which cannot
1514	 * handle them.  The real problem is that this flag probably
1515	 * should be 0 on ARM as it is not supported on this
1516	 * platform; see CONFIG_HUGETLBFS.
1517	 */
1518	gfp &= ~(__GFP_COMP);
1519
1520	pages = __iommu_alloc_buffer(dev, size, gfp, attrs);
1521	if (!pages)
1522		return NULL;
1523
1524	*handle = __iommu_create_mapping(dev, pages, size);
1525	if (*handle == DMA_ERROR_CODE)
1526		goto err_buffer;
1527
1528	if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1529		return pages;
1530
1531	addr = __iommu_alloc_remap(pages, size, gfp, prot,
1532				   __builtin_return_address(0));
1533	if (!addr)
1534		goto err_mapping;
1535
1536	return addr;
1537
1538err_mapping:
1539	__iommu_remove_mapping(dev, *handle, size);
1540err_buffer:
1541	__iommu_free_buffer(dev, pages, size, attrs);
1542	return NULL;
1543}
1544
1545static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1546		    void *cpu_addr, dma_addr_t dma_addr, size_t size,
1547		    struct dma_attrs *attrs)
1548{
1549	unsigned long uaddr = vma->vm_start;
1550	unsigned long usize = vma->vm_end - vma->vm_start;
1551	struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1552	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1553	unsigned long off = vma->vm_pgoff;
1554
1555	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
1556
1557	if (!pages)
1558		return -ENXIO;
1559
1560	if (off >= nr_pages || (usize >> PAGE_SHIFT) > nr_pages - off)
1561		return -ENXIO;
1562
1563	pages += off;
 
1564
1565	do {
1566		int ret = vm_insert_page(vma, uaddr, *pages++);
1567		if (ret) {
1568			pr_err("Remapping memory failed: %d\n", ret);
1569			return ret;
1570		}
1571		uaddr += PAGE_SIZE;
1572		usize -= PAGE_SIZE;
1573	} while (usize > 0);
1574
1575	return 0;
1576}
1577
1578/*
1579 * free a page as defined by the above mapping.
1580 * Must not be called with IRQs disabled.
1581 */
1582void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1583			  dma_addr_t handle, struct dma_attrs *attrs)
1584{
 
1585	struct page **pages;
1586	size = PAGE_ALIGN(size);
1587
1588	if (__in_atomic_pool(cpu_addr, size)) {
1589		__iommu_free_atomic(dev, cpu_addr, handle, size);
1590		return;
1591	}
1592
1593	pages = __iommu_get_pages(cpu_addr, attrs);
1594	if (!pages) {
1595		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1596		return;
1597	}
1598
1599	if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
1600		dma_common_free_remap(cpu_addr, size,
1601			VM_ARM_DMA_CONSISTENT | VM_USERMAP);
1602	}
1603
1604	__iommu_remove_mapping(dev, handle, size);
1605	__iommu_free_buffer(dev, pages, size, attrs);
1606}
1607
1608static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
1609				 void *cpu_addr, dma_addr_t dma_addr,
1610				 size_t size, struct dma_attrs *attrs)
1611{
1612	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1613	struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1614
1615	if (!pages)
1616		return -ENXIO;
1617
1618	return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
1619					 GFP_KERNEL);
1620}
1621
1622static int __dma_direction_to_prot(enum dma_data_direction dir)
1623{
1624	int prot;
1625
1626	switch (dir) {
1627	case DMA_BIDIRECTIONAL:
1628		prot = IOMMU_READ | IOMMU_WRITE;
1629		break;
1630	case DMA_TO_DEVICE:
1631		prot = IOMMU_READ;
1632		break;
1633	case DMA_FROM_DEVICE:
1634		prot = IOMMU_WRITE;
1635		break;
1636	default:
1637		prot = 0;
1638	}
1639
1640	return prot;
1641}
1642
1643/*
1644 * Map a part of the scatter-gather list into contiguous io address space
1645 */
1646static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1647			  size_t size, dma_addr_t *handle,
1648			  enum dma_data_direction dir, struct dma_attrs *attrs,
1649			  bool is_coherent)
1650{
1651	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1652	dma_addr_t iova, iova_base;
1653	int ret = 0;
1654	unsigned int count;
1655	struct scatterlist *s;
1656	int prot;
1657
1658	size = PAGE_ALIGN(size);
1659	*handle = DMA_ERROR_CODE;
1660
1661	iova_base = iova = __alloc_iova(mapping, size);
1662	if (iova == DMA_ERROR_CODE)
1663		return -ENOMEM;
1664
1665	for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1666		phys_addr_t phys = page_to_phys(sg_page(s));
1667		unsigned int len = PAGE_ALIGN(s->offset + s->length);
1668
1669		if (!is_coherent &&
1670			!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1671			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1672
1673		prot = __dma_direction_to_prot(dir);
1674
1675		ret = iommu_map(mapping->domain, iova, phys, len, prot);
 
1676		if (ret < 0)
1677			goto fail;
1678		count += len >> PAGE_SHIFT;
1679		iova += len;
1680	}
1681	*handle = iova_base;
1682
1683	return 0;
1684fail:
1685	iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1686	__free_iova(mapping, iova_base, size);
1687	return ret;
1688}
1689
1690static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1691		     enum dma_data_direction dir, struct dma_attrs *attrs,
1692		     bool is_coherent)
 
 
 
 
 
 
 
 
 
 
 
1693{
1694	struct scatterlist *s = sg, *dma = sg, *start = sg;
1695	int i, count = 0;
1696	unsigned int offset = s->offset;
1697	unsigned int size = s->offset + s->length;
1698	unsigned int max = dma_get_max_seg_size(dev);
1699
1700	for (i = 1; i < nents; i++) {
1701		s = sg_next(s);
1702
1703		s->dma_address = DMA_ERROR_CODE;
1704		s->dma_length = 0;
1705
1706		if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1707			if (__map_sg_chunk(dev, start, size, &dma->dma_address,
1708			    dir, attrs, is_coherent) < 0)
 
1709				goto bad_mapping;
1710
1711			dma->dma_address += offset;
1712			dma->dma_length = size - offset;
1713
1714			size = offset = s->offset;
1715			start = s;
1716			dma = sg_next(dma);
1717			count += 1;
1718		}
1719		size += s->length;
1720	}
1721	if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
1722		is_coherent) < 0)
1723		goto bad_mapping;
1724
1725	dma->dma_address += offset;
1726	dma->dma_length = size - offset;
1727
1728	return count+1;
1729
1730bad_mapping:
1731	for_each_sg(sg, s, count, i)
1732		__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1733	return 0;
1734}
1735
1736/**
1737 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1738 * @dev: valid struct device pointer
1739 * @sg: list of buffers
1740 * @nents: number of buffers to map
1741 * @dir: DMA transfer direction
1742 *
1743 * Map a set of i/o coherent buffers described by scatterlist in streaming
1744 * mode for DMA. The scatter gather list elements are merged together (if
1745 * possible) and tagged with the appropriate dma address and length. They are
1746 * obtained via sg_dma_{address,length}.
1747 */
1748int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1749		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1750{
1751	return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
1752}
1753
1754/**
1755 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1756 * @dev: valid struct device pointer
1757 * @sg: list of buffers
1758 * @nents: number of buffers to map
1759 * @dir: DMA transfer direction
1760 *
1761 * Map a set of buffers described by scatterlist in streaming mode for DMA.
1762 * The scatter gather list elements are merged together (if possible) and
1763 * tagged with the appropriate dma address and length. They are obtained via
1764 * sg_dma_{address,length}.
1765 */
1766int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1767		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1768{
1769	return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
1770}
1771
1772static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1773		int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
1774		bool is_coherent)
1775{
1776	struct scatterlist *s;
1777	int i;
1778
1779	for_each_sg(sg, s, nents, i) {
1780		if (sg_dma_len(s))
1781			__iommu_remove_mapping(dev, sg_dma_address(s),
1782					       sg_dma_len(s));
1783		if (!is_coherent &&
1784		    !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1785			__dma_page_dev_to_cpu(sg_page(s), s->offset,
1786					      s->length, dir);
1787	}
1788}
1789
1790/**
1791 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1792 * @dev: valid struct device pointer
1793 * @sg: list of buffers
1794 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1795 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1796 *
1797 * Unmap a set of streaming mode DMA translations.  Again, CPU access
1798 * rules concerning calls here are the same as for dma_unmap_single().
1799 */
1800void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1801		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1802{
1803	__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
1804}
1805
1806/**
1807 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1808 * @dev: valid struct device pointer
1809 * @sg: list of buffers
1810 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1811 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1812 *
1813 * Unmap a set of streaming mode DMA translations.  Again, CPU access
1814 * rules concerning calls here are the same as for dma_unmap_single().
1815 */
1816void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1817			enum dma_data_direction dir, struct dma_attrs *attrs)
1818{
1819	__iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
1820}
1821
1822/**
1823 * arm_iommu_sync_sg_for_cpu
1824 * @dev: valid struct device pointer
1825 * @sg: list of buffers
1826 * @nents: number of buffers to map (returned from dma_map_sg)
1827 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1828 */
1829void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 
1830			int nents, enum dma_data_direction dir)
1831{
1832	struct scatterlist *s;
1833	int i;
1834
 
 
 
1835	for_each_sg(sg, s, nents, i)
1836		__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1837
1838}
1839
1840/**
1841 * arm_iommu_sync_sg_for_device
1842 * @dev: valid struct device pointer
1843 * @sg: list of buffers
1844 * @nents: number of buffers to map (returned from dma_map_sg)
1845 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1846 */
1847void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 
1848			int nents, enum dma_data_direction dir)
1849{
1850	struct scatterlist *s;
1851	int i;
1852
 
 
 
1853	for_each_sg(sg, s, nents, i)
1854		__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1855}
1856
1857
1858/**
1859 * arm_coherent_iommu_map_page
1860 * @dev: valid struct device pointer
1861 * @page: page that buffer resides in
1862 * @offset: offset into page for start of buffer
1863 * @size: size of buffer to map
1864 * @dir: DMA transfer direction
1865 *
1866 * Coherent IOMMU aware version of arm_dma_map_page()
1867 */
1868static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1869	     unsigned long offset, size_t size, enum dma_data_direction dir,
1870	     struct dma_attrs *attrs)
1871{
1872	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1873	dma_addr_t dma_addr;
1874	int ret, prot, len = PAGE_ALIGN(size + offset);
1875
 
 
 
1876	dma_addr = __alloc_iova(mapping, len);
1877	if (dma_addr == DMA_ERROR_CODE)
1878		return dma_addr;
1879
1880	prot = __dma_direction_to_prot(dir);
1881
1882	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
 
1883	if (ret < 0)
1884		goto fail;
1885
1886	return dma_addr + offset;
1887fail:
1888	__free_iova(mapping, dma_addr, len);
1889	return DMA_ERROR_CODE;
1890}
1891
1892/**
1893 * arm_iommu_map_page
1894 * @dev: valid struct device pointer
1895 * @page: page that buffer resides in
1896 * @offset: offset into page for start of buffer
1897 * @size: size of buffer to map
1898 * @dir: DMA transfer direction
1899 *
1900 * IOMMU aware version of arm_dma_map_page()
1901 */
1902static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1903	     unsigned long offset, size_t size, enum dma_data_direction dir,
1904	     struct dma_attrs *attrs)
1905{
1906	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1907		__dma_page_cpu_to_dev(page, offset, size, dir);
1908
1909	return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
1910}
1911
1912/**
1913 * arm_coherent_iommu_unmap_page
1914 * @dev: valid struct device pointer
1915 * @handle: DMA address of buffer
1916 * @size: size of buffer (same as passed to dma_map_page)
1917 * @dir: DMA transfer direction (same as passed to dma_map_page)
1918 *
1919 * Coherent IOMMU aware version of arm_dma_unmap_page()
1920 */
1921static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1922		size_t size, enum dma_data_direction dir,
1923		struct dma_attrs *attrs)
1924{
1925	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1926	dma_addr_t iova = handle & PAGE_MASK;
 
1927	int offset = handle & ~PAGE_MASK;
1928	int len = PAGE_ALIGN(size + offset);
1929
1930	if (!iova)
1931		return;
1932
 
 
 
 
 
1933	iommu_unmap(mapping->domain, iova, len);
1934	__free_iova(mapping, iova, len);
1935}
1936
1937/**
1938 * arm_iommu_unmap_page
1939 * @dev: valid struct device pointer
1940 * @handle: DMA address of buffer
1941 * @size: size of buffer (same as passed to dma_map_page)
1942 * @dir: DMA transfer direction (same as passed to dma_map_page)
1943 *
1944 * IOMMU aware version of arm_dma_unmap_page()
1945 */
1946static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1947		size_t size, enum dma_data_direction dir,
1948		struct dma_attrs *attrs)
1949{
1950	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1951	dma_addr_t iova = handle & PAGE_MASK;
1952	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1953	int offset = handle & ~PAGE_MASK;
1954	int len = PAGE_ALIGN(size + offset);
1955
1956	if (!iova)
1957		return;
1958
1959	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1960		__dma_page_dev_to_cpu(page, offset, size, dir);
1961
1962	iommu_unmap(mapping->domain, iova, len);
1963	__free_iova(mapping, iova, len);
1964}
1965
1966static void arm_iommu_sync_single_for_cpu(struct device *dev,
1967		dma_addr_t handle, size_t size, enum dma_data_direction dir)
1968{
1969	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1970	dma_addr_t iova = handle & PAGE_MASK;
1971	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1972	unsigned int offset = handle & ~PAGE_MASK;
1973
1974	if (!iova)
1975		return;
1976
 
1977	__dma_page_dev_to_cpu(page, offset, size, dir);
1978}
1979
1980static void arm_iommu_sync_single_for_device(struct device *dev,
1981		dma_addr_t handle, size_t size, enum dma_data_direction dir)
1982{
1983	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1984	dma_addr_t iova = handle & PAGE_MASK;
1985	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1986	unsigned int offset = handle & ~PAGE_MASK;
1987
1988	if (!iova)
1989		return;
1990
 
1991	__dma_page_cpu_to_dev(page, offset, size, dir);
1992}
1993
1994struct dma_map_ops iommu_ops = {
1995	.alloc		= arm_iommu_alloc_attrs,
1996	.free		= arm_iommu_free_attrs,
1997	.mmap		= arm_iommu_mmap_attrs,
1998	.get_sgtable	= arm_iommu_get_sgtable,
1999
2000	.map_page		= arm_iommu_map_page,
2001	.unmap_page		= arm_iommu_unmap_page,
2002	.sync_single_for_cpu	= arm_iommu_sync_single_for_cpu,
2003	.sync_single_for_device	= arm_iommu_sync_single_for_device,
2004
2005	.map_sg			= arm_iommu_map_sg,
2006	.unmap_sg		= arm_iommu_unmap_sg,
2007	.sync_sg_for_cpu	= arm_iommu_sync_sg_for_cpu,
2008	.sync_sg_for_device	= arm_iommu_sync_sg_for_device,
2009
2010	.set_dma_mask		= arm_dma_set_mask,
2011};
2012
2013struct dma_map_ops iommu_coherent_ops = {
2014	.alloc		= arm_iommu_alloc_attrs,
2015	.free		= arm_iommu_free_attrs,
2016	.mmap		= arm_iommu_mmap_attrs,
2017	.get_sgtable	= arm_iommu_get_sgtable,
2018
2019	.map_page	= arm_coherent_iommu_map_page,
2020	.unmap_page	= arm_coherent_iommu_unmap_page,
2021
2022	.map_sg		= arm_coherent_iommu_map_sg,
2023	.unmap_sg	= arm_coherent_iommu_unmap_sg,
2024
2025	.set_dma_mask	= arm_dma_set_mask,
2026};
2027
2028/**
2029 * arm_iommu_create_mapping
2030 * @bus: pointer to the bus holding the client device (for IOMMU calls)
2031 * @base: start address of the valid IO address space
2032 * @size: maximum size of the valid IO address space
2033 *
2034 * Creates a mapping structure which holds information about used/unused
2035 * IO address ranges, which is required to perform memory allocation and
2036 * mapping with IOMMU aware functions.
2037 *
2038 * The client device need to be attached to the mapping with
2039 * arm_iommu_attach_device function.
2040 */
2041struct dma_iommu_mapping *
2042arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size)
2043{
2044	unsigned int bits = size >> PAGE_SHIFT;
2045	unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
2046	struct dma_iommu_mapping *mapping;
2047	int extensions = 1;
2048	int err = -ENOMEM;
2049
2050	/* currently only 32-bit DMA address space is supported */
2051	if (size > DMA_BIT_MASK(32) + 1)
2052		return ERR_PTR(-ERANGE);
2053
2054	if (!bitmap_size)
2055		return ERR_PTR(-EINVAL);
2056
2057	if (bitmap_size > PAGE_SIZE) {
2058		extensions = bitmap_size / PAGE_SIZE;
2059		bitmap_size = PAGE_SIZE;
2060	}
2061
2062	mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
2063	if (!mapping)
2064		goto err;
2065
2066	mapping->bitmap_size = bitmap_size;
2067	mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *),
2068				GFP_KERNEL);
2069	if (!mapping->bitmaps)
2070		goto err2;
2071
2072	mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);
2073	if (!mapping->bitmaps[0])
2074		goto err3;
2075
2076	mapping->nr_bitmaps = 1;
2077	mapping->extensions = extensions;
2078	mapping->base = base;
2079	mapping->bits = BITS_PER_BYTE * bitmap_size;
2080
2081	spin_lock_init(&mapping->lock);
2082
2083	mapping->domain = iommu_domain_alloc(bus);
2084	if (!mapping->domain)
2085		goto err4;
2086
2087	kref_init(&mapping->kref);
2088	return mapping;
2089err4:
2090	kfree(mapping->bitmaps[0]);
2091err3:
2092	kfree(mapping->bitmaps);
2093err2:
2094	kfree(mapping);
2095err:
2096	return ERR_PTR(err);
2097}
2098EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
2099
2100static void release_iommu_mapping(struct kref *kref)
2101{
2102	int i;
2103	struct dma_iommu_mapping *mapping =
2104		container_of(kref, struct dma_iommu_mapping, kref);
2105
2106	iommu_domain_free(mapping->domain);
2107	for (i = 0; i < mapping->nr_bitmaps; i++)
2108		kfree(mapping->bitmaps[i]);
2109	kfree(mapping->bitmaps);
2110	kfree(mapping);
2111}
2112
2113static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
2114{
2115	int next_bitmap;
2116
2117	if (mapping->nr_bitmaps >= mapping->extensions)
2118		return -EINVAL;
2119
2120	next_bitmap = mapping->nr_bitmaps;
2121	mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,
2122						GFP_ATOMIC);
2123	if (!mapping->bitmaps[next_bitmap])
2124		return -ENOMEM;
2125
2126	mapping->nr_bitmaps++;
2127
2128	return 0;
2129}
2130
2131void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
2132{
2133	if (mapping)
2134		kref_put(&mapping->kref, release_iommu_mapping);
2135}
2136EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
2137
2138static int __arm_iommu_attach_device(struct device *dev,
2139				     struct dma_iommu_mapping *mapping)
2140{
2141	int err;
2142
2143	err = iommu_attach_device(mapping->domain, dev);
2144	if (err)
2145		return err;
2146
2147	kref_get(&mapping->kref);
2148	to_dma_iommu_mapping(dev) = mapping;
2149
2150	pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
2151	return 0;
2152}
2153
2154/**
2155 * arm_iommu_attach_device
2156 * @dev: valid struct device pointer
2157 * @mapping: io address space mapping structure (returned from
2158 *	arm_iommu_create_mapping)
2159 *
2160 * Attaches specified io address space mapping to the provided device.
2161 * This replaces the dma operations (dma_map_ops pointer) with the
2162 * IOMMU aware version.
2163 *
2164 * More than one client might be attached to the same io address space
2165 * mapping.
2166 */
2167int arm_iommu_attach_device(struct device *dev,
2168			    struct dma_iommu_mapping *mapping)
2169{
2170	int err;
2171
2172	err = __arm_iommu_attach_device(dev, mapping);
2173	if (err)
2174		return err;
2175
2176	set_dma_ops(dev, &iommu_ops);
2177	return 0;
2178}
2179EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
2180
2181static void __arm_iommu_detach_device(struct device *dev)
 
 
 
 
 
 
 
2182{
2183	struct dma_iommu_mapping *mapping;
2184
2185	mapping = to_dma_iommu_mapping(dev);
2186	if (!mapping) {
2187		dev_warn(dev, "Not attached\n");
2188		return;
2189	}
2190
2191	iommu_detach_device(mapping->domain, dev);
2192	kref_put(&mapping->kref, release_iommu_mapping);
2193	to_dma_iommu_mapping(dev) = NULL;
 
2194
2195	pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
2196}
2197
2198/**
2199 * arm_iommu_detach_device
2200 * @dev: valid struct device pointer
2201 *
2202 * Detaches the provided device from a previously attached map.
2203 * This voids the dma operations (dma_map_ops pointer)
2204 */
2205void arm_iommu_detach_device(struct device *dev)
2206{
2207	__arm_iommu_detach_device(dev);
2208	set_dma_ops(dev, NULL);
2209}
2210EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
2211
2212static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
2213{
2214	return coherent ? &iommu_coherent_ops : &iommu_ops;
2215}
2216
2217static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2218				    struct iommu_ops *iommu)
2219{
2220	struct dma_iommu_mapping *mapping;
2221
2222	if (!iommu)
2223		return false;
2224
2225	mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
2226	if (IS_ERR(mapping)) {
2227		pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
2228				size, dev_name(dev));
2229		return false;
2230	}
2231
2232	if (__arm_iommu_attach_device(dev, mapping)) {
2233		pr_warn("Failed to attached device %s to IOMMU_mapping\n",
2234				dev_name(dev));
2235		arm_iommu_release_mapping(mapping);
2236		return false;
2237	}
2238
2239	return true;
2240}
2241
2242static void arm_teardown_iommu_dma_ops(struct device *dev)
2243{
2244	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
2245
2246	if (!mapping)
2247		return;
2248
2249	__arm_iommu_detach_device(dev);
2250	arm_iommu_release_mapping(mapping);
2251}
2252
2253#else
2254
2255static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2256				    struct iommu_ops *iommu)
2257{
2258	return false;
2259}
2260
2261static void arm_teardown_iommu_dma_ops(struct device *dev) { }
2262
2263#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops
2264
2265#endif	/* CONFIG_ARM_DMA_USE_IOMMU */
2266
2267static struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
2268{
2269	return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;
2270}
2271
2272void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
2273			struct iommu_ops *iommu, bool coherent)
2274{
2275	struct dma_map_ops *dma_ops;
 
 
 
 
 
 
 
2276
2277	dev->archdata.dma_coherent = coherent;
2278	if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))
2279		dma_ops = arm_get_iommu_dma_map_ops(coherent);
2280	else
2281		dma_ops = arm_get_dma_map_ops(coherent);
 
 
2282
2283	set_dma_ops(dev, dma_ops);
 
 
 
 
2284}
2285
2286void arch_teardown_dma_ops(struct device *dev)
2287{
 
 
 
2288	arm_teardown_iommu_dma_ops(dev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2289}