1/*
   2 *  linux/arch/arm/mm/dma-mapping.c
   3 *
   4 *  Copyright (C) 2000-2004 Russell King
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 *
  10 *  DMA uncached mapping support.
  11 */
  12#include <linux/bootmem.h>
  13#include <linux/module.h>
  14#include <linux/mm.h>
  15#include <linux/gfp.h>
  16#include <linux/errno.h>
  17#include <linux/list.h>
  18#include <linux/init.h>
  19#include <linux/device.h>
  20#include <linux/dma-mapping.h>
  21#include <linux/dma-contiguous.h>
  22#include <linux/highmem.h>
  23#include <linux/memblock.h>
  24#include <linux/slab.h>
  25#include <linux/iommu.h>
  26#include <linux/io.h>
  27#include <linux/vmalloc.h>
  28#include <linux/sizes.h>
  29
  30#include <asm/memory.h>
  31#include <asm/highmem.h>
  32#include <asm/cacheflush.h>
  33#include <asm/tlbflush.h>
 
  34#include <asm/mach/arch.h>
  35#include <asm/dma-iommu.h>
  36#include <asm/mach/map.h>
  37#include <asm/system_info.h>
  38#include <asm/dma-contiguous.h>
  39
  40#include "mm.h"
  41
  42/*
  43 * The DMA API is built upon the notion of "buffer ownership".  A buffer
  44 * is either exclusively owned by the CPU (and therefore may be accessed
  45 * by it) or exclusively owned by the DMA device.  These helper functions
  46 * represent the transitions between these two ownership states.
  47 *
  48 * Note, however, that on later ARMs, this notion does not work due to
  49 * speculative prefetches.  We model our approach on the assumption that
  50 * the CPU does do speculative prefetches, which means we clean caches
  51 * before transfers and delay cache invalidation until transfer completion.
  52 *
  53 */
  54static void __dma_page_cpu_to_dev(struct page *, unsigned long,
  55		size_t, enum dma_data_direction);
  56static void __dma_page_dev_to_cpu(struct page *, unsigned long,
  57		size_t, enum dma_data_direction);
  58
  59/**
  60 * arm_dma_map_page - map a portion of a page for streaming DMA
  61 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  62 * @page: page that buffer resides in
  63 * @offset: offset into page for start of buffer
  64 * @size: size of buffer to map
  65 * @dir: DMA transfer direction
  66 *
  67 * Ensure that any data held in the cache is appropriately discarded
  68 * or written back.
  69 *
  70 * The device owns this memory once this call has completed.  The CPU
  71 * can regain ownership by calling dma_unmap_page().
  72 */
  73static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
  74	     unsigned long offset, size_t size, enum dma_data_direction dir,
  75	     struct dma_attrs *attrs)
  76{
  77	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
  78		__dma_page_cpu_to_dev(page, offset, size, dir);
  79	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
  80}
  81
  82static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
  83	     unsigned long offset, size_t size, enum dma_data_direction dir,
  84	     struct dma_attrs *attrs)
  85{
  86	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
  87}
  88
  89/**
  90 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
  91 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  92 * @handle: DMA address of buffer
  93 * @size: size of buffer (same as passed to dma_map_page)
  94 * @dir: DMA transfer direction (same as passed to dma_map_page)
  95 *
  96 * Unmap a page streaming mode DMA translation.  The handle and size
  97 * must match what was provided in the previous dma_map_page() call.
  98 * All other usages are undefined.
  99 *
 100 * After this call, reads by the CPU to the buffer are guaranteed to see
 101 * whatever the device wrote there.
 102 */
 103static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
 104		size_t size, enum dma_data_direction dir,
 105		struct dma_attrs *attrs)
 106{
 107	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 108		__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
 109				      handle & ~PAGE_MASK, size, dir);
 110}
 111
 112static void arm_dma_sync_single_for_cpu(struct device *dev,
 113		dma_addr_t handle, size_t size, enum dma_data_direction dir)
 114{
 115	unsigned int offset = handle & (PAGE_SIZE - 1);
 116	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
 117	__dma_page_dev_to_cpu(page, offset, size, dir);
 
 118}
 119
 120static void arm_dma_sync_single_for_device(struct device *dev,
 121		dma_addr_t handle, size_t size, enum dma_data_direction dir)
 122{
 123	unsigned int offset = handle & (PAGE_SIZE - 1);
 124	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
 125	__dma_page_cpu_to_dev(page, offset, size, dir);
 
 126}
 127
 
 
 128struct dma_map_ops arm_dma_ops = {
 129	.alloc			= arm_dma_alloc,
 130	.free			= arm_dma_free,
 131	.mmap			= arm_dma_mmap,
 132	.get_sgtable		= arm_dma_get_sgtable,
 133	.map_page		= arm_dma_map_page,
 134	.unmap_page		= arm_dma_unmap_page,
 135	.map_sg			= arm_dma_map_sg,
 136	.unmap_sg		= arm_dma_unmap_sg,
 137	.sync_single_for_cpu	= arm_dma_sync_single_for_cpu,
 138	.sync_single_for_device	= arm_dma_sync_single_for_device,
 139	.sync_sg_for_cpu	= arm_dma_sync_sg_for_cpu,
 140	.sync_sg_for_device	= arm_dma_sync_sg_for_device,
 141	.set_dma_mask		= arm_dma_set_mask,
 142};
 143EXPORT_SYMBOL(arm_dma_ops);
 144
 145static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 146	dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);
 147static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
 148				  dma_addr_t handle, struct dma_attrs *attrs);
 149
 150struct dma_map_ops arm_coherent_dma_ops = {
 151	.alloc			= arm_coherent_dma_alloc,
 152	.free			= arm_coherent_dma_free,
 153	.mmap			= arm_dma_mmap,
 154	.get_sgtable		= arm_dma_get_sgtable,
 155	.map_page		= arm_coherent_dma_map_page,
 156	.map_sg			= arm_dma_map_sg,
 157	.set_dma_mask		= arm_dma_set_mask,
 158};
 159EXPORT_SYMBOL(arm_coherent_dma_ops);
 160
 161static int __dma_supported(struct device *dev, u64 mask, bool warn)
 162{
 163	unsigned long max_dma_pfn;
 164
 165	/*
 166	 * If the mask allows for more memory than we can address,
 167	 * and we actually have that much memory, then we must
 168	 * indicate that DMA to this device is not supported.
 169	 */
 170	if (sizeof(mask) != sizeof(dma_addr_t) &&
 171	    mask > (dma_addr_t)~0 &&
 172	    dma_to_pfn(dev, ~0) < max_pfn) {
 173		if (warn) {
 174			dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n",
 175				 mask);
 176			dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n");
 177		}
 178		return 0;
 179	}
 180
 181	max_dma_pfn = min(max_pfn, arm_dma_pfn_limit);
 182
 183	/*
 184	 * Translate the device's DMA mask to a PFN limit.  This
 185	 * PFN number includes the page which we can DMA to.
 186	 */
 187	if (dma_to_pfn(dev, mask) < max_dma_pfn) {
 188		if (warn)
 189			dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",
 190				 mask,
 191				 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,
 192				 max_dma_pfn + 1);
 193		return 0;
 194	}
 195
 196	return 1;
 197}
 198
 199static u64 get_coherent_dma_mask(struct device *dev)
 200{
 201	u64 mask = (u64)DMA_BIT_MASK(32);
 202
 203	if (dev) {
 204		mask = dev->coherent_dma_mask;
 205
 206		/*
 207		 * Sanity check the DMA mask - it must be non-zero, and
 208		 * must be able to be satisfied by a DMA allocation.
 209		 */
 210		if (mask == 0) {
 211			dev_warn(dev, "coherent DMA mask is unset\n");
 212			return 0;
 213		}
 214
 215		if (!__dma_supported(dev, mask, true))
 
 
 
 216			return 0;
 
 217	}
 218
 219	return mask;
 220}
 221
 222static void __dma_clear_buffer(struct page *page, size_t size)
 223{
 
 224	/*
 225	 * Ensure that the allocated pages are zeroed, and that any data
 226	 * lurking in the kernel direct-mapped region is invalidated.
 227	 */
 228	if (PageHighMem(page)) {
 229		phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
 230		phys_addr_t end = base + size;
 231		while (size > 0) {
 232			void *ptr = kmap_atomic(page);
 233			memset(ptr, 0, PAGE_SIZE);
 234			dmac_flush_range(ptr, ptr + PAGE_SIZE);
 235			kunmap_atomic(ptr);
 236			page++;
 237			size -= PAGE_SIZE;
 238		}
 239		outer_flush_range(base, end);
 240	} else {
 241		void *ptr = page_address(page);
 242		memset(ptr, 0, size);
 243		dmac_flush_range(ptr, ptr + size);
 244		outer_flush_range(__pa(ptr), __pa(ptr) + size);
 245	}
 246}
 247
 248/*
 249 * Allocate a DMA buffer for 'dev' of size 'size' using the
 250 * specified gfp mask.  Note that 'size' must be page aligned.
 251 */
 252static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
 253{
 254	unsigned long order = get_order(size);
 255	struct page *page, *p, *e;
 256
 257	page = alloc_pages(gfp, order);
 258	if (!page)
 259		return NULL;
 260
 261	/*
 262	 * Now split the huge page and free the excess pages
 263	 */
 264	split_page(page, order);
 265	for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
 266		__free_page(p);
 267
 268	__dma_clear_buffer(page, size);
 269
 270	return page;
 271}
 272
 273/*
 274 * Free a DMA buffer.  'size' must be page aligned.
 275 */
 276static void __dma_free_buffer(struct page *page, size_t size)
 277{
 278	struct page *e = page + (size >> PAGE_SHIFT);
 279
 280	while (page < e) {
 281		__free_page(page);
 282		page++;
 283	}
 284}
 285
 286#ifdef CONFIG_MMU
 287
 288static void *__alloc_from_contiguous(struct device *dev, size_t size,
 289				     pgprot_t prot, struct page **ret_page,
 290				     const void *caller);
 291
 292static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 293				 pgprot_t prot, struct page **ret_page,
 294				 const void *caller);
 
 295
 296static void *
 297__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
 298	const void *caller)
 
 
 299{
 300	struct vm_struct *area;
 301	unsigned long addr;
 302
 303	/*
 304	 * DMA allocation can be mapped to user space, so lets
 305	 * set VM_USERMAP flags too.
 306	 */
 307	area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
 308				  caller);
 309	if (!area)
 310		return NULL;
 311	addr = (unsigned long)area->addr;
 312	area->phys_addr = __pfn_to_phys(page_to_pfn(page));
 313
 314	if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
 315		vunmap((void *)addr);
 316		return NULL;
 317	}
 318	return (void *)addr;
 319}
 320
 321static void __dma_free_remap(void *cpu_addr, size_t size)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 322{
 323	unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP;
 324	struct vm_struct *area = find_vm_area(cpu_addr);
 325	if (!area || (area->flags & flags) != flags) {
 326		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
 327		return;
 
 
 
 
 
 
 
 
 
 
 
 328	}
 329	unmap_kernel_range((unsigned long)cpu_addr, size);
 330	vunmap(cpu_addr);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 331}
 
 332
 333#define DEFAULT_DMA_COHERENT_POOL_SIZE	SZ_256K
 
 334
 335struct dma_pool {
 336	size_t size;
 337	spinlock_t lock;
 338	unsigned long *bitmap;
 339	unsigned long nr_pages;
 340	void *vaddr;
 341	struct page **pages;
 342};
 343
 344static struct dma_pool atomic_pool = {
 345	.size = DEFAULT_DMA_COHERENT_POOL_SIZE,
 346};
 347
 348static int __init early_coherent_pool(char *p)
 349{
 350	atomic_pool.size = memparse(p, &p);
 351	return 0;
 352}
 353early_param("coherent_pool", early_coherent_pool);
 354
 355void __init init_dma_coherent_pool_size(unsigned long size)
 356{
 357	/*
 358	 * Catch any attempt to set the pool size too late.
 359	 */
 360	BUG_ON(atomic_pool.vaddr);
 361
 362	/*
 363	 * Set architecture specific coherent pool size only if
 364	 * it has not been changed by kernel command line parameter.
 365	 */
 366	if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
 367		atomic_pool.size = size;
 368}
 369
 370/*
 371 * Initialise the coherent pool for atomic allocations.
 372 */
 373static int __init atomic_pool_init(void)
 374{
 375	struct dma_pool *pool = &atomic_pool;
 376	pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
 377	gfp_t gfp = GFP_KERNEL | GFP_DMA;
 378	unsigned long nr_pages = pool->size >> PAGE_SHIFT;
 379	unsigned long *bitmap;
 380	struct page *page;
 381	struct page **pages;
 382	void *ptr;
 383	int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
 384
 385	bitmap = kzalloc(bitmap_size, GFP_KERNEL);
 386	if (!bitmap)
 387		goto no_bitmap;
 388
 389	pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
 390	if (!pages)
 391		goto no_pages;
 392
 393	if (IS_ENABLED(CONFIG_DMA_CMA))
 394		ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page,
 395					      atomic_pool_init);
 396	else
 397		ptr = __alloc_remap_buffer(NULL, pool->size, gfp, prot, &page,
 398					   atomic_pool_init);
 399	if (ptr) {
 400		int i;
 401
 402		for (i = 0; i < nr_pages; i++)
 403			pages[i] = page + i;
 404
 405		spin_lock_init(&pool->lock);
 406		pool->vaddr = ptr;
 407		pool->pages = pages;
 408		pool->bitmap = bitmap;
 409		pool->nr_pages = nr_pages;
 410		pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
 411		       (unsigned)pool->size / 1024);
 412		return 0;
 413	}
 414
 415	kfree(pages);
 416no_pages:
 417	kfree(bitmap);
 418no_bitmap:
 419	pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
 420	       (unsigned)pool->size / 1024);
 421	return -ENOMEM;
 422}
 423/*
 424 * CMA is activated by core_initcall, so we must be called after it.
 425 */
 426postcore_initcall(atomic_pool_init);
 427
 428struct dma_contig_early_reserve {
 429	phys_addr_t base;
 430	unsigned long size;
 431};
 432
 433static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
 434
 435static int dma_mmu_remap_num __initdata;
 436
 437void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
 438{
 439	dma_mmu_remap[dma_mmu_remap_num].base = base;
 440	dma_mmu_remap[dma_mmu_remap_num].size = size;
 441	dma_mmu_remap_num++;
 442}
 443
 444void __init dma_contiguous_remap(void)
 445{
 446	int i;
 447	for (i = 0; i < dma_mmu_remap_num; i++) {
 448		phys_addr_t start = dma_mmu_remap[i].base;
 449		phys_addr_t end = start + dma_mmu_remap[i].size;
 450		struct map_desc map;
 451		unsigned long addr;
 452
 453		if (end > arm_lowmem_limit)
 454			end = arm_lowmem_limit;
 455		if (start >= end)
 456			continue;
 457
 458		map.pfn = __phys_to_pfn(start);
 459		map.virtual = __phys_to_virt(start);
 460		map.length = end - start;
 461		map.type = MT_MEMORY_DMA_READY;
 462
 463		/*
 464		 * Clear previous low-memory mapping
 465		 */
 466		for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
 467		     addr += PMD_SIZE)
 468			pmd_clear(pmd_off_k(addr));
 469
 470		iotable_init(&map, 1);
 471	}
 472}
 473
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 474static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
 475			    void *data)
 476{
 477	struct page *page = virt_to_page(addr);
 478	pgprot_t prot = *(pgprot_t *)data;
 479
 480	set_pte_ext(pte, mk_pte(page, prot), 0);
 481	return 0;
 482}
 483
 484static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
 485{
 486	unsigned long start = (unsigned long) page_address(page);
 487	unsigned end = start + size;
 488
 489	apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
 
 490	flush_tlb_kernel_range(start, end);
 491}
 492
 493static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 494				 pgprot_t prot, struct page **ret_page,
 495				 const void *caller)
 496{
 497	struct page *page;
 498	void *ptr;
 499	page = __dma_alloc_buffer(dev, size, gfp);
 500	if (!page)
 501		return NULL;
 502
 503	ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
 504	if (!ptr) {
 505		__dma_free_buffer(page, size);
 506		return NULL;
 507	}
 508
 509	*ret_page = page;
 510	return ptr;
 511}
 512
 513static void *__alloc_from_pool(size_t size, struct page **ret_page)
 
 514{
 515	struct dma_pool *pool = &atomic_pool;
 516	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 517	unsigned int pageno;
 518	unsigned long flags;
 519	void *ptr = NULL;
 520	unsigned long align_mask;
 521
 522	if (!pool->vaddr) {
 523		WARN(1, "coherent pool not initialised!\n");
 
 
 524		return NULL;
 525	}
 526
 527	/*
 528	 * Align the region allocation - allocations from pool are rather
 529	 * small, so align them to their order in pages, minimum is a page
 530	 * size. This helps reduce fragmentation of the DMA space.
 531	 */
 532	align_mask = (1 << get_order(size)) - 1;
 533
 534	spin_lock_irqsave(&pool->lock, flags);
 535	pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages,
 536					    0, count, align_mask);
 537	if (pageno < pool->nr_pages) {
 538		bitmap_set(pool->bitmap, pageno, count);
 539		ptr = pool->vaddr + PAGE_SIZE * pageno;
 540		*ret_page = pool->pages[pageno];
 541	} else {
 542		pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
 543			    "Please increase it with coherent_pool= kernel parameter!\n",
 544			    (unsigned)pool->size / 1024);
 545	}
 546	spin_unlock_irqrestore(&pool->lock, flags);
 547
 548	return ptr;
 549}
 550
 551static bool __in_atomic_pool(void *start, size_t size)
 552{
 553	struct dma_pool *pool = &atomic_pool;
 554	void *end = start + size;
 555	void *pool_start = pool->vaddr;
 556	void *pool_end = pool->vaddr + pool->size;
 557
 558	if (start < pool_start || start >= pool_end)
 559		return false;
 560
 561	if (end <= pool_end)
 562		return true;
 563
 564	WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
 565	     start, end - 1, pool_start, pool_end - 1);
 566
 567	return false;
 568}
 569
 570static int __free_from_pool(void *start, size_t size)
 571{
 572	struct dma_pool *pool = &atomic_pool;
 573	unsigned long pageno, count;
 574	unsigned long flags;
 575
 576	if (!__in_atomic_pool(start, size))
 577		return 0;
 578
 579	pageno = (start - pool->vaddr) >> PAGE_SHIFT;
 580	count = size >> PAGE_SHIFT;
 581
 582	spin_lock_irqsave(&pool->lock, flags);
 583	bitmap_clear(pool->bitmap, pageno, count);
 584	spin_unlock_irqrestore(&pool->lock, flags);
 
 
 
 585
 
 586	return 1;
 587}
 588
 589static void *__alloc_from_contiguous(struct device *dev, size_t size,
 590				     pgprot_t prot, struct page **ret_page,
 591				     const void *caller)
 592{
 593	unsigned long order = get_order(size);
 594	size_t count = size >> PAGE_SHIFT;
 595	struct page *page;
 596	void *ptr;
 597
 598	page = dma_alloc_from_contiguous(dev, count, order);
 599	if (!page)
 600		return NULL;
 601
 602	__dma_clear_buffer(page, size);
 
 603
 604	if (PageHighMem(page)) {
 605		ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
 606		if (!ptr) {
 607			dma_release_from_contiguous(dev, page, count);
 608			return NULL;
 609		}
 610	} else {
 611		__dma_remap(page, size, prot);
 612		ptr = page_address(page);
 613	}
 614	*ret_page = page;
 615	return ptr;
 616}
 617
 618static void __free_from_contiguous(struct device *dev, struct page *page,
 619				   void *cpu_addr, size_t size)
 620{
 621	if (PageHighMem(page))
 622		__dma_free_remap(cpu_addr, size);
 623	else
 624		__dma_remap(page, size, PAGE_KERNEL);
 625	dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
 626}
 627
 628static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
 629{
 630	prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
 631			    pgprot_writecombine(prot) :
 632			    pgprot_dmacoherent(prot);
 633	return prot;
 634}
 635
 636#define nommu() 0
 637
 638#else	/* !CONFIG_MMU */
 639
 640#define nommu() 1
 641
 642#define __get_dma_pgprot(attrs, prot)	__pgprot(0)
 643#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c)	NULL
 644#define __alloc_from_pool(size, ret_page)			NULL
 645#define __alloc_from_contiguous(dev, size, prot, ret, c)	NULL
 646#define __free_from_pool(cpu_addr, size)			0
 647#define __free_from_contiguous(dev, page, cpu_addr, size)	do { } while (0)
 648#define __dma_free_remap(cpu_addr, size)			do { } while (0)
 649
 650#endif	/* CONFIG_MMU */
 651
 652static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
 653				   struct page **ret_page)
 654{
 655	struct page *page;
 656	page = __dma_alloc_buffer(dev, size, gfp);
 657	if (!page)
 658		return NULL;
 659
 660	*ret_page = page;
 661	return page_address(page);
 662}
 663
 664
 665
 666static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 667			 gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)
 668{
 669	u64 mask = get_coherent_dma_mask(dev);
 670	struct page *page = NULL;
 671	void *addr;
 672
 673#ifdef CONFIG_DMA_API_DEBUG
 674	u64 limit = (mask + 1) & ~mask;
 675	if (limit && size >= limit) {
 676		dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
 677			size, mask);
 678		return NULL;
 679	}
 680#endif
 681
 682	if (!mask)
 683		return NULL;
 684
 685	if (mask < 0xffffffffULL)
 686		gfp |= GFP_DMA;
 687
 688	/*
 689	 * Following is a work-around (a.k.a. hack) to prevent pages
 690	 * with __GFP_COMP being passed to split_page() which cannot
 691	 * handle them.  The real problem is that this flag probably
 692	 * should be 0 on ARM as it is not supported on this
 693	 * platform; see CONFIG_HUGETLBFS.
 694	 */
 695	gfp &= ~(__GFP_COMP);
 696
 697	*handle = DMA_ERROR_CODE;
 698	size = PAGE_ALIGN(size);
 699
 700	if (is_coherent || nommu())
 701		addr = __alloc_simple_buffer(dev, size, gfp, &page);
 702	else if (!(gfp & __GFP_WAIT))
 703		addr = __alloc_from_pool(size, &page);
 704	else if (!IS_ENABLED(CONFIG_DMA_CMA))
 705		addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
 
 
 706	else
 707		addr = __alloc_from_contiguous(dev, size, prot, &page, caller);
 708
 709	if (addr)
 710		*handle = pfn_to_dma(dev, page_to_pfn(page));
 711
 712	return addr;
 713}
 714
 715/*
 716 * Allocate DMA-coherent memory space and return both the kernel remapped
 717 * virtual and bus address for that space.
 718 */
 719void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 720		    gfp_t gfp, struct dma_attrs *attrs)
 721{
 722	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
 723	void *memory;
 724
 725	if (dma_alloc_from_coherent(dev, size, handle, &memory))
 726		return memory;
 727
 728	return __dma_alloc(dev, size, handle, gfp, prot, false,
 729			   __builtin_return_address(0));
 730}
 731
 732static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 733	dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
 734{
 735	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
 736	void *memory;
 737
 738	if (dma_alloc_from_coherent(dev, size, handle, &memory))
 739		return memory;
 740
 741	return __dma_alloc(dev, size, handle, gfp, prot, true,
 742			   __builtin_return_address(0));
 743}
 744
 745/*
 746 * Create userspace mapping for the DMA-coherent memory.
 747 */
 748int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 749		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
 750		 struct dma_attrs *attrs)
 751{
 752	int ret = -ENXIO;
 753#ifdef CONFIG_MMU
 754	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 755	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 756	unsigned long pfn = dma_to_pfn(dev, dma_addr);
 757	unsigned long off = vma->vm_pgoff;
 758
 759	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
 760
 761	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
 762		return ret;
 763
 764	if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
 765		ret = remap_pfn_range(vma, vma->vm_start,
 766				      pfn + off,
 767				      vma->vm_end - vma->vm_start,
 768				      vma->vm_page_prot);
 769	}
 770#endif	/* CONFIG_MMU */
 771
 772	return ret;
 773}
 774
 775/*
 776 * Free a buffer as defined by the above mapping.
 777 */
 778static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 779			   dma_addr_t handle, struct dma_attrs *attrs,
 780			   bool is_coherent)
 781{
 782	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
 783
 784	if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
 785		return;
 786
 787	size = PAGE_ALIGN(size);
 788
 789	if (is_coherent || nommu()) {
 790		__dma_free_buffer(page, size);
 791	} else if (__free_from_pool(cpu_addr, size)) {
 792		return;
 793	} else if (!IS_ENABLED(CONFIG_DMA_CMA)) {
 794		__dma_free_remap(cpu_addr, size);
 795		__dma_free_buffer(page, size);
 796	} else {
 
 
 797		/*
 798		 * Non-atomic allocations cannot be freed with IRQs disabled
 799		 */
 800		WARN_ON(irqs_disabled());
 801		__free_from_contiguous(dev, page, cpu_addr, size);
 802	}
 803}
 804
 805void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 806		  dma_addr_t handle, struct dma_attrs *attrs)
 807{
 808	__arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
 809}
 810
 811static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
 812				  dma_addr_t handle, struct dma_attrs *attrs)
 813{
 814	__arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
 815}
 816
 817int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
 818		 void *cpu_addr, dma_addr_t handle, size_t size,
 819		 struct dma_attrs *attrs)
 820{
 821	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
 822	int ret;
 823
 824	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
 825	if (unlikely(ret))
 826		return ret;
 827
 828	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
 829	return 0;
 830}
 831
 832static void dma_cache_maint_page(struct page *page, unsigned long offset,
 833	size_t size, enum dma_data_direction dir,
 834	void (*op)(const void *, size_t, int))
 835{
 836	unsigned long pfn;
 837	size_t left = size;
 838
 839	pfn = page_to_pfn(page) + offset / PAGE_SIZE;
 840	offset %= PAGE_SIZE;
 841
 842	/*
 843	 * A single sg entry may refer to multiple physically contiguous
 844	 * pages.  But we still need to process highmem pages individually.
 845	 * If highmem is not configured then the bulk of this loop gets
 846	 * optimized out.
 847	 */
 
 848	do {
 849		size_t len = left;
 850		void *vaddr;
 851
 852		page = pfn_to_page(pfn);
 853
 854		if (PageHighMem(page)) {
 855			if (len + offset > PAGE_SIZE)
 
 
 
 
 856				len = PAGE_SIZE - offset;
 857
 858			if (cache_is_vipt_nonaliasing()) {
 
 
 
 
 
 
 859				vaddr = kmap_atomic(page);
 860				op(vaddr + offset, len, dir);
 861				kunmap_atomic(vaddr);
 862			} else {
 863				vaddr = kmap_high_get(page);
 864				if (vaddr) {
 865					op(vaddr + offset, len, dir);
 866					kunmap_high(page);
 867				}
 868			}
 869		} else {
 870			vaddr = page_address(page) + offset;
 871			op(vaddr, len, dir);
 872		}
 873		offset = 0;
 874		pfn++;
 875		left -= len;
 876	} while (left);
 877}
 878
 879/*
 880 * Make an area consistent for devices.
 881 * Note: Drivers should NOT use this function directly, as it will break
 882 * platforms with CONFIG_DMABOUNCE.
 883 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
 884 */
 885static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
 886	size_t size, enum dma_data_direction dir)
 887{
 888	unsigned long paddr;
 889
 890	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
 891
 892	paddr = page_to_phys(page) + off;
 893	if (dir == DMA_FROM_DEVICE) {
 894		outer_inv_range(paddr, paddr + size);
 895	} else {
 896		outer_clean_range(paddr, paddr + size);
 897	}
 898	/* FIXME: non-speculating: flush on bidirectional mappings? */
 899}
 900
 901static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
 902	size_t size, enum dma_data_direction dir)
 903{
 904	unsigned long paddr = page_to_phys(page) + off;
 905
 906	/* FIXME: non-speculating: not required */
 907	/* don't bother invalidating if DMA to device */
 908	if (dir != DMA_TO_DEVICE)
 909		outer_inv_range(paddr, paddr + size);
 910
 911	dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
 912
 913	/*
 914	 * Mark the D-cache clean for these pages to avoid extra flushing.
 915	 */
 916	if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
 917		unsigned long pfn;
 918		size_t left = size;
 919
 920		pfn = page_to_pfn(page) + off / PAGE_SIZE;
 921		off %= PAGE_SIZE;
 922		if (off) {
 923			pfn++;
 924			left -= PAGE_SIZE - off;
 925		}
 926		while (left >= PAGE_SIZE) {
 927			page = pfn_to_page(pfn++);
 928			set_bit(PG_dcache_clean, &page->flags);
 929			left -= PAGE_SIZE;
 930		}
 931	}
 932}
 933
 934/**
 935 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
 936 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 937 * @sg: list of buffers
 938 * @nents: number of buffers to map
 939 * @dir: DMA transfer direction
 940 *
 941 * Map a set of buffers described by scatterlist in streaming mode for DMA.
 942 * This is the scatter-gather version of the dma_map_single interface.
 943 * Here the scatter gather list elements are each tagged with the
 944 * appropriate dma address and length.  They are obtained via
 945 * sg_dma_{address,length}.
 946 *
 947 * Device ownership issues as mentioned for dma_map_single are the same
 948 * here.
 949 */
 950int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 951		enum dma_data_direction dir, struct dma_attrs *attrs)
 952{
 953	struct dma_map_ops *ops = get_dma_ops(dev);
 954	struct scatterlist *s;
 955	int i, j;
 956
 957	for_each_sg(sg, s, nents, i) {
 958#ifdef CONFIG_NEED_SG_DMA_LENGTH
 959		s->dma_length = s->length;
 960#endif
 961		s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
 962						s->length, dir, attrs);
 963		if (dma_mapping_error(dev, s->dma_address))
 964			goto bad_mapping;
 965	}
 966	return nents;
 967
 968 bad_mapping:
 969	for_each_sg(sg, s, i, j)
 970		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
 971	return 0;
 972}
 973
 974/**
 975 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
 976 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 977 * @sg: list of buffers
 978 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
 979 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 980 *
 981 * Unmap a set of streaming mode DMA translations.  Again, CPU access
 982 * rules concerning calls here are the same as for dma_unmap_single().
 983 */
 984void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 985		enum dma_data_direction dir, struct dma_attrs *attrs)
 986{
 987	struct dma_map_ops *ops = get_dma_ops(dev);
 988	struct scatterlist *s;
 989
 990	int i;
 991
 992	for_each_sg(sg, s, nents, i)
 993		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
 994}
 995
 996/**
 997 * arm_dma_sync_sg_for_cpu
 998 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 999 * @sg: list of buffers
1000 * @nents: number of buffers to map (returned from dma_map_sg)
1001 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1002 */
1003void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1004			int nents, enum dma_data_direction dir)
1005{
1006	struct dma_map_ops *ops = get_dma_ops(dev);
1007	struct scatterlist *s;
1008	int i;
1009
1010	for_each_sg(sg, s, nents, i)
1011		ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
1012					 dir);
1013}
1014
1015/**
1016 * arm_dma_sync_sg_for_device
1017 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1018 * @sg: list of buffers
1019 * @nents: number of buffers to map (returned from dma_map_sg)
1020 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1021 */
1022void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1023			int nents, enum dma_data_direction dir)
1024{
1025	struct dma_map_ops *ops = get_dma_ops(dev);
1026	struct scatterlist *s;
1027	int i;
1028
1029	for_each_sg(sg, s, nents, i)
1030		ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
1031					    dir);
1032}
1033
1034/*
1035 * Return whether the given device DMA address mask can be supported
1036 * properly.  For example, if your device can only drive the low 24-bits
1037 * during bus mastering, then you would pass 0x00ffffff as the mask
1038 * to this function.
1039 */
1040int dma_supported(struct device *dev, u64 mask)
1041{
1042	return __dma_supported(dev, mask, false);
 
 
1043}
1044EXPORT_SYMBOL(dma_supported);
1045
1046int arm_dma_set_mask(struct device *dev, u64 dma_mask)
1047{
1048	if (!dev->dma_mask || !dma_supported(dev, dma_mask))
1049		return -EIO;
1050
1051	*dev->dma_mask = dma_mask;
1052
1053	return 0;
1054}
1055
1056#define PREALLOC_DMA_DEBUG_ENTRIES	4096
1057
1058static int __init dma_debug_do_init(void)
1059{
 
 
 
1060	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
1061	return 0;
1062}
1063fs_initcall(dma_debug_do_init);
1064
1065#ifdef CONFIG_ARM_DMA_USE_IOMMU
1066
1067/* IOMMU */
1068
1069static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
1070
1071static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1072				      size_t size)
1073{
1074	unsigned int order = get_order(size);
1075	unsigned int align = 0;
1076	unsigned int count, start;
1077	unsigned long flags;
1078	dma_addr_t iova;
1079	int i;
1080
1081	if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
1082		order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
1083
1084	count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1085	align = (1 << order) - 1;
1086
1087	spin_lock_irqsave(&mapping->lock, flags);
1088	for (i = 0; i < mapping->nr_bitmaps; i++) {
1089		start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1090				mapping->bits, 0, count, align);
1091
1092		if (start > mapping->bits)
1093			continue;
1094
1095		bitmap_set(mapping->bitmaps[i], start, count);
1096		break;
1097	}
1098
1099	/*
1100	 * No unused range found. Try to extend the existing mapping
1101	 * and perform a second attempt to reserve an IO virtual
1102	 * address range of size bytes.
1103	 */
1104	if (i == mapping->nr_bitmaps) {
1105		if (extend_iommu_mapping(mapping)) {
1106			spin_unlock_irqrestore(&mapping->lock, flags);
1107			return DMA_ERROR_CODE;
1108		}
1109
1110		start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1111				mapping->bits, 0, count, align);
1112
1113		if (start > mapping->bits) {
1114			spin_unlock_irqrestore(&mapping->lock, flags);
1115			return DMA_ERROR_CODE;
1116		}
1117
1118		bitmap_set(mapping->bitmaps[i], start, count);
1119	}
1120	spin_unlock_irqrestore(&mapping->lock, flags);
1121
1122	iova = mapping->base + (mapping->size * i);
1123	iova += start << PAGE_SHIFT;
1124
1125	return iova;
1126}
1127
1128static inline void __free_iova(struct dma_iommu_mapping *mapping,
1129			       dma_addr_t addr, size_t size)
1130{
1131	unsigned int start, count;
 
 
 
1132	unsigned long flags;
1133	dma_addr_t bitmap_base;
1134	u32 bitmap_index;
1135
1136	if (!size)
1137		return;
1138
1139	bitmap_index = (u32) (addr - mapping->base) / (u32) mapping->size;
1140	BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);
1141
1142	bitmap_base = mapping->base + mapping->size * bitmap_index;
1143
1144	start = (addr - bitmap_base) >>	PAGE_SHIFT;
1145
1146	if (addr + size > bitmap_base + mapping->size) {
1147		/*
1148		 * The address range to be freed reaches into the iova
1149		 * range of the next bitmap. This should not happen as
1150		 * we don't allow this in __alloc_iova (at the
1151		 * moment).
1152		 */
1153		BUG();
1154	} else
1155		count = size >> PAGE_SHIFT;
1156
1157	spin_lock_irqsave(&mapping->lock, flags);
1158	bitmap_clear(mapping->bitmaps[bitmap_index], start, count);
1159	spin_unlock_irqrestore(&mapping->lock, flags);
1160}
1161
1162static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1163					  gfp_t gfp, struct dma_attrs *attrs)
1164{
1165	struct page **pages;
1166	int count = size >> PAGE_SHIFT;
1167	int array_size = count * sizeof(struct page *);
1168	int i = 0;
1169
1170	if (array_size <= PAGE_SIZE)
1171		pages = kzalloc(array_size, gfp);
1172	else
1173		pages = vzalloc(array_size);
1174	if (!pages)
1175		return NULL;
1176
1177	if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))
1178	{
1179		unsigned long order = get_order(size);
1180		struct page *page;
1181
1182		page = dma_alloc_from_contiguous(dev, count, order);
1183		if (!page)
1184			goto error;
1185
1186		__dma_clear_buffer(page, size);
1187
1188		for (i = 0; i < count; i++)
1189			pages[i] = page + i;
1190
1191		return pages;
1192	}
1193
1194	/*
1195	 * IOMMU can map any pages, so himem can also be used here
1196	 */
1197	gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1198
1199	while (count) {
1200		int j, order = __fls(count);
1201
1202		pages[i] = alloc_pages(gfp, order);
1203		while (!pages[i] && order)
1204			pages[i] = alloc_pages(gfp, --order);
1205		if (!pages[i])
1206			goto error;
1207
1208		if (order) {
1209			split_page(pages[i], order);
1210			j = 1 << order;
1211			while (--j)
1212				pages[i + j] = pages[i] + j;
1213		}
1214
1215		__dma_clear_buffer(pages[i], PAGE_SIZE << order);
1216		i += 1 << order;
1217		count -= 1 << order;
1218	}
1219
1220	return pages;
1221error:
1222	while (i--)
1223		if (pages[i])
1224			__free_pages(pages[i], 0);
1225	if (array_size <= PAGE_SIZE)
1226		kfree(pages);
1227	else
1228		vfree(pages);
1229	return NULL;
1230}
1231
1232static int __iommu_free_buffer(struct device *dev, struct page **pages,
1233			       size_t size, struct dma_attrs *attrs)
1234{
1235	int count = size >> PAGE_SHIFT;
1236	int array_size = count * sizeof(struct page *);
1237	int i;
1238
1239	if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {
1240		dma_release_from_contiguous(dev, pages[0], count);
1241	} else {
1242		for (i = 0; i < count; i++)
1243			if (pages[i])
1244				__free_pages(pages[i], 0);
1245	}
1246
1247	if (array_size <= PAGE_SIZE)
1248		kfree(pages);
1249	else
1250		vfree(pages);
1251	return 0;
1252}
1253
1254/*
1255 * Create a CPU mapping for a specified pages
1256 */
1257static void *
1258__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
1259		    const void *caller)
1260{
1261	unsigned int i, nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1262	struct vm_struct *area;
1263	unsigned long p;
1264
1265	area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
1266				  caller);
1267	if (!area)
 
1268		return NULL;
 
 
 
 
 
 
 
 
 
 
 
 
1269
1270	area->pages = pages;
1271	area->nr_pages = nr_pages;
1272	p = (unsigned long)area->addr;
1273
1274	for (i = 0; i < nr_pages; i++) {
1275		phys_addr_t phys = __pfn_to_phys(page_to_pfn(pages[i]));
1276		if (ioremap_page_range(p, p + PAGE_SIZE, phys, prot))
1277			goto err;
1278		p += PAGE_SIZE;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1279	}
1280	return area->addr;
1281err:
1282	unmap_kernel_range((unsigned long)area->addr, size);
1283	vunmap(area->addr);
1284	return NULL;
1285}
1286
1287/*
1288 * Create a mapping in device IO address space for specified pages
1289 */
1290static dma_addr_t
1291__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
1292{
1293	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1294	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1295	dma_addr_t dma_addr, iova;
1296	int i, ret = DMA_ERROR_CODE;
1297
1298	dma_addr = __alloc_iova(mapping, size);
1299	if (dma_addr == DMA_ERROR_CODE)
1300		return dma_addr;
1301
1302	iova = dma_addr;
1303	for (i = 0; i < count; ) {
1304		unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1305		phys_addr_t phys = page_to_phys(pages[i]);
1306		unsigned int len, j;
1307
1308		for (j = i + 1; j < count; j++, next_pfn++)
1309			if (page_to_pfn(pages[j]) != next_pfn)
1310				break;
1311
1312		len = (j - i) << PAGE_SHIFT;
1313		ret = iommu_map(mapping->domain, iova, phys, len,
1314				IOMMU_READ|IOMMU_WRITE);
1315		if (ret < 0)
1316			goto fail;
1317		iova += len;
1318		i = j;
1319	}
1320	return dma_addr;
1321fail:
1322	iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1323	__free_iova(mapping, dma_addr, size);
1324	return DMA_ERROR_CODE;
1325}
1326
1327static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1328{
1329	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1330
1331	/*
1332	 * add optional in-page offset from iova to size and align
1333	 * result to page size
1334	 */
1335	size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1336	iova &= PAGE_MASK;
1337
1338	iommu_unmap(mapping->domain, iova, size);
1339	__free_iova(mapping, iova, size);
1340	return 0;
1341}
1342
1343static struct page **__atomic_get_pages(void *addr)
1344{
1345	struct dma_pool *pool = &atomic_pool;
1346	struct page **pages = pool->pages;
1347	int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
1348
1349	return pages + offs;
1350}
1351
1352static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
1353{
1354	struct vm_struct *area;
1355
1356	if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
1357		return __atomic_get_pages(cpu_addr);
1358
1359	if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1360		return cpu_addr;
1361
1362	area = find_vm_area(cpu_addr);
1363	if (area && (area->flags & VM_ARM_DMA_CONSISTENT))
1364		return area->pages;
1365	return NULL;
1366}
1367
1368static void *__iommu_alloc_atomic(struct device *dev, size_t size,
1369				  dma_addr_t *handle)
1370{
1371	struct page *page;
1372	void *addr;
1373
1374	addr = __alloc_from_pool(size, &page);
1375	if (!addr)
1376		return NULL;
1377
1378	*handle = __iommu_create_mapping(dev, &page, size);
1379	if (*handle == DMA_ERROR_CODE)
1380		goto err_mapping;
1381
1382	return addr;
1383
1384err_mapping:
1385	__free_from_pool(addr, size);
1386	return NULL;
1387}
1388
1389static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1390				dma_addr_t handle, size_t size)
1391{
1392	__iommu_remove_mapping(dev, handle, size);
1393	__free_from_pool(cpu_addr, size);
1394}
1395
1396static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1397	    dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
1398{
1399	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
1400	struct page **pages;
1401	void *addr = NULL;
1402
1403	*handle = DMA_ERROR_CODE;
1404	size = PAGE_ALIGN(size);
1405
1406	if (!(gfp & __GFP_WAIT))
1407		return __iommu_alloc_atomic(dev, size, handle);
1408
1409	/*
1410	 * Following is a work-around (a.k.a. hack) to prevent pages
1411	 * with __GFP_COMP being passed to split_page() which cannot
1412	 * handle them.  The real problem is that this flag probably
1413	 * should be 0 on ARM as it is not supported on this
1414	 * platform; see CONFIG_HUGETLBFS.
1415	 */
1416	gfp &= ~(__GFP_COMP);
1417
1418	pages = __iommu_alloc_buffer(dev, size, gfp, attrs);
1419	if (!pages)
1420		return NULL;
1421
1422	*handle = __iommu_create_mapping(dev, pages, size);
1423	if (*handle == DMA_ERROR_CODE)
1424		goto err_buffer;
1425
1426	if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1427		return pages;
1428
1429	addr = __iommu_alloc_remap(pages, size, gfp, prot,
1430				   __builtin_return_address(0));
1431	if (!addr)
1432		goto err_mapping;
1433
1434	return addr;
1435
1436err_mapping:
1437	__iommu_remove_mapping(dev, *handle, size);
1438err_buffer:
1439	__iommu_free_buffer(dev, pages, size, attrs);
1440	return NULL;
1441}
1442
1443static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1444		    void *cpu_addr, dma_addr_t dma_addr, size_t size,
1445		    struct dma_attrs *attrs)
1446{
1447	unsigned long uaddr = vma->vm_start;
1448	unsigned long usize = vma->vm_end - vma->vm_start;
1449	struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1450
1451	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
 
1452
1453	if (!pages)
1454		return -ENXIO;
1455
1456	do {
1457		int ret = vm_insert_page(vma, uaddr, *pages++);
1458		if (ret) {
1459			pr_err("Remapping memory failed: %d\n", ret);
1460			return ret;
1461		}
1462		uaddr += PAGE_SIZE;
1463		usize -= PAGE_SIZE;
1464	} while (usize > 0);
 
 
 
1465
 
 
 
 
1466	return 0;
1467}
1468
1469/*
1470 * free a page as defined by the above mapping.
1471 * Must not be called with IRQs disabled.
1472 */
1473void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1474			  dma_addr_t handle, struct dma_attrs *attrs)
1475{
1476	struct page **pages;
1477	size = PAGE_ALIGN(size);
1478
1479	if (__in_atomic_pool(cpu_addr, size)) {
1480		__iommu_free_atomic(dev, cpu_addr, handle, size);
1481		return;
1482	}
1483
1484	pages = __iommu_get_pages(cpu_addr, attrs);
1485	if (!pages) {
1486		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1487		return;
1488	}
1489
1490	if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
1491		unmap_kernel_range((unsigned long)cpu_addr, size);
1492		vunmap(cpu_addr);
1493	}
1494
1495	__iommu_remove_mapping(dev, handle, size);
1496	__iommu_free_buffer(dev, pages, size, attrs);
1497}
1498
1499static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
1500				 void *cpu_addr, dma_addr_t dma_addr,
1501				 size_t size, struct dma_attrs *attrs)
1502{
1503	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1504	struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1505
1506	if (!pages)
1507		return -ENXIO;
1508
1509	return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
1510					 GFP_KERNEL);
1511}
1512
1513static int __dma_direction_to_prot(enum dma_data_direction dir)
1514{
1515	int prot;
1516
1517	switch (dir) {
1518	case DMA_BIDIRECTIONAL:
1519		prot = IOMMU_READ | IOMMU_WRITE;
1520		break;
1521	case DMA_TO_DEVICE:
1522		prot = IOMMU_READ;
1523		break;
1524	case DMA_FROM_DEVICE:
1525		prot = IOMMU_WRITE;
1526		break;
1527	default:
1528		prot = 0;
1529	}
1530
1531	return prot;
1532}
1533
1534/*
1535 * Map a part of the scatter-gather list into contiguous io address space
1536 */
1537static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1538			  size_t size, dma_addr_t *handle,
1539			  enum dma_data_direction dir, struct dma_attrs *attrs,
1540			  bool is_coherent)
1541{
1542	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1543	dma_addr_t iova, iova_base;
1544	int ret = 0;
1545	unsigned int count;
1546	struct scatterlist *s;
1547	int prot;
1548
1549	size = PAGE_ALIGN(size);
1550	*handle = DMA_ERROR_CODE;
1551
1552	iova_base = iova = __alloc_iova(mapping, size);
1553	if (iova == DMA_ERROR_CODE)
1554		return -ENOMEM;
1555
1556	for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1557		phys_addr_t phys = page_to_phys(sg_page(s));
1558		unsigned int len = PAGE_ALIGN(s->offset + s->length);
1559
1560		if (!is_coherent &&
1561			!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1562			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1563
1564		prot = __dma_direction_to_prot(dir);
1565
1566		ret = iommu_map(mapping->domain, iova, phys, len, prot);
1567		if (ret < 0)
1568			goto fail;
1569		count += len >> PAGE_SHIFT;
1570		iova += len;
1571	}
1572	*handle = iova_base;
1573
1574	return 0;
1575fail:
1576	iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1577	__free_iova(mapping, iova_base, size);
1578	return ret;
1579}
1580
1581static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1582		     enum dma_data_direction dir, struct dma_attrs *attrs,
1583		     bool is_coherent)
 
 
 
 
 
 
 
 
 
 
 
1584{
1585	struct scatterlist *s = sg, *dma = sg, *start = sg;
1586	int i, count = 0;
1587	unsigned int offset = s->offset;
1588	unsigned int size = s->offset + s->length;
1589	unsigned int max = dma_get_max_seg_size(dev);
1590
1591	for (i = 1; i < nents; i++) {
1592		s = sg_next(s);
1593
1594		s->dma_address = DMA_ERROR_CODE;
1595		s->dma_length = 0;
1596
1597		if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1598			if (__map_sg_chunk(dev, start, size, &dma->dma_address,
1599			    dir, attrs, is_coherent) < 0)
1600				goto bad_mapping;
1601
1602			dma->dma_address += offset;
1603			dma->dma_length = size - offset;
1604
1605			size = offset = s->offset;
1606			start = s;
1607			dma = sg_next(dma);
1608			count += 1;
1609		}
1610		size += s->length;
1611	}
1612	if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
1613		is_coherent) < 0)
1614		goto bad_mapping;
1615
1616	dma->dma_address += offset;
1617	dma->dma_length = size - offset;
1618
1619	return count+1;
1620
1621bad_mapping:
1622	for_each_sg(sg, s, count, i)
1623		__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1624	return 0;
1625}
1626
1627/**
1628 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1629 * @dev: valid struct device pointer
1630 * @sg: list of buffers
1631 * @nents: number of buffers to map
1632 * @dir: DMA transfer direction
1633 *
1634 * Map a set of i/o coherent buffers described by scatterlist in streaming
1635 * mode for DMA. The scatter gather list elements are merged together (if
1636 * possible) and tagged with the appropriate dma address and length. They are
1637 * obtained via sg_dma_{address,length}.
1638 */
1639int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1640		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1641{
1642	return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
1643}
1644
1645/**
1646 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1647 * @dev: valid struct device pointer
1648 * @sg: list of buffers
1649 * @nents: number of buffers to map
1650 * @dir: DMA transfer direction
1651 *
1652 * Map a set of buffers described by scatterlist in streaming mode for DMA.
1653 * The scatter gather list elements are merged together (if possible) and
1654 * tagged with the appropriate dma address and length. They are obtained via
1655 * sg_dma_{address,length}.
1656 */
1657int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1658		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1659{
1660	return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
1661}
1662
1663static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1664		int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
1665		bool is_coherent)
1666{
1667	struct scatterlist *s;
1668	int i;
1669
1670	for_each_sg(sg, s, nents, i) {
1671		if (sg_dma_len(s))
1672			__iommu_remove_mapping(dev, sg_dma_address(s),
1673					       sg_dma_len(s));
1674		if (!is_coherent &&
1675		    !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1676			__dma_page_dev_to_cpu(sg_page(s), s->offset,
1677					      s->length, dir);
1678	}
1679}
1680
1681/**
1682 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1683 * @dev: valid struct device pointer
1684 * @sg: list of buffers
1685 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1686 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1687 *
1688 * Unmap a set of streaming mode DMA translations.  Again, CPU access
1689 * rules concerning calls here are the same as for dma_unmap_single().
1690 */
1691void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1692		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1693{
1694	__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
1695}
1696
1697/**
1698 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1699 * @dev: valid struct device pointer
1700 * @sg: list of buffers
1701 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1702 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1703 *
1704 * Unmap a set of streaming mode DMA translations.  Again, CPU access
1705 * rules concerning calls here are the same as for dma_unmap_single().
1706 */
1707void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1708			enum dma_data_direction dir, struct dma_attrs *attrs)
1709{
1710	__iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
1711}
1712
1713/**
1714 * arm_iommu_sync_sg_for_cpu
1715 * @dev: valid struct device pointer
1716 * @sg: list of buffers
1717 * @nents: number of buffers to map (returned from dma_map_sg)
1718 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1719 */
1720void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1721			int nents, enum dma_data_direction dir)
1722{
1723	struct scatterlist *s;
1724	int i;
1725
1726	for_each_sg(sg, s, nents, i)
1727		__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
 
1728
1729}
1730
1731/**
1732 * arm_iommu_sync_sg_for_device
1733 * @dev: valid struct device pointer
1734 * @sg: list of buffers
1735 * @nents: number of buffers to map (returned from dma_map_sg)
1736 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1737 */
1738void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1739			int nents, enum dma_data_direction dir)
1740{
1741	struct scatterlist *s;
1742	int i;
1743
1744	for_each_sg(sg, s, nents, i)
1745		__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
 
1746}
1747
1748
1749/**
1750 * arm_coherent_iommu_map_page
1751 * @dev: valid struct device pointer
1752 * @page: page that buffer resides in
1753 * @offset: offset into page for start of buffer
1754 * @size: size of buffer to map
1755 * @dir: DMA transfer direction
1756 *
1757 * Coherent IOMMU aware version of arm_dma_map_page()
1758 */
1759static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1760	     unsigned long offset, size_t size, enum dma_data_direction dir,
1761	     struct dma_attrs *attrs)
1762{
1763	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1764	dma_addr_t dma_addr;
1765	int ret, prot, len = PAGE_ALIGN(size + offset);
 
 
 
1766
1767	dma_addr = __alloc_iova(mapping, len);
1768	if (dma_addr == DMA_ERROR_CODE)
1769		return dma_addr;
1770
1771	prot = __dma_direction_to_prot(dir);
1772
1773	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
1774	if (ret < 0)
1775		goto fail;
1776
1777	return dma_addr + offset;
1778fail:
1779	__free_iova(mapping, dma_addr, len);
1780	return DMA_ERROR_CODE;
1781}
1782
1783/**
1784 * arm_iommu_map_page
1785 * @dev: valid struct device pointer
1786 * @page: page that buffer resides in
1787 * @offset: offset into page for start of buffer
1788 * @size: size of buffer to map
1789 * @dir: DMA transfer direction
1790 *
1791 * IOMMU aware version of arm_dma_map_page()
1792 */
1793static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1794	     unsigned long offset, size_t size, enum dma_data_direction dir,
1795	     struct dma_attrs *attrs)
1796{
1797	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1798		__dma_page_cpu_to_dev(page, offset, size, dir);
1799
1800	return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
1801}
1802
1803/**
1804 * arm_coherent_iommu_unmap_page
1805 * @dev: valid struct device pointer
1806 * @handle: DMA address of buffer
1807 * @size: size of buffer (same as passed to dma_map_page)
1808 * @dir: DMA transfer direction (same as passed to dma_map_page)
1809 *
1810 * Coherent IOMMU aware version of arm_dma_unmap_page()
1811 */
1812static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1813		size_t size, enum dma_data_direction dir,
1814		struct dma_attrs *attrs)
1815{
1816	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1817	dma_addr_t iova = handle & PAGE_MASK;
1818	int offset = handle & ~PAGE_MASK;
1819	int len = PAGE_ALIGN(size + offset);
1820
1821	if (!iova)
1822		return;
1823
1824	iommu_unmap(mapping->domain, iova, len);
1825	__free_iova(mapping, iova, len);
1826}
1827
1828/**
1829 * arm_iommu_unmap_page
1830 * @dev: valid struct device pointer
1831 * @handle: DMA address of buffer
1832 * @size: size of buffer (same as passed to dma_map_page)
1833 * @dir: DMA transfer direction (same as passed to dma_map_page)
1834 *
1835 * IOMMU aware version of arm_dma_unmap_page()
1836 */
1837static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1838		size_t size, enum dma_data_direction dir,
1839		struct dma_attrs *attrs)
1840{
1841	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1842	dma_addr_t iova = handle & PAGE_MASK;
1843	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1844	int offset = handle & ~PAGE_MASK;
1845	int len = PAGE_ALIGN(size + offset);
1846
1847	if (!iova)
1848		return;
1849
1850	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1851		__dma_page_dev_to_cpu(page, offset, size, dir);
1852
1853	iommu_unmap(mapping->domain, iova, len);
1854	__free_iova(mapping, iova, len);
1855}
1856
1857static void arm_iommu_sync_single_for_cpu(struct device *dev,
1858		dma_addr_t handle, size_t size, enum dma_data_direction dir)
1859{
1860	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1861	dma_addr_t iova = handle & PAGE_MASK;
1862	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1863	unsigned int offset = handle & ~PAGE_MASK;
1864
1865	if (!iova)
1866		return;
1867
1868	__dma_page_dev_to_cpu(page, offset, size, dir);
 
1869}
1870
1871static void arm_iommu_sync_single_for_device(struct device *dev,
1872		dma_addr_t handle, size_t size, enum dma_data_direction dir)
1873{
1874	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1875	dma_addr_t iova = handle & PAGE_MASK;
1876	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1877	unsigned int offset = handle & ~PAGE_MASK;
1878
1879	if (!iova)
1880		return;
1881
1882	__dma_page_cpu_to_dev(page, offset, size, dir);
1883}
1884
1885struct dma_map_ops iommu_ops = {
1886	.alloc		= arm_iommu_alloc_attrs,
1887	.free		= arm_iommu_free_attrs,
1888	.mmap		= arm_iommu_mmap_attrs,
1889	.get_sgtable	= arm_iommu_get_sgtable,
1890
1891	.map_page		= arm_iommu_map_page,
1892	.unmap_page		= arm_iommu_unmap_page,
1893	.sync_single_for_cpu	= arm_iommu_sync_single_for_cpu,
1894	.sync_single_for_device	= arm_iommu_sync_single_for_device,
1895
1896	.map_sg			= arm_iommu_map_sg,
1897	.unmap_sg		= arm_iommu_unmap_sg,
1898	.sync_sg_for_cpu	= arm_iommu_sync_sg_for_cpu,
1899	.sync_sg_for_device	= arm_iommu_sync_sg_for_device,
1900
1901	.set_dma_mask		= arm_dma_set_mask,
1902};
1903
1904struct dma_map_ops iommu_coherent_ops = {
1905	.alloc		= arm_iommu_alloc_attrs,
1906	.free		= arm_iommu_free_attrs,
1907	.mmap		= arm_iommu_mmap_attrs,
1908	.get_sgtable	= arm_iommu_get_sgtable,
1909
1910	.map_page	= arm_coherent_iommu_map_page,
1911	.unmap_page	= arm_coherent_iommu_unmap_page,
1912
1913	.map_sg		= arm_coherent_iommu_map_sg,
1914	.unmap_sg	= arm_coherent_iommu_unmap_sg,
1915
1916	.set_dma_mask	= arm_dma_set_mask,
1917};
1918
1919/**
1920 * arm_iommu_create_mapping
1921 * @bus: pointer to the bus holding the client device (for IOMMU calls)
1922 * @base: start address of the valid IO address space
1923 * @size: maximum size of the valid IO address space
 
1924 *
1925 * Creates a mapping structure which holds information about used/unused
1926 * IO address ranges, which is required to perform memory allocation and
1927 * mapping with IOMMU aware functions.
1928 *
1929 * The client device need to be attached to the mapping with
1930 * arm_iommu_attach_device function.
1931 */
1932struct dma_iommu_mapping *
1933arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size)
 
1934{
1935	unsigned int bits = size >> PAGE_SHIFT;
1936	unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
1937	struct dma_iommu_mapping *mapping;
1938	int extensions = 1;
1939	int err = -ENOMEM;
1940
1941	if (!bitmap_size)
1942		return ERR_PTR(-EINVAL);
1943
1944	if (bitmap_size > PAGE_SIZE) {
1945		extensions = bitmap_size / PAGE_SIZE;
1946		bitmap_size = PAGE_SIZE;
1947	}
1948
1949	mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
1950	if (!mapping)
1951		goto err;
1952
1953	mapping->bitmap_size = bitmap_size;
1954	mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *),
1955				GFP_KERNEL);
1956	if (!mapping->bitmaps)
1957		goto err2;
1958
1959	mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);
1960	if (!mapping->bitmaps[0])
1961		goto err3;
1962
1963	mapping->nr_bitmaps = 1;
1964	mapping->extensions = extensions;
1965	mapping->base = base;
1966	mapping->bits = BITS_PER_BYTE * bitmap_size;
1967	mapping->size = mapping->bits << PAGE_SHIFT;
1968
1969	spin_lock_init(&mapping->lock);
1970
1971	mapping->domain = iommu_domain_alloc(bus);
1972	if (!mapping->domain)
1973		goto err4;
1974
1975	kref_init(&mapping->kref);
1976	return mapping;
1977err4:
1978	kfree(mapping->bitmaps[0]);
1979err3:
1980	kfree(mapping->bitmaps);
1981err2:
1982	kfree(mapping);
1983err:
1984	return ERR_PTR(err);
1985}
1986EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
1987
1988static void release_iommu_mapping(struct kref *kref)
1989{
1990	int i;
1991	struct dma_iommu_mapping *mapping =
1992		container_of(kref, struct dma_iommu_mapping, kref);
1993
1994	iommu_domain_free(mapping->domain);
1995	for (i = 0; i < mapping->nr_bitmaps; i++)
1996		kfree(mapping->bitmaps[i]);
1997	kfree(mapping->bitmaps);
1998	kfree(mapping);
1999}
2000
2001static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
2002{
2003	int next_bitmap;
2004
2005	if (mapping->nr_bitmaps > mapping->extensions)
2006		return -EINVAL;
2007
2008	next_bitmap = mapping->nr_bitmaps;
2009	mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,
2010						GFP_ATOMIC);
2011	if (!mapping->bitmaps[next_bitmap])
2012		return -ENOMEM;
2013
2014	mapping->nr_bitmaps++;
2015
2016	return 0;
2017}
2018
2019void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
2020{
2021	if (mapping)
2022		kref_put(&mapping->kref, release_iommu_mapping);
2023}
2024EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
2025
2026/**
2027 * arm_iommu_attach_device
2028 * @dev: valid struct device pointer
2029 * @mapping: io address space mapping structure (returned from
2030 *	arm_iommu_create_mapping)
2031 *
2032 * Attaches specified io address space mapping to the provided device,
2033 * this replaces the dma operations (dma_map_ops pointer) with the
2034 * IOMMU aware version. More than one client might be attached to
2035 * the same io address space mapping.
2036 */
2037int arm_iommu_attach_device(struct device *dev,
2038			    struct dma_iommu_mapping *mapping)
2039{
2040	int err;
2041
2042	err = iommu_attach_device(mapping->domain, dev);
2043	if (err)
2044		return err;
2045
2046	kref_get(&mapping->kref);
2047	dev->archdata.mapping = mapping;
2048	set_dma_ops(dev, &iommu_ops);
2049
2050	pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
2051	return 0;
2052}
2053EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
2054
2055/**
2056 * arm_iommu_detach_device
2057 * @dev: valid struct device pointer
2058 *
2059 * Detaches the provided device from a previously attached map.
2060 * This voids the dma operations (dma_map_ops pointer)
2061 */
2062void arm_iommu_detach_device(struct device *dev)
2063{
2064	struct dma_iommu_mapping *mapping;
2065
2066	mapping = to_dma_iommu_mapping(dev);
2067	if (!mapping) {
2068		dev_warn(dev, "Not attached\n");
2069		return;
2070	}
2071
2072	iommu_detach_device(mapping->domain, dev);
2073	kref_put(&mapping->kref, release_iommu_mapping);
2074	dev->archdata.mapping = NULL;
2075	set_dma_ops(dev, NULL);
2076
2077	pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
2078}
2079EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
2080
2081#endif