1/*
  2 *  linux/arch/arm/mm/dma-mapping.c
  3 *
  4 *  Copyright (C) 2000-2004 Russell King
  5 *
  6 * This program is free software; you can redistribute it and/or modify
  7 * it under the terms of the GNU General Public License version 2 as
  8 * published by the Free Software Foundation.
  9 *
 10 *  DMA uncached mapping support.
 11 */
 
 12#include <linux/module.h>
 13#include <linux/mm.h>
 
 14#include <linux/gfp.h>
 15#include <linux/errno.h>
 16#include <linux/list.h>
 17#include <linux/init.h>
 18#include <linux/device.h>
 19#include <linux/dma-mapping.h>
 
 20#include <linux/highmem.h>
 
 
 
 
 
 
 
 21
 22#include <asm/memory.h>
 23#include <asm/highmem.h>
 24#include <asm/cacheflush.h>
 25#include <asm/tlbflush.h>
 26#include <asm/sizes.h>
 
 
 
 
 27
 
 28#include "mm.h"
 29
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 30static u64 get_coherent_dma_mask(struct device *dev)
 31{
 32	u64 mask = (u64)arm_dma_limit;
 33
 34	if (dev) {
 35		mask = dev->coherent_dma_mask;
 36
 37		/*
 38		 * Sanity check the DMA mask - it must be non-zero, and
 39		 * must be able to be satisfied by a DMA allocation.
 40		 */
 41		if (mask == 0) {
 42			dev_warn(dev, "coherent DMA mask is unset\n");
 43			return 0;
 44		}
 45
 46		if ((~mask) & (u64)arm_dma_limit) {
 47			dev_warn(dev, "coherent DMA mask %#llx is smaller "
 48				 "than system GFP_DMA mask %#llx\n",
 49				 mask, (u64)arm_dma_limit);
 50			return 0;
 51		}
 52	}
 53
 54	return mask;
 55}
 56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 57/*
 58 * Allocate a DMA buffer for 'dev' of size 'size' using the
 59 * specified gfp mask.  Note that 'size' must be page aligned.
 60 */
 61static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
 
 62{
 63	unsigned long order = get_order(size);
 64	struct page *page, *p, *e;
 65	void *ptr;
 66	u64 mask = get_coherent_dma_mask(dev);
 67
 68#ifdef CONFIG_DMA_API_DEBUG
 69	u64 limit = (mask + 1) & ~mask;
 70	if (limit && size >= limit) {
 71		dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
 72			size, mask);
 73		return NULL;
 74	}
 75#endif
 76
 77	if (!mask)
 78		return NULL;
 79
 80	if (mask < 0xffffffffULL)
 81		gfp |= GFP_DMA;
 82
 83	page = alloc_pages(gfp, order);
 84	if (!page)
 85		return NULL;
 86
 87	/*
 88	 * Now split the huge page and free the excess pages
 89	 */
 90	split_page(page, order);
 91	for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
 92		__free_page(p);
 93
 94	/*
 95	 * Ensure that the allocated pages are zeroed, and that any data
 96	 * lurking in the kernel direct-mapped region is invalidated.
 97	 */
 98	ptr = page_address(page);
 99	memset(ptr, 0, size);
100	dmac_flush_range(ptr, ptr + size);
101	outer_flush_range(__pa(ptr), __pa(ptr) + size);
102
103	return page;
104}
105
106/*
107 * Free a DMA buffer.  'size' must be page aligned.
108 */
109static void __dma_free_buffer(struct page *page, size_t size)
110{
111	struct page *e = page + (size >> PAGE_SHIFT);
112
113	while (page < e) {
114		__free_page(page);
115		page++;
116	}
117}
118
119#ifdef CONFIG_MMU
120/* Sanity check size */
121#if (CONSISTENT_DMA_SIZE % SZ_2M)
122#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
123#endif
124
125#define CONSISTENT_OFFSET(x)	(((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
126#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
127#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
 
 
 
 
 
128
129/*
130 * These are the page tables (2MB each) covering uncached, DMA consistent allocations
131 */
132static pte_t *consistent_pte[NUM_CONSISTENT_PTES];
 
 
 
 
 
 
 
 
133
134#include "vmregion.h"
 
 
 
 
135
136static struct arm_vmregion_head consistent_head = {
137	.vm_lock	= __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock),
138	.vm_list	= LIST_HEAD_INIT(consistent_head.vm_list),
139	.vm_start	= CONSISTENT_BASE,
140	.vm_end		= CONSISTENT_END,
141};
142
143#ifdef CONFIG_HUGETLB_PAGE
144#error ARM Coherent DMA allocator does not (yet) support huge TLB
145#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
147/*
148 * Initialise the consistent memory allocation.
149 */
150static int __init consistent_init(void)
151{
152	int ret = 0;
153	pgd_t *pgd;
154	pud_t *pud;
155	pmd_t *pmd;
156	pte_t *pte;
157	int i = 0;
158	u32 base = CONSISTENT_BASE;
159
160	do {
161		pgd = pgd_offset(&init_mm, base);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
162
163		pud = pud_alloc(&init_mm, pgd, base);
164		if (!pud) {
165			printk(KERN_ERR "%s: no pud tables\n", __func__);
166			ret = -ENOMEM;
167			break;
168		}
 
 
 
 
 
 
169
170		pmd = pmd_alloc(&init_mm, pud, base);
171		if (!pmd) {
172			printk(KERN_ERR "%s: no pmd tables\n", __func__);
173			ret = -ENOMEM;
174			break;
175		}
176		WARN_ON(!pmd_none(*pmd));
177
178		pte = pte_alloc_kernel(pmd, base);
179		if (!pte) {
180			printk(KERN_ERR "%s: no pte tables\n", __func__);
181			ret = -ENOMEM;
182			break;
183		}
184
185		consistent_pte[i++] = pte;
186		base += (1 << PGDIR_SHIFT);
187	} while (base < CONSISTENT_END);
188
189	return ret;
 
 
 
 
190}
191
192core_initcall(consistent_init);
193
194static void *
195__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot)
196{
197	struct arm_vmregion *c;
198	size_t align;
199	int bit;
 
 
 
 
 
 
 
 
200
201	if (!consistent_pte[0]) {
202		printk(KERN_ERR "%s: not initialised\n", __func__);
203		dump_stack();
204		return NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
205	}
 
206
207	/*
208	 * Align the virtual region allocation - maximum alignment is
209	 * a section size, minimum is a page size.  This helps reduce
210	 * fragmentation of the DMA space, and also prevents allocations
211	 * smaller than a section from crossing a section boundary.
212	 */
213	bit = fls(size - 1);
214	if (bit > SECTION_SHIFT)
215		bit = SECTION_SHIFT;
216	align = 1 << bit;
217
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
218	/*
219	 * Allocate a virtual address in the consistent mapping region.
 
220	 */
221	c = arm_vmregion_alloc(&consistent_head, align, size,
222			    gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
223	if (c) {
224		pte_t *pte;
225		int idx = CONSISTENT_PTE_INDEX(c->vm_start);
226		u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
227
228		pte = consistent_pte[idx] + off;
229		c->vm_pages = page;
 
 
 
230
231		do {
232			BUG_ON(!pte_none(*pte));
 
 
233
234			set_pte_ext(pte, mk_pte(page, prot), 0);
235			page++;
236			pte++;
237			off++;
238			if (off >= PTRS_PER_PTE) {
239				off = 0;
240				pte = consistent_pte[++idx];
241			}
242		} while (size -= PAGE_SIZE);
243
244		dsb();
 
 
245
246		return (void *)c->vm_start;
 
247	}
248	return NULL;
 
249}
250
251static void __dma_free_remap(void *cpu_addr, size_t size)
252{
253	struct arm_vmregion *c;
254	unsigned long addr;
255	pte_t *ptep;
256	int idx;
257	u32 off;
258
259	c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr);
260	if (!c) {
261		printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
262		       __func__, cpu_addr);
263		dump_stack();
264		return;
265	}
266
267	if ((c->vm_end - c->vm_start) != size) {
268		printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
269		       __func__, c->vm_end - c->vm_start, size);
270		dump_stack();
271		size = c->vm_end - c->vm_start;
272	}
273
274	idx = CONSISTENT_PTE_INDEX(c->vm_start);
275	off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
276	ptep = consistent_pte[idx] + off;
277	addr = c->vm_start;
278	do {
279		pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
280
281		ptep++;
282		addr += PAGE_SIZE;
283		off++;
284		if (off >= PTRS_PER_PTE) {
285			off = 0;
286			ptep = consistent_pte[++idx];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
287		}
 
 
 
 
288
289		if (pte_none(pte) || !pte_present(pte))
290			printk(KERN_CRIT "%s: bad page in kernel page table\n",
291			       __func__);
292	} while (size -= PAGE_SIZE);
293
294	flush_tlb_kernel_range(c->vm_start, c->vm_end);
 
 
 
 
 
 
 
 
 
 
295
296	arm_vmregion_free(&consistent_head, c);
 
 
 
 
 
297}
298
 
 
299#else	/* !CONFIG_MMU */
300
301#define __dma_alloc_remap(page, size, gfp, prot)	page_address(page)
302#define __dma_free_remap(addr, size)			do { } while (0)
 
 
 
 
 
 
 
303
304#endif	/* CONFIG_MMU */
305
306static void *
307__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
308	    pgprot_t prot)
309{
310	struct page *page;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
311	void *addr;
 
 
 
 
 
 
 
 
 
 
 
312
313	*handle = ~0;
314	size = PAGE_ALIGN(size);
 
 
 
 
 
 
315
316	page = __dma_alloc_buffer(dev, size, gfp);
317	if (!page)
 
 
 
 
318		return NULL;
319
320	if (!arch_is_coherent())
321		addr = __dma_alloc_remap(page, size, gfp, prot);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
322	else
323		addr = page_address(page);
 
 
 
 
 
324
325	if (addr)
326		*handle = pfn_to_dma(dev, page_to_pfn(page));
327	else
328		__dma_free_buffer(page, size);
329
330	return addr;
 
 
 
 
 
 
 
331}
332
333/*
334 * Allocate DMA-coherent memory space and return both the kernel remapped
335 * virtual and bus address for that space.
336 */
337void *
338dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
339{
340	void *memory;
341
342	if (dma_alloc_from_coherent(dev, size, handle, &memory))
343		return memory;
344
345	return __dma_alloc(dev, size, handle, gfp,
346			   pgprot_dmacoherent(pgprot_kernel));
347}
348EXPORT_SYMBOL(dma_alloc_coherent);
349
350/*
351 * Allocate a writecombining region, in much the same way as
352 * dma_alloc_coherent above.
353 */
354void *
355dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
356{
357	return __dma_alloc(dev, size, handle, gfp,
358			   pgprot_writecombine(pgprot_kernel));
359}
360EXPORT_SYMBOL(dma_alloc_writecombine);
361
362static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
363		    void *cpu_addr, dma_addr_t dma_addr, size_t size)
 
364{
365	int ret = -ENXIO;
366#ifdef CONFIG_MMU
367	unsigned long user_size, kern_size;
368	struct arm_vmregion *c;
369
370	user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
371
372	c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
373	if (c) {
374		unsigned long off = vma->vm_pgoff;
375
376		kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
377
378		if (off < kern_size &&
379		    user_size <= (kern_size - off)) {
380			ret = remap_pfn_range(vma, vma->vm_start,
381					      page_to_pfn(c->vm_pages) + off,
382					      user_size << PAGE_SHIFT,
383					      vma->vm_page_prot);
384		}
385	}
386#endif	/* CONFIG_MMU */
387
388	return ret;
389}
390
391int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
392		      void *cpu_addr, dma_addr_t dma_addr, size_t size)
 
 
 
 
393{
394	vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
395	return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
396}
397EXPORT_SYMBOL(dma_mmap_coherent);
398
399int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
400			  void *cpu_addr, dma_addr_t dma_addr, size_t size)
 
401{
402	vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
403	return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
 
 
404}
405EXPORT_SYMBOL(dma_mmap_writecombine);
406
407/*
408 * free a page as defined by the above mapping.
409 * Must not be called with IRQs disabled.
410 */
411void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
412{
413	WARN_ON(irqs_disabled());
 
 
 
 
 
 
 
 
 
 
414
415	if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
 
416		return;
417
418	size = PAGE_ALIGN(size);
419
420	if (!arch_is_coherent())
421		__dma_free_remap(cpu_addr, size);
422
423	__dma_free_buffer(pfn_to_page(dma_to_pfn(dev, handle)), size);
424}
425EXPORT_SYMBOL(dma_free_coherent);
426
427/*
428 * Make an area consistent for devices.
429 * Note: Drivers should NOT use this function directly, as it will break
430 * platforms with CONFIG_DMABOUNCE.
431 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
432 */
433void ___dma_single_cpu_to_dev(const void *kaddr, size_t size,
434	enum dma_data_direction dir)
435{
436	unsigned long paddr;
437
438	BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
439
440	dmac_map_area(kaddr, size, dir);
441
442	paddr = __pa(kaddr);
443	if (dir == DMA_FROM_DEVICE) {
444		outer_inv_range(paddr, paddr + size);
445	} else {
446		outer_clean_range(paddr, paddr + size);
447	}
448	/* FIXME: non-speculating: flush on bidirectional mappings? */
449}
450EXPORT_SYMBOL(___dma_single_cpu_to_dev);
451
452void ___dma_single_dev_to_cpu(const void *kaddr, size_t size,
453	enum dma_data_direction dir)
 
454{
455	BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
 
456
457	/* FIXME: non-speculating: not required */
458	/* don't bother invalidating if DMA to device */
459	if (dir != DMA_TO_DEVICE) {
460		unsigned long paddr = __pa(kaddr);
461		outer_inv_range(paddr, paddr + size);
462	}
463
464	dmac_unmap_area(kaddr, size, dir);
 
465}
466EXPORT_SYMBOL(___dma_single_dev_to_cpu);
467
468static void dma_cache_maint_page(struct page *page, unsigned long offset,
469	size_t size, enum dma_data_direction dir,
470	void (*op)(const void *, size_t, int))
471{
 
 
 
 
 
 
472	/*
473	 * A single sg entry may refer to multiple physically contiguous
474	 * pages.  But we still need to process highmem pages individually.
475	 * If highmem is not configured then the bulk of this loop gets
476	 * optimized out.
477	 */
478	size_t left = size;
479	do {
480		size_t len = left;
481		void *vaddr;
482
 
 
483		if (PageHighMem(page)) {
484			if (len + offset > PAGE_SIZE) {
485				if (offset >= PAGE_SIZE) {
486					page += offset / PAGE_SIZE;
487					offset %= PAGE_SIZE;
488				}
489				len = PAGE_SIZE - offset;
490			}
491			vaddr = kmap_high_get(page);
492			if (vaddr) {
493				vaddr += offset;
494				op(vaddr, len, dir);
495				kunmap_high(page);
496			} else if (cache_is_vipt()) {
497				/* unmapped pages might still be cached */
498				vaddr = kmap_atomic(page);
499				op(vaddr + offset, len, dir);
500				kunmap_atomic(vaddr);
 
 
 
 
 
 
501			}
502		} else {
503			vaddr = page_address(page) + offset;
504			op(vaddr, len, dir);
505		}
506		offset = 0;
507		page++;
508		left -= len;
509	} while (left);
510}
511
512void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
 
 
 
 
 
 
513	size_t size, enum dma_data_direction dir)
514{
515	unsigned long paddr;
516
517	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
518
519	paddr = page_to_phys(page) + off;
520	if (dir == DMA_FROM_DEVICE) {
521		outer_inv_range(paddr, paddr + size);
522	} else {
523		outer_clean_range(paddr, paddr + size);
524	}
525	/* FIXME: non-speculating: flush on bidirectional mappings? */
526}
527EXPORT_SYMBOL(___dma_page_cpu_to_dev);
528
529void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
530	size_t size, enum dma_data_direction dir)
531{
532	unsigned long paddr = page_to_phys(page) + off;
533
534	/* FIXME: non-speculating: not required */
535	/* don't bother invalidating if DMA to device */
536	if (dir != DMA_TO_DEVICE)
537		outer_inv_range(paddr, paddr + size);
538
539	dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
 
540
541	/*
542	 * Mark the D-cache clean for this page to avoid extra flushing.
543	 */
544	if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
545		set_bit(PG_dcache_clean, &page->flags);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
546}
547EXPORT_SYMBOL(___dma_page_dev_to_cpu);
548
549/**
550 * dma_map_sg - map a set of SG buffers for streaming mode DMA
551 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
552 * @sg: list of buffers
553 * @nents: number of buffers to map
554 * @dir: DMA transfer direction
555 *
556 * Map a set of buffers described by scatterlist in streaming mode for DMA.
557 * This is the scatter-gather version of the dma_map_single interface.
558 * Here the scatter gather list elements are each tagged with the
559 * appropriate dma address and length.  They are obtained via
560 * sg_dma_{address,length}.
561 *
562 * Device ownership issues as mentioned for dma_map_single are the same
563 * here.
564 */
565int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
566		enum dma_data_direction dir)
567{
 
568	struct scatterlist *s;
569	int i, j;
570
571	BUG_ON(!valid_dma_direction(dir));
572
573	for_each_sg(sg, s, nents, i) {
574		s->dma_address = __dma_map_page(dev, sg_page(s), s->offset,
575						s->length, dir);
 
 
 
576		if (dma_mapping_error(dev, s->dma_address))
577			goto bad_mapping;
578	}
579	debug_dma_map_sg(dev, sg, nents, nents, dir);
580	return nents;
581
582 bad_mapping:
583	for_each_sg(sg, s, i, j)
584		__dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
585	return 0;
586}
587EXPORT_SYMBOL(dma_map_sg);
588
589/**
590 * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
591 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
592 * @sg: list of buffers
593 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
594 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
595 *
596 * Unmap a set of streaming mode DMA translations.  Again, CPU access
597 * rules concerning calls here are the same as for dma_unmap_single().
598 */
599void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
600		enum dma_data_direction dir)
601{
 
602	struct scatterlist *s;
603	int i;
604
605	debug_dma_unmap_sg(dev, sg, nents, dir);
606
607	for_each_sg(sg, s, nents, i)
608		__dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
609}
610EXPORT_SYMBOL(dma_unmap_sg);
611
612/**
613 * dma_sync_sg_for_cpu
614 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
615 * @sg: list of buffers
616 * @nents: number of buffers to map (returned from dma_map_sg)
617 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
618 */
619void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
620			int nents, enum dma_data_direction dir)
621{
 
622	struct scatterlist *s;
623	int i;
624
625	for_each_sg(sg, s, nents, i) {
626		if (!dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0,
627					    sg_dma_len(s), dir))
628			continue;
629
630		__dma_page_dev_to_cpu(sg_page(s), s->offset,
631				      s->length, dir);
632	}
633
634	debug_dma_sync_sg_for_cpu(dev, sg, nents, dir);
635}
636EXPORT_SYMBOL(dma_sync_sg_for_cpu);
637
638/**
639 * dma_sync_sg_for_device
640 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
641 * @sg: list of buffers
642 * @nents: number of buffers to map (returned from dma_map_sg)
643 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
644 */
645void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
646			int nents, enum dma_data_direction dir)
647{
 
648	struct scatterlist *s;
649	int i;
650
651	for_each_sg(sg, s, nents, i) {
652		if (!dmabounce_sync_for_device(dev, sg_dma_address(s), 0,
653					sg_dma_len(s), dir))
654			continue;
655
656		__dma_page_cpu_to_dev(sg_page(s), s->offset,
657				      s->length, dir);
658	}
659
660	debug_dma_sync_sg_for_device(dev, sg, nents, dir);
661}
662EXPORT_SYMBOL(dma_sync_sg_for_device);
663
664/*
665 * Return whether the given device DMA address mask can be supported
666 * properly.  For example, if your device can only drive the low 24-bits
667 * during bus mastering, then you would pass 0x00ffffff as the mask
668 * to this function.
669 */
670int dma_supported(struct device *dev, u64 mask)
671{
672	if (mask < (u64)arm_dma_limit)
673		return 0;
674	return 1;
675}
676EXPORT_SYMBOL(dma_supported);
677
678int dma_set_mask(struct device *dev, u64 dma_mask)
 
 
679{
680	if (!dev->dma_mask || !dma_supported(dev, dma_mask))
681		return -EIO;
 
 
682
683#ifndef CONFIG_DMABOUNCE
684	*dev->dma_mask = dma_mask;
685#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
686
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
687	return 0;
688}
689EXPORT_SYMBOL(dma_set_mask);
690
691#define PREALLOC_DMA_DEBUG_ENTRIES	4096
 
 
 
 
 
 
 
 
 
692
693static int __init dma_debug_do_init(void)
 
 
 
 
694{
695	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
696	return 0;
697}
698fs_initcall(dma_debug_do_init);