Loading...
1/*
2 * Meta version derived from arch/powerpc/lib/dma-noncoherent.c
3 * Copyright (C) 2008 Imagination Technologies Ltd.
4 *
5 * PowerPC version derived from arch/arm/mm/consistent.c
6 * Copyright (C) 2001 Dan Malek (dmalek@jlc.net)
7 *
8 * Copyright (C) 2000 Russell King
9 *
10 * Consistent memory allocators. Used for DMA devices that want to
11 * share uncached memory with the processor core. The function return
12 * is the virtual address and 'dma_handle' is the physical address.
13 * Mostly stolen from the ARM port, with some changes for PowerPC.
14 * -- Dan
15 *
16 * Reorganized to get rid of the arch-specific consistent_* functions
17 * and provide non-coherent implementations for the DMA API. -Matt
18 *
19 * Added in_interrupt() safe dma_alloc_coherent()/dma_free_coherent()
20 * implementation. This is pulled straight from ARM and barely
21 * modified. -Matt
22 *
23 * This program is free software; you can redistribute it and/or modify
24 * it under the terms of the GNU General Public License version 2 as
25 * published by the Free Software Foundation.
26 */
27
28#include <linux/sched.h>
29#include <linux/kernel.h>
30#include <linux/errno.h>
31#include <linux/export.h>
32#include <linux/string.h>
33#include <linux/types.h>
34#include <linux/highmem.h>
35#include <linux/dma-mapping.h>
36#include <linux/slab.h>
37
38#include <asm/tlbflush.h>
39#include <asm/mmu.h>
40
41#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_START) \
42 >> PAGE_SHIFT)
43
44static u64 get_coherent_dma_mask(struct device *dev)
45{
46 u64 mask = ~0ULL;
47
48 if (dev) {
49 mask = dev->coherent_dma_mask;
50
51 /*
52 * Sanity check the DMA mask - it must be non-zero, and
53 * must be able to be satisfied by a DMA allocation.
54 */
55 if (mask == 0) {
56 dev_warn(dev, "coherent DMA mask is unset\n");
57 return 0;
58 }
59 }
60
61 return mask;
62}
63/*
64 * This is the page table (2MB) covering uncached, DMA consistent allocations
65 */
66static pte_t *consistent_pte;
67static DEFINE_SPINLOCK(consistent_lock);
68
69/*
70 * VM region handling support.
71 *
72 * This should become something generic, handling VM region allocations for
73 * vmalloc and similar (ioremap, module space, etc).
74 *
75 * I envisage vmalloc()'s supporting vm_struct becoming:
76 *
77 * struct vm_struct {
78 * struct metag_vm_region region;
79 * unsigned long flags;
80 * struct page **pages;
81 * unsigned int nr_pages;
82 * unsigned long phys_addr;
83 * };
84 *
85 * get_vm_area() would then call metag_vm_region_alloc with an appropriate
86 * struct metag_vm_region head (eg):
87 *
88 * struct metag_vm_region vmalloc_head = {
89 * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list),
90 * .vm_start = VMALLOC_START,
91 * .vm_end = VMALLOC_END,
92 * };
93 *
94 * However, vmalloc_head.vm_start is variable (typically, it is dependent on
95 * the amount of RAM found at boot time.) I would imagine that get_vm_area()
96 * would have to initialise this each time prior to calling
97 * metag_vm_region_alloc().
98 */
99struct metag_vm_region {
100 struct list_head vm_list;
101 unsigned long vm_start;
102 unsigned long vm_end;
103 struct page *vm_pages;
104 int vm_active;
105};
106
107static struct metag_vm_region consistent_head = {
108 .vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
109 .vm_start = CONSISTENT_START,
110 .vm_end = CONSISTENT_END,
111};
112
113static struct metag_vm_region *metag_vm_region_alloc(struct metag_vm_region
114 *head, size_t size,
115 gfp_t gfp)
116{
117 unsigned long addr = head->vm_start, end = head->vm_end - size;
118 unsigned long flags;
119 struct metag_vm_region *c, *new;
120
121 new = kmalloc(sizeof(struct metag_vm_region), gfp);
122 if (!new)
123 goto out;
124
125 spin_lock_irqsave(&consistent_lock, flags);
126
127 list_for_each_entry(c, &head->vm_list, vm_list) {
128 if ((addr + size) < addr)
129 goto nospc;
130 if ((addr + size) <= c->vm_start)
131 goto found;
132 addr = c->vm_end;
133 if (addr > end)
134 goto nospc;
135 }
136
137found:
138 /*
139 * Insert this entry _before_ the one we found.
140 */
141 list_add_tail(&new->vm_list, &c->vm_list);
142 new->vm_start = addr;
143 new->vm_end = addr + size;
144 new->vm_active = 1;
145
146 spin_unlock_irqrestore(&consistent_lock, flags);
147 return new;
148
149nospc:
150 spin_unlock_irqrestore(&consistent_lock, flags);
151 kfree(new);
152out:
153 return NULL;
154}
155
156static struct metag_vm_region *metag_vm_region_find(struct metag_vm_region
157 *head, unsigned long addr)
158{
159 struct metag_vm_region *c;
160
161 list_for_each_entry(c, &head->vm_list, vm_list) {
162 if (c->vm_active && c->vm_start == addr)
163 goto out;
164 }
165 c = NULL;
166out:
167 return c;
168}
169
170/*
171 * Allocate DMA-coherent memory space and return both the kernel remapped
172 * virtual and bus address for that space.
173 */
174static void *metag_dma_alloc(struct device *dev, size_t size,
175 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
176{
177 struct page *page;
178 struct metag_vm_region *c;
179 unsigned long order;
180 u64 mask = get_coherent_dma_mask(dev);
181 u64 limit;
182
183 if (!consistent_pte) {
184 pr_err("%s: not initialised\n", __func__);
185 dump_stack();
186 return NULL;
187 }
188
189 if (!mask)
190 goto no_page;
191 size = PAGE_ALIGN(size);
192 limit = (mask + 1) & ~mask;
193 if ((limit && size >= limit)
194 || size >= (CONSISTENT_END - CONSISTENT_START)) {
195 pr_warn("coherent allocation too big (requested %#x mask %#Lx)\n",
196 size, mask);
197 return NULL;
198 }
199
200 order = get_order(size);
201
202 if (mask != 0xffffffff)
203 gfp |= GFP_DMA;
204
205 page = alloc_pages(gfp, order);
206 if (!page)
207 goto no_page;
208
209 /*
210 * Invalidate any data that might be lurking in the
211 * kernel direct-mapped region for device DMA.
212 */
213 {
214 void *kaddr = page_address(page);
215 memset(kaddr, 0, size);
216 flush_dcache_region(kaddr, size);
217 }
218
219 /*
220 * Allocate a virtual address in the consistent mapping region.
221 */
222 c = metag_vm_region_alloc(&consistent_head, size,
223 gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
224 if (c) {
225 unsigned long vaddr = c->vm_start;
226 pte_t *pte = consistent_pte + CONSISTENT_OFFSET(vaddr);
227 struct page *end = page + (1 << order);
228
229 c->vm_pages = page;
230 split_page(page, order);
231
232 /*
233 * Set the "dma handle"
234 */
235 *handle = page_to_bus(page);
236
237 do {
238 BUG_ON(!pte_none(*pte));
239
240 SetPageReserved(page);
241 set_pte_at(&init_mm, vaddr,
242 pte, mk_pte(page,
243 pgprot_writecombine
244 (PAGE_KERNEL)));
245 page++;
246 pte++;
247 vaddr += PAGE_SIZE;
248 } while (size -= PAGE_SIZE);
249
250 /*
251 * Free the otherwise unused pages.
252 */
253 while (page < end) {
254 __free_page(page);
255 page++;
256 }
257
258 return (void *)c->vm_start;
259 }
260
261 if (page)
262 __free_pages(page, order);
263no_page:
264 return NULL;
265}
266
267/*
268 * free a page as defined by the above mapping.
269 */
270static void metag_dma_free(struct device *dev, size_t size, void *vaddr,
271 dma_addr_t dma_handle, struct dma_attrs *attrs)
272{
273 struct metag_vm_region *c;
274 unsigned long flags, addr;
275 pte_t *ptep;
276
277 size = PAGE_ALIGN(size);
278
279 spin_lock_irqsave(&consistent_lock, flags);
280
281 c = metag_vm_region_find(&consistent_head, (unsigned long)vaddr);
282 if (!c)
283 goto no_area;
284
285 c->vm_active = 0;
286 if ((c->vm_end - c->vm_start) != size) {
287 pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
288 __func__, c->vm_end - c->vm_start, size);
289 dump_stack();
290 size = c->vm_end - c->vm_start;
291 }
292
293 ptep = consistent_pte + CONSISTENT_OFFSET(c->vm_start);
294 addr = c->vm_start;
295 do {
296 pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
297 unsigned long pfn;
298
299 ptep++;
300 addr += PAGE_SIZE;
301
302 if (!pte_none(pte) && pte_present(pte)) {
303 pfn = pte_pfn(pte);
304
305 if (pfn_valid(pfn)) {
306 struct page *page = pfn_to_page(pfn);
307 __free_reserved_page(page);
308 continue;
309 }
310 }
311
312 pr_crit("%s: bad page in kernel page table\n",
313 __func__);
314 } while (size -= PAGE_SIZE);
315
316 flush_tlb_kernel_range(c->vm_start, c->vm_end);
317
318 list_del(&c->vm_list);
319
320 spin_unlock_irqrestore(&consistent_lock, flags);
321
322 kfree(c);
323 return;
324
325no_area:
326 spin_unlock_irqrestore(&consistent_lock, flags);
327 pr_err("%s: trying to free invalid coherent area: %p\n",
328 __func__, vaddr);
329 dump_stack();
330}
331
332static int metag_dma_mmap(struct device *dev, struct vm_area_struct *vma,
333 void *cpu_addr, dma_addr_t dma_addr, size_t size,
334 struct dma_attrs *attrs)
335{
336 unsigned long flags, user_size, kern_size;
337 struct metag_vm_region *c;
338 int ret = -ENXIO;
339
340 if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
341 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
342 else
343 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
344
345 user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
346
347 spin_lock_irqsave(&consistent_lock, flags);
348 c = metag_vm_region_find(&consistent_head, (unsigned long)cpu_addr);
349 spin_unlock_irqrestore(&consistent_lock, flags);
350
351 if (c) {
352 unsigned long off = vma->vm_pgoff;
353
354 kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
355
356 if (off < kern_size &&
357 user_size <= (kern_size - off)) {
358 ret = remap_pfn_range(vma, vma->vm_start,
359 page_to_pfn(c->vm_pages) + off,
360 user_size << PAGE_SHIFT,
361 vma->vm_page_prot);
362 }
363 }
364
365
366 return ret;
367}
368
369/*
370 * Initialise the consistent memory allocation.
371 */
372static int __init dma_alloc_init(void)
373{
374 pgd_t *pgd, *pgd_k;
375 pud_t *pud, *pud_k;
376 pmd_t *pmd, *pmd_k;
377 pte_t *pte;
378 int ret = 0;
379
380 do {
381 int offset = pgd_index(CONSISTENT_START);
382 pgd = pgd_offset(&init_mm, CONSISTENT_START);
383 pud = pud_alloc(&init_mm, pgd, CONSISTENT_START);
384 pmd = pmd_alloc(&init_mm, pud, CONSISTENT_START);
385 WARN_ON(!pmd_none(*pmd));
386
387 pte = pte_alloc_kernel(pmd, CONSISTENT_START);
388 if (!pte) {
389 pr_err("%s: no pte tables\n", __func__);
390 ret = -ENOMEM;
391 break;
392 }
393
394 pgd_k = ((pgd_t *) mmu_get_base()) + offset;
395 pud_k = pud_offset(pgd_k, CONSISTENT_START);
396 pmd_k = pmd_offset(pud_k, CONSISTENT_START);
397 set_pmd(pmd_k, *pmd);
398
399 consistent_pte = pte;
400 } while (0);
401
402 return ret;
403}
404early_initcall(dma_alloc_init);
405
406/*
407 * make an area consistent to devices.
408 */
409static void dma_sync_for_device(void *vaddr, size_t size, int dma_direction)
410{
411 /*
412 * Ensure any writes get through the write combiner. This is necessary
413 * even with DMA_FROM_DEVICE, or the write may dirty the cache after
414 * we've invalidated it and get written back during the DMA.
415 */
416
417 barrier();
418
419 switch (dma_direction) {
420 case DMA_BIDIRECTIONAL:
421 /*
422 * Writeback to ensure the device can see our latest changes and
423 * so that we have no dirty lines, and invalidate the cache
424 * lines too in preparation for receiving the buffer back
425 * (dma_sync_for_cpu) later.
426 */
427 flush_dcache_region(vaddr, size);
428 break;
429 case DMA_TO_DEVICE:
430 /*
431 * Writeback to ensure the device can see our latest changes.
432 * There's no need to invalidate as the device shouldn't write
433 * to the buffer.
434 */
435 writeback_dcache_region(vaddr, size);
436 break;
437 case DMA_FROM_DEVICE:
438 /*
439 * Invalidate to ensure we have no dirty lines that could get
440 * written back during the DMA. It's also safe to flush
441 * (writeback) here if necessary.
442 */
443 invalidate_dcache_region(vaddr, size);
444 break;
445 case DMA_NONE:
446 BUG();
447 }
448
449 wmb();
450}
451
452/*
453 * make an area consistent to the core.
454 */
455static void dma_sync_for_cpu(void *vaddr, size_t size, int dma_direction)
456{
457 /*
458 * Hardware L2 cache prefetch doesn't occur across 4K physical
459 * boundaries, however according to Documentation/DMA-API-HOWTO.txt
460 * kmalloc'd memory is DMA'able, so accesses in nearby memory could
461 * trigger a cache fill in the DMA buffer.
462 *
463 * This should never cause dirty lines, so a flush or invalidate should
464 * be safe to allow us to see data from the device.
465 */
466 if (_meta_l2c_pf_is_enabled()) {
467 switch (dma_direction) {
468 case DMA_BIDIRECTIONAL:
469 case DMA_FROM_DEVICE:
470 invalidate_dcache_region(vaddr, size);
471 break;
472 case DMA_TO_DEVICE:
473 /* The device shouldn't have written to the buffer */
474 break;
475 case DMA_NONE:
476 BUG();
477 }
478 }
479
480 rmb();
481}
482
483static dma_addr_t metag_dma_map_page(struct device *dev, struct page *page,
484 unsigned long offset, size_t size,
485 enum dma_data_direction direction, struct dma_attrs *attrs)
486{
487 dma_sync_for_device((void *)(page_to_phys(page) + offset), size,
488 direction);
489 return page_to_phys(page) + offset;
490}
491
492static void metag_dma_unmap_page(struct device *dev, dma_addr_t dma_address,
493 size_t size, enum dma_data_direction direction,
494 struct dma_attrs *attrs)
495{
496 dma_sync_for_cpu(phys_to_virt(dma_address), size, direction);
497}
498
499static int metag_dma_map_sg(struct device *dev, struct scatterlist *sglist,
500 int nents, enum dma_data_direction direction,
501 struct dma_attrs *attrs)
502{
503 struct scatterlist *sg;
504 int i;
505
506 for_each_sg(sglist, sg, nents, i) {
507 BUG_ON(!sg_page(sg));
508
509 sg->dma_address = sg_phys(sg);
510 dma_sync_for_device(sg_virt(sg), sg->length, direction);
511 }
512
513 return nents;
514}
515
516
517static void metag_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
518 int nhwentries, enum dma_data_direction direction,
519 struct dma_attrs *attrs)
520{
521 struct scatterlist *sg;
522 int i;
523
524 for_each_sg(sglist, sg, nhwentries, i) {
525 BUG_ON(!sg_page(sg));
526
527 sg->dma_address = sg_phys(sg);
528 dma_sync_for_cpu(sg_virt(sg), sg->length, direction);
529 }
530}
531
532static void metag_dma_sync_single_for_cpu(struct device *dev,
533 dma_addr_t dma_handle, size_t size,
534 enum dma_data_direction direction)
535{
536 dma_sync_for_cpu(phys_to_virt(dma_handle), size, direction);
537}
538
539static void metag_dma_sync_single_for_device(struct device *dev,
540 dma_addr_t dma_handle, size_t size,
541 enum dma_data_direction direction)
542{
543 dma_sync_for_device(phys_to_virt(dma_handle), size, direction);
544}
545
546static void metag_dma_sync_sg_for_cpu(struct device *dev,
547 struct scatterlist *sglist, int nelems,
548 enum dma_data_direction direction)
549{
550 int i;
551 struct scatterlist *sg;
552
553 for_each_sg(sglist, sg, nelems, i)
554 dma_sync_for_cpu(sg_virt(sg), sg->length, direction);
555}
556
557static void metag_dma_sync_sg_for_device(struct device *dev,
558 struct scatterlist *sglist, int nelems,
559 enum dma_data_direction direction)
560{
561 int i;
562 struct scatterlist *sg;
563
564 for_each_sg(sglist, sg, nelems, i)
565 dma_sync_for_device(sg_virt(sg), sg->length, direction);
566}
567
568struct dma_map_ops metag_dma_ops = {
569 .alloc = metag_dma_alloc,
570 .free = metag_dma_free,
571 .map_page = metag_dma_map_page,
572 .map_sg = metag_dma_map_sg,
573 .sync_single_for_device = metag_dma_sync_single_for_device,
574 .sync_single_for_cpu = metag_dma_sync_single_for_cpu,
575 .sync_sg_for_cpu = metag_dma_sync_sg_for_cpu,
576 .mmap = metag_dma_mmap,
577};
578EXPORT_SYMBOL(metag_dma_ops);
1/*
2 * Meta version derived from arch/powerpc/lib/dma-noncoherent.c
3 * Copyright (C) 2008 Imagination Technologies Ltd.
4 *
5 * PowerPC version derived from arch/arm/mm/consistent.c
6 * Copyright (C) 2001 Dan Malek (dmalek@jlc.net)
7 *
8 * Copyright (C) 2000 Russell King
9 *
10 * Consistent memory allocators. Used for DMA devices that want to
11 * share uncached memory with the processor core. The function return
12 * is the virtual address and 'dma_handle' is the physical address.
13 * Mostly stolen from the ARM port, with some changes for PowerPC.
14 * -- Dan
15 *
16 * Reorganized to get rid of the arch-specific consistent_* functions
17 * and provide non-coherent implementations for the DMA API. -Matt
18 *
19 * Added in_interrupt() safe dma_alloc_coherent()/dma_free_coherent()
20 * implementation. This is pulled straight from ARM and barely
21 * modified. -Matt
22 *
23 * This program is free software; you can redistribute it and/or modify
24 * it under the terms of the GNU General Public License version 2 as
25 * published by the Free Software Foundation.
26 */
27
28#include <linux/sched.h>
29#include <linux/kernel.h>
30#include <linux/errno.h>
31#include <linux/export.h>
32#include <linux/string.h>
33#include <linux/types.h>
34#include <linux/highmem.h>
35#include <linux/dma-mapping.h>
36#include <linux/slab.h>
37
38#include <asm/tlbflush.h>
39#include <asm/mmu.h>
40
41#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_START) \
42 >> PAGE_SHIFT)
43
44static u64 get_coherent_dma_mask(struct device *dev)
45{
46 u64 mask = ~0ULL;
47
48 if (dev) {
49 mask = dev->coherent_dma_mask;
50
51 /*
52 * Sanity check the DMA mask - it must be non-zero, and
53 * must be able to be satisfied by a DMA allocation.
54 */
55 if (mask == 0) {
56 dev_warn(dev, "coherent DMA mask is unset\n");
57 return 0;
58 }
59 }
60
61 return mask;
62}
63/*
64 * This is the page table (2MB) covering uncached, DMA consistent allocations
65 */
66static pte_t *consistent_pte;
67static DEFINE_SPINLOCK(consistent_lock);
68
69/*
70 * VM region handling support.
71 *
72 * This should become something generic, handling VM region allocations for
73 * vmalloc and similar (ioremap, module space, etc).
74 *
75 * I envisage vmalloc()'s supporting vm_struct becoming:
76 *
77 * struct vm_struct {
78 * struct metag_vm_region region;
79 * unsigned long flags;
80 * struct page **pages;
81 * unsigned int nr_pages;
82 * unsigned long phys_addr;
83 * };
84 *
85 * get_vm_area() would then call metag_vm_region_alloc with an appropriate
86 * struct metag_vm_region head (eg):
87 *
88 * struct metag_vm_region vmalloc_head = {
89 * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list),
90 * .vm_start = VMALLOC_START,
91 * .vm_end = VMALLOC_END,
92 * };
93 *
94 * However, vmalloc_head.vm_start is variable (typically, it is dependent on
95 * the amount of RAM found at boot time.) I would imagine that get_vm_area()
96 * would have to initialise this each time prior to calling
97 * metag_vm_region_alloc().
98 */
99struct metag_vm_region {
100 struct list_head vm_list;
101 unsigned long vm_start;
102 unsigned long vm_end;
103 struct page *vm_pages;
104 int vm_active;
105};
106
107static struct metag_vm_region consistent_head = {
108 .vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
109 .vm_start = CONSISTENT_START,
110 .vm_end = CONSISTENT_END,
111};
112
113static struct metag_vm_region *metag_vm_region_alloc(struct metag_vm_region
114 *head, size_t size,
115 gfp_t gfp)
116{
117 unsigned long addr = head->vm_start, end = head->vm_end - size;
118 unsigned long flags;
119 struct metag_vm_region *c, *new;
120
121 new = kmalloc(sizeof(struct metag_vm_region), gfp);
122 if (!new)
123 goto out;
124
125 spin_lock_irqsave(&consistent_lock, flags);
126
127 list_for_each_entry(c, &head->vm_list, vm_list) {
128 if ((addr + size) < addr)
129 goto nospc;
130 if ((addr + size) <= c->vm_start)
131 goto found;
132 addr = c->vm_end;
133 if (addr > end)
134 goto nospc;
135 }
136
137found:
138 /*
139 * Insert this entry _before_ the one we found.
140 */
141 list_add_tail(&new->vm_list, &c->vm_list);
142 new->vm_start = addr;
143 new->vm_end = addr + size;
144 new->vm_active = 1;
145
146 spin_unlock_irqrestore(&consistent_lock, flags);
147 return new;
148
149nospc:
150 spin_unlock_irqrestore(&consistent_lock, flags);
151 kfree(new);
152out:
153 return NULL;
154}
155
156static struct metag_vm_region *metag_vm_region_find(struct metag_vm_region
157 *head, unsigned long addr)
158{
159 struct metag_vm_region *c;
160
161 list_for_each_entry(c, &head->vm_list, vm_list) {
162 if (c->vm_active && c->vm_start == addr)
163 goto out;
164 }
165 c = NULL;
166out:
167 return c;
168}
169
170/*
171 * Allocate DMA-coherent memory space and return both the kernel remapped
172 * virtual and bus address for that space.
173 */
174void *dma_alloc_coherent(struct device *dev, size_t size,
175 dma_addr_t *handle, gfp_t gfp)
176{
177 struct page *page;
178 struct metag_vm_region *c;
179 unsigned long order;
180 u64 mask = get_coherent_dma_mask(dev);
181 u64 limit;
182
183 if (!consistent_pte) {
184 pr_err("%s: not initialised\n", __func__);
185 dump_stack();
186 return NULL;
187 }
188
189 if (!mask)
190 goto no_page;
191 size = PAGE_ALIGN(size);
192 limit = (mask + 1) & ~mask;
193 if ((limit && size >= limit)
194 || size >= (CONSISTENT_END - CONSISTENT_START)) {
195 pr_warn("coherent allocation too big (requested %#x mask %#Lx)\n",
196 size, mask);
197 return NULL;
198 }
199
200 order = get_order(size);
201
202 if (mask != 0xffffffff)
203 gfp |= GFP_DMA;
204
205 page = alloc_pages(gfp, order);
206 if (!page)
207 goto no_page;
208
209 /*
210 * Invalidate any data that might be lurking in the
211 * kernel direct-mapped region for device DMA.
212 */
213 {
214 void *kaddr = page_address(page);
215 memset(kaddr, 0, size);
216 flush_dcache_region(kaddr, size);
217 }
218
219 /*
220 * Allocate a virtual address in the consistent mapping region.
221 */
222 c = metag_vm_region_alloc(&consistent_head, size,
223 gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
224 if (c) {
225 unsigned long vaddr = c->vm_start;
226 pte_t *pte = consistent_pte + CONSISTENT_OFFSET(vaddr);
227 struct page *end = page + (1 << order);
228
229 c->vm_pages = page;
230 split_page(page, order);
231
232 /*
233 * Set the "dma handle"
234 */
235 *handle = page_to_bus(page);
236
237 do {
238 BUG_ON(!pte_none(*pte));
239
240 SetPageReserved(page);
241 set_pte_at(&init_mm, vaddr,
242 pte, mk_pte(page,
243 pgprot_writecombine
244 (PAGE_KERNEL)));
245 page++;
246 pte++;
247 vaddr += PAGE_SIZE;
248 } while (size -= PAGE_SIZE);
249
250 /*
251 * Free the otherwise unused pages.
252 */
253 while (page < end) {
254 __free_page(page);
255 page++;
256 }
257
258 return (void *)c->vm_start;
259 }
260
261 if (page)
262 __free_pages(page, order);
263no_page:
264 return NULL;
265}
266EXPORT_SYMBOL(dma_alloc_coherent);
267
268/*
269 * free a page as defined by the above mapping.
270 */
271void dma_free_coherent(struct device *dev, size_t size,
272 void *vaddr, dma_addr_t dma_handle)
273{
274 struct metag_vm_region *c;
275 unsigned long flags, addr;
276 pte_t *ptep;
277
278 size = PAGE_ALIGN(size);
279
280 spin_lock_irqsave(&consistent_lock, flags);
281
282 c = metag_vm_region_find(&consistent_head, (unsigned long)vaddr);
283 if (!c)
284 goto no_area;
285
286 c->vm_active = 0;
287 if ((c->vm_end - c->vm_start) != size) {
288 pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
289 __func__, c->vm_end - c->vm_start, size);
290 dump_stack();
291 size = c->vm_end - c->vm_start;
292 }
293
294 ptep = consistent_pte + CONSISTENT_OFFSET(c->vm_start);
295 addr = c->vm_start;
296 do {
297 pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
298 unsigned long pfn;
299
300 ptep++;
301 addr += PAGE_SIZE;
302
303 if (!pte_none(pte) && pte_present(pte)) {
304 pfn = pte_pfn(pte);
305
306 if (pfn_valid(pfn)) {
307 struct page *page = pfn_to_page(pfn);
308 __free_reserved_page(page);
309 continue;
310 }
311 }
312
313 pr_crit("%s: bad page in kernel page table\n",
314 __func__);
315 } while (size -= PAGE_SIZE);
316
317 flush_tlb_kernel_range(c->vm_start, c->vm_end);
318
319 list_del(&c->vm_list);
320
321 spin_unlock_irqrestore(&consistent_lock, flags);
322
323 kfree(c);
324 return;
325
326no_area:
327 spin_unlock_irqrestore(&consistent_lock, flags);
328 pr_err("%s: trying to free invalid coherent area: %p\n",
329 __func__, vaddr);
330 dump_stack();
331}
332EXPORT_SYMBOL(dma_free_coherent);
333
334
335static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
336 void *cpu_addr, dma_addr_t dma_addr, size_t size)
337{
338 int ret = -ENXIO;
339
340 unsigned long flags, user_size, kern_size;
341 struct metag_vm_region *c;
342
343 user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
344
345 spin_lock_irqsave(&consistent_lock, flags);
346 c = metag_vm_region_find(&consistent_head, (unsigned long)cpu_addr);
347 spin_unlock_irqrestore(&consistent_lock, flags);
348
349 if (c) {
350 unsigned long off = vma->vm_pgoff;
351
352 kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
353
354 if (off < kern_size &&
355 user_size <= (kern_size - off)) {
356 ret = remap_pfn_range(vma, vma->vm_start,
357 page_to_pfn(c->vm_pages) + off,
358 user_size << PAGE_SHIFT,
359 vma->vm_page_prot);
360 }
361 }
362
363
364 return ret;
365}
366
367int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
368 void *cpu_addr, dma_addr_t dma_addr, size_t size)
369{
370 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
371 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
372}
373EXPORT_SYMBOL(dma_mmap_coherent);
374
375int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
376 void *cpu_addr, dma_addr_t dma_addr, size_t size)
377{
378 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
379 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
380}
381EXPORT_SYMBOL(dma_mmap_writecombine);
382
383
384
385
386/*
387 * Initialise the consistent memory allocation.
388 */
389static int __init dma_alloc_init(void)
390{
391 pgd_t *pgd, *pgd_k;
392 pud_t *pud, *pud_k;
393 pmd_t *pmd, *pmd_k;
394 pte_t *pte;
395 int ret = 0;
396
397 do {
398 int offset = pgd_index(CONSISTENT_START);
399 pgd = pgd_offset(&init_mm, CONSISTENT_START);
400 pud = pud_alloc(&init_mm, pgd, CONSISTENT_START);
401 pmd = pmd_alloc(&init_mm, pud, CONSISTENT_START);
402 WARN_ON(!pmd_none(*pmd));
403
404 pte = pte_alloc_kernel(pmd, CONSISTENT_START);
405 if (!pte) {
406 pr_err("%s: no pte tables\n", __func__);
407 ret = -ENOMEM;
408 break;
409 }
410
411 pgd_k = ((pgd_t *) mmu_get_base()) + offset;
412 pud_k = pud_offset(pgd_k, CONSISTENT_START);
413 pmd_k = pmd_offset(pud_k, CONSISTENT_START);
414 set_pmd(pmd_k, *pmd);
415
416 consistent_pte = pte;
417 } while (0);
418
419 return ret;
420}
421early_initcall(dma_alloc_init);
422
423/*
424 * make an area consistent to devices.
425 */
426void dma_sync_for_device(void *vaddr, size_t size, int dma_direction)
427{
428 /*
429 * Ensure any writes get through the write combiner. This is necessary
430 * even with DMA_FROM_DEVICE, or the write may dirty the cache after
431 * we've invalidated it and get written back during the DMA.
432 */
433
434 barrier();
435
436 switch (dma_direction) {
437 case DMA_BIDIRECTIONAL:
438 /*
439 * Writeback to ensure the device can see our latest changes and
440 * so that we have no dirty lines, and invalidate the cache
441 * lines too in preparation for receiving the buffer back
442 * (dma_sync_for_cpu) later.
443 */
444 flush_dcache_region(vaddr, size);
445 break;
446 case DMA_TO_DEVICE:
447 /*
448 * Writeback to ensure the device can see our latest changes.
449 * There's no need to invalidate as the device shouldn't write
450 * to the buffer.
451 */
452 writeback_dcache_region(vaddr, size);
453 break;
454 case DMA_FROM_DEVICE:
455 /*
456 * Invalidate to ensure we have no dirty lines that could get
457 * written back during the DMA. It's also safe to flush
458 * (writeback) here if necessary.
459 */
460 invalidate_dcache_region(vaddr, size);
461 break;
462 case DMA_NONE:
463 BUG();
464 }
465
466 wmb();
467}
468EXPORT_SYMBOL(dma_sync_for_device);
469
470/*
471 * make an area consistent to the core.
472 */
473void dma_sync_for_cpu(void *vaddr, size_t size, int dma_direction)
474{
475 /*
476 * Hardware L2 cache prefetch doesn't occur across 4K physical
477 * boundaries, however according to Documentation/DMA-API-HOWTO.txt
478 * kmalloc'd memory is DMA'able, so accesses in nearby memory could
479 * trigger a cache fill in the DMA buffer.
480 *
481 * This should never cause dirty lines, so a flush or invalidate should
482 * be safe to allow us to see data from the device.
483 */
484 if (_meta_l2c_pf_is_enabled()) {
485 switch (dma_direction) {
486 case DMA_BIDIRECTIONAL:
487 case DMA_FROM_DEVICE:
488 invalidate_dcache_region(vaddr, size);
489 break;
490 case DMA_TO_DEVICE:
491 /* The device shouldn't have written to the buffer */
492 break;
493 case DMA_NONE:
494 BUG();
495 }
496 }
497
498 rmb();
499}
500EXPORT_SYMBOL(dma_sync_for_cpu);