Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
10
11#include <linux/types.h>
12#include <linux/dma-mapping.h>
13#include <linux/mm.h>
14#include <linux/export.h>
15#include <linux/scatterlist.h>
16#include <linux/string.h>
17#include <linux/gfp.h>
18#include <linux/highmem.h>
19#include <linux/dma-contiguous.h>
20
21#include <asm/cache.h>
22#include <asm/cpu-type.h>
23#include <asm/io.h>
24
25#include <dma-coherence.h>
26
27#if defined(CONFIG_DMA_MAYBE_COHERENT) && !defined(CONFIG_DMA_PERDEV_COHERENT)
28/* User defined DMA coherency from command line. */
29enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT;
30EXPORT_SYMBOL_GPL(coherentio);
31int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
32
33static int __init setcoherentio(char *str)
34{
35 coherentio = IO_COHERENCE_ENABLED;
36 pr_info("Hardware DMA cache coherency (command line)\n");
37 return 0;
38}
39early_param("coherentio", setcoherentio);
40
41static int __init setnocoherentio(char *str)
42{
43 coherentio = IO_COHERENCE_DISABLED;
44 pr_info("Software DMA cache coherency (command line)\n");
45 return 0;
46}
47early_param("nocoherentio", setnocoherentio);
48#endif
49
50static inline struct page *dma_addr_to_page(struct device *dev,
51 dma_addr_t dma_addr)
52{
53 return pfn_to_page(
54 plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
55}
56
57/*
58 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
59 * speculatively fill random cachelines with stale data at any time,
60 * requiring an extra flush post-DMA.
61 *
62 * Warning on the terminology - Linux calls an uncached area coherent;
63 * MIPS terminology calls memory areas with hardware maintained coherency
64 * coherent.
65 *
66 * Note that the R14000 and R16000 should also be checked for in this
67 * condition. However this function is only called on non-I/O-coherent
68 * systems and only the R10000 and R12000 are used in such systems, the
69 * SGI IP28 Indigo² rsp. SGI IP32 aka O2.
70 */
71static inline int cpu_needs_post_dma_flush(struct device *dev)
72{
73 return !plat_device_is_coherent(dev) &&
74 (boot_cpu_type() == CPU_R10000 ||
75 boot_cpu_type() == CPU_R12000 ||
76 boot_cpu_type() == CPU_BMIPS5000);
77}
78
79static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
80{
81 gfp_t dma_flag;
82
83 /* ignore region specifiers */
84 gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
85
86#ifdef CONFIG_ISA
87 if (dev == NULL)
88 dma_flag = __GFP_DMA;
89 else
90#endif
91#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
92 if (dev == NULL || dev->coherent_dma_mask < DMA_BIT_MASK(32))
93 dma_flag = __GFP_DMA;
94 else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
95 dma_flag = __GFP_DMA32;
96 else
97#endif
98#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
99 if (dev == NULL || dev->coherent_dma_mask < DMA_BIT_MASK(64))
100 dma_flag = __GFP_DMA32;
101 else
102#endif
103#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
104 if (dev == NULL ||
105 dev->coherent_dma_mask < DMA_BIT_MASK(sizeof(phys_addr_t) * 8))
106 dma_flag = __GFP_DMA;
107 else
108#endif
109 dma_flag = 0;
110
111 /* Don't invoke OOM killer */
112 gfp |= __GFP_NORETRY;
113
114 return gfp | dma_flag;
115}
116
117static void *mips_dma_alloc_noncoherent(struct device *dev, size_t size,
118 dma_addr_t * dma_handle, gfp_t gfp)
119{
120 void *ret;
121
122 gfp = massage_gfp_flags(dev, gfp);
123
124 ret = (void *) __get_free_pages(gfp, get_order(size));
125
126 if (ret != NULL) {
127 memset(ret, 0, size);
128 *dma_handle = plat_map_dma_mem(dev, ret, size);
129 }
130
131 return ret;
132}
133
134static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
135 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
136{
137 void *ret;
138 struct page *page = NULL;
139 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
140
141 /*
142 * XXX: seems like the coherent and non-coherent implementations could
143 * be consolidated.
144 */
145 if (attrs & DMA_ATTR_NON_CONSISTENT)
146 return mips_dma_alloc_noncoherent(dev, size, dma_handle, gfp);
147
148 gfp = massage_gfp_flags(dev, gfp);
149
150 if (IS_ENABLED(CONFIG_DMA_CMA) && gfpflags_allow_blocking(gfp))
151 page = dma_alloc_from_contiguous(dev,
152 count, get_order(size));
153 if (!page)
154 page = alloc_pages(gfp, get_order(size));
155
156 if (!page)
157 return NULL;
158
159 ret = page_address(page);
160 memset(ret, 0, size);
161 *dma_handle = plat_map_dma_mem(dev, ret, size);
162 if (!plat_device_is_coherent(dev)) {
163 dma_cache_wback_inv((unsigned long) ret, size);
164 ret = UNCAC_ADDR(ret);
165 }
166
167 return ret;
168}
169
170
171static void mips_dma_free_noncoherent(struct device *dev, size_t size,
172 void *vaddr, dma_addr_t dma_handle)
173{
174 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
175 free_pages((unsigned long) vaddr, get_order(size));
176}
177
178static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
179 dma_addr_t dma_handle, unsigned long attrs)
180{
181 unsigned long addr = (unsigned long) vaddr;
182 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
183 struct page *page = NULL;
184
185 if (attrs & DMA_ATTR_NON_CONSISTENT) {
186 mips_dma_free_noncoherent(dev, size, vaddr, dma_handle);
187 return;
188 }
189
190 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
191
192 if (!plat_device_is_coherent(dev))
193 addr = CAC_ADDR(addr);
194
195 page = virt_to_page((void *) addr);
196
197 if (!dma_release_from_contiguous(dev, page, count))
198 __free_pages(page, get_order(size));
199}
200
201static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma,
202 void *cpu_addr, dma_addr_t dma_addr, size_t size,
203 unsigned long attrs)
204{
205 unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
206 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
207 unsigned long addr = (unsigned long)cpu_addr;
208 unsigned long off = vma->vm_pgoff;
209 unsigned long pfn;
210 int ret = -ENXIO;
211
212 if (!plat_device_is_coherent(dev))
213 addr = CAC_ADDR(addr);
214
215 pfn = page_to_pfn(virt_to_page((void *)addr));
216
217 if (attrs & DMA_ATTR_WRITE_COMBINE)
218 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
219 else
220 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
221
222 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
223 return ret;
224
225 if (off < count && user_count <= (count - off)) {
226 ret = remap_pfn_range(vma, vma->vm_start,
227 pfn + off,
228 user_count << PAGE_SHIFT,
229 vma->vm_page_prot);
230 }
231
232 return ret;
233}
234
235static inline void __dma_sync_virtual(void *addr, size_t size,
236 enum dma_data_direction direction)
237{
238 switch (direction) {
239 case DMA_TO_DEVICE:
240 dma_cache_wback((unsigned long)addr, size);
241 break;
242
243 case DMA_FROM_DEVICE:
244 dma_cache_inv((unsigned long)addr, size);
245 break;
246
247 case DMA_BIDIRECTIONAL:
248 dma_cache_wback_inv((unsigned long)addr, size);
249 break;
250
251 default:
252 BUG();
253 }
254}
255
256/*
257 * A single sg entry may refer to multiple physically contiguous
258 * pages. But we still need to process highmem pages individually.
259 * If highmem is not configured then the bulk of this loop gets
260 * optimized out.
261 */
262static inline void __dma_sync(struct page *page,
263 unsigned long offset, size_t size, enum dma_data_direction direction)
264{
265 size_t left = size;
266
267 do {
268 size_t len = left;
269
270 if (PageHighMem(page)) {
271 void *addr;
272
273 if (offset + len > PAGE_SIZE) {
274 if (offset >= PAGE_SIZE) {
275 page += offset >> PAGE_SHIFT;
276 offset &= ~PAGE_MASK;
277 }
278 len = PAGE_SIZE - offset;
279 }
280
281 addr = kmap_atomic(page);
282 __dma_sync_virtual(addr + offset, len, direction);
283 kunmap_atomic(addr);
284 } else
285 __dma_sync_virtual(page_address(page) + offset,
286 size, direction);
287 offset = 0;
288 page++;
289 left -= len;
290 } while (left);
291}
292
293static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
294 size_t size, enum dma_data_direction direction, unsigned long attrs)
295{
296 if (cpu_needs_post_dma_flush(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
297 __dma_sync(dma_addr_to_page(dev, dma_addr),
298 dma_addr & ~PAGE_MASK, size, direction);
299 plat_post_dma_flush(dev);
300 plat_unmap_dma_mem(dev, dma_addr, size, direction);
301}
302
303static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist,
304 int nents, enum dma_data_direction direction, unsigned long attrs)
305{
306 int i;
307 struct scatterlist *sg;
308
309 for_each_sg(sglist, sg, nents, i) {
310 if (!plat_device_is_coherent(dev) &&
311 !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
312 __dma_sync(sg_page(sg), sg->offset, sg->length,
313 direction);
314#ifdef CONFIG_NEED_SG_DMA_LENGTH
315 sg->dma_length = sg->length;
316#endif
317 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
318 sg->offset;
319 }
320
321 return nents;
322}
323
324static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
325 unsigned long offset, size_t size, enum dma_data_direction direction,
326 unsigned long attrs)
327{
328 if (!plat_device_is_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
329 __dma_sync(page, offset, size, direction);
330
331 return plat_map_dma_mem_page(dev, page) + offset;
332}
333
334static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
335 int nhwentries, enum dma_data_direction direction,
336 unsigned long attrs)
337{
338 int i;
339 struct scatterlist *sg;
340
341 for_each_sg(sglist, sg, nhwentries, i) {
342 if (!plat_device_is_coherent(dev) &&
343 !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
344 direction != DMA_TO_DEVICE)
345 __dma_sync(sg_page(sg), sg->offset, sg->length,
346 direction);
347 plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
348 }
349}
350
351static void mips_dma_sync_single_for_cpu(struct device *dev,
352 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
353{
354 if (cpu_needs_post_dma_flush(dev))
355 __dma_sync(dma_addr_to_page(dev, dma_handle),
356 dma_handle & ~PAGE_MASK, size, direction);
357 plat_post_dma_flush(dev);
358}
359
360static void mips_dma_sync_single_for_device(struct device *dev,
361 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
362{
363 if (!plat_device_is_coherent(dev))
364 __dma_sync(dma_addr_to_page(dev, dma_handle),
365 dma_handle & ~PAGE_MASK, size, direction);
366}
367
368static void mips_dma_sync_sg_for_cpu(struct device *dev,
369 struct scatterlist *sglist, int nelems,
370 enum dma_data_direction direction)
371{
372 int i;
373 struct scatterlist *sg;
374
375 if (cpu_needs_post_dma_flush(dev)) {
376 for_each_sg(sglist, sg, nelems, i) {
377 __dma_sync(sg_page(sg), sg->offset, sg->length,
378 direction);
379 }
380 }
381 plat_post_dma_flush(dev);
382}
383
384static void mips_dma_sync_sg_for_device(struct device *dev,
385 struct scatterlist *sglist, int nelems,
386 enum dma_data_direction direction)
387{
388 int i;
389 struct scatterlist *sg;
390
391 if (!plat_device_is_coherent(dev)) {
392 for_each_sg(sglist, sg, nelems, i) {
393 __dma_sync(sg_page(sg), sg->offset, sg->length,
394 direction);
395 }
396 }
397}
398
399int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
400{
401 return 0;
402}
403
404int mips_dma_supported(struct device *dev, u64 mask)
405{
406 return plat_dma_supported(dev, mask);
407}
408
409void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
410 enum dma_data_direction direction)
411{
412 BUG_ON(direction == DMA_NONE);
413
414 if (!plat_device_is_coherent(dev))
415 __dma_sync_virtual(vaddr, size, direction);
416}
417
418EXPORT_SYMBOL(dma_cache_sync);
419
420static struct dma_map_ops mips_default_dma_map_ops = {
421 .alloc = mips_dma_alloc_coherent,
422 .free = mips_dma_free_coherent,
423 .mmap = mips_dma_mmap,
424 .map_page = mips_dma_map_page,
425 .unmap_page = mips_dma_unmap_page,
426 .map_sg = mips_dma_map_sg,
427 .unmap_sg = mips_dma_unmap_sg,
428 .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
429 .sync_single_for_device = mips_dma_sync_single_for_device,
430 .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
431 .sync_sg_for_device = mips_dma_sync_sg_for_device,
432 .mapping_error = mips_dma_mapping_error,
433 .dma_supported = mips_dma_supported
434};
435
436struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
437EXPORT_SYMBOL(mips_dma_map_ops);
438
439#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
440
441static int __init mips_dma_init(void)
442{
443 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
444
445 return 0;
446}
447fs_initcall(mips_dma_init);
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
10
11#include <linux/types.h>
12#include <linux/dma-mapping.h>
13#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/scatterlist.h>
16#include <linux/string.h>
17#include <linux/gfp.h>
18#include <linux/highmem.h>
19
20#include <asm/cache.h>
21#include <asm/cpu-type.h>
22#include <asm/io.h>
23
24#include <dma-coherence.h>
25
26#ifdef CONFIG_DMA_MAYBE_COHERENT
27int coherentio = 0; /* User defined DMA coherency from command line. */
28EXPORT_SYMBOL_GPL(coherentio);
29int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
30
31static int __init setcoherentio(char *str)
32{
33 coherentio = 1;
34 pr_info("Hardware DMA cache coherency (command line)\n");
35 return 0;
36}
37early_param("coherentio", setcoherentio);
38
39static int __init setnocoherentio(char *str)
40{
41 coherentio = 0;
42 pr_info("Software DMA cache coherency (command line)\n");
43 return 0;
44}
45early_param("nocoherentio", setnocoherentio);
46#endif
47
48static inline struct page *dma_addr_to_page(struct device *dev,
49 dma_addr_t dma_addr)
50{
51 return pfn_to_page(
52 plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
53}
54
55/*
56 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
57 * speculatively fill random cachelines with stale data at any time,
58 * requiring an extra flush post-DMA.
59 *
60 * Warning on the terminology - Linux calls an uncached area coherent;
61 * MIPS terminology calls memory areas with hardware maintained coherency
62 * coherent.
63 */
64static inline int cpu_needs_post_dma_flush(struct device *dev)
65{
66 return !plat_device_is_coherent(dev) &&
67 (boot_cpu_type() == CPU_R10000 ||
68 boot_cpu_type() == CPU_R12000 ||
69 boot_cpu_type() == CPU_BMIPS5000);
70}
71
72static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
73{
74 gfp_t dma_flag;
75
76 /* ignore region specifiers */
77 gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
78
79#ifdef CONFIG_ISA
80 if (dev == NULL)
81 dma_flag = __GFP_DMA;
82 else
83#endif
84#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
85 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
86 dma_flag = __GFP_DMA;
87 else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
88 dma_flag = __GFP_DMA32;
89 else
90#endif
91#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
92 if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
93 dma_flag = __GFP_DMA32;
94 else
95#endif
96#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
97 if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
98 dma_flag = __GFP_DMA;
99 else
100#endif
101 dma_flag = 0;
102
103 /* Don't invoke OOM killer */
104 gfp |= __GFP_NORETRY;
105
106 return gfp | dma_flag;
107}
108
109void *dma_alloc_noncoherent(struct device *dev, size_t size,
110 dma_addr_t * dma_handle, gfp_t gfp)
111{
112 void *ret;
113
114 gfp = massage_gfp_flags(dev, gfp);
115
116 ret = (void *) __get_free_pages(gfp, get_order(size));
117
118 if (ret != NULL) {
119 memset(ret, 0, size);
120 *dma_handle = plat_map_dma_mem(dev, ret, size);
121 }
122
123 return ret;
124}
125EXPORT_SYMBOL(dma_alloc_noncoherent);
126
127static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
128 dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
129{
130 void *ret;
131
132 if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
133 return ret;
134
135 gfp = massage_gfp_flags(dev, gfp);
136
137 ret = (void *) __get_free_pages(gfp, get_order(size));
138
139 if (ret) {
140 memset(ret, 0, size);
141 *dma_handle = plat_map_dma_mem(dev, ret, size);
142
143 if (!plat_device_is_coherent(dev)) {
144 dma_cache_wback_inv((unsigned long) ret, size);
145 if (!hw_coherentio)
146 ret = UNCAC_ADDR(ret);
147 }
148 }
149
150 return ret;
151}
152
153
154void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
155 dma_addr_t dma_handle)
156{
157 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
158 free_pages((unsigned long) vaddr, get_order(size));
159}
160EXPORT_SYMBOL(dma_free_noncoherent);
161
162static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
163 dma_addr_t dma_handle, struct dma_attrs *attrs)
164{
165 unsigned long addr = (unsigned long) vaddr;
166 int order = get_order(size);
167
168 if (dma_release_from_coherent(dev, order, vaddr))
169 return;
170
171 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
172
173 if (!plat_device_is_coherent(dev) && !hw_coherentio)
174 addr = CAC_ADDR(addr);
175
176 free_pages(addr, get_order(size));
177}
178
179static inline void __dma_sync_virtual(void *addr, size_t size,
180 enum dma_data_direction direction)
181{
182 switch (direction) {
183 case DMA_TO_DEVICE:
184 dma_cache_wback((unsigned long)addr, size);
185 break;
186
187 case DMA_FROM_DEVICE:
188 dma_cache_inv((unsigned long)addr, size);
189 break;
190
191 case DMA_BIDIRECTIONAL:
192 dma_cache_wback_inv((unsigned long)addr, size);
193 break;
194
195 default:
196 BUG();
197 }
198}
199
200/*
201 * A single sg entry may refer to multiple physically contiguous
202 * pages. But we still need to process highmem pages individually.
203 * If highmem is not configured then the bulk of this loop gets
204 * optimized out.
205 */
206static inline void __dma_sync(struct page *page,
207 unsigned long offset, size_t size, enum dma_data_direction direction)
208{
209 size_t left = size;
210
211 do {
212 size_t len = left;
213
214 if (PageHighMem(page)) {
215 void *addr;
216
217 if (offset + len > PAGE_SIZE) {
218 if (offset >= PAGE_SIZE) {
219 page += offset >> PAGE_SHIFT;
220 offset &= ~PAGE_MASK;
221 }
222 len = PAGE_SIZE - offset;
223 }
224
225 addr = kmap_atomic(page);
226 __dma_sync_virtual(addr + offset, len, direction);
227 kunmap_atomic(addr);
228 } else
229 __dma_sync_virtual(page_address(page) + offset,
230 size, direction);
231 offset = 0;
232 page++;
233 left -= len;
234 } while (left);
235}
236
237static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
238 size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
239{
240 if (cpu_needs_post_dma_flush(dev))
241 __dma_sync(dma_addr_to_page(dev, dma_addr),
242 dma_addr & ~PAGE_MASK, size, direction);
243
244 plat_unmap_dma_mem(dev, dma_addr, size, direction);
245}
246
247static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
248 int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
249{
250 int i;
251
252 for (i = 0; i < nents; i++, sg++) {
253 if (!plat_device_is_coherent(dev))
254 __dma_sync(sg_page(sg), sg->offset, sg->length,
255 direction);
256#ifdef CONFIG_NEED_SG_DMA_LENGTH
257 sg->dma_length = sg->length;
258#endif
259 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
260 sg->offset;
261 }
262
263 return nents;
264}
265
266static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
267 unsigned long offset, size_t size, enum dma_data_direction direction,
268 struct dma_attrs *attrs)
269{
270 if (!plat_device_is_coherent(dev))
271 __dma_sync(page, offset, size, direction);
272
273 return plat_map_dma_mem_page(dev, page) + offset;
274}
275
276static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
277 int nhwentries, enum dma_data_direction direction,
278 struct dma_attrs *attrs)
279{
280 int i;
281
282 for (i = 0; i < nhwentries; i++, sg++) {
283 if (!plat_device_is_coherent(dev) &&
284 direction != DMA_TO_DEVICE)
285 __dma_sync(sg_page(sg), sg->offset, sg->length,
286 direction);
287 plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
288 }
289}
290
291static void mips_dma_sync_single_for_cpu(struct device *dev,
292 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
293{
294 if (cpu_needs_post_dma_flush(dev))
295 __dma_sync(dma_addr_to_page(dev, dma_handle),
296 dma_handle & ~PAGE_MASK, size, direction);
297}
298
299static void mips_dma_sync_single_for_device(struct device *dev,
300 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
301{
302 if (!plat_device_is_coherent(dev))
303 __dma_sync(dma_addr_to_page(dev, dma_handle),
304 dma_handle & ~PAGE_MASK, size, direction);
305}
306
307static void mips_dma_sync_sg_for_cpu(struct device *dev,
308 struct scatterlist *sg, int nelems, enum dma_data_direction direction)
309{
310 int i;
311
312 if (cpu_needs_post_dma_flush(dev))
313 for (i = 0; i < nelems; i++, sg++)
314 __dma_sync(sg_page(sg), sg->offset, sg->length,
315 direction);
316}
317
318static void mips_dma_sync_sg_for_device(struct device *dev,
319 struct scatterlist *sg, int nelems, enum dma_data_direction direction)
320{
321 int i;
322
323 if (!plat_device_is_coherent(dev))
324 for (i = 0; i < nelems; i++, sg++)
325 __dma_sync(sg_page(sg), sg->offset, sg->length,
326 direction);
327}
328
329int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
330{
331 return 0;
332}
333
334int mips_dma_supported(struct device *dev, u64 mask)
335{
336 return plat_dma_supported(dev, mask);
337}
338
339void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
340 enum dma_data_direction direction)
341{
342 BUG_ON(direction == DMA_NONE);
343
344 if (!plat_device_is_coherent(dev))
345 __dma_sync_virtual(vaddr, size, direction);
346}
347
348EXPORT_SYMBOL(dma_cache_sync);
349
350static struct dma_map_ops mips_default_dma_map_ops = {
351 .alloc = mips_dma_alloc_coherent,
352 .free = mips_dma_free_coherent,
353 .map_page = mips_dma_map_page,
354 .unmap_page = mips_dma_unmap_page,
355 .map_sg = mips_dma_map_sg,
356 .unmap_sg = mips_dma_unmap_sg,
357 .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
358 .sync_single_for_device = mips_dma_sync_single_for_device,
359 .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
360 .sync_sg_for_device = mips_dma_sync_sg_for_device,
361 .mapping_error = mips_dma_mapping_error,
362 .dma_supported = mips_dma_supported
363};
364
365struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
366EXPORT_SYMBOL(mips_dma_map_ops);
367
368#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
369
370static int __init mips_dma_init(void)
371{
372 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
373
374 return 0;
375}
376fs_initcall(mips_dma_init);