Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
10
11#include <linux/types.h>
12#include <linux/dma-mapping.h>
13#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/scatterlist.h>
16#include <linux/string.h>
17#include <linux/gfp.h>
18#include <linux/highmem.h>
19
20#include <asm/cache.h>
21#include <asm/io.h>
22
23#include <dma-coherence.h>
24
25static inline struct page *dma_addr_to_page(struct device *dev,
26 dma_addr_t dma_addr)
27{
28 return pfn_to_page(
29 plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
30}
31
32/*
33 * Warning on the terminology - Linux calls an uncached area coherent;
34 * MIPS terminology calls memory areas with hardware maintained coherency
35 * coherent.
36 */
37
38static inline int cpu_is_noncoherent_r10000(struct device *dev)
39{
40 return !plat_device_is_coherent(dev) &&
41 (current_cpu_type() == CPU_R10000 ||
42 current_cpu_type() == CPU_R12000);
43}
44
45static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
46{
47 gfp_t dma_flag;
48
49 /* ignore region specifiers */
50 gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
51
52#ifdef CONFIG_ISA
53 if (dev == NULL)
54 dma_flag = __GFP_DMA;
55 else
56#endif
57#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
58 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
59 dma_flag = __GFP_DMA;
60 else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
61 dma_flag = __GFP_DMA32;
62 else
63#endif
64#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
65 if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
66 dma_flag = __GFP_DMA32;
67 else
68#endif
69#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
70 if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
71 dma_flag = __GFP_DMA;
72 else
73#endif
74 dma_flag = 0;
75
76 /* Don't invoke OOM killer */
77 gfp |= __GFP_NORETRY;
78
79 return gfp | dma_flag;
80}
81
82void *dma_alloc_noncoherent(struct device *dev, size_t size,
83 dma_addr_t * dma_handle, gfp_t gfp)
84{
85 void *ret;
86
87 gfp = massage_gfp_flags(dev, gfp);
88
89 ret = (void *) __get_free_pages(gfp, get_order(size));
90
91 if (ret != NULL) {
92 memset(ret, 0, size);
93 *dma_handle = plat_map_dma_mem(dev, ret, size);
94 }
95
96 return ret;
97}
98EXPORT_SYMBOL(dma_alloc_noncoherent);
99
100static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
101 dma_addr_t * dma_handle, gfp_t gfp)
102{
103 void *ret;
104
105 if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
106 return ret;
107
108 gfp = massage_gfp_flags(dev, gfp);
109
110 ret = (void *) __get_free_pages(gfp, get_order(size));
111
112 if (ret) {
113 memset(ret, 0, size);
114 *dma_handle = plat_map_dma_mem(dev, ret, size);
115
116 if (!plat_device_is_coherent(dev)) {
117 dma_cache_wback_inv((unsigned long) ret, size);
118 ret = UNCAC_ADDR(ret);
119 }
120 }
121
122 return ret;
123}
124
125
126void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
127 dma_addr_t dma_handle)
128{
129 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
130 free_pages((unsigned long) vaddr, get_order(size));
131}
132EXPORT_SYMBOL(dma_free_noncoherent);
133
134static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
135 dma_addr_t dma_handle)
136{
137 unsigned long addr = (unsigned long) vaddr;
138 int order = get_order(size);
139
140 if (dma_release_from_coherent(dev, order, vaddr))
141 return;
142
143 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
144
145 if (!plat_device_is_coherent(dev))
146 addr = CAC_ADDR(addr);
147
148 free_pages(addr, get_order(size));
149}
150
151static inline void __dma_sync_virtual(void *addr, size_t size,
152 enum dma_data_direction direction)
153{
154 switch (direction) {
155 case DMA_TO_DEVICE:
156 dma_cache_wback((unsigned long)addr, size);
157 break;
158
159 case DMA_FROM_DEVICE:
160 dma_cache_inv((unsigned long)addr, size);
161 break;
162
163 case DMA_BIDIRECTIONAL:
164 dma_cache_wback_inv((unsigned long)addr, size);
165 break;
166
167 default:
168 BUG();
169 }
170}
171
172/*
173 * A single sg entry may refer to multiple physically contiguous
174 * pages. But we still need to process highmem pages individually.
175 * If highmem is not configured then the bulk of this loop gets
176 * optimized out.
177 */
178static inline void __dma_sync(struct page *page,
179 unsigned long offset, size_t size, enum dma_data_direction direction)
180{
181 size_t left = size;
182
183 do {
184 size_t len = left;
185
186 if (PageHighMem(page)) {
187 void *addr;
188
189 if (offset + len > PAGE_SIZE) {
190 if (offset >= PAGE_SIZE) {
191 page += offset >> PAGE_SHIFT;
192 offset &= ~PAGE_MASK;
193 }
194 len = PAGE_SIZE - offset;
195 }
196
197 addr = kmap_atomic(page);
198 __dma_sync_virtual(addr + offset, len, direction);
199 kunmap_atomic(addr);
200 } else
201 __dma_sync_virtual(page_address(page) + offset,
202 size, direction);
203 offset = 0;
204 page++;
205 left -= len;
206 } while (left);
207}
208
209static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
210 size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
211{
212 if (cpu_is_noncoherent_r10000(dev))
213 __dma_sync(dma_addr_to_page(dev, dma_addr),
214 dma_addr & ~PAGE_MASK, size, direction);
215
216 plat_unmap_dma_mem(dev, dma_addr, size, direction);
217}
218
219static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
220 int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
221{
222 int i;
223
224 for (i = 0; i < nents; i++, sg++) {
225 if (!plat_device_is_coherent(dev))
226 __dma_sync(sg_page(sg), sg->offset, sg->length,
227 direction);
228 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
229 sg->offset;
230 }
231
232 return nents;
233}
234
235static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
236 unsigned long offset, size_t size, enum dma_data_direction direction,
237 struct dma_attrs *attrs)
238{
239 if (!plat_device_is_coherent(dev))
240 __dma_sync(page, offset, size, direction);
241
242 return plat_map_dma_mem_page(dev, page) + offset;
243}
244
245static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
246 int nhwentries, enum dma_data_direction direction,
247 struct dma_attrs *attrs)
248{
249 int i;
250
251 for (i = 0; i < nhwentries; i++, sg++) {
252 if (!plat_device_is_coherent(dev) &&
253 direction != DMA_TO_DEVICE)
254 __dma_sync(sg_page(sg), sg->offset, sg->length,
255 direction);
256 plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
257 }
258}
259
260static void mips_dma_sync_single_for_cpu(struct device *dev,
261 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
262{
263 if (cpu_is_noncoherent_r10000(dev))
264 __dma_sync(dma_addr_to_page(dev, dma_handle),
265 dma_handle & ~PAGE_MASK, size, direction);
266}
267
268static void mips_dma_sync_single_for_device(struct device *dev,
269 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
270{
271 plat_extra_sync_for_device(dev);
272 if (!plat_device_is_coherent(dev))
273 __dma_sync(dma_addr_to_page(dev, dma_handle),
274 dma_handle & ~PAGE_MASK, size, direction);
275}
276
277static void mips_dma_sync_sg_for_cpu(struct device *dev,
278 struct scatterlist *sg, int nelems, enum dma_data_direction direction)
279{
280 int i;
281
282 /* Make sure that gcc doesn't leave the empty loop body. */
283 for (i = 0; i < nelems; i++, sg++) {
284 if (cpu_is_noncoherent_r10000(dev))
285 __dma_sync(sg_page(sg), sg->offset, sg->length,
286 direction);
287 }
288}
289
290static void mips_dma_sync_sg_for_device(struct device *dev,
291 struct scatterlist *sg, int nelems, enum dma_data_direction direction)
292{
293 int i;
294
295 /* Make sure that gcc doesn't leave the empty loop body. */
296 for (i = 0; i < nelems; i++, sg++) {
297 if (!plat_device_is_coherent(dev))
298 __dma_sync(sg_page(sg), sg->offset, sg->length,
299 direction);
300 }
301}
302
303int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
304{
305 return plat_dma_mapping_error(dev, dma_addr);
306}
307
308int mips_dma_supported(struct device *dev, u64 mask)
309{
310 return plat_dma_supported(dev, mask);
311}
312
313void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
314 enum dma_data_direction direction)
315{
316 BUG_ON(direction == DMA_NONE);
317
318 plat_extra_sync_for_device(dev);
319 if (!plat_device_is_coherent(dev))
320 __dma_sync_virtual(vaddr, size, direction);
321}
322
323EXPORT_SYMBOL(dma_cache_sync);
324
325static struct dma_map_ops mips_default_dma_map_ops = {
326 .alloc_coherent = mips_dma_alloc_coherent,
327 .free_coherent = mips_dma_free_coherent,
328 .map_page = mips_dma_map_page,
329 .unmap_page = mips_dma_unmap_page,
330 .map_sg = mips_dma_map_sg,
331 .unmap_sg = mips_dma_unmap_sg,
332 .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
333 .sync_single_for_device = mips_dma_sync_single_for_device,
334 .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
335 .sync_sg_for_device = mips_dma_sync_sg_for_device,
336 .mapping_error = mips_dma_mapping_error,
337 .dma_supported = mips_dma_supported
338};
339
340struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
341EXPORT_SYMBOL(mips_dma_map_ops);
342
343#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
344
345static int __init mips_dma_init(void)
346{
347 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
348
349 return 0;
350}
351fs_initcall(mips_dma_init);
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
10
11#include <linux/types.h>
12#include <linux/dma-mapping.h>
13#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/scatterlist.h>
16#include <linux/string.h>
17#include <linux/gfp.h>
18#include <linux/highmem.h>
19#include <linux/dma-contiguous.h>
20
21#include <asm/cache.h>
22#include <asm/cpu-type.h>
23#include <asm/io.h>
24
25#include <dma-coherence.h>
26
27#ifdef CONFIG_DMA_MAYBE_COHERENT
28int coherentio = 0; /* User defined DMA coherency from command line. */
29EXPORT_SYMBOL_GPL(coherentio);
30int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
31
32static int __init setcoherentio(char *str)
33{
34 coherentio = 1;
35 pr_info("Hardware DMA cache coherency (command line)\n");
36 return 0;
37}
38early_param("coherentio", setcoherentio);
39
40static int __init setnocoherentio(char *str)
41{
42 coherentio = 0;
43 pr_info("Software DMA cache coherency (command line)\n");
44 return 0;
45}
46early_param("nocoherentio", setnocoherentio);
47#endif
48
49static inline struct page *dma_addr_to_page(struct device *dev,
50 dma_addr_t dma_addr)
51{
52 return pfn_to_page(
53 plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
54}
55
56/*
57 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
58 * speculatively fill random cachelines with stale data at any time,
59 * requiring an extra flush post-DMA.
60 *
61 * Warning on the terminology - Linux calls an uncached area coherent;
62 * MIPS terminology calls memory areas with hardware maintained coherency
63 * coherent.
64 *
65 * Note that the R14000 and R16000 should also be checked for in this
66 * condition. However this function is only called on non-I/O-coherent
67 * systems and only the R10000 and R12000 are used in such systems, the
68 * SGI IP28 Indigo² rsp. SGI IP32 aka O2.
69 */
70static inline int cpu_needs_post_dma_flush(struct device *dev)
71{
72 return !plat_device_is_coherent(dev) &&
73 (boot_cpu_type() == CPU_R10000 ||
74 boot_cpu_type() == CPU_R12000 ||
75 boot_cpu_type() == CPU_BMIPS5000);
76}
77
78static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
79{
80 gfp_t dma_flag;
81
82 /* ignore region specifiers */
83 gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
84
85#ifdef CONFIG_ISA
86 if (dev == NULL)
87 dma_flag = __GFP_DMA;
88 else
89#endif
90#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
91 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
92 dma_flag = __GFP_DMA;
93 else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
94 dma_flag = __GFP_DMA32;
95 else
96#endif
97#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
98 if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
99 dma_flag = __GFP_DMA32;
100 else
101#endif
102#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
103 if (dev->coherent_dma_mask < DMA_BIT_MASK(sizeof(phys_addr_t) * 8))
104 dma_flag = __GFP_DMA;
105 else
106#endif
107 dma_flag = 0;
108
109 /* Don't invoke OOM killer */
110 gfp |= __GFP_NORETRY;
111
112 return gfp | dma_flag;
113}
114
115static void *mips_dma_alloc_noncoherent(struct device *dev, size_t size,
116 dma_addr_t * dma_handle, gfp_t gfp)
117{
118 void *ret;
119
120 gfp = massage_gfp_flags(dev, gfp);
121
122 ret = (void *) __get_free_pages(gfp, get_order(size));
123
124 if (ret != NULL) {
125 memset(ret, 0, size);
126 *dma_handle = plat_map_dma_mem(dev, ret, size);
127 }
128
129 return ret;
130}
131
132static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
133 dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
134{
135 void *ret;
136 struct page *page = NULL;
137 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
138
139 /*
140 * XXX: seems like the coherent and non-coherent implementations could
141 * be consolidated.
142 */
143 if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs))
144 return mips_dma_alloc_noncoherent(dev, size, dma_handle, gfp);
145
146 gfp = massage_gfp_flags(dev, gfp);
147
148 if (IS_ENABLED(CONFIG_DMA_CMA) && gfpflags_allow_blocking(gfp))
149 page = dma_alloc_from_contiguous(dev,
150 count, get_order(size));
151 if (!page)
152 page = alloc_pages(gfp, get_order(size));
153
154 if (!page)
155 return NULL;
156
157 ret = page_address(page);
158 memset(ret, 0, size);
159 *dma_handle = plat_map_dma_mem(dev, ret, size);
160 if (!plat_device_is_coherent(dev)) {
161 dma_cache_wback_inv((unsigned long) ret, size);
162 if (!hw_coherentio)
163 ret = UNCAC_ADDR(ret);
164 }
165
166 return ret;
167}
168
169
170static void mips_dma_free_noncoherent(struct device *dev, size_t size,
171 void *vaddr, dma_addr_t dma_handle)
172{
173 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
174 free_pages((unsigned long) vaddr, get_order(size));
175}
176
177static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
178 dma_addr_t dma_handle, struct dma_attrs *attrs)
179{
180 unsigned long addr = (unsigned long) vaddr;
181 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
182 struct page *page = NULL;
183
184 if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) {
185 mips_dma_free_noncoherent(dev, size, vaddr, dma_handle);
186 return;
187 }
188
189 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
190
191 if (!plat_device_is_coherent(dev) && !hw_coherentio)
192 addr = CAC_ADDR(addr);
193
194 page = virt_to_page((void *) addr);
195
196 if (!dma_release_from_contiguous(dev, page, count))
197 __free_pages(page, get_order(size));
198}
199
200static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma,
201 void *cpu_addr, dma_addr_t dma_addr, size_t size,
202 struct dma_attrs *attrs)
203{
204 unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
205 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
206 unsigned long addr = (unsigned long)cpu_addr;
207 unsigned long off = vma->vm_pgoff;
208 unsigned long pfn;
209 int ret = -ENXIO;
210
211 if (!plat_device_is_coherent(dev) && !hw_coherentio)
212 addr = CAC_ADDR(addr);
213
214 pfn = page_to_pfn(virt_to_page((void *)addr));
215
216 if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
217 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
218 else
219 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
220
221 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
222 return ret;
223
224 if (off < count && user_count <= (count - off)) {
225 ret = remap_pfn_range(vma, vma->vm_start,
226 pfn + off,
227 user_count << PAGE_SHIFT,
228 vma->vm_page_prot);
229 }
230
231 return ret;
232}
233
234static inline void __dma_sync_virtual(void *addr, size_t size,
235 enum dma_data_direction direction)
236{
237 switch (direction) {
238 case DMA_TO_DEVICE:
239 dma_cache_wback((unsigned long)addr, size);
240 break;
241
242 case DMA_FROM_DEVICE:
243 dma_cache_inv((unsigned long)addr, size);
244 break;
245
246 case DMA_BIDIRECTIONAL:
247 dma_cache_wback_inv((unsigned long)addr, size);
248 break;
249
250 default:
251 BUG();
252 }
253}
254
255/*
256 * A single sg entry may refer to multiple physically contiguous
257 * pages. But we still need to process highmem pages individually.
258 * If highmem is not configured then the bulk of this loop gets
259 * optimized out.
260 */
261static inline void __dma_sync(struct page *page,
262 unsigned long offset, size_t size, enum dma_data_direction direction)
263{
264 size_t left = size;
265
266 do {
267 size_t len = left;
268
269 if (PageHighMem(page)) {
270 void *addr;
271
272 if (offset + len > PAGE_SIZE) {
273 if (offset >= PAGE_SIZE) {
274 page += offset >> PAGE_SHIFT;
275 offset &= ~PAGE_MASK;
276 }
277 len = PAGE_SIZE - offset;
278 }
279
280 addr = kmap_atomic(page);
281 __dma_sync_virtual(addr + offset, len, direction);
282 kunmap_atomic(addr);
283 } else
284 __dma_sync_virtual(page_address(page) + offset,
285 size, direction);
286 offset = 0;
287 page++;
288 left -= len;
289 } while (left);
290}
291
292static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
293 size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
294{
295 if (cpu_needs_post_dma_flush(dev))
296 __dma_sync(dma_addr_to_page(dev, dma_addr),
297 dma_addr & ~PAGE_MASK, size, direction);
298 plat_post_dma_flush(dev);
299 plat_unmap_dma_mem(dev, dma_addr, size, direction);
300}
301
302static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist,
303 int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
304{
305 int i;
306 struct scatterlist *sg;
307
308 for_each_sg(sglist, sg, nents, i) {
309 if (!plat_device_is_coherent(dev))
310 __dma_sync(sg_page(sg), sg->offset, sg->length,
311 direction);
312#ifdef CONFIG_NEED_SG_DMA_LENGTH
313 sg->dma_length = sg->length;
314#endif
315 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
316 sg->offset;
317 }
318
319 return nents;
320}
321
322static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
323 unsigned long offset, size_t size, enum dma_data_direction direction,
324 struct dma_attrs *attrs)
325{
326 if (!plat_device_is_coherent(dev))
327 __dma_sync(page, offset, size, direction);
328
329 return plat_map_dma_mem_page(dev, page) + offset;
330}
331
332static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
333 int nhwentries, enum dma_data_direction direction,
334 struct dma_attrs *attrs)
335{
336 int i;
337 struct scatterlist *sg;
338
339 for_each_sg(sglist, sg, nhwentries, i) {
340 if (!plat_device_is_coherent(dev) &&
341 direction != DMA_TO_DEVICE)
342 __dma_sync(sg_page(sg), sg->offset, sg->length,
343 direction);
344 plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
345 }
346}
347
348static void mips_dma_sync_single_for_cpu(struct device *dev,
349 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
350{
351 if (cpu_needs_post_dma_flush(dev))
352 __dma_sync(dma_addr_to_page(dev, dma_handle),
353 dma_handle & ~PAGE_MASK, size, direction);
354 plat_post_dma_flush(dev);
355}
356
357static void mips_dma_sync_single_for_device(struct device *dev,
358 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
359{
360 if (!plat_device_is_coherent(dev))
361 __dma_sync(dma_addr_to_page(dev, dma_handle),
362 dma_handle & ~PAGE_MASK, size, direction);
363}
364
365static void mips_dma_sync_sg_for_cpu(struct device *dev,
366 struct scatterlist *sglist, int nelems,
367 enum dma_data_direction direction)
368{
369 int i;
370 struct scatterlist *sg;
371
372 if (cpu_needs_post_dma_flush(dev)) {
373 for_each_sg(sglist, sg, nelems, i) {
374 __dma_sync(sg_page(sg), sg->offset, sg->length,
375 direction);
376 }
377 }
378 plat_post_dma_flush(dev);
379}
380
381static void mips_dma_sync_sg_for_device(struct device *dev,
382 struct scatterlist *sglist, int nelems,
383 enum dma_data_direction direction)
384{
385 int i;
386 struct scatterlist *sg;
387
388 if (!plat_device_is_coherent(dev)) {
389 for_each_sg(sglist, sg, nelems, i) {
390 __dma_sync(sg_page(sg), sg->offset, sg->length,
391 direction);
392 }
393 }
394}
395
396int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
397{
398 return 0;
399}
400
401int mips_dma_supported(struct device *dev, u64 mask)
402{
403 return plat_dma_supported(dev, mask);
404}
405
406void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
407 enum dma_data_direction direction)
408{
409 BUG_ON(direction == DMA_NONE);
410
411 if (!plat_device_is_coherent(dev))
412 __dma_sync_virtual(vaddr, size, direction);
413}
414
415EXPORT_SYMBOL(dma_cache_sync);
416
417static struct dma_map_ops mips_default_dma_map_ops = {
418 .alloc = mips_dma_alloc_coherent,
419 .free = mips_dma_free_coherent,
420 .mmap = mips_dma_mmap,
421 .map_page = mips_dma_map_page,
422 .unmap_page = mips_dma_unmap_page,
423 .map_sg = mips_dma_map_sg,
424 .unmap_sg = mips_dma_unmap_sg,
425 .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
426 .sync_single_for_device = mips_dma_sync_single_for_device,
427 .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
428 .sync_sg_for_device = mips_dma_sync_sg_for_device,
429 .mapping_error = mips_dma_mapping_error,
430 .dma_supported = mips_dma_supported
431};
432
433struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
434EXPORT_SYMBOL(mips_dma_map_ops);
435
436#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
437
438static int __init mips_dma_init(void)
439{
440 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
441
442 return 0;
443}
444fs_initcall(mips_dma_init);