Loading...
Note: File does not exist in v6.8.
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
10
11#include <linux/types.h>
12#include <linux/dma-mapping.h>
13#include <linux/mm.h>
14#include <linux/export.h>
15#include <linux/scatterlist.h>
16#include <linux/string.h>
17#include <linux/gfp.h>
18#include <linux/highmem.h>
19#include <linux/dma-contiguous.h>
20
21#include <asm/cache.h>
22#include <asm/cpu-type.h>
23#include <asm/io.h>
24
25#include <dma-coherence.h>
26
27#if defined(CONFIG_DMA_MAYBE_COHERENT) && !defined(CONFIG_DMA_PERDEV_COHERENT)
28/* User defined DMA coherency from command line. */
29enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT;
30EXPORT_SYMBOL_GPL(coherentio);
31int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
32
33static int __init setcoherentio(char *str)
34{
35 coherentio = IO_COHERENCE_ENABLED;
36 pr_info("Hardware DMA cache coherency (command line)\n");
37 return 0;
38}
39early_param("coherentio", setcoherentio);
40
41static int __init setnocoherentio(char *str)
42{
43 coherentio = IO_COHERENCE_DISABLED;
44 pr_info("Software DMA cache coherency (command line)\n");
45 return 0;
46}
47early_param("nocoherentio", setnocoherentio);
48#endif
49
50static inline struct page *dma_addr_to_page(struct device *dev,
51 dma_addr_t dma_addr)
52{
53 return pfn_to_page(
54 plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
55}
56
57/*
58 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
59 * speculatively fill random cachelines with stale data at any time,
60 * requiring an extra flush post-DMA.
61 *
62 * Warning on the terminology - Linux calls an uncached area coherent;
63 * MIPS terminology calls memory areas with hardware maintained coherency
64 * coherent.
65 *
66 * Note that the R14000 and R16000 should also be checked for in this
67 * condition. However this function is only called on non-I/O-coherent
68 * systems and only the R10000 and R12000 are used in such systems, the
69 * SGI IP28 Indigo² rsp. SGI IP32 aka O2.
70 */
71static inline bool cpu_needs_post_dma_flush(struct device *dev)
72{
73 if (plat_device_is_coherent(dev))
74 return false;
75
76 switch (boot_cpu_type()) {
77 case CPU_R10000:
78 case CPU_R12000:
79 case CPU_BMIPS5000:
80 return true;
81
82 default:
83 /*
84 * Presence of MAARs suggests that the CPU supports
85 * speculatively prefetching data, and therefore requires
86 * the post-DMA flush/invalidate.
87 */
88 return cpu_has_maar;
89 }
90}
91
92static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
93{
94 gfp_t dma_flag;
95
96#ifdef CONFIG_ISA
97 if (dev == NULL)
98 dma_flag = __GFP_DMA;
99 else
100#endif
101#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
102 if (dev == NULL || dev->coherent_dma_mask < DMA_BIT_MASK(32))
103 dma_flag = __GFP_DMA;
104 else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
105 dma_flag = __GFP_DMA32;
106 else
107#endif
108#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
109 if (dev == NULL || dev->coherent_dma_mask < DMA_BIT_MASK(64))
110 dma_flag = __GFP_DMA32;
111 else
112#endif
113#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
114 if (dev == NULL ||
115 dev->coherent_dma_mask < DMA_BIT_MASK(sizeof(phys_addr_t) * 8))
116 dma_flag = __GFP_DMA;
117 else
118#endif
119 dma_flag = 0;
120
121 /* Don't invoke OOM killer */
122 gfp |= __GFP_NORETRY;
123
124 return gfp | dma_flag;
125}
126
127static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
128 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
129{
130 void *ret;
131 struct page *page = NULL;
132 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
133
134 gfp = massage_gfp_flags(dev, gfp);
135
136 if (IS_ENABLED(CONFIG_DMA_CMA) && gfpflags_allow_blocking(gfp))
137 page = dma_alloc_from_contiguous(dev, count, get_order(size),
138 gfp);
139 if (!page)
140 page = alloc_pages(gfp, get_order(size));
141
142 if (!page)
143 return NULL;
144
145 ret = page_address(page);
146 memset(ret, 0, size);
147 *dma_handle = plat_map_dma_mem(dev, ret, size);
148 if (!(attrs & DMA_ATTR_NON_CONSISTENT) &&
149 !plat_device_is_coherent(dev)) {
150 dma_cache_wback_inv((unsigned long) ret, size);
151 ret = UNCAC_ADDR(ret);
152 }
153
154 return ret;
155}
156
157static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
158 dma_addr_t dma_handle, unsigned long attrs)
159{
160 unsigned long addr = (unsigned long) vaddr;
161 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
162 struct page *page = NULL;
163
164 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
165
166 if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !plat_device_is_coherent(dev))
167 addr = CAC_ADDR(addr);
168
169 page = virt_to_page((void *) addr);
170
171 if (!dma_release_from_contiguous(dev, page, count))
172 __free_pages(page, get_order(size));
173}
174
175static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma,
176 void *cpu_addr, dma_addr_t dma_addr, size_t size,
177 unsigned long attrs)
178{
179 unsigned long user_count = vma_pages(vma);
180 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
181 unsigned long addr = (unsigned long)cpu_addr;
182 unsigned long off = vma->vm_pgoff;
183 unsigned long pfn;
184 int ret = -ENXIO;
185
186 if (!plat_device_is_coherent(dev))
187 addr = CAC_ADDR(addr);
188
189 pfn = page_to_pfn(virt_to_page((void *)addr));
190
191 if (attrs & DMA_ATTR_WRITE_COMBINE)
192 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
193 else
194 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
195
196 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
197 return ret;
198
199 if (off < count && user_count <= (count - off)) {
200 ret = remap_pfn_range(vma, vma->vm_start,
201 pfn + off,
202 user_count << PAGE_SHIFT,
203 vma->vm_page_prot);
204 }
205
206 return ret;
207}
208
209static inline void __dma_sync_virtual(void *addr, size_t size,
210 enum dma_data_direction direction)
211{
212 switch (direction) {
213 case DMA_TO_DEVICE:
214 dma_cache_wback((unsigned long)addr, size);
215 break;
216
217 case DMA_FROM_DEVICE:
218 dma_cache_inv((unsigned long)addr, size);
219 break;
220
221 case DMA_BIDIRECTIONAL:
222 dma_cache_wback_inv((unsigned long)addr, size);
223 break;
224
225 default:
226 BUG();
227 }
228}
229
230/*
231 * A single sg entry may refer to multiple physically contiguous
232 * pages. But we still need to process highmem pages individually.
233 * If highmem is not configured then the bulk of this loop gets
234 * optimized out.
235 */
236static inline void __dma_sync(struct page *page,
237 unsigned long offset, size_t size, enum dma_data_direction direction)
238{
239 size_t left = size;
240
241 do {
242 size_t len = left;
243
244 if (PageHighMem(page)) {
245 void *addr;
246
247 if (offset + len > PAGE_SIZE) {
248 if (offset >= PAGE_SIZE) {
249 page += offset >> PAGE_SHIFT;
250 offset &= ~PAGE_MASK;
251 }
252 len = PAGE_SIZE - offset;
253 }
254
255 addr = kmap_atomic(page);
256 __dma_sync_virtual(addr + offset, len, direction);
257 kunmap_atomic(addr);
258 } else
259 __dma_sync_virtual(page_address(page) + offset,
260 size, direction);
261 offset = 0;
262 page++;
263 left -= len;
264 } while (left);
265}
266
267static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
268 size_t size, enum dma_data_direction direction, unsigned long attrs)
269{
270 if (cpu_needs_post_dma_flush(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
271 __dma_sync(dma_addr_to_page(dev, dma_addr),
272 dma_addr & ~PAGE_MASK, size, direction);
273 plat_post_dma_flush(dev);
274 plat_unmap_dma_mem(dev, dma_addr, size, direction);
275}
276
277static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist,
278 int nents, enum dma_data_direction direction, unsigned long attrs)
279{
280 int i;
281 struct scatterlist *sg;
282
283 for_each_sg(sglist, sg, nents, i) {
284 if (!plat_device_is_coherent(dev) &&
285 !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
286 __dma_sync(sg_page(sg), sg->offset, sg->length,
287 direction);
288#ifdef CONFIG_NEED_SG_DMA_LENGTH
289 sg->dma_length = sg->length;
290#endif
291 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
292 sg->offset;
293 }
294
295 return nents;
296}
297
298static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
299 unsigned long offset, size_t size, enum dma_data_direction direction,
300 unsigned long attrs)
301{
302 if (!plat_device_is_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
303 __dma_sync(page, offset, size, direction);
304
305 return plat_map_dma_mem_page(dev, page) + offset;
306}
307
308static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
309 int nhwentries, enum dma_data_direction direction,
310 unsigned long attrs)
311{
312 int i;
313 struct scatterlist *sg;
314
315 for_each_sg(sglist, sg, nhwentries, i) {
316 if (!plat_device_is_coherent(dev) &&
317 !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
318 direction != DMA_TO_DEVICE)
319 __dma_sync(sg_page(sg), sg->offset, sg->length,
320 direction);
321 plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
322 }
323}
324
325static void mips_dma_sync_single_for_cpu(struct device *dev,
326 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
327{
328 if (cpu_needs_post_dma_flush(dev))
329 __dma_sync(dma_addr_to_page(dev, dma_handle),
330 dma_handle & ~PAGE_MASK, size, direction);
331 plat_post_dma_flush(dev);
332}
333
334static void mips_dma_sync_single_for_device(struct device *dev,
335 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
336{
337 if (!plat_device_is_coherent(dev))
338 __dma_sync(dma_addr_to_page(dev, dma_handle),
339 dma_handle & ~PAGE_MASK, size, direction);
340}
341
342static void mips_dma_sync_sg_for_cpu(struct device *dev,
343 struct scatterlist *sglist, int nelems,
344 enum dma_data_direction direction)
345{
346 int i;
347 struct scatterlist *sg;
348
349 if (cpu_needs_post_dma_flush(dev)) {
350 for_each_sg(sglist, sg, nelems, i) {
351 __dma_sync(sg_page(sg), sg->offset, sg->length,
352 direction);
353 }
354 }
355 plat_post_dma_flush(dev);
356}
357
358static void mips_dma_sync_sg_for_device(struct device *dev,
359 struct scatterlist *sglist, int nelems,
360 enum dma_data_direction direction)
361{
362 int i;
363 struct scatterlist *sg;
364
365 if (!plat_device_is_coherent(dev)) {
366 for_each_sg(sglist, sg, nelems, i) {
367 __dma_sync(sg_page(sg), sg->offset, sg->length,
368 direction);
369 }
370 }
371}
372
373static int mips_dma_supported(struct device *dev, u64 mask)
374{
375 return plat_dma_supported(dev, mask);
376}
377
378static void mips_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
379 enum dma_data_direction direction)
380{
381 BUG_ON(direction == DMA_NONE);
382
383 if (!plat_device_is_coherent(dev))
384 __dma_sync_virtual(vaddr, size, direction);
385}
386
387static const struct dma_map_ops mips_default_dma_map_ops = {
388 .alloc = mips_dma_alloc_coherent,
389 .free = mips_dma_free_coherent,
390 .mmap = mips_dma_mmap,
391 .map_page = mips_dma_map_page,
392 .unmap_page = mips_dma_unmap_page,
393 .map_sg = mips_dma_map_sg,
394 .unmap_sg = mips_dma_unmap_sg,
395 .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
396 .sync_single_for_device = mips_dma_sync_single_for_device,
397 .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
398 .sync_sg_for_device = mips_dma_sync_sg_for_device,
399 .dma_supported = mips_dma_supported,
400 .cache_sync = mips_dma_cache_sync,
401};
402
403const struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
404EXPORT_SYMBOL(mips_dma_map_ops);
405
406#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
407
408static int __init mips_dma_init(void)
409{
410 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
411
412 return 0;
413}
414fs_initcall(mips_dma_init);