1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 *  Based on linux/arch/arm/mm/dma-mapping.c
  4 *
  5 *  Copyright (C) 2000-2004 Russell King
  6 */
  7
  8#include <linux/export.h>
  9#include <linux/mm.h>
 10#include <linux/dma-direct.h>
 11#include <linux/dma-map-ops.h>
 12#include <linux/scatterlist.h>
 13
 14#include <asm/cachetype.h>
 15#include <asm/cacheflush.h>
 16#include <asm/outercache.h>
 17#include <asm/cp15.h>
 18
 19#include "dma.h"
 20
 21/*
 22 *  The generic direct mapping code is used if
 23 *   - MMU/MPU is off
 24 *   - cpu is v7m w/o cache support
 25 *   - device is coherent
 26 *  otherwise arm_nommu_dma_ops is used.
 27 *
 28 *  arm_nommu_dma_ops rely on consistent DMA memory (please, refer to
 29 *  [1] on how to declare such memory).
 30 *
 31 *  [1] Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
 32 */
 33
 34static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
 35				 dma_addr_t *dma_handle, gfp_t gfp,
 36				 unsigned long attrs)
 37
 38{
 39	void *ret = dma_alloc_from_global_coherent(dev, size, dma_handle);
 40
 41	/*
 42	 * dma_alloc_from_global_coherent() may fail because:
 43	 *
 44	 * - no consistent DMA region has been defined, so we can't
 45	 *   continue.
 46	 * - there is no space left in consistent DMA region, so we
 47	 *   only can fallback to generic allocator if we are
 48	 *   advertised that consistency is not required.
 49	 */
 50
 51	WARN_ON_ONCE(ret == NULL);
 52	return ret;
 53}
 54
 55static void arm_nommu_dma_free(struct device *dev, size_t size,
 56			       void *cpu_addr, dma_addr_t dma_addr,
 57			       unsigned long attrs)
 58{
 59	int ret = dma_release_from_global_coherent(get_order(size), cpu_addr);
 60
 61	WARN_ON_ONCE(ret == 0);
 62}
 63
 64static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 65			      void *cpu_addr, dma_addr_t dma_addr, size_t size,
 66			      unsigned long attrs)
 67{
 68	int ret;
 69
 70	if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))
 71		return ret;
 72	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
 73		return ret;
 74	return -ENXIO;
 75}
 76
 77
 78static void __dma_page_cpu_to_dev(phys_addr_t paddr, size_t size,
 79				  enum dma_data_direction dir)
 80{
 81	dmac_map_area(__va(paddr), size, dir);
 82
 83	if (dir == DMA_FROM_DEVICE)
 84		outer_inv_range(paddr, paddr + size);
 85	else
 86		outer_clean_range(paddr, paddr + size);
 87}
 88
 89static void __dma_page_dev_to_cpu(phys_addr_t paddr, size_t size,
 90				  enum dma_data_direction dir)
 91{
 92	if (dir != DMA_TO_DEVICE) {
 93		outer_inv_range(paddr, paddr + size);
 94		dmac_unmap_area(__va(paddr), size, dir);
 95	}
 96}
 97
 98static dma_addr_t arm_nommu_dma_map_page(struct device *dev, struct page *page,
 99					 unsigned long offset, size_t size,
100					 enum dma_data_direction dir,
101					 unsigned long attrs)
102{
103	dma_addr_t handle = page_to_phys(page) + offset;
104
105	__dma_page_cpu_to_dev(handle, size, dir);
106
107	return handle;
108}
109
110static void arm_nommu_dma_unmap_page(struct device *dev, dma_addr_t handle,
111				     size_t size, enum dma_data_direction dir,
112				     unsigned long attrs)
113{
114	__dma_page_dev_to_cpu(handle, size, dir);
115}
116
117
118static int arm_nommu_dma_map_sg(struct device *dev, struct scatterlist *sgl,
119				int nents, enum dma_data_direction dir,
120				unsigned long attrs)
121{
122	int i;
123	struct scatterlist *sg;
124
125	for_each_sg(sgl, sg, nents, i) {
126		sg_dma_address(sg) = sg_phys(sg);
127		sg_dma_len(sg) = sg->length;
128		__dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir);
129	}
130
131	return nents;
132}
133
134static void arm_nommu_dma_unmap_sg(struct device *dev, struct scatterlist *sgl,
135				   int nents, enum dma_data_direction dir,
136				   unsigned long attrs)
137{
138	struct scatterlist *sg;
139	int i;
140
141	for_each_sg(sgl, sg, nents, i)
142		__dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir);
143}
144
145static void arm_nommu_dma_sync_single_for_device(struct device *dev,
146		dma_addr_t handle, size_t size, enum dma_data_direction dir)
147{
148	__dma_page_cpu_to_dev(handle, size, dir);
149}
150
151static void arm_nommu_dma_sync_single_for_cpu(struct device *dev,
152		dma_addr_t handle, size_t size, enum dma_data_direction dir)
153{
154	__dma_page_cpu_to_dev(handle, size, dir);
155}
156
157static void arm_nommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
158					     int nents, enum dma_data_direction dir)
159{
160	struct scatterlist *sg;
161	int i;
162
163	for_each_sg(sgl, sg, nents, i)
164		__dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir);
165}
166
167static void arm_nommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
168					  int nents, enum dma_data_direction dir)
169{
170	struct scatterlist *sg;
171	int i;
172
173	for_each_sg(sgl, sg, nents, i)
174		__dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir);
175}
176
177const struct dma_map_ops arm_nommu_dma_ops = {
178	.alloc			= arm_nommu_dma_alloc,
179	.free			= arm_nommu_dma_free,
180	.alloc_pages		= dma_direct_alloc_pages,
181	.free_pages		= dma_direct_free_pages,
182	.mmap			= arm_nommu_dma_mmap,
183	.map_page		= arm_nommu_dma_map_page,
184	.unmap_page		= arm_nommu_dma_unmap_page,
185	.map_sg			= arm_nommu_dma_map_sg,
186	.unmap_sg		= arm_nommu_dma_unmap_sg,
187	.sync_single_for_device	= arm_nommu_dma_sync_single_for_device,
188	.sync_single_for_cpu	= arm_nommu_dma_sync_single_for_cpu,
189	.sync_sg_for_device	= arm_nommu_dma_sync_sg_for_device,
190	.sync_sg_for_cpu	= arm_nommu_dma_sync_sg_for_cpu,
191};
192EXPORT_SYMBOL(arm_nommu_dma_ops);
193
194void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
195			const struct iommu_ops *iommu, bool coherent)
196{
197	if (IS_ENABLED(CONFIG_CPU_V7M)) {
198		/*
199		 * Cache support for v7m is optional, so can be treated as
200		 * coherent if no cache has been detected. Note that it is not
201		 * enough to check if MPU is in use or not since in absense of
202		 * MPU system memory map is used.
203		 */
204		dev->archdata.dma_coherent = (cacheid) ? coherent : true;
205	} else {
206		/*
207		 * Assume coherent DMA in case MMU/MPU has not been set up.
208		 */
209		dev->archdata.dma_coherent = (get_cr() & CR_M) ? coherent : true;
210	}
211
212	if (!dev->archdata.dma_coherent)
213		set_dma_ops(dev, &arm_nommu_dma_ops);
214}