Linux Audio

Check our new training course

Loading...
v3.1
 
  1/*
  2 * iommu.c:  IOMMU specific routines for memory management.
  3 *
  4 * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
  5 * Copyright (C) 1995,2002 Pete Zaitcev     (zaitcev@yahoo.com)
  6 * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
  7 * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
  8 */
  9 
 10#include <linux/kernel.h>
 11#include <linux/init.h>
 12#include <linux/mm.h>
 13#include <linux/slab.h>
 14#include <linux/highmem.h>	/* pte_offset_map => kmap_atomic */
 15#include <linux/scatterlist.h>
 16#include <linux/of.h>
 17#include <linux/of_device.h>
 18
 19#include <asm/pgalloc.h>
 20#include <asm/pgtable.h>
 21#include <asm/io.h>
 22#include <asm/mxcc.h>
 23#include <asm/mbus.h>
 24#include <asm/cacheflush.h>
 25#include <asm/tlbflush.h>
 26#include <asm/bitext.h>
 27#include <asm/iommu.h>
 28#include <asm/dma.h>
 29
 
 
 30/*
 31 * This can be sized dynamically, but we will do this
 32 * only when we have a guidance about actual I/O pressures.
 33 */
 34#define IOMMU_RNGE	IOMMU_RNGE_256MB
 35#define IOMMU_START	0xF0000000
 36#define IOMMU_WINSIZE	(256*1024*1024U)
 37#define IOMMU_NPTES	(IOMMU_WINSIZE/PAGE_SIZE)	/* 64K PTEs, 265KB */
 38#define IOMMU_ORDER	6				/* 4096 * (1<<6) */
 39
 40/* srmmu.c */
 41extern int viking_mxcc_present;
 42BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
 43#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)
 44extern int flush_page_for_dma_global;
 45static int viking_flush;
 46/* viking.S */
 47extern void viking_flush_page(unsigned long page);
 48extern void viking_mxcc_flush_page(unsigned long page);
 49
 50/*
 51 * Values precomputed according to CPU type.
 52 */
 53static unsigned int ioperm_noc;		/* Consistent mapping iopte flags */
 54static pgprot_t dvma_prot;		/* Consistent mapping pte flags */
 55
 56#define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
 57#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
 58
 59static void __init sbus_iommu_init(struct platform_device *op)
 60{
 61	struct iommu_struct *iommu;
 62	unsigned int impl, vers;
 63	unsigned long *bitmap;
 
 
 64	unsigned long tmp;
 65
 66	iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
 67	if (!iommu) {
 68		prom_printf("Unable to allocate iommu structure\n");
 69		prom_halt();
 70	}
 71
 72	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
 73				 "iommu_regs");
 74	if (!iommu->regs) {
 75		prom_printf("Cannot map IOMMU registers\n");
 76		prom_halt();
 77	}
 78	impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28;
 79	vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24;
 80	tmp = iommu->regs->control;
 81	tmp &= ~(IOMMU_CTRL_RNGE);
 82	tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
 83	iommu->regs->control = tmp;
 
 
 84	iommu_invalidate(iommu->regs);
 85	iommu->start = IOMMU_START;
 86	iommu->end = 0xffffffff;
 87
 88	/* Allocate IOMMU page table */
 89	/* Stupid alignment constraints give me a headache. 
 90	   We need 256K or 512K or 1M or 2M area aligned to
 91           its size and current gfp will fortunately give
 92           it to us. */
 93        tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
 94	if (!tmp) {
 95		prom_printf("Unable to allocate iommu table [0x%08x]\n",
 96			    IOMMU_NPTES*sizeof(iopte_t));
 97		prom_halt();
 98	}
 99	iommu->page_table = (iopte_t *)tmp;
100
101	/* Initialize new table. */
102	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
103	flush_cache_all();
104	flush_tlb_all();
105	iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4;
 
 
106	iommu_invalidate(iommu->regs);
107
108	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
109	if (!bitmap) {
110		prom_printf("Unable to allocate iommu bitmap [%d]\n",
111			    (int)(IOMMU_NPTES>>3));
112		prom_halt();
113	}
114	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
115	/* To be coherent on HyperSparc, the page color of DVMA
116	 * and physical addresses must match.
117	 */
118	if (srmmu_modtype == HyperSparc)
119		iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
120	else
121		iommu->usemap.num_colors = 1;
122
123	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
124	       impl, vers, iommu->page_table,
125	       (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
126
127	op->dev.archdata.iommu = iommu;
128}
129
130static int __init iommu_init(void)
131{
132	struct device_node *dp;
133
134	for_each_node_by_name(dp, "iommu") {
135		struct platform_device *op = of_find_device_by_node(dp);
136
137		sbus_iommu_init(op);
138		of_propagate_archdata(op);
139	}
140
141	return 0;
142}
143
144subsys_initcall(iommu_init);
145
146/* This begs to be btfixup-ed by srmmu. */
147/* Flush the iotlb entries to ram. */
148/* This could be better if we didn't have to flush whole pages. */
149static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
150{
151	unsigned long start;
152	unsigned long end;
153
154	start = (unsigned long)iopte;
155	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
156	start &= PAGE_MASK;
157	if (viking_mxcc_present) {
158		while(start < end) {
159			viking_mxcc_flush_page(start);
160			start += PAGE_SIZE;
161		}
162	} else if (viking_flush) {
163		while(start < end) {
164			viking_flush_page(start);
165			start += PAGE_SIZE;
166		}
167	} else {
168		while(start < end) {
169			__flush_page_to_ram(start);
170			start += PAGE_SIZE;
171		}
172	}
173}
174
175static u32 iommu_get_one(struct device *dev, struct page *page, int npages)
 
176{
177	struct iommu_struct *iommu = dev->archdata.iommu;
178	int ioptex;
179	iopte_t *iopte, *iopte0;
 
 
180	unsigned int busa, busa0;
181	int i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
182
183	/* page color = pfn of page */
184	ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page));
185	if (ioptex < 0)
186		panic("iommu out");
187	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
188	iopte0 = &iommu->page_table[ioptex];
189
190	busa = busa0;
191	iopte = iopte0;
192	for (i = 0; i < npages; i++) {
193		iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM);
194		iommu_invalidate_page(iommu->regs, busa);
195		busa += PAGE_SIZE;
196		iopte++;
197		page++;
198	}
199
200	iommu_flush_iotlb(iopte0, npages);
201
202	return busa0;
203}
204
205static u32 iommu_get_scsi_one(struct device *dev, char *vaddr, unsigned int len)
206{
207	unsigned long off;
208	int npages;
209	struct page *page;
210	u32 busa;
211
212	off = (unsigned long)vaddr & ~PAGE_MASK;
213	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
214	page = virt_to_page((unsigned long)vaddr & PAGE_MASK);
215	busa = iommu_get_one(dev, page, npages);
216	return busa + off;
217}
218
219static __u32 iommu_get_scsi_one_noflush(struct device *dev, char *vaddr, unsigned long len)
220{
221	return iommu_get_scsi_one(dev, vaddr, len);
222}
223
224static __u32 iommu_get_scsi_one_gflush(struct device *dev, char *vaddr, unsigned long len)
 
 
225{
226	flush_page_for_dma(0);
227	return iommu_get_scsi_one(dev, vaddr, len);
228}
229
230static __u32 iommu_get_scsi_one_pflush(struct device *dev, char *vaddr, unsigned long len)
 
 
231{
232	unsigned long page = ((unsigned long) vaddr) & PAGE_MASK;
233
234	while(page < ((unsigned long)(vaddr + len))) {
235		flush_page_for_dma(page);
236		page += PAGE_SIZE;
237	}
238	return iommu_get_scsi_one(dev, vaddr, len);
239}
240
241static void iommu_get_scsi_sgl_noflush(struct device *dev, struct scatterlist *sg, int sz)
 
 
242{
243	int n;
 
244
245	while (sz != 0) {
246		--sz;
247		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
248		sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
 
249		sg->dma_length = sg->length;
250		sg = sg_next(sg);
251	}
 
 
252}
253
254static void iommu_get_scsi_sgl_gflush(struct device *dev, struct scatterlist *sg, int sz)
 
255{
256	int n;
257
258	flush_page_for_dma(0);
259	while (sz != 0) {
260		--sz;
261		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
262		sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
263		sg->dma_length = sg->length;
264		sg = sg_next(sg);
265	}
266}
267
268static void iommu_get_scsi_sgl_pflush(struct device *dev, struct scatterlist *sg, int sz)
 
269{
270	unsigned long page, oldpage = 0;
271	int n, i;
272
273	while(sz != 0) {
274		--sz;
275
276		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
277
278		/*
279		 * We expect unmapped highmem pages to be not in the cache.
280		 * XXX Is this a good assumption?
281		 * XXX What if someone else unmaps it here and races us?
282		 */
283		if ((page = (unsigned long) page_address(sg_page(sg))) != 0) {
284			for (i = 0; i < n; i++) {
285				if (page != oldpage) {	/* Already flushed? */
286					flush_page_for_dma(page);
287					oldpage = page;
288				}
289				page += PAGE_SIZE;
290			}
291		}
292
293		sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
294		sg->dma_length = sg->length;
295		sg = sg_next(sg);
296	}
297}
298
299static void iommu_release_one(struct device *dev, u32 busa, int npages)
 
300{
301	struct iommu_struct *iommu = dev->archdata.iommu;
302	int ioptex;
303	int i;
 
 
 
304
305	BUG_ON(busa < iommu->start);
306	ioptex = (busa - iommu->start) >> PAGE_SHIFT;
307	for (i = 0; i < npages; i++) {
308		iopte_val(iommu->page_table[ioptex + i]) = 0;
309		iommu_invalidate_page(iommu->regs, busa);
310		busa += PAGE_SIZE;
311	}
312	bit_map_clear(&iommu->usemap, ioptex, npages);
313}
314
315static void iommu_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len)
 
316{
317	unsigned long off;
318	int npages;
319
320	off = vaddr & ~PAGE_MASK;
321	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
322	iommu_release_one(dev, vaddr & PAGE_MASK, npages);
323}
324
325static void iommu_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
326{
327	int n;
328
329	while(sz != 0) {
330		--sz;
331
332		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
333		iommu_release_one(dev, sg->dma_address & PAGE_MASK, n);
 
334		sg->dma_address = 0x21212121;
335		sg = sg_next(sg);
336	}
337}
338
339#ifdef CONFIG_SBUS
340static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va,
341			      unsigned long addr, int len)
342{
343	struct iommu_struct *iommu = dev->archdata.iommu;
344	unsigned long page, end;
345	iopte_t *iopte = iommu->page_table;
346	iopte_t *first;
347	int ioptex;
348
 
 
 
 
 
 
 
 
 
 
 
 
 
349	BUG_ON((va & ~PAGE_MASK) != 0);
350	BUG_ON((addr & ~PAGE_MASK) != 0);
351	BUG_ON((len & ~PAGE_MASK) != 0);
352
353	/* page color = physical address */
354	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
355		addr >> PAGE_SHIFT);
356	if (ioptex < 0)
357		panic("iommu out");
358
359	iopte += ioptex;
360	first = iopte;
361	end = addr + len;
362	while(addr < end) {
363		page = va;
364		{
365			pgd_t *pgdp;
366			pmd_t *pmdp;
367			pte_t *ptep;
368
369			if (viking_mxcc_present)
370				viking_mxcc_flush_page(page);
371			else if (viking_flush)
372				viking_flush_page(page);
373			else
374				__flush_page_to_ram(page);
375
376			pgdp = pgd_offset(&init_mm, addr);
377			pmdp = pmd_offset(pgdp, addr);
378			ptep = pte_offset_map(pmdp, addr);
379
380			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
381		}
382		iopte_val(*iopte++) =
383		    MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
384		addr += PAGE_SIZE;
385		va += PAGE_SIZE;
386	}
387	/* P3: why do we need this?
388	 *
389	 * DAVEM: Because there are several aspects, none of which
390	 *        are handled by a single interface.  Some cpus are
391	 *        completely not I/O DMA coherent, and some have
392	 *        virtually indexed caches.  The driver DMA flushing
393	 *        methods handle the former case, but here during
394	 *        IOMMU page table modifications, and usage of non-cacheable
395	 *        cpu mappings of pages potentially in the cpu caches, we have
396	 *        to handle the latter case as well.
397	 */
398	flush_cache_all();
399	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
400	flush_tlb_all();
401	iommu_invalidate(iommu->regs);
402
403	*pba = iommu->start + (ioptex << PAGE_SHIFT);
404	return 0;
 
 
 
 
405}
406
407static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len)
 
408{
409	struct iommu_struct *iommu = dev->archdata.iommu;
410	iopte_t *iopte = iommu->page_table;
411	unsigned long end;
412	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
 
 
 
 
413
414	BUG_ON((busa & ~PAGE_MASK) != 0);
415	BUG_ON((len & ~PAGE_MASK) != 0);
416
417	iopte += ioptex;
418	end = busa + len;
419	while (busa < end) {
420		iopte_val(*iopte++) = 0;
421		busa += PAGE_SIZE;
422	}
423	flush_tlb_all();
424	iommu_invalidate(iommu->regs);
425	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
 
 
426}
427#endif
428
429static char *iommu_lockarea(char *vaddr, unsigned long len)
430{
431	return vaddr;
432}
 
 
 
 
 
 
433
434static void iommu_unlockarea(char *vaddr, unsigned long len)
435{
436}
 
 
 
 
 
 
 
437
438void __init ld_mmu_iommu(void)
439{
440	viking_flush = (BTFIXUPVAL_CALL(flush_page_for_dma) == (unsigned long)viking_flush_page);
441	BTFIXUPSET_CALL(mmu_lockarea, iommu_lockarea, BTFIXUPCALL_RETO0);
442	BTFIXUPSET_CALL(mmu_unlockarea, iommu_unlockarea, BTFIXUPCALL_NOP);
443
444	if (!BTFIXUPVAL_CALL(flush_page_for_dma)) {
445		/* IO coherent chip */
446		BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_noflush, BTFIXUPCALL_RETO0);
447		BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_noflush, BTFIXUPCALL_NORM);
448	} else if (flush_page_for_dma_global) {
449		/* flush_page_for_dma flushes everything, no matter of what page is it */
450		BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_gflush, BTFIXUPCALL_NORM);
451		BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_gflush, BTFIXUPCALL_NORM);
452	} else {
453		BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_pflush, BTFIXUPCALL_NORM);
454		BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_pflush, BTFIXUPCALL_NORM);
455	}
456	BTFIXUPSET_CALL(mmu_release_scsi_one, iommu_release_scsi_one, BTFIXUPCALL_NORM);
457	BTFIXUPSET_CALL(mmu_release_scsi_sgl, iommu_release_scsi_sgl, BTFIXUPCALL_NORM);
458
459#ifdef CONFIG_SBUS
460	BTFIXUPSET_CALL(mmu_map_dma_area, iommu_map_dma_area, BTFIXUPCALL_NORM);
461	BTFIXUPSET_CALL(mmu_unmap_dma_area, iommu_unmap_dma_area, BTFIXUPCALL_NORM);
462#endif
463
464	if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
465		dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
466		ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
467	} else {
468		dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
469		ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
470	}
471}
v5.4
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * iommu.c:  IOMMU specific routines for memory management.
  4 *
  5 * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
  6 * Copyright (C) 1995,2002 Pete Zaitcev     (zaitcev@yahoo.com)
  7 * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
  8 * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
  9 */
 10 
 11#include <linux/kernel.h>
 12#include <linux/init.h>
 13#include <linux/mm.h>
 14#include <linux/slab.h>
 15#include <linux/highmem.h>	/* pte_offset_map => kmap_atomic */
 16#include <linux/dma-mapping.h>
 17#include <linux/of.h>
 18#include <linux/of_device.h>
 19
 20#include <asm/pgalloc.h>
 21#include <asm/pgtable.h>
 22#include <asm/io.h>
 23#include <asm/mxcc.h>
 24#include <asm/mbus.h>
 25#include <asm/cacheflush.h>
 26#include <asm/tlbflush.h>
 27#include <asm/bitext.h>
 28#include <asm/iommu.h>
 29#include <asm/dma.h>
 30
 31#include "mm_32.h"
 32
 33/*
 34 * This can be sized dynamically, but we will do this
 35 * only when we have a guidance about actual I/O pressures.
 36 */
 37#define IOMMU_RNGE	IOMMU_RNGE_256MB
 38#define IOMMU_START	0xF0000000
 39#define IOMMU_WINSIZE	(256*1024*1024U)
 40#define IOMMU_NPTES	(IOMMU_WINSIZE/PAGE_SIZE)	/* 64K PTEs, 256KB */
 41#define IOMMU_ORDER	6				/* 4096 * (1<<6) */
 42
 
 
 
 
 
 43static int viking_flush;
 44/* viking.S */
 45extern void viking_flush_page(unsigned long page);
 46extern void viking_mxcc_flush_page(unsigned long page);
 47
 48/*
 49 * Values precomputed according to CPU type.
 50 */
 51static unsigned int ioperm_noc;		/* Consistent mapping iopte flags */
 52static pgprot_t dvma_prot;		/* Consistent mapping pte flags */
 53
 54#define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
 55#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
 56
 57static void __init sbus_iommu_init(struct platform_device *op)
 58{
 59	struct iommu_struct *iommu;
 60	unsigned int impl, vers;
 61	unsigned long *bitmap;
 62	unsigned long control;
 63	unsigned long base;
 64	unsigned long tmp;
 65
 66	iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
 67	if (!iommu) {
 68		prom_printf("Unable to allocate iommu structure\n");
 69		prom_halt();
 70	}
 71
 72	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
 73				 "iommu_regs");
 74	if (!iommu->regs) {
 75		prom_printf("Cannot map IOMMU registers\n");
 76		prom_halt();
 77	}
 78
 79	control = sbus_readl(&iommu->regs->control);
 80	impl = (control & IOMMU_CTRL_IMPL) >> 28;
 81	vers = (control & IOMMU_CTRL_VERS) >> 24;
 82	control &= ~(IOMMU_CTRL_RNGE);
 83	control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
 84	sbus_writel(control, &iommu->regs->control);
 85
 86	iommu_invalidate(iommu->regs);
 87	iommu->start = IOMMU_START;
 88	iommu->end = 0xffffffff;
 89
 90	/* Allocate IOMMU page table */
 91	/* Stupid alignment constraints give me a headache. 
 92	   We need 256K or 512K or 1M or 2M area aligned to
 93           its size and current gfp will fortunately give
 94           it to us. */
 95        tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
 96	if (!tmp) {
 97		prom_printf("Unable to allocate iommu table [0x%lx]\n",
 98			    IOMMU_NPTES * sizeof(iopte_t));
 99		prom_halt();
100	}
101	iommu->page_table = (iopte_t *)tmp;
102
103	/* Initialize new table. */
104	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
105	flush_cache_all();
106	flush_tlb_all();
107
108	base = __pa((unsigned long)iommu->page_table) >> 4;
109	sbus_writel(base, &iommu->regs->base);
110	iommu_invalidate(iommu->regs);
111
112	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
113	if (!bitmap) {
114		prom_printf("Unable to allocate iommu bitmap [%d]\n",
115			    (int)(IOMMU_NPTES>>3));
116		prom_halt();
117	}
118	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
119	/* To be coherent on HyperSparc, the page color of DVMA
120	 * and physical addresses must match.
121	 */
122	if (srmmu_modtype == HyperSparc)
123		iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
124	else
125		iommu->usemap.num_colors = 1;
126
127	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
128	       impl, vers, iommu->page_table,
129	       (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
130
131	op->dev.archdata.iommu = iommu;
132}
133
134static int __init iommu_init(void)
135{
136	struct device_node *dp;
137
138	for_each_node_by_name(dp, "iommu") {
139		struct platform_device *op = of_find_device_by_node(dp);
140
141		sbus_iommu_init(op);
142		of_propagate_archdata(op);
143	}
144
145	return 0;
146}
147
148subsys_initcall(iommu_init);
149
 
150/* Flush the iotlb entries to ram. */
151/* This could be better if we didn't have to flush whole pages. */
152static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
153{
154	unsigned long start;
155	unsigned long end;
156
157	start = (unsigned long)iopte;
158	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
159	start &= PAGE_MASK;
160	if (viking_mxcc_present) {
161		while(start < end) {
162			viking_mxcc_flush_page(start);
163			start += PAGE_SIZE;
164		}
165	} else if (viking_flush) {
166		while(start < end) {
167			viking_flush_page(start);
168			start += PAGE_SIZE;
169		}
170	} else {
171		while(start < end) {
172			__flush_page_to_ram(start);
173			start += PAGE_SIZE;
174		}
175	}
176}
177
178static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
179		unsigned long offset, size_t len, bool per_page_flush)
180{
181	struct iommu_struct *iommu = dev->archdata.iommu;
182	phys_addr_t paddr = page_to_phys(page) + offset;
183	unsigned long off = paddr & ~PAGE_MASK;
184	unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
185	unsigned long pfn = __phys_to_pfn(paddr);
186	unsigned int busa, busa0;
187	iopte_t *iopte, *iopte0;
188	int ioptex, i;
189
190	/* XXX So what is maxphys for us and how do drivers know it? */
191	if (!len || len > 256 * 1024)
192		return DMA_MAPPING_ERROR;
193
194	/*
195	 * We expect unmapped highmem pages to be not in the cache.
196	 * XXX Is this a good assumption?
197	 * XXX What if someone else unmaps it here and races us?
198	 */
199	if (per_page_flush && !PageHighMem(page)) {
200		unsigned long vaddr, p;
201
202		vaddr = (unsigned long)page_address(page) + offset;
203		for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
204			flush_page_for_dma(p);
205	}
206
207	/* page color = pfn of page */
208	ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
209	if (ioptex < 0)
210		panic("iommu out");
211	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
212	iopte0 = &iommu->page_table[ioptex];
213
214	busa = busa0;
215	iopte = iopte0;
216	for (i = 0; i < npages; i++) {
217		iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
218		iommu_invalidate_page(iommu->regs, busa);
219		busa += PAGE_SIZE;
220		iopte++;
221		pfn++;
222	}
223
224	iommu_flush_iotlb(iopte0, npages);
225	return busa0 + off;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
226}
227
228static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
229		struct page *page, unsigned long offset, size_t len,
230		enum dma_data_direction dir, unsigned long attrs)
231{
232	flush_page_for_dma(0);
233	return __sbus_iommu_map_page(dev, page, offset, len, false);
234}
235
236static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
237		struct page *page, unsigned long offset, size_t len,
238		enum dma_data_direction dir, unsigned long attrs)
239{
240	return __sbus_iommu_map_page(dev, page, offset, len, true);
 
 
 
 
 
 
241}
242
243static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
244		int nents, enum dma_data_direction dir, unsigned long attrs,
245		bool per_page_flush)
246{
247	struct scatterlist *sg;
248	int j;
249
250	for_each_sg(sgl, sg, nents, j) {
251		sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
252				sg->offset, sg->length, per_page_flush);
253		if (sg->dma_address == DMA_MAPPING_ERROR)
254			return 0;
255		sg->dma_length = sg->length;
 
256	}
257
258	return nents;
259}
260
261static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
262		int nents, enum dma_data_direction dir, unsigned long attrs)
263{
 
 
264	flush_page_for_dma(0);
265	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
 
 
 
 
 
 
266}
267
268static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
269		int nents, enum dma_data_direction dir, unsigned long attrs)
270{
271	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
272}
273
274static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
275		size_t len, enum dma_data_direction dir, unsigned long attrs)
276{
277	struct iommu_struct *iommu = dev->archdata.iommu;
278	unsigned int busa = dma_addr & PAGE_MASK;
279	unsigned long off = dma_addr & ~PAGE_MASK;
280	unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
281	unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
282	unsigned int i;
283
284	BUG_ON(busa < iommu->start);
 
285	for (i = 0; i < npages; i++) {
286		iopte_val(iommu->page_table[ioptex + i]) = 0;
287		iommu_invalidate_page(iommu->regs, busa);
288		busa += PAGE_SIZE;
289	}
290	bit_map_clear(&iommu->usemap, ioptex, npages);
291}
292
293static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
294		int nents, enum dma_data_direction dir, unsigned long attrs)
295{
296	struct scatterlist *sg;
297	int i;
 
 
 
 
 
 
 
 
 
 
 
 
298
299	for_each_sg(sgl, sg, nents, i) {
300		sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
301				attrs);
302		sg->dma_address = 0x21212121;
 
303	}
304}
305
306#ifdef CONFIG_SBUS
307static void *sbus_iommu_alloc(struct device *dev, size_t len,
308		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
309{
310	struct iommu_struct *iommu = dev->archdata.iommu;
311	unsigned long va, addr, page, end, ret;
312	iopte_t *iopte = iommu->page_table;
313	iopte_t *first;
314	int ioptex;
315
316	/* XXX So what is maxphys for us and how do drivers know it? */
317	if (!len || len > 256 * 1024)
318		return NULL;
319
320	len = PAGE_ALIGN(len);
321	va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
322	if (va == 0)
323		return NULL;
324
325	addr = ret = sparc_dma_alloc_resource(dev, len);
326	if (!addr)
327		goto out_free_pages;
328
329	BUG_ON((va & ~PAGE_MASK) != 0);
330	BUG_ON((addr & ~PAGE_MASK) != 0);
331	BUG_ON((len & ~PAGE_MASK) != 0);
332
333	/* page color = physical address */
334	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
335		addr >> PAGE_SHIFT);
336	if (ioptex < 0)
337		panic("iommu out");
338
339	iopte += ioptex;
340	first = iopte;
341	end = addr + len;
342	while(addr < end) {
343		page = va;
344		{
345			pgd_t *pgdp;
346			pmd_t *pmdp;
347			pte_t *ptep;
348
349			if (viking_mxcc_present)
350				viking_mxcc_flush_page(page);
351			else if (viking_flush)
352				viking_flush_page(page);
353			else
354				__flush_page_to_ram(page);
355
356			pgdp = pgd_offset(&init_mm, addr);
357			pmdp = pmd_offset(pgdp, addr);
358			ptep = pte_offset_map(pmdp, addr);
359
360			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
361		}
362		iopte_val(*iopte++) =
363		    MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
364		addr += PAGE_SIZE;
365		va += PAGE_SIZE;
366	}
367	/* P3: why do we need this?
368	 *
369	 * DAVEM: Because there are several aspects, none of which
370	 *        are handled by a single interface.  Some cpus are
371	 *        completely not I/O DMA coherent, and some have
372	 *        virtually indexed caches.  The driver DMA flushing
373	 *        methods handle the former case, but here during
374	 *        IOMMU page table modifications, and usage of non-cacheable
375	 *        cpu mappings of pages potentially in the cpu caches, we have
376	 *        to handle the latter case as well.
377	 */
378	flush_cache_all();
379	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
380	flush_tlb_all();
381	iommu_invalidate(iommu->regs);
382
383	*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
384	return (void *)ret;
385
386out_free_pages:
387	free_pages(va, get_order(len));
388	return NULL;
389}
390
391static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
392			       dma_addr_t busa, unsigned long attrs)
393{
394	struct iommu_struct *iommu = dev->archdata.iommu;
395	iopte_t *iopte = iommu->page_table;
396	struct page *page = virt_to_page(cpu_addr);
397	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
398	unsigned long end;
399
400	if (!sparc_dma_free_resource(cpu_addr, len))
401		return;
402
403	BUG_ON((busa & ~PAGE_MASK) != 0);
404	BUG_ON((len & ~PAGE_MASK) != 0);
405
406	iopte += ioptex;
407	end = busa + len;
408	while (busa < end) {
409		iopte_val(*iopte++) = 0;
410		busa += PAGE_SIZE;
411	}
412	flush_tlb_all();
413	iommu_invalidate(iommu->regs);
414	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
415
416	__free_pages(page, get_order(len));
417}
418#endif
419
420static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
421#ifdef CONFIG_SBUS
422	.alloc			= sbus_iommu_alloc,
423	.free			= sbus_iommu_free,
424#endif
425	.map_page		= sbus_iommu_map_page_gflush,
426	.unmap_page		= sbus_iommu_unmap_page,
427	.map_sg			= sbus_iommu_map_sg_gflush,
428	.unmap_sg		= sbus_iommu_unmap_sg,
429};
430
431static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
432#ifdef CONFIG_SBUS
433	.alloc			= sbus_iommu_alloc,
434	.free			= sbus_iommu_free,
435#endif
436	.map_page		= sbus_iommu_map_page_pflush,
437	.unmap_page		= sbus_iommu_unmap_page,
438	.map_sg			= sbus_iommu_map_sg_pflush,
439	.unmap_sg		= sbus_iommu_unmap_sg,
440};
441
442void __init ld_mmu_iommu(void)
443{
444	if (flush_page_for_dma_global) {
 
 
 
 
 
 
 
 
445		/* flush_page_for_dma flushes everything, no matter of what page is it */
446		dma_ops = &sbus_iommu_dma_gflush_ops;
 
447	} else {
448		dma_ops = &sbus_iommu_dma_pflush_ops;
 
449	}
 
 
 
 
 
 
 
450
451	if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
452		dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
453		ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
454	} else {
455		dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
456		ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
457	}
458}