1/**************************************************************************
  2 * Copyright (c) 2007, Intel Corporation.
  3 *
  4 * This program is free software; you can redistribute it and/or modify it
  5 * under the terms and conditions of the GNU General Public License,
  6 * version 2, as published by the Free Software Foundation.
  7 *
  8 * This program is distributed in the hope it will be useful, but WITHOUT
  9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 10 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 11 * more details.
 12 *
 13 * You should have received a copy of the GNU General Public License along with
 14 * this program; if not, write to the Free Software Foundation, Inc.,
 15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 16 *
 17 **************************************************************************/
 18#include <drm/drmP.h>
 19#include "psb_drv.h"
 20#include "psb_reg.h"
 21#include "mmu.h"
 22
 23/*
 24 * Code for the SGX MMU:
 25 */
 26
 27/*
 28 * clflush on one processor only:
 29 * clflush should apparently flush the cache line on all processors in an
 30 * SMP system.
 31 */
 32
 33/*
 34 * kmap atomic:
 35 * The usage of the slots must be completely encapsulated within a spinlock, and
 36 * no other functions that may be using the locks for other purposed may be
 37 * called from within the locked region.
 38 * Since the slots are per processor, this will guarantee that we are the only
 39 * user.
 40 */
 41
 42/*
 43 * TODO: Inserting ptes from an interrupt handler:
 44 * This may be desirable for some SGX functionality where the GPU can fault in
 45 * needed pages. For that, we need to make an atomic insert_pages function, that
 46 * may fail.
 47 * If it fails, the caller need to insert the page using a workqueue function,
 48 * but on average it should be fast.
 49 */
 50
 51static inline uint32_t psb_mmu_pt_index(uint32_t offset)
 52{
 53	return (offset >> PSB_PTE_SHIFT) & 0x3FF;
 54}
 55
 56static inline uint32_t psb_mmu_pd_index(uint32_t offset)
 57{
 58	return offset >> PSB_PDE_SHIFT;
 59}
 60
 61#if defined(CONFIG_X86)
 62static inline void psb_clflush(void *addr)
 63{
 64	__asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
 65}
 66
 67static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
 68{
 69	if (!driver->has_clflush)
 70		return;
 71
 72	mb();
 73	psb_clflush(addr);
 74	mb();
 75}
 76#else
 77
 78static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
 79{;
 80}
 81
 82#endif
 83
 84static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
 85{
 86	struct drm_device *dev = driver->dev;
 87	struct drm_psb_private *dev_priv = dev->dev_private;
 88
 89	if (atomic_read(&driver->needs_tlbflush) || force) {
 90		uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
 91		PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
 92
 93		/* Make sure data cache is turned off before enabling it */
 94		wmb();
 95		PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
 96		(void)PSB_RSGX32(PSB_CR_BIF_CTRL);
 97		if (driver->msvdx_mmu_invaldc)
 98			atomic_set(driver->msvdx_mmu_invaldc, 1);
 99	}
100	atomic_set(&driver->needs_tlbflush, 0);
101}
102
103#if 0
104static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
105{
106	down_write(&driver->sem);
107	psb_mmu_flush_pd_locked(driver, force);
108	up_write(&driver->sem);
109}
110#endif
111
112void psb_mmu_flush(struct psb_mmu_driver *driver)
113{
114	struct drm_device *dev = driver->dev;
115	struct drm_psb_private *dev_priv = dev->dev_private;
116	uint32_t val;
117
118	down_write(&driver->sem);
119	val = PSB_RSGX32(PSB_CR_BIF_CTRL);
120	if (atomic_read(&driver->needs_tlbflush))
121		PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
122	else
123		PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
124
125	/* Make sure data cache is turned off and MMU is flushed before
126	   restoring bank interface control register */
127	wmb();
128	PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
129		   PSB_CR_BIF_CTRL);
130	(void)PSB_RSGX32(PSB_CR_BIF_CTRL);
131
132	atomic_set(&driver->needs_tlbflush, 0);
133	if (driver->msvdx_mmu_invaldc)
134		atomic_set(driver->msvdx_mmu_invaldc, 1);
135	up_write(&driver->sem);
136}
137
138void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
139{
140	struct drm_device *dev = pd->driver->dev;
141	struct drm_psb_private *dev_priv = dev->dev_private;
142	uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
143			  PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
144
145	down_write(&pd->driver->sem);
146	PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
147	wmb();
148	psb_mmu_flush_pd_locked(pd->driver, 1);
149	pd->hw_context = hw_context;
150	up_write(&pd->driver->sem);
151
152}
153
154static inline unsigned long psb_pd_addr_end(unsigned long addr,
155					    unsigned long end)
156{
157	addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
158	return (addr < end) ? addr : end;
159}
160
161static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
162{
163	uint32_t mask = PSB_PTE_VALID;
164
165	if (type & PSB_MMU_CACHED_MEMORY)
166		mask |= PSB_PTE_CACHED;
167	if (type & PSB_MMU_RO_MEMORY)
168		mask |= PSB_PTE_RO;
169	if (type & PSB_MMU_WO_MEMORY)
170		mask |= PSB_PTE_WO;
171
172	return (pfn << PAGE_SHIFT) | mask;
173}
174
175struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
176				    int trap_pagefaults, int invalid_type)
177{
178	struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
179	uint32_t *v;
180	int i;
181
182	if (!pd)
183		return NULL;
184
185	pd->p = alloc_page(GFP_DMA32);
186	if (!pd->p)
187		goto out_err1;
188	pd->dummy_pt = alloc_page(GFP_DMA32);
189	if (!pd->dummy_pt)
190		goto out_err2;
191	pd->dummy_page = alloc_page(GFP_DMA32);
192	if (!pd->dummy_page)
193		goto out_err3;
194
195	if (!trap_pagefaults) {
196		pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
197						   invalid_type);
198		pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
199						   invalid_type);
200	} else {
201		pd->invalid_pde = 0;
202		pd->invalid_pte = 0;
203	}
204
205	v = kmap(pd->dummy_pt);
206	for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
207		v[i] = pd->invalid_pte;
208
209	kunmap(pd->dummy_pt);
210
211	v = kmap(pd->p);
212	for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
213		v[i] = pd->invalid_pde;
214
215	kunmap(pd->p);
216
217	clear_page(kmap(pd->dummy_page));
218	kunmap(pd->dummy_page);
219
220	pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
221	if (!pd->tables)
222		goto out_err4;
223
224	pd->hw_context = -1;
225	pd->pd_mask = PSB_PTE_VALID;
226	pd->driver = driver;
227
228	return pd;
229
230out_err4:
231	__free_page(pd->dummy_page);
232out_err3:
233	__free_page(pd->dummy_pt);
234out_err2:
235	__free_page(pd->p);
236out_err1:
237	kfree(pd);
238	return NULL;
239}
240
241static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
242{
243	__free_page(pt->p);
244	kfree(pt);
245}
246
247void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
248{
249	struct psb_mmu_driver *driver = pd->driver;
250	struct drm_device *dev = driver->dev;
251	struct drm_psb_private *dev_priv = dev->dev_private;
252	struct psb_mmu_pt *pt;
253	int i;
254
255	down_write(&driver->sem);
256	if (pd->hw_context != -1) {
257		PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
258		psb_mmu_flush_pd_locked(driver, 1);
259	}
260
261	/* Should take the spinlock here, but we don't need to do that
262	   since we have the semaphore in write mode. */
263
264	for (i = 0; i < 1024; ++i) {
265		pt = pd->tables[i];
266		if (pt)
267			psb_mmu_free_pt(pt);
268	}
269
270	vfree(pd->tables);
271	__free_page(pd->dummy_page);
272	__free_page(pd->dummy_pt);
273	__free_page(pd->p);
274	kfree(pd);
275	up_write(&driver->sem);
276}
277
278static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
279{
280	struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
281	void *v;
282	uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
283	uint32_t clflush_count = PAGE_SIZE / clflush_add;
284	spinlock_t *lock = &pd->driver->lock;
285	uint8_t *clf;
286	uint32_t *ptes;
287	int i;
288
289	if (!pt)
290		return NULL;
291
292	pt->p = alloc_page(GFP_DMA32);
293	if (!pt->p) {
294		kfree(pt);
295		return NULL;
296	}
297
298	spin_lock(lock);
299
300	v = kmap_atomic(pt->p);
301	clf = (uint8_t *) v;
302	ptes = (uint32_t *) v;
303	for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
304		*ptes++ = pd->invalid_pte;
305
306#if defined(CONFIG_X86)
307	if (pd->driver->has_clflush && pd->hw_context != -1) {
308		mb();
309		for (i = 0; i < clflush_count; ++i) {
310			psb_clflush(clf);
311			clf += clflush_add;
312		}
313		mb();
314	}
315#endif
316	kunmap_atomic(v);
317	spin_unlock(lock);
318
319	pt->count = 0;
320	pt->pd = pd;
321	pt->index = 0;
322
323	return pt;
324}
325
326struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
327					     unsigned long addr)
328{
329	uint32_t index = psb_mmu_pd_index(addr);
330	struct psb_mmu_pt *pt;
331	uint32_t *v;
332	spinlock_t *lock = &pd->driver->lock;
333
334	spin_lock(lock);
335	pt = pd->tables[index];
336	while (!pt) {
337		spin_unlock(lock);
338		pt = psb_mmu_alloc_pt(pd);
339		if (!pt)
340			return NULL;
341		spin_lock(lock);
342
343		if (pd->tables[index]) {
344			spin_unlock(lock);
345			psb_mmu_free_pt(pt);
346			spin_lock(lock);
347			pt = pd->tables[index];
348			continue;
349		}
350
351		v = kmap_atomic(pd->p);
352		pd->tables[index] = pt;
353		v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
354		pt->index = index;
355		kunmap_atomic((void *) v);
356
357		if (pd->hw_context != -1) {
358			psb_mmu_clflush(pd->driver, (void *)&v[index]);
359			atomic_set(&pd->driver->needs_tlbflush, 1);
360		}
361	}
362	pt->v = kmap_atomic(pt->p);
363	return pt;
364}
365
366static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
367					      unsigned long addr)
368{
369	uint32_t index = psb_mmu_pd_index(addr);
370	struct psb_mmu_pt *pt;
371	spinlock_t *lock = &pd->driver->lock;
372
373	spin_lock(lock);
374	pt = pd->tables[index];
375	if (!pt) {
376		spin_unlock(lock);
377		return NULL;
378	}
379	pt->v = kmap_atomic(pt->p);
380	return pt;
381}
382
383static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
384{
385	struct psb_mmu_pd *pd = pt->pd;
386	uint32_t *v;
387
388	kunmap_atomic(pt->v);
389	if (pt->count == 0) {
390		v = kmap_atomic(pd->p);
391		v[pt->index] = pd->invalid_pde;
392		pd->tables[pt->index] = NULL;
393
394		if (pd->hw_context != -1) {
395			psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
396			atomic_set(&pd->driver->needs_tlbflush, 1);
397		}
398		kunmap_atomic(pt->v);
399		spin_unlock(&pd->driver->lock);
400		psb_mmu_free_pt(pt);
401		return;
402	}
403	spin_unlock(&pd->driver->lock);
404}
405
406static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
407				   uint32_t pte)
408{
409	pt->v[psb_mmu_pt_index(addr)] = pte;
410}
411
412static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
413					  unsigned long addr)
414{
415	pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
416}
417
418struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
419{
420	struct psb_mmu_pd *pd;
421
422	down_read(&driver->sem);
423	pd = driver->default_pd;
424	up_read(&driver->sem);
425
426	return pd;
427}
428
429/* Returns the physical address of the PD shared by sgx/msvdx */
430uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)
431{
432	struct psb_mmu_pd *pd;
433
434	pd = psb_mmu_get_default_pd(driver);
435	return page_to_pfn(pd->p) << PAGE_SHIFT;
436}
437
438void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
439{
440	struct drm_device *dev = driver->dev;
441	struct drm_psb_private *dev_priv = dev->dev_private;
442
443	PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
444	psb_mmu_free_pagedir(driver->default_pd);
445	kfree(driver);
446}
447
448struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
449					   int trap_pagefaults,
450					   int invalid_type,
451					   atomic_t *msvdx_mmu_invaldc)
452{
453	struct psb_mmu_driver *driver;
454	struct drm_psb_private *dev_priv = dev->dev_private;
455
456	driver = kmalloc(sizeof(*driver), GFP_KERNEL);
457
458	if (!driver)
459		return NULL;
460
461	driver->dev = dev;
462	driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
463					      invalid_type);
464	if (!driver->default_pd)
465		goto out_err1;
466
467	spin_lock_init(&driver->lock);
468	init_rwsem(&driver->sem);
469	down_write(&driver->sem);
470	atomic_set(&driver->needs_tlbflush, 1);
471	driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
472
473	driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
474	PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
475		   PSB_CR_BIF_CTRL);
476	PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
477		   PSB_CR_BIF_CTRL);
478
479	driver->has_clflush = 0;
480
481#if defined(CONFIG_X86)
482	if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
483		uint32_t tfms, misc, cap0, cap4, clflush_size;
484
485		/*
486		 * clflush size is determined at kernel setup for x86_64 but not
487		 * for i386. We have to do it here.
488		 */
489
490		cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
491		clflush_size = ((misc >> 8) & 0xff) * 8;
492		driver->has_clflush = 1;
493		driver->clflush_add =
494		    PAGE_SIZE * clflush_size / sizeof(uint32_t);
495		driver->clflush_mask = driver->clflush_add - 1;
496		driver->clflush_mask = ~driver->clflush_mask;
497	}
498#endif
499
500	up_write(&driver->sem);
501	return driver;
502
503out_err1:
504	kfree(driver);
505	return NULL;
506}
507
508#if defined(CONFIG_X86)
509static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
510			       uint32_t num_pages, uint32_t desired_tile_stride,
511			       uint32_t hw_tile_stride)
512{
513	struct psb_mmu_pt *pt;
514	uint32_t rows = 1;
515	uint32_t i;
516	unsigned long addr;
517	unsigned long end;
518	unsigned long next;
519	unsigned long add;
520	unsigned long row_add;
521	unsigned long clflush_add = pd->driver->clflush_add;
522	unsigned long clflush_mask = pd->driver->clflush_mask;
523
524	if (!pd->driver->has_clflush)
525		return;
526
527	if (hw_tile_stride)
528		rows = num_pages / desired_tile_stride;
529	else
530		desired_tile_stride = num_pages;
531
532	add = desired_tile_stride << PAGE_SHIFT;
533	row_add = hw_tile_stride << PAGE_SHIFT;
534	mb();
535	for (i = 0; i < rows; ++i) {
536
537		addr = address;
538		end = addr + add;
539
540		do {
541			next = psb_pd_addr_end(addr, end);
542			pt = psb_mmu_pt_map_lock(pd, addr);
543			if (!pt)
544				continue;
545			do {
546				psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
547			} while (addr += clflush_add,
548				 (addr & clflush_mask) < next);
549
550			psb_mmu_pt_unmap_unlock(pt);
551		} while (addr = next, next != end);
552		address += row_add;
553	}
554	mb();
555}
556#else
557static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
558			       uint32_t num_pages, uint32_t desired_tile_stride,
559			       uint32_t hw_tile_stride)
560{
561	drm_ttm_cache_flush();
562}
563#endif
564
565void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
566				 unsigned long address, uint32_t num_pages)
567{
568	struct psb_mmu_pt *pt;
569	unsigned long addr;
570	unsigned long end;
571	unsigned long next;
572	unsigned long f_address = address;
573
574	down_read(&pd->driver->sem);
575
576	addr = address;
577	end = addr + (num_pages << PAGE_SHIFT);
578
579	do {
580		next = psb_pd_addr_end(addr, end);
581		pt = psb_mmu_pt_alloc_map_lock(pd, addr);
582		if (!pt)
583			goto out;
584		do {
585			psb_mmu_invalidate_pte(pt, addr);
586			--pt->count;
587		} while (addr += PAGE_SIZE, addr < next);
588		psb_mmu_pt_unmap_unlock(pt);
589
590	} while (addr = next, next != end);
591
592out:
593	if (pd->hw_context != -1)
594		psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
595
596	up_read(&pd->driver->sem);
597
598	if (pd->hw_context != -1)
599		psb_mmu_flush(pd->driver);
600
601	return;
602}
603
604void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
605			  uint32_t num_pages, uint32_t desired_tile_stride,
606			  uint32_t hw_tile_stride)
607{
608	struct psb_mmu_pt *pt;
609	uint32_t rows = 1;
610	uint32_t i;
611	unsigned long addr;
612	unsigned long end;
613	unsigned long next;
614	unsigned long add;
615	unsigned long row_add;
616	unsigned long f_address = address;
617
618	if (hw_tile_stride)
619		rows = num_pages / desired_tile_stride;
620	else
621		desired_tile_stride = num_pages;
622
623	add = desired_tile_stride << PAGE_SHIFT;
624	row_add = hw_tile_stride << PAGE_SHIFT;
625
626	down_read(&pd->driver->sem);
627
628	/* Make sure we only need to flush this processor's cache */
629
630	for (i = 0; i < rows; ++i) {
631
632		addr = address;
633		end = addr + add;
634
635		do {
636			next = psb_pd_addr_end(addr, end);
637			pt = psb_mmu_pt_map_lock(pd, addr);
638			if (!pt)
639				continue;
640			do {
641				psb_mmu_invalidate_pte(pt, addr);
642				--pt->count;
643
644			} while (addr += PAGE_SIZE, addr < next);
645			psb_mmu_pt_unmap_unlock(pt);
646
647		} while (addr = next, next != end);
648		address += row_add;
649	}
650	if (pd->hw_context != -1)
651		psb_mmu_flush_ptes(pd, f_address, num_pages,
652				   desired_tile_stride, hw_tile_stride);
653
654	up_read(&pd->driver->sem);
655
656	if (pd->hw_context != -1)
657		psb_mmu_flush(pd->driver);
658}
659
660int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
661				unsigned long address, uint32_t num_pages,
662				int type)
663{
664	struct psb_mmu_pt *pt;
665	uint32_t pte;
666	unsigned long addr;
667	unsigned long end;
668	unsigned long next;
669	unsigned long f_address = address;
670	int ret = -ENOMEM;
671
672	down_read(&pd->driver->sem);
673
674	addr = address;
675	end = addr + (num_pages << PAGE_SHIFT);
676
677	do {
678		next = psb_pd_addr_end(addr, end);
679		pt = psb_mmu_pt_alloc_map_lock(pd, addr);
680		if (!pt) {
681			ret = -ENOMEM;
682			goto out;
683		}
684		do {
685			pte = psb_mmu_mask_pte(start_pfn++, type);
686			psb_mmu_set_pte(pt, addr, pte);
687			pt->count++;
688		} while (addr += PAGE_SIZE, addr < next);
689		psb_mmu_pt_unmap_unlock(pt);
690
691	} while (addr = next, next != end);
692	ret = 0;
693
694out:
695	if (pd->hw_context != -1)
696		psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
697
698	up_read(&pd->driver->sem);
699
700	if (pd->hw_context != -1)
701		psb_mmu_flush(pd->driver);
702
703	return 0;
704}
705
706int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
707			 unsigned long address, uint32_t num_pages,
708			 uint32_t desired_tile_stride, uint32_t hw_tile_stride,
709			 int type)
710{
711	struct psb_mmu_pt *pt;
712	uint32_t rows = 1;
713	uint32_t i;
714	uint32_t pte;
715	unsigned long addr;
716	unsigned long end;
717	unsigned long next;
718	unsigned long add;
719	unsigned long row_add;
720	unsigned long f_address = address;
721	int ret = -ENOMEM;
722
723	if (hw_tile_stride) {
724		if (num_pages % desired_tile_stride != 0)
725			return -EINVAL;
726		rows = num_pages / desired_tile_stride;
727	} else {
728		desired_tile_stride = num_pages;
729	}
730
731	add = desired_tile_stride << PAGE_SHIFT;
732	row_add = hw_tile_stride << PAGE_SHIFT;
733
734	down_read(&pd->driver->sem);
735
736	for (i = 0; i < rows; ++i) {
737
738		addr = address;
739		end = addr + add;
740
741		do {
742			next = psb_pd_addr_end(addr, end);
743			pt = psb_mmu_pt_alloc_map_lock(pd, addr);
744			if (!pt)
745				goto out;
746			do {
747				pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
748						       type);
749				psb_mmu_set_pte(pt, addr, pte);
750				pt->count++;
751			} while (addr += PAGE_SIZE, addr < next);
752			psb_mmu_pt_unmap_unlock(pt);
753
754		} while (addr = next, next != end);
755
756		address += row_add;
757	}
758
759	ret = 0;
760out:
761	if (pd->hw_context != -1)
762		psb_mmu_flush_ptes(pd, f_address, num_pages,
763				   desired_tile_stride, hw_tile_stride);
764
765	up_read(&pd->driver->sem);
766
767	if (pd->hw_context != -1)
768		psb_mmu_flush(pd->driver);
769
770	return ret;
771}
772
773int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
774			   unsigned long *pfn)
775{
776	int ret;
777	struct psb_mmu_pt *pt;
778	uint32_t tmp;
779	spinlock_t *lock = &pd->driver->lock;
780
781	down_read(&pd->driver->sem);
782	pt = psb_mmu_pt_map_lock(pd, virtual);
783	if (!pt) {
784		uint32_t *v;
785
786		spin_lock(lock);
787		v = kmap_atomic(pd->p);
788		tmp = v[psb_mmu_pd_index(virtual)];
789		kunmap_atomic(v);
790		spin_unlock(lock);
791
792		if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
793		    !(pd->invalid_pte & PSB_PTE_VALID)) {
794			ret = -EINVAL;
795			goto out;
796		}
797		ret = 0;
798		*pfn = pd->invalid_pte >> PAGE_SHIFT;
799		goto out;
800	}
801	tmp = pt->v[psb_mmu_pt_index(virtual)];
802	if (!(tmp & PSB_PTE_VALID)) {
803		ret = -EINVAL;
804	} else {
805		ret = 0;
806		*pfn = tmp >> PAGE_SHIFT;
807	}
808	psb_mmu_pt_unmap_unlock(pt);
809out:
810	up_read(&pd->driver->sem);
811	return ret;
812}