Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/**************************************************************************
3 * Copyright (c) 2007, Intel Corporation.
4 *
5 **************************************************************************/
6
7#include <linux/highmem.h>
8
9#include "mmu.h"
10#include "psb_drv.h"
11#include "psb_reg.h"
12
13/*
14 * Code for the SGX MMU:
15 */
16
17/*
18 * clflush on one processor only:
19 * clflush should apparently flush the cache line on all processors in an
20 * SMP system.
21 */
22
23/*
24 * kmap atomic:
25 * The usage of the slots must be completely encapsulated within a spinlock, and
26 * no other functions that may be using the locks for other purposed may be
27 * called from within the locked region.
28 * Since the slots are per processor, this will guarantee that we are the only
29 * user.
30 */
31
32/*
33 * TODO: Inserting ptes from an interrupt handler:
34 * This may be desirable for some SGX functionality where the GPU can fault in
35 * needed pages. For that, we need to make an atomic insert_pages function, that
36 * may fail.
37 * If it fails, the caller need to insert the page using a workqueue function,
38 * but on average it should be fast.
39 */
40
41static inline uint32_t psb_mmu_pt_index(uint32_t offset)
42{
43 return (offset >> PSB_PTE_SHIFT) & 0x3FF;
44}
45
46static inline uint32_t psb_mmu_pd_index(uint32_t offset)
47{
48 return offset >> PSB_PDE_SHIFT;
49}
50
51#if defined(CONFIG_X86)
52static inline void psb_clflush(void *addr)
53{
54 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
55}
56
57static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
58{
59 if (!driver->has_clflush)
60 return;
61
62 mb();
63 psb_clflush(addr);
64 mb();
65}
66#else
67
68static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
69{;
70}
71
72#endif
73
74static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
75{
76 struct drm_device *dev = driver->dev;
77 struct drm_psb_private *dev_priv = dev->dev_private;
78
79 if (atomic_read(&driver->needs_tlbflush) || force) {
80 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
81 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
82
83 /* Make sure data cache is turned off before enabling it */
84 wmb();
85 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
86 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
87 if (driver->msvdx_mmu_invaldc)
88 atomic_set(driver->msvdx_mmu_invaldc, 1);
89 }
90 atomic_set(&driver->needs_tlbflush, 0);
91}
92
93#if 0
94static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
95{
96 down_write(&driver->sem);
97 psb_mmu_flush_pd_locked(driver, force);
98 up_write(&driver->sem);
99}
100#endif
101
102void psb_mmu_flush(struct psb_mmu_driver *driver)
103{
104 struct drm_device *dev = driver->dev;
105 struct drm_psb_private *dev_priv = dev->dev_private;
106 uint32_t val;
107
108 down_write(&driver->sem);
109 val = PSB_RSGX32(PSB_CR_BIF_CTRL);
110 if (atomic_read(&driver->needs_tlbflush))
111 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
112 else
113 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
114
115 /* Make sure data cache is turned off and MMU is flushed before
116 restoring bank interface control register */
117 wmb();
118 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
119 PSB_CR_BIF_CTRL);
120 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
121
122 atomic_set(&driver->needs_tlbflush, 0);
123 if (driver->msvdx_mmu_invaldc)
124 atomic_set(driver->msvdx_mmu_invaldc, 1);
125 up_write(&driver->sem);
126}
127
128void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
129{
130 struct drm_device *dev = pd->driver->dev;
131 struct drm_psb_private *dev_priv = dev->dev_private;
132 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
133 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
134
135 down_write(&pd->driver->sem);
136 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
137 wmb();
138 psb_mmu_flush_pd_locked(pd->driver, 1);
139 pd->hw_context = hw_context;
140 up_write(&pd->driver->sem);
141
142}
143
144static inline unsigned long psb_pd_addr_end(unsigned long addr,
145 unsigned long end)
146{
147 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
148 return (addr < end) ? addr : end;
149}
150
151static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
152{
153 uint32_t mask = PSB_PTE_VALID;
154
155 if (type & PSB_MMU_CACHED_MEMORY)
156 mask |= PSB_PTE_CACHED;
157 if (type & PSB_MMU_RO_MEMORY)
158 mask |= PSB_PTE_RO;
159 if (type & PSB_MMU_WO_MEMORY)
160 mask |= PSB_PTE_WO;
161
162 return (pfn << PAGE_SHIFT) | mask;
163}
164
165struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
166 int trap_pagefaults, int invalid_type)
167{
168 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
169 uint32_t *v;
170 int i;
171
172 if (!pd)
173 return NULL;
174
175 pd->p = alloc_page(GFP_DMA32);
176 if (!pd->p)
177 goto out_err1;
178 pd->dummy_pt = alloc_page(GFP_DMA32);
179 if (!pd->dummy_pt)
180 goto out_err2;
181 pd->dummy_page = alloc_page(GFP_DMA32);
182 if (!pd->dummy_page)
183 goto out_err3;
184
185 if (!trap_pagefaults) {
186 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
187 invalid_type);
188 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
189 invalid_type);
190 } else {
191 pd->invalid_pde = 0;
192 pd->invalid_pte = 0;
193 }
194
195 v = kmap(pd->dummy_pt);
196 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
197 v[i] = pd->invalid_pte;
198
199 kunmap(pd->dummy_pt);
200
201 v = kmap(pd->p);
202 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
203 v[i] = pd->invalid_pde;
204
205 kunmap(pd->p);
206
207 clear_page(kmap(pd->dummy_page));
208 kunmap(pd->dummy_page);
209
210 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
211 if (!pd->tables)
212 goto out_err4;
213
214 pd->hw_context = -1;
215 pd->pd_mask = PSB_PTE_VALID;
216 pd->driver = driver;
217
218 return pd;
219
220out_err4:
221 __free_page(pd->dummy_page);
222out_err3:
223 __free_page(pd->dummy_pt);
224out_err2:
225 __free_page(pd->p);
226out_err1:
227 kfree(pd);
228 return NULL;
229}
230
231static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
232{
233 __free_page(pt->p);
234 kfree(pt);
235}
236
237void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
238{
239 struct psb_mmu_driver *driver = pd->driver;
240 struct drm_device *dev = driver->dev;
241 struct drm_psb_private *dev_priv = dev->dev_private;
242 struct psb_mmu_pt *pt;
243 int i;
244
245 down_write(&driver->sem);
246 if (pd->hw_context != -1) {
247 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
248 psb_mmu_flush_pd_locked(driver, 1);
249 }
250
251 /* Should take the spinlock here, but we don't need to do that
252 since we have the semaphore in write mode. */
253
254 for (i = 0; i < 1024; ++i) {
255 pt = pd->tables[i];
256 if (pt)
257 psb_mmu_free_pt(pt);
258 }
259
260 vfree(pd->tables);
261 __free_page(pd->dummy_page);
262 __free_page(pd->dummy_pt);
263 __free_page(pd->p);
264 kfree(pd);
265 up_write(&driver->sem);
266}
267
268static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
269{
270 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
271 void *v;
272 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
273 uint32_t clflush_count = PAGE_SIZE / clflush_add;
274 spinlock_t *lock = &pd->driver->lock;
275 uint8_t *clf;
276 uint32_t *ptes;
277 int i;
278
279 if (!pt)
280 return NULL;
281
282 pt->p = alloc_page(GFP_DMA32);
283 if (!pt->p) {
284 kfree(pt);
285 return NULL;
286 }
287
288 spin_lock(lock);
289
290 v = kmap_atomic(pt->p);
291 clf = (uint8_t *) v;
292 ptes = (uint32_t *) v;
293 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
294 *ptes++ = pd->invalid_pte;
295
296#if defined(CONFIG_X86)
297 if (pd->driver->has_clflush && pd->hw_context != -1) {
298 mb();
299 for (i = 0; i < clflush_count; ++i) {
300 psb_clflush(clf);
301 clf += clflush_add;
302 }
303 mb();
304 }
305#endif
306 kunmap_atomic(v);
307 spin_unlock(lock);
308
309 pt->count = 0;
310 pt->pd = pd;
311 pt->index = 0;
312
313 return pt;
314}
315
316struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
317 unsigned long addr)
318{
319 uint32_t index = psb_mmu_pd_index(addr);
320 struct psb_mmu_pt *pt;
321 uint32_t *v;
322 spinlock_t *lock = &pd->driver->lock;
323
324 spin_lock(lock);
325 pt = pd->tables[index];
326 while (!pt) {
327 spin_unlock(lock);
328 pt = psb_mmu_alloc_pt(pd);
329 if (!pt)
330 return NULL;
331 spin_lock(lock);
332
333 if (pd->tables[index]) {
334 spin_unlock(lock);
335 psb_mmu_free_pt(pt);
336 spin_lock(lock);
337 pt = pd->tables[index];
338 continue;
339 }
340
341 v = kmap_atomic(pd->p);
342 pd->tables[index] = pt;
343 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
344 pt->index = index;
345 kunmap_atomic((void *) v);
346
347 if (pd->hw_context != -1) {
348 psb_mmu_clflush(pd->driver, (void *)&v[index]);
349 atomic_set(&pd->driver->needs_tlbflush, 1);
350 }
351 }
352 pt->v = kmap_atomic(pt->p);
353 return pt;
354}
355
356static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
357 unsigned long addr)
358{
359 uint32_t index = psb_mmu_pd_index(addr);
360 struct psb_mmu_pt *pt;
361 spinlock_t *lock = &pd->driver->lock;
362
363 spin_lock(lock);
364 pt = pd->tables[index];
365 if (!pt) {
366 spin_unlock(lock);
367 return NULL;
368 }
369 pt->v = kmap_atomic(pt->p);
370 return pt;
371}
372
373static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
374{
375 struct psb_mmu_pd *pd = pt->pd;
376 uint32_t *v;
377
378 kunmap_atomic(pt->v);
379 if (pt->count == 0) {
380 v = kmap_atomic(pd->p);
381 v[pt->index] = pd->invalid_pde;
382 pd->tables[pt->index] = NULL;
383
384 if (pd->hw_context != -1) {
385 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
386 atomic_set(&pd->driver->needs_tlbflush, 1);
387 }
388 kunmap_atomic(v);
389 spin_unlock(&pd->driver->lock);
390 psb_mmu_free_pt(pt);
391 return;
392 }
393 spin_unlock(&pd->driver->lock);
394}
395
396static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
397 uint32_t pte)
398{
399 pt->v[psb_mmu_pt_index(addr)] = pte;
400}
401
402static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
403 unsigned long addr)
404{
405 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
406}
407
408struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
409{
410 struct psb_mmu_pd *pd;
411
412 down_read(&driver->sem);
413 pd = driver->default_pd;
414 up_read(&driver->sem);
415
416 return pd;
417}
418
419/* Returns the physical address of the PD shared by sgx/msvdx */
420uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)
421{
422 struct psb_mmu_pd *pd;
423
424 pd = psb_mmu_get_default_pd(driver);
425 return page_to_pfn(pd->p) << PAGE_SHIFT;
426}
427
428void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
429{
430 struct drm_device *dev = driver->dev;
431 struct drm_psb_private *dev_priv = dev->dev_private;
432
433 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
434 psb_mmu_free_pagedir(driver->default_pd);
435 kfree(driver);
436}
437
438struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
439 int trap_pagefaults,
440 int invalid_type,
441 atomic_t *msvdx_mmu_invaldc)
442{
443 struct psb_mmu_driver *driver;
444 struct drm_psb_private *dev_priv = dev->dev_private;
445
446 driver = kmalloc(sizeof(*driver), GFP_KERNEL);
447
448 if (!driver)
449 return NULL;
450
451 driver->dev = dev;
452 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
453 invalid_type);
454 if (!driver->default_pd)
455 goto out_err1;
456
457 spin_lock_init(&driver->lock);
458 init_rwsem(&driver->sem);
459 down_write(&driver->sem);
460 atomic_set(&driver->needs_tlbflush, 1);
461 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
462
463 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
464 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
465 PSB_CR_BIF_CTRL);
466 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
467 PSB_CR_BIF_CTRL);
468
469 driver->has_clflush = 0;
470
471#if defined(CONFIG_X86)
472 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
473 uint32_t tfms, misc, cap0, cap4, clflush_size;
474
475 /*
476 * clflush size is determined at kernel setup for x86_64 but not
477 * for i386. We have to do it here.
478 */
479
480 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
481 clflush_size = ((misc >> 8) & 0xff) * 8;
482 driver->has_clflush = 1;
483 driver->clflush_add =
484 PAGE_SIZE * clflush_size / sizeof(uint32_t);
485 driver->clflush_mask = driver->clflush_add - 1;
486 driver->clflush_mask = ~driver->clflush_mask;
487 }
488#endif
489
490 up_write(&driver->sem);
491 return driver;
492
493out_err1:
494 kfree(driver);
495 return NULL;
496}
497
498#if defined(CONFIG_X86)
499static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
500 uint32_t num_pages, uint32_t desired_tile_stride,
501 uint32_t hw_tile_stride)
502{
503 struct psb_mmu_pt *pt;
504 uint32_t rows = 1;
505 uint32_t i;
506 unsigned long addr;
507 unsigned long end;
508 unsigned long next;
509 unsigned long add;
510 unsigned long row_add;
511 unsigned long clflush_add = pd->driver->clflush_add;
512 unsigned long clflush_mask = pd->driver->clflush_mask;
513
514 if (!pd->driver->has_clflush)
515 return;
516
517 if (hw_tile_stride)
518 rows = num_pages / desired_tile_stride;
519 else
520 desired_tile_stride = num_pages;
521
522 add = desired_tile_stride << PAGE_SHIFT;
523 row_add = hw_tile_stride << PAGE_SHIFT;
524 mb();
525 for (i = 0; i < rows; ++i) {
526
527 addr = address;
528 end = addr + add;
529
530 do {
531 next = psb_pd_addr_end(addr, end);
532 pt = psb_mmu_pt_map_lock(pd, addr);
533 if (!pt)
534 continue;
535 do {
536 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
537 } while (addr += clflush_add,
538 (addr & clflush_mask) < next);
539
540 psb_mmu_pt_unmap_unlock(pt);
541 } while (addr = next, next != end);
542 address += row_add;
543 }
544 mb();
545}
546#else
547static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
548 uint32_t num_pages, uint32_t desired_tile_stride,
549 uint32_t hw_tile_stride)
550{
551 drm_ttm_cache_flush();
552}
553#endif
554
555void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
556 unsigned long address, uint32_t num_pages)
557{
558 struct psb_mmu_pt *pt;
559 unsigned long addr;
560 unsigned long end;
561 unsigned long next;
562 unsigned long f_address = address;
563
564 down_read(&pd->driver->sem);
565
566 addr = address;
567 end = addr + (num_pages << PAGE_SHIFT);
568
569 do {
570 next = psb_pd_addr_end(addr, end);
571 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
572 if (!pt)
573 goto out;
574 do {
575 psb_mmu_invalidate_pte(pt, addr);
576 --pt->count;
577 } while (addr += PAGE_SIZE, addr < next);
578 psb_mmu_pt_unmap_unlock(pt);
579
580 } while (addr = next, next != end);
581
582out:
583 if (pd->hw_context != -1)
584 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
585
586 up_read(&pd->driver->sem);
587
588 if (pd->hw_context != -1)
589 psb_mmu_flush(pd->driver);
590
591 return;
592}
593
594void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
595 uint32_t num_pages, uint32_t desired_tile_stride,
596 uint32_t hw_tile_stride)
597{
598 struct psb_mmu_pt *pt;
599 uint32_t rows = 1;
600 uint32_t i;
601 unsigned long addr;
602 unsigned long end;
603 unsigned long next;
604 unsigned long add;
605 unsigned long row_add;
606 unsigned long f_address = address;
607
608 if (hw_tile_stride)
609 rows = num_pages / desired_tile_stride;
610 else
611 desired_tile_stride = num_pages;
612
613 add = desired_tile_stride << PAGE_SHIFT;
614 row_add = hw_tile_stride << PAGE_SHIFT;
615
616 down_read(&pd->driver->sem);
617
618 /* Make sure we only need to flush this processor's cache */
619
620 for (i = 0; i < rows; ++i) {
621
622 addr = address;
623 end = addr + add;
624
625 do {
626 next = psb_pd_addr_end(addr, end);
627 pt = psb_mmu_pt_map_lock(pd, addr);
628 if (!pt)
629 continue;
630 do {
631 psb_mmu_invalidate_pte(pt, addr);
632 --pt->count;
633
634 } while (addr += PAGE_SIZE, addr < next);
635 psb_mmu_pt_unmap_unlock(pt);
636
637 } while (addr = next, next != end);
638 address += row_add;
639 }
640 if (pd->hw_context != -1)
641 psb_mmu_flush_ptes(pd, f_address, num_pages,
642 desired_tile_stride, hw_tile_stride);
643
644 up_read(&pd->driver->sem);
645
646 if (pd->hw_context != -1)
647 psb_mmu_flush(pd->driver);
648}
649
650int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
651 unsigned long address, uint32_t num_pages,
652 int type)
653{
654 struct psb_mmu_pt *pt;
655 uint32_t pte;
656 unsigned long addr;
657 unsigned long end;
658 unsigned long next;
659 unsigned long f_address = address;
660 int ret = -ENOMEM;
661
662 down_read(&pd->driver->sem);
663
664 addr = address;
665 end = addr + (num_pages << PAGE_SHIFT);
666
667 do {
668 next = psb_pd_addr_end(addr, end);
669 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
670 if (!pt) {
671 ret = -ENOMEM;
672 goto out;
673 }
674 do {
675 pte = psb_mmu_mask_pte(start_pfn++, type);
676 psb_mmu_set_pte(pt, addr, pte);
677 pt->count++;
678 } while (addr += PAGE_SIZE, addr < next);
679 psb_mmu_pt_unmap_unlock(pt);
680
681 } while (addr = next, next != end);
682 ret = 0;
683
684out:
685 if (pd->hw_context != -1)
686 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
687
688 up_read(&pd->driver->sem);
689
690 if (pd->hw_context != -1)
691 psb_mmu_flush(pd->driver);
692
693 return 0;
694}
695
696int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
697 unsigned long address, uint32_t num_pages,
698 uint32_t desired_tile_stride, uint32_t hw_tile_stride,
699 int type)
700{
701 struct psb_mmu_pt *pt;
702 uint32_t rows = 1;
703 uint32_t i;
704 uint32_t pte;
705 unsigned long addr;
706 unsigned long end;
707 unsigned long next;
708 unsigned long add;
709 unsigned long row_add;
710 unsigned long f_address = address;
711 int ret = -ENOMEM;
712
713 if (hw_tile_stride) {
714 if (num_pages % desired_tile_stride != 0)
715 return -EINVAL;
716 rows = num_pages / desired_tile_stride;
717 } else {
718 desired_tile_stride = num_pages;
719 }
720
721 add = desired_tile_stride << PAGE_SHIFT;
722 row_add = hw_tile_stride << PAGE_SHIFT;
723
724 down_read(&pd->driver->sem);
725
726 for (i = 0; i < rows; ++i) {
727
728 addr = address;
729 end = addr + add;
730
731 do {
732 next = psb_pd_addr_end(addr, end);
733 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
734 if (!pt)
735 goto out;
736 do {
737 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
738 type);
739 psb_mmu_set_pte(pt, addr, pte);
740 pt->count++;
741 } while (addr += PAGE_SIZE, addr < next);
742 psb_mmu_pt_unmap_unlock(pt);
743
744 } while (addr = next, next != end);
745
746 address += row_add;
747 }
748
749 ret = 0;
750out:
751 if (pd->hw_context != -1)
752 psb_mmu_flush_ptes(pd, f_address, num_pages,
753 desired_tile_stride, hw_tile_stride);
754
755 up_read(&pd->driver->sem);
756
757 if (pd->hw_context != -1)
758 psb_mmu_flush(pd->driver);
759
760 return ret;
761}
762
763int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
764 unsigned long *pfn)
765{
766 int ret;
767 struct psb_mmu_pt *pt;
768 uint32_t tmp;
769 spinlock_t *lock = &pd->driver->lock;
770
771 down_read(&pd->driver->sem);
772 pt = psb_mmu_pt_map_lock(pd, virtual);
773 if (!pt) {
774 uint32_t *v;
775
776 spin_lock(lock);
777 v = kmap_atomic(pd->p);
778 tmp = v[psb_mmu_pd_index(virtual)];
779 kunmap_atomic(v);
780 spin_unlock(lock);
781
782 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
783 !(pd->invalid_pte & PSB_PTE_VALID)) {
784 ret = -EINVAL;
785 goto out;
786 }
787 ret = 0;
788 *pfn = pd->invalid_pte >> PAGE_SHIFT;
789 goto out;
790 }
791 tmp = pt->v[psb_mmu_pt_index(virtual)];
792 if (!(tmp & PSB_PTE_VALID)) {
793 ret = -EINVAL;
794 } else {
795 ret = 0;
796 *pfn = tmp >> PAGE_SHIFT;
797 }
798 psb_mmu_pt_unmap_unlock(pt);
799out:
800 up_read(&pd->driver->sem);
801 return ret;
802}
1// SPDX-License-Identifier: GPL-2.0-only
2/**************************************************************************
3 * Copyright (c) 2007, Intel Corporation.
4 *
5 **************************************************************************/
6
7#include <linux/highmem.h>
8
9#include "mmu.h"
10#include "psb_drv.h"
11#include "psb_reg.h"
12
13/*
14 * Code for the SGX MMU:
15 */
16
17/*
18 * clflush on one processor only:
19 * clflush should apparently flush the cache line on all processors in an
20 * SMP system.
21 */
22
23/*
24 * kmap atomic:
25 * The usage of the slots must be completely encapsulated within a spinlock, and
26 * no other functions that may be using the locks for other purposed may be
27 * called from within the locked region.
28 * Since the slots are per processor, this will guarantee that we are the only
29 * user.
30 */
31
32/*
33 * TODO: Inserting ptes from an interrupt handler:
34 * This may be desirable for some SGX functionality where the GPU can fault in
35 * needed pages. For that, we need to make an atomic insert_pages function, that
36 * may fail.
37 * If it fails, the caller need to insert the page using a workqueue function,
38 * but on average it should be fast.
39 */
40
41static inline uint32_t psb_mmu_pt_index(uint32_t offset)
42{
43 return (offset >> PSB_PTE_SHIFT) & 0x3FF;
44}
45
46static inline uint32_t psb_mmu_pd_index(uint32_t offset)
47{
48 return offset >> PSB_PDE_SHIFT;
49}
50
51static inline void psb_clflush(void *addr)
52{
53 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
54}
55
56static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
57{
58 if (!driver->has_clflush)
59 return;
60
61 mb();
62 psb_clflush(addr);
63 mb();
64}
65
66static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
67{
68 struct drm_device *dev = driver->dev;
69 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
70
71 if (atomic_read(&driver->needs_tlbflush) || force) {
72 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
73 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
74
75 /* Make sure data cache is turned off before enabling it */
76 wmb();
77 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
78 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
79 if (driver->msvdx_mmu_invaldc)
80 atomic_set(driver->msvdx_mmu_invaldc, 1);
81 }
82 atomic_set(&driver->needs_tlbflush, 0);
83}
84
85#if 0
86static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
87{
88 down_write(&driver->sem);
89 psb_mmu_flush_pd_locked(driver, force);
90 up_write(&driver->sem);
91}
92#endif
93
94void psb_mmu_flush(struct psb_mmu_driver *driver)
95{
96 struct drm_device *dev = driver->dev;
97 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
98 uint32_t val;
99
100 down_write(&driver->sem);
101 val = PSB_RSGX32(PSB_CR_BIF_CTRL);
102 if (atomic_read(&driver->needs_tlbflush))
103 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
104 else
105 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
106
107 /* Make sure data cache is turned off and MMU is flushed before
108 restoring bank interface control register */
109 wmb();
110 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
111 PSB_CR_BIF_CTRL);
112 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
113
114 atomic_set(&driver->needs_tlbflush, 0);
115 if (driver->msvdx_mmu_invaldc)
116 atomic_set(driver->msvdx_mmu_invaldc, 1);
117 up_write(&driver->sem);
118}
119
120void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
121{
122 struct drm_device *dev = pd->driver->dev;
123 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
124 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
125 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
126
127 down_write(&pd->driver->sem);
128 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
129 wmb();
130 psb_mmu_flush_pd_locked(pd->driver, 1);
131 pd->hw_context = hw_context;
132 up_write(&pd->driver->sem);
133
134}
135
136static inline unsigned long psb_pd_addr_end(unsigned long addr,
137 unsigned long end)
138{
139 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
140 return (addr < end) ? addr : end;
141}
142
143static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
144{
145 uint32_t mask = PSB_PTE_VALID;
146
147 if (type & PSB_MMU_CACHED_MEMORY)
148 mask |= PSB_PTE_CACHED;
149 if (type & PSB_MMU_RO_MEMORY)
150 mask |= PSB_PTE_RO;
151 if (type & PSB_MMU_WO_MEMORY)
152 mask |= PSB_PTE_WO;
153
154 return (pfn << PAGE_SHIFT) | mask;
155}
156
157struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
158 int trap_pagefaults, int invalid_type)
159{
160 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
161 uint32_t *v;
162 int i;
163
164 if (!pd)
165 return NULL;
166
167 pd->p = alloc_page(GFP_DMA32);
168 if (!pd->p)
169 goto out_err1;
170 pd->dummy_pt = alloc_page(GFP_DMA32);
171 if (!pd->dummy_pt)
172 goto out_err2;
173 pd->dummy_page = alloc_page(GFP_DMA32);
174 if (!pd->dummy_page)
175 goto out_err3;
176
177 if (!trap_pagefaults) {
178 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
179 invalid_type);
180 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
181 invalid_type);
182 } else {
183 pd->invalid_pde = 0;
184 pd->invalid_pte = 0;
185 }
186
187 v = kmap_local_page(pd->dummy_pt);
188 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
189 v[i] = pd->invalid_pte;
190
191 kunmap_local(v);
192
193 v = kmap_local_page(pd->p);
194 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
195 v[i] = pd->invalid_pde;
196
197 kunmap_local(v);
198
199 clear_page(kmap(pd->dummy_page));
200 kunmap(pd->dummy_page);
201
202 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
203 if (!pd->tables)
204 goto out_err4;
205
206 pd->hw_context = -1;
207 pd->pd_mask = PSB_PTE_VALID;
208 pd->driver = driver;
209
210 return pd;
211
212out_err4:
213 __free_page(pd->dummy_page);
214out_err3:
215 __free_page(pd->dummy_pt);
216out_err2:
217 __free_page(pd->p);
218out_err1:
219 kfree(pd);
220 return NULL;
221}
222
223static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
224{
225 __free_page(pt->p);
226 kfree(pt);
227}
228
229void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
230{
231 struct psb_mmu_driver *driver = pd->driver;
232 struct drm_device *dev = driver->dev;
233 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
234 struct psb_mmu_pt *pt;
235 int i;
236
237 down_write(&driver->sem);
238 if (pd->hw_context != -1) {
239 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
240 psb_mmu_flush_pd_locked(driver, 1);
241 }
242
243 /* Should take the spinlock here, but we don't need to do that
244 since we have the semaphore in write mode. */
245
246 for (i = 0; i < 1024; ++i) {
247 pt = pd->tables[i];
248 if (pt)
249 psb_mmu_free_pt(pt);
250 }
251
252 vfree(pd->tables);
253 __free_page(pd->dummy_page);
254 __free_page(pd->dummy_pt);
255 __free_page(pd->p);
256 kfree(pd);
257 up_write(&driver->sem);
258}
259
260static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
261{
262 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
263 void *v;
264 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
265 uint32_t clflush_count = PAGE_SIZE / clflush_add;
266 spinlock_t *lock = &pd->driver->lock;
267 uint8_t *clf;
268 uint32_t *ptes;
269 int i;
270
271 if (!pt)
272 return NULL;
273
274 pt->p = alloc_page(GFP_DMA32);
275 if (!pt->p) {
276 kfree(pt);
277 return NULL;
278 }
279
280 spin_lock(lock);
281
282 v = kmap_atomic(pt->p);
283 clf = (uint8_t *) v;
284 ptes = (uint32_t *) v;
285 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
286 *ptes++ = pd->invalid_pte;
287
288 if (pd->driver->has_clflush && pd->hw_context != -1) {
289 mb();
290 for (i = 0; i < clflush_count; ++i) {
291 psb_clflush(clf);
292 clf += clflush_add;
293 }
294 mb();
295 }
296 kunmap_atomic(v);
297 spin_unlock(lock);
298
299 pt->count = 0;
300 pt->pd = pd;
301 pt->index = 0;
302
303 return pt;
304}
305
306static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
307 unsigned long addr)
308{
309 uint32_t index = psb_mmu_pd_index(addr);
310 struct psb_mmu_pt *pt;
311 uint32_t *v;
312 spinlock_t *lock = &pd->driver->lock;
313
314 spin_lock(lock);
315 pt = pd->tables[index];
316 while (!pt) {
317 spin_unlock(lock);
318 pt = psb_mmu_alloc_pt(pd);
319 if (!pt)
320 return NULL;
321 spin_lock(lock);
322
323 if (pd->tables[index]) {
324 spin_unlock(lock);
325 psb_mmu_free_pt(pt);
326 spin_lock(lock);
327 pt = pd->tables[index];
328 continue;
329 }
330
331 v = kmap_atomic(pd->p);
332 pd->tables[index] = pt;
333 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
334 pt->index = index;
335 kunmap_atomic((void *) v);
336
337 if (pd->hw_context != -1) {
338 psb_mmu_clflush(pd->driver, (void *)&v[index]);
339 atomic_set(&pd->driver->needs_tlbflush, 1);
340 }
341 }
342 pt->v = kmap_atomic(pt->p);
343 return pt;
344}
345
346static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
347 unsigned long addr)
348{
349 uint32_t index = psb_mmu_pd_index(addr);
350 struct psb_mmu_pt *pt;
351 spinlock_t *lock = &pd->driver->lock;
352
353 spin_lock(lock);
354 pt = pd->tables[index];
355 if (!pt) {
356 spin_unlock(lock);
357 return NULL;
358 }
359 pt->v = kmap_atomic(pt->p);
360 return pt;
361}
362
363static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
364{
365 struct psb_mmu_pd *pd = pt->pd;
366 uint32_t *v;
367
368 kunmap_atomic(pt->v);
369 if (pt->count == 0) {
370 v = kmap_atomic(pd->p);
371 v[pt->index] = pd->invalid_pde;
372 pd->tables[pt->index] = NULL;
373
374 if (pd->hw_context != -1) {
375 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
376 atomic_set(&pd->driver->needs_tlbflush, 1);
377 }
378 kunmap_atomic(v);
379 spin_unlock(&pd->driver->lock);
380 psb_mmu_free_pt(pt);
381 return;
382 }
383 spin_unlock(&pd->driver->lock);
384}
385
386static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
387 uint32_t pte)
388{
389 pt->v[psb_mmu_pt_index(addr)] = pte;
390}
391
392static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
393 unsigned long addr)
394{
395 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
396}
397
398struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
399{
400 struct psb_mmu_pd *pd;
401
402 down_read(&driver->sem);
403 pd = driver->default_pd;
404 up_read(&driver->sem);
405
406 return pd;
407}
408
409void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
410{
411 struct drm_device *dev = driver->dev;
412 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
413
414 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
415 psb_mmu_free_pagedir(driver->default_pd);
416 kfree(driver);
417}
418
419struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
420 int trap_pagefaults,
421 int invalid_type,
422 atomic_t *msvdx_mmu_invaldc)
423{
424 struct psb_mmu_driver *driver;
425 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
426
427 driver = kmalloc(sizeof(*driver), GFP_KERNEL);
428
429 if (!driver)
430 return NULL;
431
432 driver->dev = dev;
433 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
434 invalid_type);
435 if (!driver->default_pd)
436 goto out_err1;
437
438 spin_lock_init(&driver->lock);
439 init_rwsem(&driver->sem);
440 down_write(&driver->sem);
441 atomic_set(&driver->needs_tlbflush, 1);
442 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
443
444 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
445 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
446 PSB_CR_BIF_CTRL);
447 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
448 PSB_CR_BIF_CTRL);
449
450 driver->has_clflush = 0;
451
452 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
453 uint32_t tfms, misc, cap0, cap4, clflush_size;
454
455 /*
456 * clflush size is determined at kernel setup for x86_64 but not
457 * for i386. We have to do it here.
458 */
459
460 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
461 clflush_size = ((misc >> 8) & 0xff) * 8;
462 driver->has_clflush = 1;
463 driver->clflush_add =
464 PAGE_SIZE * clflush_size / sizeof(uint32_t);
465 driver->clflush_mask = driver->clflush_add - 1;
466 driver->clflush_mask = ~driver->clflush_mask;
467 }
468
469 up_write(&driver->sem);
470 return driver;
471
472out_err1:
473 kfree(driver);
474 return NULL;
475}
476
477static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
478 uint32_t num_pages, uint32_t desired_tile_stride,
479 uint32_t hw_tile_stride)
480{
481 struct psb_mmu_pt *pt;
482 uint32_t rows = 1;
483 uint32_t i;
484 unsigned long addr;
485 unsigned long end;
486 unsigned long next;
487 unsigned long add;
488 unsigned long row_add;
489 unsigned long clflush_add = pd->driver->clflush_add;
490 unsigned long clflush_mask = pd->driver->clflush_mask;
491
492 if (!pd->driver->has_clflush)
493 return;
494
495 if (hw_tile_stride)
496 rows = num_pages / desired_tile_stride;
497 else
498 desired_tile_stride = num_pages;
499
500 add = desired_tile_stride << PAGE_SHIFT;
501 row_add = hw_tile_stride << PAGE_SHIFT;
502 mb();
503 for (i = 0; i < rows; ++i) {
504
505 addr = address;
506 end = addr + add;
507
508 do {
509 next = psb_pd_addr_end(addr, end);
510 pt = psb_mmu_pt_map_lock(pd, addr);
511 if (!pt)
512 continue;
513 do {
514 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
515 } while (addr += clflush_add,
516 (addr & clflush_mask) < next);
517
518 psb_mmu_pt_unmap_unlock(pt);
519 } while (addr = next, next != end);
520 address += row_add;
521 }
522 mb();
523}
524
525void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
526 unsigned long address, uint32_t num_pages)
527{
528 struct psb_mmu_pt *pt;
529 unsigned long addr;
530 unsigned long end;
531 unsigned long next;
532 unsigned long f_address = address;
533
534 down_read(&pd->driver->sem);
535
536 addr = address;
537 end = addr + (num_pages << PAGE_SHIFT);
538
539 do {
540 next = psb_pd_addr_end(addr, end);
541 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
542 if (!pt)
543 goto out;
544 do {
545 psb_mmu_invalidate_pte(pt, addr);
546 --pt->count;
547 } while (addr += PAGE_SIZE, addr < next);
548 psb_mmu_pt_unmap_unlock(pt);
549
550 } while (addr = next, next != end);
551
552out:
553 if (pd->hw_context != -1)
554 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
555
556 up_read(&pd->driver->sem);
557
558 if (pd->hw_context != -1)
559 psb_mmu_flush(pd->driver);
560
561 return;
562}
563
564void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
565 uint32_t num_pages, uint32_t desired_tile_stride,
566 uint32_t hw_tile_stride)
567{
568 struct psb_mmu_pt *pt;
569 uint32_t rows = 1;
570 uint32_t i;
571 unsigned long addr;
572 unsigned long end;
573 unsigned long next;
574 unsigned long add;
575 unsigned long row_add;
576 unsigned long f_address = address;
577
578 if (hw_tile_stride)
579 rows = num_pages / desired_tile_stride;
580 else
581 desired_tile_stride = num_pages;
582
583 add = desired_tile_stride << PAGE_SHIFT;
584 row_add = hw_tile_stride << PAGE_SHIFT;
585
586 down_read(&pd->driver->sem);
587
588 /* Make sure we only need to flush this processor's cache */
589
590 for (i = 0; i < rows; ++i) {
591
592 addr = address;
593 end = addr + add;
594
595 do {
596 next = psb_pd_addr_end(addr, end);
597 pt = psb_mmu_pt_map_lock(pd, addr);
598 if (!pt)
599 continue;
600 do {
601 psb_mmu_invalidate_pte(pt, addr);
602 --pt->count;
603
604 } while (addr += PAGE_SIZE, addr < next);
605 psb_mmu_pt_unmap_unlock(pt);
606
607 } while (addr = next, next != end);
608 address += row_add;
609 }
610 if (pd->hw_context != -1)
611 psb_mmu_flush_ptes(pd, f_address, num_pages,
612 desired_tile_stride, hw_tile_stride);
613
614 up_read(&pd->driver->sem);
615
616 if (pd->hw_context != -1)
617 psb_mmu_flush(pd->driver);
618}
619
620int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
621 unsigned long address, uint32_t num_pages,
622 int type)
623{
624 struct psb_mmu_pt *pt;
625 uint32_t pte;
626 unsigned long addr;
627 unsigned long end;
628 unsigned long next;
629 unsigned long f_address = address;
630 int ret = -ENOMEM;
631
632 down_read(&pd->driver->sem);
633
634 addr = address;
635 end = addr + (num_pages << PAGE_SHIFT);
636
637 do {
638 next = psb_pd_addr_end(addr, end);
639 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
640 if (!pt) {
641 ret = -ENOMEM;
642 goto out;
643 }
644 do {
645 pte = psb_mmu_mask_pte(start_pfn++, type);
646 psb_mmu_set_pte(pt, addr, pte);
647 pt->count++;
648 } while (addr += PAGE_SIZE, addr < next);
649 psb_mmu_pt_unmap_unlock(pt);
650
651 } while (addr = next, next != end);
652 ret = 0;
653
654out:
655 if (pd->hw_context != -1)
656 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
657
658 up_read(&pd->driver->sem);
659
660 if (pd->hw_context != -1)
661 psb_mmu_flush(pd->driver);
662
663 return ret;
664}
665
666int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
667 unsigned long address, uint32_t num_pages,
668 uint32_t desired_tile_stride, uint32_t hw_tile_stride,
669 int type)
670{
671 struct psb_mmu_pt *pt;
672 uint32_t rows = 1;
673 uint32_t i;
674 uint32_t pte;
675 unsigned long addr;
676 unsigned long end;
677 unsigned long next;
678 unsigned long add;
679 unsigned long row_add;
680 unsigned long f_address = address;
681 int ret = -ENOMEM;
682
683 if (hw_tile_stride) {
684 if (num_pages % desired_tile_stride != 0)
685 return -EINVAL;
686 rows = num_pages / desired_tile_stride;
687 } else {
688 desired_tile_stride = num_pages;
689 }
690
691 add = desired_tile_stride << PAGE_SHIFT;
692 row_add = hw_tile_stride << PAGE_SHIFT;
693
694 down_read(&pd->driver->sem);
695
696 for (i = 0; i < rows; ++i) {
697
698 addr = address;
699 end = addr + add;
700
701 do {
702 next = psb_pd_addr_end(addr, end);
703 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
704 if (!pt)
705 goto out;
706 do {
707 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
708 type);
709 psb_mmu_set_pte(pt, addr, pte);
710 pt->count++;
711 } while (addr += PAGE_SIZE, addr < next);
712 psb_mmu_pt_unmap_unlock(pt);
713
714 } while (addr = next, next != end);
715
716 address += row_add;
717 }
718
719 ret = 0;
720out:
721 if (pd->hw_context != -1)
722 psb_mmu_flush_ptes(pd, f_address, num_pages,
723 desired_tile_stride, hw_tile_stride);
724
725 up_read(&pd->driver->sem);
726
727 if (pd->hw_context != -1)
728 psb_mmu_flush(pd->driver);
729
730 return ret;
731}
732
733int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
734 unsigned long *pfn)
735{
736 int ret;
737 struct psb_mmu_pt *pt;
738 uint32_t tmp;
739 spinlock_t *lock = &pd->driver->lock;
740
741 down_read(&pd->driver->sem);
742 pt = psb_mmu_pt_map_lock(pd, virtual);
743 if (!pt) {
744 uint32_t *v;
745
746 spin_lock(lock);
747 v = kmap_atomic(pd->p);
748 tmp = v[psb_mmu_pd_index(virtual)];
749 kunmap_atomic(v);
750 spin_unlock(lock);
751
752 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
753 !(pd->invalid_pte & PSB_PTE_VALID)) {
754 ret = -EINVAL;
755 goto out;
756 }
757 ret = 0;
758 *pfn = pd->invalid_pte >> PAGE_SHIFT;
759 goto out;
760 }
761 tmp = pt->v[psb_mmu_pt_index(virtual)];
762 if (!(tmp & PSB_PTE_VALID)) {
763 ret = -EINVAL;
764 } else {
765 ret = 0;
766 *pfn = tmp >> PAGE_SHIFT;
767 }
768 psb_mmu_pt_unmap_unlock(pt);
769out:
770 up_read(&pd->driver->sem);
771 return ret;
772}