Loading...
1/**************************************************************************
2 * Copyright (c) 2007, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
16 *
17 **************************************************************************/
18#include <drm/drmP.h>
19#include "psb_drv.h"
20#include "psb_reg.h"
21#include "mmu.h"
22
23/*
24 * Code for the SGX MMU:
25 */
26
27/*
28 * clflush on one processor only:
29 * clflush should apparently flush the cache line on all processors in an
30 * SMP system.
31 */
32
33/*
34 * kmap atomic:
35 * The usage of the slots must be completely encapsulated within a spinlock, and
36 * no other functions that may be using the locks for other purposed may be
37 * called from within the locked region.
38 * Since the slots are per processor, this will guarantee that we are the only
39 * user.
40 */
41
42/*
43 * TODO: Inserting ptes from an interrupt handler:
44 * This may be desirable for some SGX functionality where the GPU can fault in
45 * needed pages. For that, we need to make an atomic insert_pages function, that
46 * may fail.
47 * If it fails, the caller need to insert the page using a workqueue function,
48 * but on average it should be fast.
49 */
50
51static inline uint32_t psb_mmu_pt_index(uint32_t offset)
52{
53 return (offset >> PSB_PTE_SHIFT) & 0x3FF;
54}
55
56static inline uint32_t psb_mmu_pd_index(uint32_t offset)
57{
58 return offset >> PSB_PDE_SHIFT;
59}
60
61#if defined(CONFIG_X86)
62static inline void psb_clflush(void *addr)
63{
64 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
65}
66
67static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
68{
69 if (!driver->has_clflush)
70 return;
71
72 mb();
73 psb_clflush(addr);
74 mb();
75}
76#else
77
78static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
79{;
80}
81
82#endif
83
84static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
85{
86 struct drm_device *dev = driver->dev;
87 struct drm_psb_private *dev_priv = dev->dev_private;
88
89 if (atomic_read(&driver->needs_tlbflush) || force) {
90 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
91 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
92
93 /* Make sure data cache is turned off before enabling it */
94 wmb();
95 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
96 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
97 if (driver->msvdx_mmu_invaldc)
98 atomic_set(driver->msvdx_mmu_invaldc, 1);
99 }
100 atomic_set(&driver->needs_tlbflush, 0);
101}
102
103#if 0
104static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
105{
106 down_write(&driver->sem);
107 psb_mmu_flush_pd_locked(driver, force);
108 up_write(&driver->sem);
109}
110#endif
111
112void psb_mmu_flush(struct psb_mmu_driver *driver)
113{
114 struct drm_device *dev = driver->dev;
115 struct drm_psb_private *dev_priv = dev->dev_private;
116 uint32_t val;
117
118 down_write(&driver->sem);
119 val = PSB_RSGX32(PSB_CR_BIF_CTRL);
120 if (atomic_read(&driver->needs_tlbflush))
121 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
122 else
123 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
124
125 /* Make sure data cache is turned off and MMU is flushed before
126 restoring bank interface control register */
127 wmb();
128 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
129 PSB_CR_BIF_CTRL);
130 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
131
132 atomic_set(&driver->needs_tlbflush, 0);
133 if (driver->msvdx_mmu_invaldc)
134 atomic_set(driver->msvdx_mmu_invaldc, 1);
135 up_write(&driver->sem);
136}
137
138void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
139{
140 struct drm_device *dev = pd->driver->dev;
141 struct drm_psb_private *dev_priv = dev->dev_private;
142 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
143 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
144
145 down_write(&pd->driver->sem);
146 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
147 wmb();
148 psb_mmu_flush_pd_locked(pd->driver, 1);
149 pd->hw_context = hw_context;
150 up_write(&pd->driver->sem);
151
152}
153
154static inline unsigned long psb_pd_addr_end(unsigned long addr,
155 unsigned long end)
156{
157 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
158 return (addr < end) ? addr : end;
159}
160
161static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
162{
163 uint32_t mask = PSB_PTE_VALID;
164
165 if (type & PSB_MMU_CACHED_MEMORY)
166 mask |= PSB_PTE_CACHED;
167 if (type & PSB_MMU_RO_MEMORY)
168 mask |= PSB_PTE_RO;
169 if (type & PSB_MMU_WO_MEMORY)
170 mask |= PSB_PTE_WO;
171
172 return (pfn << PAGE_SHIFT) | mask;
173}
174
175struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
176 int trap_pagefaults, int invalid_type)
177{
178 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
179 uint32_t *v;
180 int i;
181
182 if (!pd)
183 return NULL;
184
185 pd->p = alloc_page(GFP_DMA32);
186 if (!pd->p)
187 goto out_err1;
188 pd->dummy_pt = alloc_page(GFP_DMA32);
189 if (!pd->dummy_pt)
190 goto out_err2;
191 pd->dummy_page = alloc_page(GFP_DMA32);
192 if (!pd->dummy_page)
193 goto out_err3;
194
195 if (!trap_pagefaults) {
196 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
197 invalid_type);
198 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
199 invalid_type);
200 } else {
201 pd->invalid_pde = 0;
202 pd->invalid_pte = 0;
203 }
204
205 v = kmap(pd->dummy_pt);
206 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
207 v[i] = pd->invalid_pte;
208
209 kunmap(pd->dummy_pt);
210
211 v = kmap(pd->p);
212 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
213 v[i] = pd->invalid_pde;
214
215 kunmap(pd->p);
216
217 clear_page(kmap(pd->dummy_page));
218 kunmap(pd->dummy_page);
219
220 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
221 if (!pd->tables)
222 goto out_err4;
223
224 pd->hw_context = -1;
225 pd->pd_mask = PSB_PTE_VALID;
226 pd->driver = driver;
227
228 return pd;
229
230out_err4:
231 __free_page(pd->dummy_page);
232out_err3:
233 __free_page(pd->dummy_pt);
234out_err2:
235 __free_page(pd->p);
236out_err1:
237 kfree(pd);
238 return NULL;
239}
240
241static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
242{
243 __free_page(pt->p);
244 kfree(pt);
245}
246
247void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
248{
249 struct psb_mmu_driver *driver = pd->driver;
250 struct drm_device *dev = driver->dev;
251 struct drm_psb_private *dev_priv = dev->dev_private;
252 struct psb_mmu_pt *pt;
253 int i;
254
255 down_write(&driver->sem);
256 if (pd->hw_context != -1) {
257 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
258 psb_mmu_flush_pd_locked(driver, 1);
259 }
260
261 /* Should take the spinlock here, but we don't need to do that
262 since we have the semaphore in write mode. */
263
264 for (i = 0; i < 1024; ++i) {
265 pt = pd->tables[i];
266 if (pt)
267 psb_mmu_free_pt(pt);
268 }
269
270 vfree(pd->tables);
271 __free_page(pd->dummy_page);
272 __free_page(pd->dummy_pt);
273 __free_page(pd->p);
274 kfree(pd);
275 up_write(&driver->sem);
276}
277
278static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
279{
280 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
281 void *v;
282 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
283 uint32_t clflush_count = PAGE_SIZE / clflush_add;
284 spinlock_t *lock = &pd->driver->lock;
285 uint8_t *clf;
286 uint32_t *ptes;
287 int i;
288
289 if (!pt)
290 return NULL;
291
292 pt->p = alloc_page(GFP_DMA32);
293 if (!pt->p) {
294 kfree(pt);
295 return NULL;
296 }
297
298 spin_lock(lock);
299
300 v = kmap_atomic(pt->p);
301 clf = (uint8_t *) v;
302 ptes = (uint32_t *) v;
303 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
304 *ptes++ = pd->invalid_pte;
305
306#if defined(CONFIG_X86)
307 if (pd->driver->has_clflush && pd->hw_context != -1) {
308 mb();
309 for (i = 0; i < clflush_count; ++i) {
310 psb_clflush(clf);
311 clf += clflush_add;
312 }
313 mb();
314 }
315#endif
316 kunmap_atomic(v);
317 spin_unlock(lock);
318
319 pt->count = 0;
320 pt->pd = pd;
321 pt->index = 0;
322
323 return pt;
324}
325
326struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
327 unsigned long addr)
328{
329 uint32_t index = psb_mmu_pd_index(addr);
330 struct psb_mmu_pt *pt;
331 uint32_t *v;
332 spinlock_t *lock = &pd->driver->lock;
333
334 spin_lock(lock);
335 pt = pd->tables[index];
336 while (!pt) {
337 spin_unlock(lock);
338 pt = psb_mmu_alloc_pt(pd);
339 if (!pt)
340 return NULL;
341 spin_lock(lock);
342
343 if (pd->tables[index]) {
344 spin_unlock(lock);
345 psb_mmu_free_pt(pt);
346 spin_lock(lock);
347 pt = pd->tables[index];
348 continue;
349 }
350
351 v = kmap_atomic(pd->p);
352 pd->tables[index] = pt;
353 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
354 pt->index = index;
355 kunmap_atomic((void *) v);
356
357 if (pd->hw_context != -1) {
358 psb_mmu_clflush(pd->driver, (void *)&v[index]);
359 atomic_set(&pd->driver->needs_tlbflush, 1);
360 }
361 }
362 pt->v = kmap_atomic(pt->p);
363 return pt;
364}
365
366static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
367 unsigned long addr)
368{
369 uint32_t index = psb_mmu_pd_index(addr);
370 struct psb_mmu_pt *pt;
371 spinlock_t *lock = &pd->driver->lock;
372
373 spin_lock(lock);
374 pt = pd->tables[index];
375 if (!pt) {
376 spin_unlock(lock);
377 return NULL;
378 }
379 pt->v = kmap_atomic(pt->p);
380 return pt;
381}
382
383static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
384{
385 struct psb_mmu_pd *pd = pt->pd;
386 uint32_t *v;
387
388 kunmap_atomic(pt->v);
389 if (pt->count == 0) {
390 v = kmap_atomic(pd->p);
391 v[pt->index] = pd->invalid_pde;
392 pd->tables[pt->index] = NULL;
393
394 if (pd->hw_context != -1) {
395 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
396 atomic_set(&pd->driver->needs_tlbflush, 1);
397 }
398 kunmap_atomic(pt->v);
399 spin_unlock(&pd->driver->lock);
400 psb_mmu_free_pt(pt);
401 return;
402 }
403 spin_unlock(&pd->driver->lock);
404}
405
406static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
407 uint32_t pte)
408{
409 pt->v[psb_mmu_pt_index(addr)] = pte;
410}
411
412static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
413 unsigned long addr)
414{
415 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
416}
417
418struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
419{
420 struct psb_mmu_pd *pd;
421
422 down_read(&driver->sem);
423 pd = driver->default_pd;
424 up_read(&driver->sem);
425
426 return pd;
427}
428
429/* Returns the physical address of the PD shared by sgx/msvdx */
430uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)
431{
432 struct psb_mmu_pd *pd;
433
434 pd = psb_mmu_get_default_pd(driver);
435 return page_to_pfn(pd->p) << PAGE_SHIFT;
436}
437
438void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
439{
440 struct drm_device *dev = driver->dev;
441 struct drm_psb_private *dev_priv = dev->dev_private;
442
443 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
444 psb_mmu_free_pagedir(driver->default_pd);
445 kfree(driver);
446}
447
448struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
449 int trap_pagefaults,
450 int invalid_type,
451 atomic_t *msvdx_mmu_invaldc)
452{
453 struct psb_mmu_driver *driver;
454 struct drm_psb_private *dev_priv = dev->dev_private;
455
456 driver = kmalloc(sizeof(*driver), GFP_KERNEL);
457
458 if (!driver)
459 return NULL;
460
461 driver->dev = dev;
462 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
463 invalid_type);
464 if (!driver->default_pd)
465 goto out_err1;
466
467 spin_lock_init(&driver->lock);
468 init_rwsem(&driver->sem);
469 down_write(&driver->sem);
470 atomic_set(&driver->needs_tlbflush, 1);
471 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
472
473 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
474 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
475 PSB_CR_BIF_CTRL);
476 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
477 PSB_CR_BIF_CTRL);
478
479 driver->has_clflush = 0;
480
481#if defined(CONFIG_X86)
482 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
483 uint32_t tfms, misc, cap0, cap4, clflush_size;
484
485 /*
486 * clflush size is determined at kernel setup for x86_64 but not
487 * for i386. We have to do it here.
488 */
489
490 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
491 clflush_size = ((misc >> 8) & 0xff) * 8;
492 driver->has_clflush = 1;
493 driver->clflush_add =
494 PAGE_SIZE * clflush_size / sizeof(uint32_t);
495 driver->clflush_mask = driver->clflush_add - 1;
496 driver->clflush_mask = ~driver->clflush_mask;
497 }
498#endif
499
500 up_write(&driver->sem);
501 return driver;
502
503out_err1:
504 kfree(driver);
505 return NULL;
506}
507
508#if defined(CONFIG_X86)
509static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
510 uint32_t num_pages, uint32_t desired_tile_stride,
511 uint32_t hw_tile_stride)
512{
513 struct psb_mmu_pt *pt;
514 uint32_t rows = 1;
515 uint32_t i;
516 unsigned long addr;
517 unsigned long end;
518 unsigned long next;
519 unsigned long add;
520 unsigned long row_add;
521 unsigned long clflush_add = pd->driver->clflush_add;
522 unsigned long clflush_mask = pd->driver->clflush_mask;
523
524 if (!pd->driver->has_clflush)
525 return;
526
527 if (hw_tile_stride)
528 rows = num_pages / desired_tile_stride;
529 else
530 desired_tile_stride = num_pages;
531
532 add = desired_tile_stride << PAGE_SHIFT;
533 row_add = hw_tile_stride << PAGE_SHIFT;
534 mb();
535 for (i = 0; i < rows; ++i) {
536
537 addr = address;
538 end = addr + add;
539
540 do {
541 next = psb_pd_addr_end(addr, end);
542 pt = psb_mmu_pt_map_lock(pd, addr);
543 if (!pt)
544 continue;
545 do {
546 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
547 } while (addr += clflush_add,
548 (addr & clflush_mask) < next);
549
550 psb_mmu_pt_unmap_unlock(pt);
551 } while (addr = next, next != end);
552 address += row_add;
553 }
554 mb();
555}
556#else
557static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
558 uint32_t num_pages, uint32_t desired_tile_stride,
559 uint32_t hw_tile_stride)
560{
561 drm_ttm_cache_flush();
562}
563#endif
564
565void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
566 unsigned long address, uint32_t num_pages)
567{
568 struct psb_mmu_pt *pt;
569 unsigned long addr;
570 unsigned long end;
571 unsigned long next;
572 unsigned long f_address = address;
573
574 down_read(&pd->driver->sem);
575
576 addr = address;
577 end = addr + (num_pages << PAGE_SHIFT);
578
579 do {
580 next = psb_pd_addr_end(addr, end);
581 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
582 if (!pt)
583 goto out;
584 do {
585 psb_mmu_invalidate_pte(pt, addr);
586 --pt->count;
587 } while (addr += PAGE_SIZE, addr < next);
588 psb_mmu_pt_unmap_unlock(pt);
589
590 } while (addr = next, next != end);
591
592out:
593 if (pd->hw_context != -1)
594 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
595
596 up_read(&pd->driver->sem);
597
598 if (pd->hw_context != -1)
599 psb_mmu_flush(pd->driver);
600
601 return;
602}
603
604void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
605 uint32_t num_pages, uint32_t desired_tile_stride,
606 uint32_t hw_tile_stride)
607{
608 struct psb_mmu_pt *pt;
609 uint32_t rows = 1;
610 uint32_t i;
611 unsigned long addr;
612 unsigned long end;
613 unsigned long next;
614 unsigned long add;
615 unsigned long row_add;
616 unsigned long f_address = address;
617
618 if (hw_tile_stride)
619 rows = num_pages / desired_tile_stride;
620 else
621 desired_tile_stride = num_pages;
622
623 add = desired_tile_stride << PAGE_SHIFT;
624 row_add = hw_tile_stride << PAGE_SHIFT;
625
626 down_read(&pd->driver->sem);
627
628 /* Make sure we only need to flush this processor's cache */
629
630 for (i = 0; i < rows; ++i) {
631
632 addr = address;
633 end = addr + add;
634
635 do {
636 next = psb_pd_addr_end(addr, end);
637 pt = psb_mmu_pt_map_lock(pd, addr);
638 if (!pt)
639 continue;
640 do {
641 psb_mmu_invalidate_pte(pt, addr);
642 --pt->count;
643
644 } while (addr += PAGE_SIZE, addr < next);
645 psb_mmu_pt_unmap_unlock(pt);
646
647 } while (addr = next, next != end);
648 address += row_add;
649 }
650 if (pd->hw_context != -1)
651 psb_mmu_flush_ptes(pd, f_address, num_pages,
652 desired_tile_stride, hw_tile_stride);
653
654 up_read(&pd->driver->sem);
655
656 if (pd->hw_context != -1)
657 psb_mmu_flush(pd->driver);
658}
659
660int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
661 unsigned long address, uint32_t num_pages,
662 int type)
663{
664 struct psb_mmu_pt *pt;
665 uint32_t pte;
666 unsigned long addr;
667 unsigned long end;
668 unsigned long next;
669 unsigned long f_address = address;
670 int ret = -ENOMEM;
671
672 down_read(&pd->driver->sem);
673
674 addr = address;
675 end = addr + (num_pages << PAGE_SHIFT);
676
677 do {
678 next = psb_pd_addr_end(addr, end);
679 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
680 if (!pt) {
681 ret = -ENOMEM;
682 goto out;
683 }
684 do {
685 pte = psb_mmu_mask_pte(start_pfn++, type);
686 psb_mmu_set_pte(pt, addr, pte);
687 pt->count++;
688 } while (addr += PAGE_SIZE, addr < next);
689 psb_mmu_pt_unmap_unlock(pt);
690
691 } while (addr = next, next != end);
692 ret = 0;
693
694out:
695 if (pd->hw_context != -1)
696 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
697
698 up_read(&pd->driver->sem);
699
700 if (pd->hw_context != -1)
701 psb_mmu_flush(pd->driver);
702
703 return 0;
704}
705
706int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
707 unsigned long address, uint32_t num_pages,
708 uint32_t desired_tile_stride, uint32_t hw_tile_stride,
709 int type)
710{
711 struct psb_mmu_pt *pt;
712 uint32_t rows = 1;
713 uint32_t i;
714 uint32_t pte;
715 unsigned long addr;
716 unsigned long end;
717 unsigned long next;
718 unsigned long add;
719 unsigned long row_add;
720 unsigned long f_address = address;
721 int ret = -ENOMEM;
722
723 if (hw_tile_stride) {
724 if (num_pages % desired_tile_stride != 0)
725 return -EINVAL;
726 rows = num_pages / desired_tile_stride;
727 } else {
728 desired_tile_stride = num_pages;
729 }
730
731 add = desired_tile_stride << PAGE_SHIFT;
732 row_add = hw_tile_stride << PAGE_SHIFT;
733
734 down_read(&pd->driver->sem);
735
736 for (i = 0; i < rows; ++i) {
737
738 addr = address;
739 end = addr + add;
740
741 do {
742 next = psb_pd_addr_end(addr, end);
743 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
744 if (!pt)
745 goto out;
746 do {
747 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
748 type);
749 psb_mmu_set_pte(pt, addr, pte);
750 pt->count++;
751 } while (addr += PAGE_SIZE, addr < next);
752 psb_mmu_pt_unmap_unlock(pt);
753
754 } while (addr = next, next != end);
755
756 address += row_add;
757 }
758
759 ret = 0;
760out:
761 if (pd->hw_context != -1)
762 psb_mmu_flush_ptes(pd, f_address, num_pages,
763 desired_tile_stride, hw_tile_stride);
764
765 up_read(&pd->driver->sem);
766
767 if (pd->hw_context != -1)
768 psb_mmu_flush(pd->driver);
769
770 return ret;
771}
772
773int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
774 unsigned long *pfn)
775{
776 int ret;
777 struct psb_mmu_pt *pt;
778 uint32_t tmp;
779 spinlock_t *lock = &pd->driver->lock;
780
781 down_read(&pd->driver->sem);
782 pt = psb_mmu_pt_map_lock(pd, virtual);
783 if (!pt) {
784 uint32_t *v;
785
786 spin_lock(lock);
787 v = kmap_atomic(pd->p);
788 tmp = v[psb_mmu_pd_index(virtual)];
789 kunmap_atomic(v);
790 spin_unlock(lock);
791
792 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
793 !(pd->invalid_pte & PSB_PTE_VALID)) {
794 ret = -EINVAL;
795 goto out;
796 }
797 ret = 0;
798 *pfn = pd->invalid_pte >> PAGE_SHIFT;
799 goto out;
800 }
801 tmp = pt->v[psb_mmu_pt_index(virtual)];
802 if (!(tmp & PSB_PTE_VALID)) {
803 ret = -EINVAL;
804 } else {
805 ret = 0;
806 *pfn = tmp >> PAGE_SHIFT;
807 }
808 psb_mmu_pt_unmap_unlock(pt);
809out:
810 up_read(&pd->driver->sem);
811 return ret;
812}
1/**************************************************************************
2 * Copyright (c) 2007, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
16 *
17 **************************************************************************/
18#include <drm/drmP.h>
19#include "psb_drv.h"
20#include "psb_reg.h"
21
22/*
23 * Code for the SGX MMU:
24 */
25
26/*
27 * clflush on one processor only:
28 * clflush should apparently flush the cache line on all processors in an
29 * SMP system.
30 */
31
32/*
33 * kmap atomic:
34 * The usage of the slots must be completely encapsulated within a spinlock, and
35 * no other functions that may be using the locks for other purposed may be
36 * called from within the locked region.
37 * Since the slots are per processor, this will guarantee that we are the only
38 * user.
39 */
40
41/*
42 * TODO: Inserting ptes from an interrupt handler:
43 * This may be desirable for some SGX functionality where the GPU can fault in
44 * needed pages. For that, we need to make an atomic insert_pages function, that
45 * may fail.
46 * If it fails, the caller need to insert the page using a workqueue function,
47 * but on average it should be fast.
48 */
49
50struct psb_mmu_driver {
51 /* protects driver- and pd structures. Always take in read mode
52 * before taking the page table spinlock.
53 */
54 struct rw_semaphore sem;
55
56 /* protects page tables, directory tables and pt tables.
57 * and pt structures.
58 */
59 spinlock_t lock;
60
61 atomic_t needs_tlbflush;
62
63 uint8_t __iomem *register_map;
64 struct psb_mmu_pd *default_pd;
65 /*uint32_t bif_ctrl;*/
66 int has_clflush;
67 int clflush_add;
68 unsigned long clflush_mask;
69
70 struct drm_psb_private *dev_priv;
71};
72
73struct psb_mmu_pd;
74
75struct psb_mmu_pt {
76 struct psb_mmu_pd *pd;
77 uint32_t index;
78 uint32_t count;
79 struct page *p;
80 uint32_t *v;
81};
82
83struct psb_mmu_pd {
84 struct psb_mmu_driver *driver;
85 int hw_context;
86 struct psb_mmu_pt **tables;
87 struct page *p;
88 struct page *dummy_pt;
89 struct page *dummy_page;
90 uint32_t pd_mask;
91 uint32_t invalid_pde;
92 uint32_t invalid_pte;
93};
94
95static inline uint32_t psb_mmu_pt_index(uint32_t offset)
96{
97 return (offset >> PSB_PTE_SHIFT) & 0x3FF;
98}
99
100static inline uint32_t psb_mmu_pd_index(uint32_t offset)
101{
102 return offset >> PSB_PDE_SHIFT;
103}
104
105static inline void psb_clflush(void *addr)
106{
107 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
108}
109
110static inline void psb_mmu_clflush(struct psb_mmu_driver *driver,
111 void *addr)
112{
113 if (!driver->has_clflush)
114 return;
115
116 mb();
117 psb_clflush(addr);
118 mb();
119}
120
121static void psb_page_clflush(struct psb_mmu_driver *driver, struct page* page)
122{
123 uint32_t clflush_add = driver->clflush_add >> PAGE_SHIFT;
124 uint32_t clflush_count = PAGE_SIZE / clflush_add;
125 int i;
126 uint8_t *clf;
127
128 clf = kmap_atomic(page);
129 mb();
130 for (i = 0; i < clflush_count; ++i) {
131 psb_clflush(clf);
132 clf += clflush_add;
133 }
134 mb();
135 kunmap_atomic(clf);
136}
137
138static void psb_pages_clflush(struct psb_mmu_driver *driver,
139 struct page *page[], unsigned long num_pages)
140{
141 int i;
142
143 if (!driver->has_clflush)
144 return ;
145
146 for (i = 0; i < num_pages; i++)
147 psb_page_clflush(driver, *page++);
148}
149
150static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver,
151 int force)
152{
153 atomic_set(&driver->needs_tlbflush, 0);
154}
155
156static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
157{
158 down_write(&driver->sem);
159 psb_mmu_flush_pd_locked(driver, force);
160 up_write(&driver->sem);
161}
162
163void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot)
164{
165 if (rc_prot)
166 down_write(&driver->sem);
167 if (rc_prot)
168 up_write(&driver->sem);
169}
170
171void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
172{
173 /*ttm_tt_cache_flush(&pd->p, 1);*/
174 psb_pages_clflush(pd->driver, &pd->p, 1);
175 down_write(&pd->driver->sem);
176 wmb();
177 psb_mmu_flush_pd_locked(pd->driver, 1);
178 pd->hw_context = hw_context;
179 up_write(&pd->driver->sem);
180
181}
182
183static inline unsigned long psb_pd_addr_end(unsigned long addr,
184 unsigned long end)
185{
186
187 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
188 return (addr < end) ? addr : end;
189}
190
191static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
192{
193 uint32_t mask = PSB_PTE_VALID;
194
195 if (type & PSB_MMU_CACHED_MEMORY)
196 mask |= PSB_PTE_CACHED;
197 if (type & PSB_MMU_RO_MEMORY)
198 mask |= PSB_PTE_RO;
199 if (type & PSB_MMU_WO_MEMORY)
200 mask |= PSB_PTE_WO;
201
202 return (pfn << PAGE_SHIFT) | mask;
203}
204
205struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
206 int trap_pagefaults, int invalid_type)
207{
208 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
209 uint32_t *v;
210 int i;
211
212 if (!pd)
213 return NULL;
214
215 pd->p = alloc_page(GFP_DMA32);
216 if (!pd->p)
217 goto out_err1;
218 pd->dummy_pt = alloc_page(GFP_DMA32);
219 if (!pd->dummy_pt)
220 goto out_err2;
221 pd->dummy_page = alloc_page(GFP_DMA32);
222 if (!pd->dummy_page)
223 goto out_err3;
224
225 if (!trap_pagefaults) {
226 pd->invalid_pde =
227 psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
228 invalid_type);
229 pd->invalid_pte =
230 psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
231 invalid_type);
232 } else {
233 pd->invalid_pde = 0;
234 pd->invalid_pte = 0;
235 }
236
237 v = kmap(pd->dummy_pt);
238 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
239 v[i] = pd->invalid_pte;
240
241 kunmap(pd->dummy_pt);
242
243 v = kmap(pd->p);
244 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
245 v[i] = pd->invalid_pde;
246
247 kunmap(pd->p);
248
249 clear_page(kmap(pd->dummy_page));
250 kunmap(pd->dummy_page);
251
252 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
253 if (!pd->tables)
254 goto out_err4;
255
256 pd->hw_context = -1;
257 pd->pd_mask = PSB_PTE_VALID;
258 pd->driver = driver;
259
260 return pd;
261
262out_err4:
263 __free_page(pd->dummy_page);
264out_err3:
265 __free_page(pd->dummy_pt);
266out_err2:
267 __free_page(pd->p);
268out_err1:
269 kfree(pd);
270 return NULL;
271}
272
273static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
274{
275 __free_page(pt->p);
276 kfree(pt);
277}
278
279void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
280{
281 struct psb_mmu_driver *driver = pd->driver;
282 struct psb_mmu_pt *pt;
283 int i;
284
285 down_write(&driver->sem);
286 if (pd->hw_context != -1)
287 psb_mmu_flush_pd_locked(driver, 1);
288
289 /* Should take the spinlock here, but we don't need to do that
290 since we have the semaphore in write mode. */
291
292 for (i = 0; i < 1024; ++i) {
293 pt = pd->tables[i];
294 if (pt)
295 psb_mmu_free_pt(pt);
296 }
297
298 vfree(pd->tables);
299 __free_page(pd->dummy_page);
300 __free_page(pd->dummy_pt);
301 __free_page(pd->p);
302 kfree(pd);
303 up_write(&driver->sem);
304}
305
306static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
307{
308 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
309 void *v;
310 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
311 uint32_t clflush_count = PAGE_SIZE / clflush_add;
312 spinlock_t *lock = &pd->driver->lock;
313 uint8_t *clf;
314 uint32_t *ptes;
315 int i;
316
317 if (!pt)
318 return NULL;
319
320 pt->p = alloc_page(GFP_DMA32);
321 if (!pt->p) {
322 kfree(pt);
323 return NULL;
324 }
325
326 spin_lock(lock);
327
328 v = kmap_atomic(pt->p);
329 clf = (uint8_t *) v;
330 ptes = (uint32_t *) v;
331 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
332 *ptes++ = pd->invalid_pte;
333
334
335 if (pd->driver->has_clflush && pd->hw_context != -1) {
336 mb();
337 for (i = 0; i < clflush_count; ++i) {
338 psb_clflush(clf);
339 clf += clflush_add;
340 }
341 mb();
342 }
343
344 kunmap_atomic(v);
345 spin_unlock(lock);
346
347 pt->count = 0;
348 pt->pd = pd;
349 pt->index = 0;
350
351 return pt;
352}
353
354static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
355 unsigned long addr)
356{
357 uint32_t index = psb_mmu_pd_index(addr);
358 struct psb_mmu_pt *pt;
359 uint32_t *v;
360 spinlock_t *lock = &pd->driver->lock;
361
362 spin_lock(lock);
363 pt = pd->tables[index];
364 while (!pt) {
365 spin_unlock(lock);
366 pt = psb_mmu_alloc_pt(pd);
367 if (!pt)
368 return NULL;
369 spin_lock(lock);
370
371 if (pd->tables[index]) {
372 spin_unlock(lock);
373 psb_mmu_free_pt(pt);
374 spin_lock(lock);
375 pt = pd->tables[index];
376 continue;
377 }
378
379 v = kmap_atomic(pd->p);
380 pd->tables[index] = pt;
381 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
382 pt->index = index;
383 kunmap_atomic((void *) v);
384
385 if (pd->hw_context != -1) {
386 psb_mmu_clflush(pd->driver, (void *) &v[index]);
387 atomic_set(&pd->driver->needs_tlbflush, 1);
388 }
389 }
390 pt->v = kmap_atomic(pt->p);
391 return pt;
392}
393
394static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
395 unsigned long addr)
396{
397 uint32_t index = psb_mmu_pd_index(addr);
398 struct psb_mmu_pt *pt;
399 spinlock_t *lock = &pd->driver->lock;
400
401 spin_lock(lock);
402 pt = pd->tables[index];
403 if (!pt) {
404 spin_unlock(lock);
405 return NULL;
406 }
407 pt->v = kmap_atomic(pt->p);
408 return pt;
409}
410
411static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
412{
413 struct psb_mmu_pd *pd = pt->pd;
414 uint32_t *v;
415
416 kunmap_atomic(pt->v);
417 if (pt->count == 0) {
418 v = kmap_atomic(pd->p);
419 v[pt->index] = pd->invalid_pde;
420 pd->tables[pt->index] = NULL;
421
422 if (pd->hw_context != -1) {
423 psb_mmu_clflush(pd->driver,
424 (void *) &v[pt->index]);
425 atomic_set(&pd->driver->needs_tlbflush, 1);
426 }
427 kunmap_atomic(pt->v);
428 spin_unlock(&pd->driver->lock);
429 psb_mmu_free_pt(pt);
430 return;
431 }
432 spin_unlock(&pd->driver->lock);
433}
434
435static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt,
436 unsigned long addr, uint32_t pte)
437{
438 pt->v[psb_mmu_pt_index(addr)] = pte;
439}
440
441static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
442 unsigned long addr)
443{
444 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
445}
446
447
448void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd,
449 uint32_t mmu_offset, uint32_t gtt_start,
450 uint32_t gtt_pages)
451{
452 uint32_t *v;
453 uint32_t start = psb_mmu_pd_index(mmu_offset);
454 struct psb_mmu_driver *driver = pd->driver;
455 int num_pages = gtt_pages;
456
457 down_read(&driver->sem);
458 spin_lock(&driver->lock);
459
460 v = kmap_atomic(pd->p);
461 v += start;
462
463 while (gtt_pages--) {
464 *v++ = gtt_start | pd->pd_mask;
465 gtt_start += PAGE_SIZE;
466 }
467
468 /*ttm_tt_cache_flush(&pd->p, num_pages);*/
469 psb_pages_clflush(pd->driver, &pd->p, num_pages);
470 kunmap_atomic(v);
471 spin_unlock(&driver->lock);
472
473 if (pd->hw_context != -1)
474 atomic_set(&pd->driver->needs_tlbflush, 1);
475
476 up_read(&pd->driver->sem);
477 psb_mmu_flush_pd(pd->driver, 0);
478}
479
480struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
481{
482 struct psb_mmu_pd *pd;
483
484 /* down_read(&driver->sem); */
485 pd = driver->default_pd;
486 /* up_read(&driver->sem); */
487
488 return pd;
489}
490
491void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
492{
493 psb_mmu_free_pagedir(driver->default_pd);
494 kfree(driver);
495}
496
497struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,
498 int trap_pagefaults,
499 int invalid_type,
500 struct drm_psb_private *dev_priv)
501{
502 struct psb_mmu_driver *driver;
503
504 driver = kmalloc(sizeof(*driver), GFP_KERNEL);
505
506 if (!driver)
507 return NULL;
508 driver->dev_priv = dev_priv;
509
510 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
511 invalid_type);
512 if (!driver->default_pd)
513 goto out_err1;
514
515 spin_lock_init(&driver->lock);
516 init_rwsem(&driver->sem);
517 down_write(&driver->sem);
518 driver->register_map = registers;
519 atomic_set(&driver->needs_tlbflush, 1);
520
521 driver->has_clflush = 0;
522
523 if (boot_cpu_has(X86_FEATURE_CLFLSH)) {
524 uint32_t tfms, misc, cap0, cap4, clflush_size;
525
526 /*
527 * clflush size is determined at kernel setup for x86_64
528 * but not for i386. We have to do it here.
529 */
530
531 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
532 clflush_size = ((misc >> 8) & 0xff) * 8;
533 driver->has_clflush = 1;
534 driver->clflush_add =
535 PAGE_SIZE * clflush_size / sizeof(uint32_t);
536 driver->clflush_mask = driver->clflush_add - 1;
537 driver->clflush_mask = ~driver->clflush_mask;
538 }
539
540 up_write(&driver->sem);
541 return driver;
542
543out_err1:
544 kfree(driver);
545 return NULL;
546}
547
548static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd,
549 unsigned long address, uint32_t num_pages,
550 uint32_t desired_tile_stride,
551 uint32_t hw_tile_stride)
552{
553 struct psb_mmu_pt *pt;
554 uint32_t rows = 1;
555 uint32_t i;
556 unsigned long addr;
557 unsigned long end;
558 unsigned long next;
559 unsigned long add;
560 unsigned long row_add;
561 unsigned long clflush_add = pd->driver->clflush_add;
562 unsigned long clflush_mask = pd->driver->clflush_mask;
563
564 if (!pd->driver->has_clflush) {
565 /*ttm_tt_cache_flush(&pd->p, num_pages);*/
566 psb_pages_clflush(pd->driver, &pd->p, num_pages);
567 return;
568 }
569
570 if (hw_tile_stride)
571 rows = num_pages / desired_tile_stride;
572 else
573 desired_tile_stride = num_pages;
574
575 add = desired_tile_stride << PAGE_SHIFT;
576 row_add = hw_tile_stride << PAGE_SHIFT;
577 mb();
578 for (i = 0; i < rows; ++i) {
579
580 addr = address;
581 end = addr + add;
582
583 do {
584 next = psb_pd_addr_end(addr, end);
585 pt = psb_mmu_pt_map_lock(pd, addr);
586 if (!pt)
587 continue;
588 do {
589 psb_clflush(&pt->v
590 [psb_mmu_pt_index(addr)]);
591 } while (addr +=
592 clflush_add,
593 (addr & clflush_mask) < next);
594
595 psb_mmu_pt_unmap_unlock(pt);
596 } while (addr = next, next != end);
597 address += row_add;
598 }
599 mb();
600}
601
602void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
603 unsigned long address, uint32_t num_pages)
604{
605 struct psb_mmu_pt *pt;
606 unsigned long addr;
607 unsigned long end;
608 unsigned long next;
609 unsigned long f_address = address;
610
611 down_read(&pd->driver->sem);
612
613 addr = address;
614 end = addr + (num_pages << PAGE_SHIFT);
615
616 do {
617 next = psb_pd_addr_end(addr, end);
618 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
619 if (!pt)
620 goto out;
621 do {
622 psb_mmu_invalidate_pte(pt, addr);
623 --pt->count;
624 } while (addr += PAGE_SIZE, addr < next);
625 psb_mmu_pt_unmap_unlock(pt);
626
627 } while (addr = next, next != end);
628
629out:
630 if (pd->hw_context != -1)
631 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
632
633 up_read(&pd->driver->sem);
634
635 if (pd->hw_context != -1)
636 psb_mmu_flush(pd->driver, 0);
637
638 return;
639}
640
641void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
642 uint32_t num_pages, uint32_t desired_tile_stride,
643 uint32_t hw_tile_stride)
644{
645 struct psb_mmu_pt *pt;
646 uint32_t rows = 1;
647 uint32_t i;
648 unsigned long addr;
649 unsigned long end;
650 unsigned long next;
651 unsigned long add;
652 unsigned long row_add;
653 unsigned long f_address = address;
654
655 if (hw_tile_stride)
656 rows = num_pages / desired_tile_stride;
657 else
658 desired_tile_stride = num_pages;
659
660 add = desired_tile_stride << PAGE_SHIFT;
661 row_add = hw_tile_stride << PAGE_SHIFT;
662
663 /* down_read(&pd->driver->sem); */
664
665 /* Make sure we only need to flush this processor's cache */
666
667 for (i = 0; i < rows; ++i) {
668
669 addr = address;
670 end = addr + add;
671
672 do {
673 next = psb_pd_addr_end(addr, end);
674 pt = psb_mmu_pt_map_lock(pd, addr);
675 if (!pt)
676 continue;
677 do {
678 psb_mmu_invalidate_pte(pt, addr);
679 --pt->count;
680
681 } while (addr += PAGE_SIZE, addr < next);
682 psb_mmu_pt_unmap_unlock(pt);
683
684 } while (addr = next, next != end);
685 address += row_add;
686 }
687 if (pd->hw_context != -1)
688 psb_mmu_flush_ptes(pd, f_address, num_pages,
689 desired_tile_stride, hw_tile_stride);
690
691 /* up_read(&pd->driver->sem); */
692
693 if (pd->hw_context != -1)
694 psb_mmu_flush(pd->driver, 0);
695}
696
697int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
698 unsigned long address, uint32_t num_pages,
699 int type)
700{
701 struct psb_mmu_pt *pt;
702 uint32_t pte;
703 unsigned long addr;
704 unsigned long end;
705 unsigned long next;
706 unsigned long f_address = address;
707 int ret = 0;
708
709 down_read(&pd->driver->sem);
710
711 addr = address;
712 end = addr + (num_pages << PAGE_SHIFT);
713
714 do {
715 next = psb_pd_addr_end(addr, end);
716 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
717 if (!pt) {
718 ret = -ENOMEM;
719 goto out;
720 }
721 do {
722 pte = psb_mmu_mask_pte(start_pfn++, type);
723 psb_mmu_set_pte(pt, addr, pte);
724 pt->count++;
725 } while (addr += PAGE_SIZE, addr < next);
726 psb_mmu_pt_unmap_unlock(pt);
727
728 } while (addr = next, next != end);
729
730out:
731 if (pd->hw_context != -1)
732 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
733
734 up_read(&pd->driver->sem);
735
736 if (pd->hw_context != -1)
737 psb_mmu_flush(pd->driver, 1);
738
739 return ret;
740}
741
742int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
743 unsigned long address, uint32_t num_pages,
744 uint32_t desired_tile_stride,
745 uint32_t hw_tile_stride, int type)
746{
747 struct psb_mmu_pt *pt;
748 uint32_t rows = 1;
749 uint32_t i;
750 uint32_t pte;
751 unsigned long addr;
752 unsigned long end;
753 unsigned long next;
754 unsigned long add;
755 unsigned long row_add;
756 unsigned long f_address = address;
757 int ret = 0;
758
759 if (hw_tile_stride) {
760 if (num_pages % desired_tile_stride != 0)
761 return -EINVAL;
762 rows = num_pages / desired_tile_stride;
763 } else {
764 desired_tile_stride = num_pages;
765 }
766
767 add = desired_tile_stride << PAGE_SHIFT;
768 row_add = hw_tile_stride << PAGE_SHIFT;
769
770 down_read(&pd->driver->sem);
771
772 for (i = 0; i < rows; ++i) {
773
774 addr = address;
775 end = addr + add;
776
777 do {
778 next = psb_pd_addr_end(addr, end);
779 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
780 if (!pt) {
781 ret = -ENOMEM;
782 goto out;
783 }
784 do {
785 pte =
786 psb_mmu_mask_pte(page_to_pfn(*pages++),
787 type);
788 psb_mmu_set_pte(pt, addr, pte);
789 pt->count++;
790 } while (addr += PAGE_SIZE, addr < next);
791 psb_mmu_pt_unmap_unlock(pt);
792
793 } while (addr = next, next != end);
794
795 address += row_add;
796 }
797out:
798 if (pd->hw_context != -1)
799 psb_mmu_flush_ptes(pd, f_address, num_pages,
800 desired_tile_stride, hw_tile_stride);
801
802 up_read(&pd->driver->sem);
803
804 if (pd->hw_context != -1)
805 psb_mmu_flush(pd->driver, 1);
806
807 return ret;
808}
809
810int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
811 unsigned long *pfn)
812{
813 int ret;
814 struct psb_mmu_pt *pt;
815 uint32_t tmp;
816 spinlock_t *lock = &pd->driver->lock;
817
818 down_read(&pd->driver->sem);
819 pt = psb_mmu_pt_map_lock(pd, virtual);
820 if (!pt) {
821 uint32_t *v;
822
823 spin_lock(lock);
824 v = kmap_atomic(pd->p);
825 tmp = v[psb_mmu_pd_index(virtual)];
826 kunmap_atomic(v);
827 spin_unlock(lock);
828
829 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
830 !(pd->invalid_pte & PSB_PTE_VALID)) {
831 ret = -EINVAL;
832 goto out;
833 }
834 ret = 0;
835 *pfn = pd->invalid_pte >> PAGE_SHIFT;
836 goto out;
837 }
838 tmp = pt->v[psb_mmu_pt_index(virtual)];
839 if (!(tmp & PSB_PTE_VALID)) {
840 ret = -EINVAL;
841 } else {
842 ret = 0;
843 *pfn = tmp >> PAGE_SHIFT;
844 }
845 psb_mmu_pt_unmap_unlock(pt);
846out:
847 up_read(&pd->driver->sem);
848 return ret;
849}