Loading...
1/*
2 * Copyright (c) 2007, Intel Corporation.
3 * All Rights Reserved.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics.com>
19 * Alan Cox <alan@linux.intel.com>
20 */
21
22#include <drm/drmP.h>
23#include <linux/shmem_fs.h>
24#include "psb_drv.h"
25#include "blitter.h"
26
27
28/*
29 * GTT resource allocator - manage page mappings in GTT space
30 */
31
32/**
33 * psb_gtt_mask_pte - generate GTT pte entry
34 * @pfn: page number to encode
35 * @type: type of memory in the GTT
36 *
37 * Set the GTT entry for the appropriate memory type.
38 */
39static inline uint32_t psb_gtt_mask_pte(uint32_t pfn, int type)
40{
41 uint32_t mask = PSB_PTE_VALID;
42
43 /* Ensure we explode rather than put an invalid low mapping of
44 a high mapping page into the gtt */
45 BUG_ON(pfn & ~(0xFFFFFFFF >> PAGE_SHIFT));
46
47 if (type & PSB_MMU_CACHED_MEMORY)
48 mask |= PSB_PTE_CACHED;
49 if (type & PSB_MMU_RO_MEMORY)
50 mask |= PSB_PTE_RO;
51 if (type & PSB_MMU_WO_MEMORY)
52 mask |= PSB_PTE_WO;
53
54 return (pfn << PAGE_SHIFT) | mask;
55}
56
57/**
58 * psb_gtt_entry - find the GTT entries for a gtt_range
59 * @dev: our DRM device
60 * @r: our GTT range
61 *
62 * Given a gtt_range object return the GTT offset of the page table
63 * entries for this gtt_range
64 */
65static u32 __iomem *psb_gtt_entry(struct drm_device *dev, struct gtt_range *r)
66{
67 struct drm_psb_private *dev_priv = dev->dev_private;
68 unsigned long offset;
69
70 offset = r->resource.start - dev_priv->gtt_mem->start;
71
72 return dev_priv->gtt_map + (offset >> PAGE_SHIFT);
73}
74
75/**
76 * psb_gtt_insert - put an object into the GTT
77 * @dev: our DRM device
78 * @r: our GTT range
79 *
80 * Take our preallocated GTT range and insert the GEM object into
81 * the GTT. This is protected via the gtt mutex which the caller
82 * must hold.
83 */
84static int psb_gtt_insert(struct drm_device *dev, struct gtt_range *r,
85 int resume)
86{
87 u32 __iomem *gtt_slot;
88 u32 pte;
89 struct page **pages;
90 int i;
91
92 if (r->pages == NULL) {
93 WARN_ON(1);
94 return -EINVAL;
95 }
96
97 WARN_ON(r->stolen); /* refcount these maybe ? */
98
99 gtt_slot = psb_gtt_entry(dev, r);
100 pages = r->pages;
101
102 if (!resume) {
103 /* Make sure changes are visible to the GPU */
104 set_pages_array_wc(pages, r->npage);
105 }
106
107 /* Write our page entries into the GTT itself */
108 for (i = r->roll; i < r->npage; i++) {
109 pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
110 PSB_MMU_CACHED_MEMORY);
111 iowrite32(pte, gtt_slot++);
112 }
113 for (i = 0; i < r->roll; i++) {
114 pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
115 PSB_MMU_CACHED_MEMORY);
116 iowrite32(pte, gtt_slot++);
117 }
118 /* Make sure all the entries are set before we return */
119 ioread32(gtt_slot - 1);
120
121 return 0;
122}
123
124/**
125 * psb_gtt_remove - remove an object from the GTT
126 * @dev: our DRM device
127 * @r: our GTT range
128 *
129 * Remove a preallocated GTT range from the GTT. Overwrite all the
130 * page table entries with the dummy page. This is protected via the gtt
131 * mutex which the caller must hold.
132 */
133void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r)
134{
135 struct drm_psb_private *dev_priv = dev->dev_private;
136 u32 __iomem *gtt_slot;
137 u32 pte;
138 int i;
139
140 WARN_ON(r->stolen);
141
142 gtt_slot = psb_gtt_entry(dev, r);
143 pte = psb_gtt_mask_pte(page_to_pfn(dev_priv->scratch_page),
144 PSB_MMU_CACHED_MEMORY);
145
146 for (i = 0; i < r->npage; i++)
147 iowrite32(pte, gtt_slot++);
148 ioread32(gtt_slot - 1);
149 set_pages_array_wb(r->pages, r->npage);
150}
151
152/**
153 * psb_gtt_roll - set scrolling position
154 * @dev: our DRM device
155 * @r: the gtt mapping we are using
156 * @roll: roll offset
157 *
158 * Roll an existing pinned mapping by moving the pages through the GTT.
159 * This allows us to implement hardware scrolling on the consoles without
160 * a 2D engine
161 */
162void psb_gtt_roll(struct drm_device *dev, struct gtt_range *r, int roll)
163{
164 u32 __iomem *gtt_slot;
165 u32 pte;
166 int i;
167
168 if (roll >= r->npage) {
169 WARN_ON(1);
170 return;
171 }
172
173 r->roll = roll;
174
175 /* Not currently in the GTT - no worry we will write the mapping at
176 the right position when it gets pinned */
177 if (!r->stolen && !r->in_gart)
178 return;
179
180 gtt_slot = psb_gtt_entry(dev, r);
181
182 for (i = r->roll; i < r->npage; i++) {
183 pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
184 PSB_MMU_CACHED_MEMORY);
185 iowrite32(pte, gtt_slot++);
186 }
187 for (i = 0; i < r->roll; i++) {
188 pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
189 PSB_MMU_CACHED_MEMORY);
190 iowrite32(pte, gtt_slot++);
191 }
192 ioread32(gtt_slot - 1);
193}
194
195/**
196 * psb_gtt_attach_pages - attach and pin GEM pages
197 * @gt: the gtt range
198 *
199 * Pin and build an in kernel list of the pages that back our GEM object.
200 * While we hold this the pages cannot be swapped out. This is protected
201 * via the gtt mutex which the caller must hold.
202 */
203static int psb_gtt_attach_pages(struct gtt_range *gt)
204{
205 struct page **pages;
206
207 WARN_ON(gt->pages);
208
209 pages = drm_gem_get_pages(>->gem);
210 if (IS_ERR(pages))
211 return PTR_ERR(pages);
212
213 gt->npage = gt->gem.size / PAGE_SIZE;
214 gt->pages = pages;
215
216 return 0;
217}
218
219/**
220 * psb_gtt_detach_pages - attach and pin GEM pages
221 * @gt: the gtt range
222 *
223 * Undo the effect of psb_gtt_attach_pages. At this point the pages
224 * must have been removed from the GTT as they could now be paged out
225 * and move bus address. This is protected via the gtt mutex which the
226 * caller must hold.
227 */
228static void psb_gtt_detach_pages(struct gtt_range *gt)
229{
230 drm_gem_put_pages(>->gem, gt->pages, true, false);
231 gt->pages = NULL;
232}
233
234/**
235 * psb_gtt_pin - pin pages into the GTT
236 * @gt: range to pin
237 *
238 * Pin a set of pages into the GTT. The pins are refcounted so that
239 * multiple pins need multiple unpins to undo.
240 *
241 * Non GEM backed objects treat this as a no-op as they are always GTT
242 * backed objects.
243 */
244int psb_gtt_pin(struct gtt_range *gt)
245{
246 int ret = 0;
247 struct drm_device *dev = gt->gem.dev;
248 struct drm_psb_private *dev_priv = dev->dev_private;
249 u32 gpu_base = dev_priv->gtt.gatt_start;
250
251 mutex_lock(&dev_priv->gtt_mutex);
252
253 if (gt->in_gart == 0 && gt->stolen == 0) {
254 ret = psb_gtt_attach_pages(gt);
255 if (ret < 0)
256 goto out;
257 ret = psb_gtt_insert(dev, gt, 0);
258 if (ret < 0) {
259 psb_gtt_detach_pages(gt);
260 goto out;
261 }
262 psb_mmu_insert_pages(psb_mmu_get_default_pd(dev_priv->mmu),
263 gt->pages, (gpu_base + gt->offset),
264 gt->npage, 0, 0, PSB_MMU_CACHED_MEMORY);
265 }
266 gt->in_gart++;
267out:
268 mutex_unlock(&dev_priv->gtt_mutex);
269 return ret;
270}
271
272/**
273 * psb_gtt_unpin - Drop a GTT pin requirement
274 * @gt: range to pin
275 *
276 * Undoes the effect of psb_gtt_pin. On the last drop the GEM object
277 * will be removed from the GTT which will also drop the page references
278 * and allow the VM to clean up or page stuff.
279 *
280 * Non GEM backed objects treat this as a no-op as they are always GTT
281 * backed objects.
282 */
283void psb_gtt_unpin(struct gtt_range *gt)
284{
285 struct drm_device *dev = gt->gem.dev;
286 struct drm_psb_private *dev_priv = dev->dev_private;
287 u32 gpu_base = dev_priv->gtt.gatt_start;
288 int ret;
289
290 /* While holding the gtt_mutex no new blits can be initiated */
291 mutex_lock(&dev_priv->gtt_mutex);
292
293 /* Wait for any possible usage of the memory to be finished */
294 ret = gma_blt_wait_idle(dev_priv);
295 if (ret) {
296 DRM_ERROR("Failed to idle the blitter, unpin failed!");
297 goto out;
298 }
299
300 WARN_ON(!gt->in_gart);
301
302 gt->in_gart--;
303 if (gt->in_gart == 0 && gt->stolen == 0) {
304 psb_mmu_remove_pages(psb_mmu_get_default_pd(dev_priv->mmu),
305 (gpu_base + gt->offset), gt->npage, 0, 0);
306 psb_gtt_remove(dev, gt);
307 psb_gtt_detach_pages(gt);
308 }
309
310out:
311 mutex_unlock(&dev_priv->gtt_mutex);
312}
313
314/*
315 * GTT resource allocator - allocate and manage GTT address space
316 */
317
318/**
319 * psb_gtt_alloc_range - allocate GTT address space
320 * @dev: Our DRM device
321 * @len: length (bytes) of address space required
322 * @name: resource name
323 * @backed: resource should be backed by stolen pages
324 *
325 * Ask the kernel core to find us a suitable range of addresses
326 * to use for a GTT mapping.
327 *
328 * Returns a gtt_range structure describing the object, or NULL on
329 * error. On successful return the resource is both allocated and marked
330 * as in use.
331 */
332struct gtt_range *psb_gtt_alloc_range(struct drm_device *dev, int len,
333 const char *name, int backed, u32 align)
334{
335 struct drm_psb_private *dev_priv = dev->dev_private;
336 struct gtt_range *gt;
337 struct resource *r = dev_priv->gtt_mem;
338 int ret;
339 unsigned long start, end;
340
341 if (backed) {
342 /* The start of the GTT is the stolen pages */
343 start = r->start;
344 end = r->start + dev_priv->gtt.stolen_size - 1;
345 } else {
346 /* The rest we will use for GEM backed objects */
347 start = r->start + dev_priv->gtt.stolen_size;
348 end = r->end;
349 }
350
351 gt = kzalloc(sizeof(struct gtt_range), GFP_KERNEL);
352 if (gt == NULL)
353 return NULL;
354 gt->resource.name = name;
355 gt->stolen = backed;
356 gt->in_gart = backed;
357 gt->roll = 0;
358 /* Ensure this is set for non GEM objects */
359 gt->gem.dev = dev;
360 ret = allocate_resource(dev_priv->gtt_mem, >->resource,
361 len, start, end, align, NULL, NULL);
362 if (ret == 0) {
363 gt->offset = gt->resource.start - r->start;
364 return gt;
365 }
366 kfree(gt);
367 return NULL;
368}
369
370/**
371 * psb_gtt_free_range - release GTT address space
372 * @dev: our DRM device
373 * @gt: a mapping created with psb_gtt_alloc_range
374 *
375 * Release a resource that was allocated with psb_gtt_alloc_range. If the
376 * object has been pinned by mmap users we clean this up here currently.
377 */
378void psb_gtt_free_range(struct drm_device *dev, struct gtt_range *gt)
379{
380 /* Undo the mmap pin if we are destroying the object */
381 if (gt->mmapping) {
382 psb_gtt_unpin(gt);
383 gt->mmapping = 0;
384 }
385 WARN_ON(gt->in_gart && !gt->stolen);
386 release_resource(>->resource);
387 kfree(gt);
388}
389
390static void psb_gtt_alloc(struct drm_device *dev)
391{
392 struct drm_psb_private *dev_priv = dev->dev_private;
393 init_rwsem(&dev_priv->gtt.sem);
394}
395
396void psb_gtt_takedown(struct drm_device *dev)
397{
398 struct drm_psb_private *dev_priv = dev->dev_private;
399
400 if (dev_priv->gtt_map) {
401 iounmap(dev_priv->gtt_map);
402 dev_priv->gtt_map = NULL;
403 }
404 if (dev_priv->gtt_initialized) {
405 pci_write_config_word(dev->pdev, PSB_GMCH_CTRL,
406 dev_priv->gmch_ctrl);
407 PSB_WVDC32(dev_priv->pge_ctl, PSB_PGETBL_CTL);
408 (void) PSB_RVDC32(PSB_PGETBL_CTL);
409 }
410 if (dev_priv->vram_addr)
411 iounmap(dev_priv->gtt_map);
412}
413
414int psb_gtt_init(struct drm_device *dev, int resume)
415{
416 struct drm_psb_private *dev_priv = dev->dev_private;
417 unsigned gtt_pages;
418 unsigned long stolen_size, vram_stolen_size;
419 unsigned i, num_pages;
420 unsigned pfn_base;
421 struct psb_gtt *pg;
422
423 int ret = 0;
424 uint32_t pte;
425
426 if (!resume) {
427 mutex_init(&dev_priv->gtt_mutex);
428 mutex_init(&dev_priv->mmap_mutex);
429 psb_gtt_alloc(dev);
430 }
431
432 pg = &dev_priv->gtt;
433
434 /* Enable the GTT */
435 pci_read_config_word(dev->pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl);
436 pci_write_config_word(dev->pdev, PSB_GMCH_CTRL,
437 dev_priv->gmch_ctrl | _PSB_GMCH_ENABLED);
438
439 dev_priv->pge_ctl = PSB_RVDC32(PSB_PGETBL_CTL);
440 PSB_WVDC32(dev_priv->pge_ctl | _PSB_PGETBL_ENABLED, PSB_PGETBL_CTL);
441 (void) PSB_RVDC32(PSB_PGETBL_CTL);
442
443 /* The root resource we allocate address space from */
444 dev_priv->gtt_initialized = 1;
445
446 pg->gtt_phys_start = dev_priv->pge_ctl & PAGE_MASK;
447
448 /*
449 * The video mmu has a hw bug when accessing 0x0D0000000.
450 * Make gatt start at 0x0e000,0000. This doesn't actually
451 * matter for us but may do if the video acceleration ever
452 * gets opened up.
453 */
454 pg->mmu_gatt_start = 0xE0000000;
455
456 pg->gtt_start = pci_resource_start(dev->pdev, PSB_GTT_RESOURCE);
457 gtt_pages = pci_resource_len(dev->pdev, PSB_GTT_RESOURCE)
458 >> PAGE_SHIFT;
459 /* CDV doesn't report this. In which case the system has 64 gtt pages */
460 if (pg->gtt_start == 0 || gtt_pages == 0) {
461 dev_dbg(dev->dev, "GTT PCI BAR not initialized.\n");
462 gtt_pages = 64;
463 pg->gtt_start = dev_priv->pge_ctl;
464 }
465
466 pg->gatt_start = pci_resource_start(dev->pdev, PSB_GATT_RESOURCE);
467 pg->gatt_pages = pci_resource_len(dev->pdev, PSB_GATT_RESOURCE)
468 >> PAGE_SHIFT;
469 dev_priv->gtt_mem = &dev->pdev->resource[PSB_GATT_RESOURCE];
470
471 if (pg->gatt_pages == 0 || pg->gatt_start == 0) {
472 static struct resource fudge; /* Preferably peppermint */
473 /* This can occur on CDV systems. Fudge it in this case.
474 We really don't care what imaginary space is being allocated
475 at this point */
476 dev_dbg(dev->dev, "GATT PCI BAR not initialized.\n");
477 pg->gatt_start = 0x40000000;
478 pg->gatt_pages = (128 * 1024 * 1024) >> PAGE_SHIFT;
479 /* This is a little confusing but in fact the GTT is providing
480 a view from the GPU into memory and not vice versa. As such
481 this is really allocating space that is not the same as the
482 CPU address space on CDV */
483 fudge.start = 0x40000000;
484 fudge.end = 0x40000000 + 128 * 1024 * 1024 - 1;
485 fudge.name = "fudge";
486 fudge.flags = IORESOURCE_MEM;
487 dev_priv->gtt_mem = &fudge;
488 }
489
490 pci_read_config_dword(dev->pdev, PSB_BSM, &dev_priv->stolen_base);
491 vram_stolen_size = pg->gtt_phys_start - dev_priv->stolen_base
492 - PAGE_SIZE;
493
494 stolen_size = vram_stolen_size;
495
496 dev_dbg(dev->dev, "Stolen memory base 0x%x, size %luK\n",
497 dev_priv->stolen_base, vram_stolen_size / 1024);
498
499 if (resume && (gtt_pages != pg->gtt_pages) &&
500 (stolen_size != pg->stolen_size)) {
501 dev_err(dev->dev, "GTT resume error.\n");
502 ret = -EINVAL;
503 goto out_err;
504 }
505
506 pg->gtt_pages = gtt_pages;
507 pg->stolen_size = stolen_size;
508 dev_priv->vram_stolen_size = vram_stolen_size;
509
510 /*
511 * Map the GTT and the stolen memory area
512 */
513 if (!resume)
514 dev_priv->gtt_map = ioremap_nocache(pg->gtt_phys_start,
515 gtt_pages << PAGE_SHIFT);
516 if (!dev_priv->gtt_map) {
517 dev_err(dev->dev, "Failure to map gtt.\n");
518 ret = -ENOMEM;
519 goto out_err;
520 }
521
522 if (!resume)
523 dev_priv->vram_addr = ioremap_wc(dev_priv->stolen_base,
524 stolen_size);
525
526 if (!dev_priv->vram_addr) {
527 dev_err(dev->dev, "Failure to map stolen base.\n");
528 ret = -ENOMEM;
529 goto out_err;
530 }
531
532 /*
533 * Insert vram stolen pages into the GTT
534 */
535
536 pfn_base = dev_priv->stolen_base >> PAGE_SHIFT;
537 num_pages = vram_stolen_size >> PAGE_SHIFT;
538 dev_dbg(dev->dev, "Set up %d stolen pages starting at 0x%08x, GTT offset %dK\n",
539 num_pages, pfn_base << PAGE_SHIFT, 0);
540 for (i = 0; i < num_pages; ++i) {
541 pte = psb_gtt_mask_pte(pfn_base + i, PSB_MMU_CACHED_MEMORY);
542 iowrite32(pte, dev_priv->gtt_map + i);
543 }
544
545 /*
546 * Init rest of GTT to the scratch page to avoid accidents or scribbles
547 */
548
549 pfn_base = page_to_pfn(dev_priv->scratch_page);
550 pte = psb_gtt_mask_pte(pfn_base, PSB_MMU_CACHED_MEMORY);
551 for (; i < gtt_pages; ++i)
552 iowrite32(pte, dev_priv->gtt_map + i);
553
554 (void) ioread32(dev_priv->gtt_map + i - 1);
555 return 0;
556
557out_err:
558 psb_gtt_takedown(dev);
559 return ret;
560}
561
562int psb_gtt_restore(struct drm_device *dev)
563{
564 struct drm_psb_private *dev_priv = dev->dev_private;
565 struct resource *r = dev_priv->gtt_mem->child;
566 struct gtt_range *range;
567 unsigned int restored = 0, total = 0, size = 0;
568
569 /* On resume, the gtt_mutex is already initialized */
570 mutex_lock(&dev_priv->gtt_mutex);
571 psb_gtt_init(dev, 1);
572
573 while (r != NULL) {
574 range = container_of(r, struct gtt_range, resource);
575 if (range->pages) {
576 psb_gtt_insert(dev, range, 1);
577 size += range->resource.end - range->resource.start;
578 restored++;
579 }
580 r = r->sibling;
581 total++;
582 }
583 mutex_unlock(&dev_priv->gtt_mutex);
584 DRM_DEBUG_DRIVER("Restored %u of %u gtt ranges (%u KB)", restored,
585 total, (size / 1024));
586
587 return 0;
588}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2007, Intel Corporation.
4 * All Rights Reserved.
5 *
6 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics.com>
7 * Alan Cox <alan@linux.intel.com>
8 */
9
10#include "gem.h" /* TODO: for struct psb_gem_object, see psb_gtt_restore() */
11#include "psb_drv.h"
12
13
14/*
15 * GTT resource allocator - manage page mappings in GTT space
16 */
17
18int psb_gtt_allocate_resource(struct drm_psb_private *pdev, struct resource *res,
19 const char *name, resource_size_t size, resource_size_t align,
20 bool stolen, u32 *offset)
21{
22 struct resource *root = pdev->gtt_mem;
23 resource_size_t start, end;
24 int ret;
25
26 if (stolen) {
27 /* The start of the GTT is backed by stolen pages. */
28 start = root->start;
29 end = root->start + pdev->gtt.stolen_size - 1;
30 } else {
31 /* The rest is backed by system pages. */
32 start = root->start + pdev->gtt.stolen_size;
33 end = root->end;
34 }
35
36 res->name = name;
37 ret = allocate_resource(root, res, size, start, end, align, NULL, NULL);
38 if (ret)
39 return ret;
40 *offset = res->start - root->start;
41
42 return 0;
43}
44
45/**
46 * psb_gtt_mask_pte - generate GTT pte entry
47 * @pfn: page number to encode
48 * @type: type of memory in the GTT
49 *
50 * Set the GTT entry for the appropriate memory type.
51 */
52uint32_t psb_gtt_mask_pte(uint32_t pfn, int type)
53{
54 uint32_t mask = PSB_PTE_VALID;
55
56 /* Ensure we explode rather than put an invalid low mapping of
57 a high mapping page into the gtt */
58 BUG_ON(pfn & ~(0xFFFFFFFF >> PAGE_SHIFT));
59
60 if (type & PSB_MMU_CACHED_MEMORY)
61 mask |= PSB_PTE_CACHED;
62 if (type & PSB_MMU_RO_MEMORY)
63 mask |= PSB_PTE_RO;
64 if (type & PSB_MMU_WO_MEMORY)
65 mask |= PSB_PTE_WO;
66
67 return (pfn << PAGE_SHIFT) | mask;
68}
69
70static u32 __iomem *psb_gtt_entry(struct drm_psb_private *pdev, const struct resource *res)
71{
72 unsigned long offset = res->start - pdev->gtt_mem->start;
73
74 return pdev->gtt_map + (offset >> PAGE_SHIFT);
75}
76
77/* Acquires GTT mutex internally. */
78void psb_gtt_insert_pages(struct drm_psb_private *pdev, const struct resource *res,
79 struct page **pages)
80{
81 resource_size_t npages, i;
82 u32 __iomem *gtt_slot;
83 u32 pte;
84
85 mutex_lock(&pdev->gtt_mutex);
86
87 /* Write our page entries into the GTT itself */
88
89 npages = resource_size(res) >> PAGE_SHIFT;
90 gtt_slot = psb_gtt_entry(pdev, res);
91
92 for (i = 0; i < npages; ++i, ++gtt_slot) {
93 pte = psb_gtt_mask_pte(page_to_pfn(pages[i]), PSB_MMU_CACHED_MEMORY);
94 iowrite32(pte, gtt_slot);
95 }
96
97 /* Make sure all the entries are set before we return */
98 ioread32(gtt_slot - 1);
99
100 mutex_unlock(&pdev->gtt_mutex);
101}
102
103/* Acquires GTT mutex internally. */
104void psb_gtt_remove_pages(struct drm_psb_private *pdev, const struct resource *res)
105{
106 resource_size_t npages, i;
107 u32 __iomem *gtt_slot;
108 u32 pte;
109
110 mutex_lock(&pdev->gtt_mutex);
111
112 /* Install scratch page for the resource */
113
114 pte = psb_gtt_mask_pte(page_to_pfn(pdev->scratch_page), PSB_MMU_CACHED_MEMORY);
115
116 npages = resource_size(res) >> PAGE_SHIFT;
117 gtt_slot = psb_gtt_entry(pdev, res);
118
119 for (i = 0; i < npages; ++i, ++gtt_slot)
120 iowrite32(pte, gtt_slot);
121
122 /* Make sure all the entries are set before we return */
123 ioread32(gtt_slot - 1);
124
125 mutex_unlock(&pdev->gtt_mutex);
126}
127
128static int psb_gtt_enable(struct drm_psb_private *dev_priv)
129{
130 struct drm_device *dev = &dev_priv->dev;
131 struct pci_dev *pdev = to_pci_dev(dev->dev);
132 int ret;
133
134 ret = pci_read_config_word(pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl);
135 if (ret)
136 return pcibios_err_to_errno(ret);
137 ret = pci_write_config_word(pdev, PSB_GMCH_CTRL, dev_priv->gmch_ctrl | _PSB_GMCH_ENABLED);
138 if (ret)
139 return pcibios_err_to_errno(ret);
140
141 dev_priv->pge_ctl = PSB_RVDC32(PSB_PGETBL_CTL);
142 PSB_WVDC32(dev_priv->pge_ctl | _PSB_PGETBL_ENABLED, PSB_PGETBL_CTL);
143
144 (void)PSB_RVDC32(PSB_PGETBL_CTL);
145
146 return 0;
147}
148
149static void psb_gtt_disable(struct drm_psb_private *dev_priv)
150{
151 struct drm_device *dev = &dev_priv->dev;
152 struct pci_dev *pdev = to_pci_dev(dev->dev);
153
154 pci_write_config_word(pdev, PSB_GMCH_CTRL, dev_priv->gmch_ctrl);
155 PSB_WVDC32(dev_priv->pge_ctl, PSB_PGETBL_CTL);
156
157 (void)PSB_RVDC32(PSB_PGETBL_CTL);
158}
159
160void psb_gtt_fini(struct drm_device *dev)
161{
162 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
163
164 iounmap(dev_priv->gtt_map);
165 psb_gtt_disable(dev_priv);
166 mutex_destroy(&dev_priv->gtt_mutex);
167}
168
169/* Clear GTT. Use a scratch page to avoid accidents or scribbles. */
170static void psb_gtt_clear(struct drm_psb_private *pdev)
171{
172 resource_size_t pfn_base;
173 unsigned long i;
174 uint32_t pte;
175
176 pfn_base = page_to_pfn(pdev->scratch_page);
177 pte = psb_gtt_mask_pte(pfn_base, PSB_MMU_CACHED_MEMORY);
178
179 for (i = 0; i < pdev->gtt.gtt_pages; ++i)
180 iowrite32(pte, pdev->gtt_map + i);
181
182 (void)ioread32(pdev->gtt_map + i - 1);
183}
184
185static void psb_gtt_init_ranges(struct drm_psb_private *dev_priv)
186{
187 struct drm_device *dev = &dev_priv->dev;
188 struct pci_dev *pdev = to_pci_dev(dev->dev);
189 struct psb_gtt *pg = &dev_priv->gtt;
190 resource_size_t gtt_phys_start, mmu_gatt_start, gtt_start, gtt_pages,
191 gatt_start, gatt_pages;
192 struct resource *gtt_mem;
193
194 /* The root resource we allocate address space from */
195 gtt_phys_start = dev_priv->pge_ctl & PAGE_MASK;
196
197 /*
198 * The video MMU has a HW bug when accessing 0x0d0000000. Make
199 * GATT start at 0x0e0000000. This doesn't actually matter for
200 * us now, but maybe will if the video acceleration ever gets
201 * opened up.
202 */
203 mmu_gatt_start = 0xe0000000;
204
205 gtt_start = pci_resource_start(pdev, PSB_GTT_RESOURCE);
206 gtt_pages = pci_resource_len(pdev, PSB_GTT_RESOURCE) >> PAGE_SHIFT;
207
208 /* CDV doesn't report this. In which case the system has 64 gtt pages */
209 if (!gtt_start || !gtt_pages) {
210 dev_dbg(dev->dev, "GTT PCI BAR not initialized.\n");
211 gtt_pages = 64;
212 gtt_start = dev_priv->pge_ctl;
213 }
214
215 gatt_start = pci_resource_start(pdev, PSB_GATT_RESOURCE);
216 gatt_pages = pci_resource_len(pdev, PSB_GATT_RESOURCE) >> PAGE_SHIFT;
217
218 if (!gatt_pages || !gatt_start) {
219 static struct resource fudge; /* Preferably peppermint */
220
221 /*
222 * This can occur on CDV systems. Fudge it in this case. We
223 * really don't care what imaginary space is being allocated
224 * at this point.
225 */
226 dev_dbg(dev->dev, "GATT PCI BAR not initialized.\n");
227 gatt_start = 0x40000000;
228 gatt_pages = (128 * 1024 * 1024) >> PAGE_SHIFT;
229
230 /*
231 * This is a little confusing but in fact the GTT is providing
232 * a view from the GPU into memory and not vice versa. As such
233 * this is really allocating space that is not the same as the
234 * CPU address space on CDV.
235 */
236 fudge.start = 0x40000000;
237 fudge.end = 0x40000000 + 128 * 1024 * 1024 - 1;
238 fudge.name = "fudge";
239 fudge.flags = IORESOURCE_MEM;
240
241 gtt_mem = &fudge;
242 } else {
243 gtt_mem = &pdev->resource[PSB_GATT_RESOURCE];
244 }
245
246 pg->gtt_phys_start = gtt_phys_start;
247 pg->mmu_gatt_start = mmu_gatt_start;
248 pg->gtt_start = gtt_start;
249 pg->gtt_pages = gtt_pages;
250 pg->gatt_start = gatt_start;
251 pg->gatt_pages = gatt_pages;
252 dev_priv->gtt_mem = gtt_mem;
253}
254
255int psb_gtt_init(struct drm_device *dev)
256{
257 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
258 struct psb_gtt *pg = &dev_priv->gtt;
259 int ret;
260
261 mutex_init(&dev_priv->gtt_mutex);
262
263 ret = psb_gtt_enable(dev_priv);
264 if (ret)
265 goto err_mutex_destroy;
266
267 psb_gtt_init_ranges(dev_priv);
268
269 dev_priv->gtt_map = ioremap(pg->gtt_phys_start, pg->gtt_pages << PAGE_SHIFT);
270 if (!dev_priv->gtt_map) {
271 dev_err(dev->dev, "Failure to map gtt.\n");
272 ret = -ENOMEM;
273 goto err_psb_gtt_disable;
274 }
275
276 psb_gtt_clear(dev_priv);
277
278 return 0;
279
280err_psb_gtt_disable:
281 psb_gtt_disable(dev_priv);
282err_mutex_destroy:
283 mutex_destroy(&dev_priv->gtt_mutex);
284 return ret;
285}
286
287int psb_gtt_resume(struct drm_device *dev)
288{
289 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
290 struct psb_gtt *pg = &dev_priv->gtt;
291 unsigned int old_gtt_pages = pg->gtt_pages;
292 int ret;
293
294 /* Enable the GTT */
295 ret = psb_gtt_enable(dev_priv);
296 if (ret)
297 return ret;
298
299 psb_gtt_init_ranges(dev_priv);
300
301 if (old_gtt_pages != pg->gtt_pages) {
302 dev_err(dev->dev, "GTT resume error.\n");
303 ret = -ENODEV;
304 goto err_psb_gtt_disable;
305 }
306
307 psb_gtt_clear(dev_priv);
308
309err_psb_gtt_disable:
310 psb_gtt_disable(dev_priv);
311 return ret;
312}