Loading...
1#include "drmP.h"
2#include "nouveau_drv.h"
3#include <linux/pagemap.h>
4#include <linux/slab.h>
5
6#define NV_CTXDMA_PAGE_SHIFT 12
7#define NV_CTXDMA_PAGE_SIZE (1 << NV_CTXDMA_PAGE_SHIFT)
8#define NV_CTXDMA_PAGE_MASK (NV_CTXDMA_PAGE_SIZE - 1)
9
10struct nouveau_sgdma_be {
11 /* this has to be the first field so populate/unpopulated in
12 * nouve_bo.c works properly, otherwise have to move them here
13 */
14 struct ttm_dma_tt ttm;
15 struct drm_device *dev;
16 u64 offset;
17};
18
19static void
20nouveau_sgdma_destroy(struct ttm_tt *ttm)
21{
22 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
23
24 if (ttm) {
25 NV_DEBUG(nvbe->dev, "\n");
26 ttm_dma_tt_fini(&nvbe->ttm);
27 kfree(nvbe);
28 }
29}
30
31static int
32nv04_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
33{
34 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
35 struct drm_device *dev = nvbe->dev;
36 struct drm_nouveau_private *dev_priv = dev->dev_private;
37 struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
38 unsigned i, j, pte;
39
40 NV_DEBUG(dev, "pg=0x%lx\n", mem->start);
41
42 nvbe->offset = mem->start << PAGE_SHIFT;
43 pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
44 for (i = 0; i < ttm->num_pages; i++) {
45 dma_addr_t dma_offset = nvbe->ttm.dma_address[i];
46 uint32_t offset_l = lower_32_bits(dma_offset);
47
48 for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++) {
49 nv_wo32(gpuobj, (pte * 4) + 0, offset_l | 3);
50 offset_l += NV_CTXDMA_PAGE_SIZE;
51 }
52 }
53
54 return 0;
55}
56
57static int
58nv04_sgdma_unbind(struct ttm_tt *ttm)
59{
60 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
61 struct drm_device *dev = nvbe->dev;
62 struct drm_nouveau_private *dev_priv = dev->dev_private;
63 struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
64 unsigned i, j, pte;
65
66 NV_DEBUG(dev, "\n");
67
68 if (ttm->state != tt_bound)
69 return 0;
70
71 pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
72 for (i = 0; i < ttm->num_pages; i++) {
73 for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++)
74 nv_wo32(gpuobj, (pte * 4) + 0, 0x00000000);
75 }
76
77 return 0;
78}
79
80static struct ttm_backend_func nv04_sgdma_backend = {
81 .bind = nv04_sgdma_bind,
82 .unbind = nv04_sgdma_unbind,
83 .destroy = nouveau_sgdma_destroy
84};
85
86static void
87nv41_sgdma_flush(struct nouveau_sgdma_be *nvbe)
88{
89 struct drm_device *dev = nvbe->dev;
90
91 nv_wr32(dev, 0x100810, 0x00000022);
92 if (!nv_wait(dev, 0x100810, 0x00000100, 0x00000100))
93 NV_ERROR(dev, "vm flush timeout: 0x%08x\n",
94 nv_rd32(dev, 0x100810));
95 nv_wr32(dev, 0x100810, 0x00000000);
96}
97
98static int
99nv41_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
100{
101 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
102 struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
103 struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
104 dma_addr_t *list = nvbe->ttm.dma_address;
105 u32 pte = mem->start << 2;
106 u32 cnt = ttm->num_pages;
107
108 nvbe->offset = mem->start << PAGE_SHIFT;
109
110 while (cnt--) {
111 nv_wo32(pgt, pte, (*list++ >> 7) | 1);
112 pte += 4;
113 }
114
115 nv41_sgdma_flush(nvbe);
116 return 0;
117}
118
119static int
120nv41_sgdma_unbind(struct ttm_tt *ttm)
121{
122 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
123 struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
124 struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
125 u32 pte = (nvbe->offset >> 12) << 2;
126 u32 cnt = ttm->num_pages;
127
128 while (cnt--) {
129 nv_wo32(pgt, pte, 0x00000000);
130 pte += 4;
131 }
132
133 nv41_sgdma_flush(nvbe);
134 return 0;
135}
136
137static struct ttm_backend_func nv41_sgdma_backend = {
138 .bind = nv41_sgdma_bind,
139 .unbind = nv41_sgdma_unbind,
140 .destroy = nouveau_sgdma_destroy
141};
142
143static void
144nv44_sgdma_flush(struct ttm_tt *ttm)
145{
146 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
147 struct drm_device *dev = nvbe->dev;
148
149 nv_wr32(dev, 0x100814, (ttm->num_pages - 1) << 12);
150 nv_wr32(dev, 0x100808, nvbe->offset | 0x20);
151 if (!nv_wait(dev, 0x100808, 0x00000001, 0x00000001))
152 NV_ERROR(dev, "gart flush timeout: 0x%08x\n",
153 nv_rd32(dev, 0x100808));
154 nv_wr32(dev, 0x100808, 0x00000000);
155}
156
157static void
158nv44_sgdma_fill(struct nouveau_gpuobj *pgt, dma_addr_t *list, u32 base, u32 cnt)
159{
160 struct drm_nouveau_private *dev_priv = pgt->dev->dev_private;
161 dma_addr_t dummy = dev_priv->gart_info.dummy.addr;
162 u32 pte, tmp[4];
163
164 pte = base >> 2;
165 base &= ~0x0000000f;
166
167 tmp[0] = nv_ro32(pgt, base + 0x0);
168 tmp[1] = nv_ro32(pgt, base + 0x4);
169 tmp[2] = nv_ro32(pgt, base + 0x8);
170 tmp[3] = nv_ro32(pgt, base + 0xc);
171 while (cnt--) {
172 u32 addr = list ? (*list++ >> 12) : (dummy >> 12);
173 switch (pte++ & 0x3) {
174 case 0:
175 tmp[0] &= ~0x07ffffff;
176 tmp[0] |= addr;
177 break;
178 case 1:
179 tmp[0] &= ~0xf8000000;
180 tmp[0] |= addr << 27;
181 tmp[1] &= ~0x003fffff;
182 tmp[1] |= addr >> 5;
183 break;
184 case 2:
185 tmp[1] &= ~0xffc00000;
186 tmp[1] |= addr << 22;
187 tmp[2] &= ~0x0001ffff;
188 tmp[2] |= addr >> 10;
189 break;
190 case 3:
191 tmp[2] &= ~0xfffe0000;
192 tmp[2] |= addr << 17;
193 tmp[3] &= ~0x00000fff;
194 tmp[3] |= addr >> 15;
195 break;
196 }
197 }
198
199 tmp[3] |= 0x40000000;
200
201 nv_wo32(pgt, base + 0x0, tmp[0]);
202 nv_wo32(pgt, base + 0x4, tmp[1]);
203 nv_wo32(pgt, base + 0x8, tmp[2]);
204 nv_wo32(pgt, base + 0xc, tmp[3]);
205}
206
207static int
208nv44_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
209{
210 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
211 struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
212 struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
213 dma_addr_t *list = nvbe->ttm.dma_address;
214 u32 pte = mem->start << 2, tmp[4];
215 u32 cnt = ttm->num_pages;
216 int i;
217
218 nvbe->offset = mem->start << PAGE_SHIFT;
219
220 if (pte & 0x0000000c) {
221 u32 max = 4 - ((pte >> 2) & 0x3);
222 u32 part = (cnt > max) ? max : cnt;
223 nv44_sgdma_fill(pgt, list, pte, part);
224 pte += (part << 2);
225 list += part;
226 cnt -= part;
227 }
228
229 while (cnt >= 4) {
230 for (i = 0; i < 4; i++)
231 tmp[i] = *list++ >> 12;
232 nv_wo32(pgt, pte + 0x0, tmp[0] >> 0 | tmp[1] << 27);
233 nv_wo32(pgt, pte + 0x4, tmp[1] >> 5 | tmp[2] << 22);
234 nv_wo32(pgt, pte + 0x8, tmp[2] >> 10 | tmp[3] << 17);
235 nv_wo32(pgt, pte + 0xc, tmp[3] >> 15 | 0x40000000);
236 pte += 0x10;
237 cnt -= 4;
238 }
239
240 if (cnt)
241 nv44_sgdma_fill(pgt, list, pte, cnt);
242
243 nv44_sgdma_flush(ttm);
244 return 0;
245}
246
247static int
248nv44_sgdma_unbind(struct ttm_tt *ttm)
249{
250 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
251 struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
252 struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
253 u32 pte = (nvbe->offset >> 12) << 2;
254 u32 cnt = ttm->num_pages;
255
256 if (pte & 0x0000000c) {
257 u32 max = 4 - ((pte >> 2) & 0x3);
258 u32 part = (cnt > max) ? max : cnt;
259 nv44_sgdma_fill(pgt, NULL, pte, part);
260 pte += (part << 2);
261 cnt -= part;
262 }
263
264 while (cnt >= 4) {
265 nv_wo32(pgt, pte + 0x0, 0x00000000);
266 nv_wo32(pgt, pte + 0x4, 0x00000000);
267 nv_wo32(pgt, pte + 0x8, 0x00000000);
268 nv_wo32(pgt, pte + 0xc, 0x00000000);
269 pte += 0x10;
270 cnt -= 4;
271 }
272
273 if (cnt)
274 nv44_sgdma_fill(pgt, NULL, pte, cnt);
275
276 nv44_sgdma_flush(ttm);
277 return 0;
278}
279
280static struct ttm_backend_func nv44_sgdma_backend = {
281 .bind = nv44_sgdma_bind,
282 .unbind = nv44_sgdma_unbind,
283 .destroy = nouveau_sgdma_destroy
284};
285
286static int
287nv50_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
288{
289 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
290 struct nouveau_mem *node = mem->mm_node;
291
292 /* noop: bound in move_notify() */
293 if (ttm->sg) {
294 node->sg = ttm->sg;
295 } else
296 node->pages = nvbe->ttm.dma_address;
297 return 0;
298}
299
300static int
301nv50_sgdma_unbind(struct ttm_tt *ttm)
302{
303 /* noop: unbound in move_notify() */
304 return 0;
305}
306
307static struct ttm_backend_func nv50_sgdma_backend = {
308 .bind = nv50_sgdma_bind,
309 .unbind = nv50_sgdma_unbind,
310 .destroy = nouveau_sgdma_destroy
311};
312
313struct ttm_tt *
314nouveau_sgdma_create_ttm(struct ttm_bo_device *bdev,
315 unsigned long size, uint32_t page_flags,
316 struct page *dummy_read_page)
317{
318 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
319 struct drm_device *dev = dev_priv->dev;
320 struct nouveau_sgdma_be *nvbe;
321
322 nvbe = kzalloc(sizeof(*nvbe), GFP_KERNEL);
323 if (!nvbe)
324 return NULL;
325
326 nvbe->dev = dev;
327 nvbe->ttm.ttm.func = dev_priv->gart_info.func;
328
329 if (ttm_dma_tt_init(&nvbe->ttm, bdev, size, page_flags, dummy_read_page)) {
330 kfree(nvbe);
331 return NULL;
332 }
333 return &nvbe->ttm.ttm;
334}
335
336int
337nouveau_sgdma_init(struct drm_device *dev)
338{
339 struct drm_nouveau_private *dev_priv = dev->dev_private;
340 struct nouveau_gpuobj *gpuobj = NULL;
341 u32 aper_size, align;
342 int ret;
343
344 if (dev_priv->card_type >= NV_40)
345 aper_size = 512 * 1024 * 1024;
346 else
347 aper_size = 128 * 1024 * 1024;
348
349 /* Dear NVIDIA, NV44+ would like proper present bits in PTEs for
350 * christmas. The cards before it have them, the cards after
351 * it have them, why is NV44 so unloved?
352 */
353 dev_priv->gart_info.dummy.page = alloc_page(GFP_DMA32 | GFP_KERNEL);
354 if (!dev_priv->gart_info.dummy.page)
355 return -ENOMEM;
356
357 dev_priv->gart_info.dummy.addr =
358 pci_map_page(dev->pdev, dev_priv->gart_info.dummy.page,
359 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
360 if (pci_dma_mapping_error(dev->pdev, dev_priv->gart_info.dummy.addr)) {
361 NV_ERROR(dev, "error mapping dummy page\n");
362 __free_page(dev_priv->gart_info.dummy.page);
363 dev_priv->gart_info.dummy.page = NULL;
364 return -ENOMEM;
365 }
366
367 if (dev_priv->card_type >= NV_50) {
368 dev_priv->gart_info.aper_base = 0;
369 dev_priv->gart_info.aper_size = aper_size;
370 dev_priv->gart_info.type = NOUVEAU_GART_HW;
371 dev_priv->gart_info.func = &nv50_sgdma_backend;
372 } else
373 if (0 && pci_is_pcie(dev->pdev) &&
374 dev_priv->chipset > 0x40 && dev_priv->chipset != 0x45) {
375 if (nv44_graph_class(dev)) {
376 dev_priv->gart_info.func = &nv44_sgdma_backend;
377 align = 512 * 1024;
378 } else {
379 dev_priv->gart_info.func = &nv41_sgdma_backend;
380 align = 16;
381 }
382
383 ret = nouveau_gpuobj_new(dev, NULL, aper_size / 1024, align,
384 NVOBJ_FLAG_ZERO_ALLOC |
385 NVOBJ_FLAG_ZERO_FREE, &gpuobj);
386 if (ret) {
387 NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
388 return ret;
389 }
390
391 dev_priv->gart_info.sg_ctxdma = gpuobj;
392 dev_priv->gart_info.aper_base = 0;
393 dev_priv->gart_info.aper_size = aper_size;
394 dev_priv->gart_info.type = NOUVEAU_GART_HW;
395 } else {
396 ret = nouveau_gpuobj_new(dev, NULL, (aper_size / 1024) + 8, 16,
397 NVOBJ_FLAG_ZERO_ALLOC |
398 NVOBJ_FLAG_ZERO_FREE, &gpuobj);
399 if (ret) {
400 NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
401 return ret;
402 }
403
404 nv_wo32(gpuobj, 0, NV_CLASS_DMA_IN_MEMORY |
405 (1 << 12) /* PT present */ |
406 (0 << 13) /* PT *not* linear */ |
407 (0 << 14) /* RW */ |
408 (2 << 16) /* PCI */);
409 nv_wo32(gpuobj, 4, aper_size - 1);
410
411 dev_priv->gart_info.sg_ctxdma = gpuobj;
412 dev_priv->gart_info.aper_base = 0;
413 dev_priv->gart_info.aper_size = aper_size;
414 dev_priv->gart_info.type = NOUVEAU_GART_PDMA;
415 dev_priv->gart_info.func = &nv04_sgdma_backend;
416 }
417
418 return 0;
419}
420
421void
422nouveau_sgdma_takedown(struct drm_device *dev)
423{
424 struct drm_nouveau_private *dev_priv = dev->dev_private;
425
426 nouveau_gpuobj_ref(NULL, &dev_priv->gart_info.sg_ctxdma);
427
428 if (dev_priv->gart_info.dummy.page) {
429 pci_unmap_page(dev->pdev, dev_priv->gart_info.dummy.addr,
430 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
431 __free_page(dev_priv->gart_info.dummy.page);
432 dev_priv->gart_info.dummy.page = NULL;
433 }
434}
435
436uint32_t
437nouveau_sgdma_get_physical(struct drm_device *dev, uint32_t offset)
438{
439 struct drm_nouveau_private *dev_priv = dev->dev_private;
440 struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
441 int pte = (offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
442
443 BUG_ON(dev_priv->card_type >= NV_50);
444
445 return (nv_ro32(gpuobj, 4 * pte) & ~NV_CTXDMA_PAGE_MASK) |
446 (offset & NV_CTXDMA_PAGE_MASK);
447}