Loading...
1/**************************************************************************
2 *
3 * Copyright © 2009-2015 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28#include "vmwgfx_drv.h"
29#include <drm/ttm/ttm_bo_driver.h>
30#include <drm/ttm/ttm_placement.h>
31#include <drm/ttm/ttm_page_alloc.h>
32
33static struct ttm_place vram_placement_flags = {
34 .fpfn = 0,
35 .lpfn = 0,
36 .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED
37};
38
39static struct ttm_place vram_ne_placement_flags = {
40 .fpfn = 0,
41 .lpfn = 0,
42 .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT
43};
44
45static struct ttm_place sys_placement_flags = {
46 .fpfn = 0,
47 .lpfn = 0,
48 .flags = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED
49};
50
51static struct ttm_place sys_ne_placement_flags = {
52 .fpfn = 0,
53 .lpfn = 0,
54 .flags = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT
55};
56
57static struct ttm_place gmr_placement_flags = {
58 .fpfn = 0,
59 .lpfn = 0,
60 .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED
61};
62
63static struct ttm_place gmr_ne_placement_flags = {
64 .fpfn = 0,
65 .lpfn = 0,
66 .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT
67};
68
69static struct ttm_place mob_placement_flags = {
70 .fpfn = 0,
71 .lpfn = 0,
72 .flags = VMW_PL_FLAG_MOB | TTM_PL_FLAG_CACHED
73};
74
75static struct ttm_place mob_ne_placement_flags = {
76 .fpfn = 0,
77 .lpfn = 0,
78 .flags = VMW_PL_FLAG_MOB | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT
79};
80
81struct ttm_placement vmw_vram_placement = {
82 .num_placement = 1,
83 .placement = &vram_placement_flags,
84 .num_busy_placement = 1,
85 .busy_placement = &vram_placement_flags
86};
87
88static struct ttm_place vram_gmr_placement_flags[] = {
89 {
90 .fpfn = 0,
91 .lpfn = 0,
92 .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED
93 }, {
94 .fpfn = 0,
95 .lpfn = 0,
96 .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED
97 }
98};
99
100static struct ttm_place gmr_vram_placement_flags[] = {
101 {
102 .fpfn = 0,
103 .lpfn = 0,
104 .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED
105 }, {
106 .fpfn = 0,
107 .lpfn = 0,
108 .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED
109 }
110};
111
112struct ttm_placement vmw_vram_gmr_placement = {
113 .num_placement = 2,
114 .placement = vram_gmr_placement_flags,
115 .num_busy_placement = 1,
116 .busy_placement = &gmr_placement_flags
117};
118
119static struct ttm_place vram_gmr_ne_placement_flags[] = {
120 {
121 .fpfn = 0,
122 .lpfn = 0,
123 .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED |
124 TTM_PL_FLAG_NO_EVICT
125 }, {
126 .fpfn = 0,
127 .lpfn = 0,
128 .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED |
129 TTM_PL_FLAG_NO_EVICT
130 }
131};
132
133struct ttm_placement vmw_vram_gmr_ne_placement = {
134 .num_placement = 2,
135 .placement = vram_gmr_ne_placement_flags,
136 .num_busy_placement = 1,
137 .busy_placement = &gmr_ne_placement_flags
138};
139
140struct ttm_placement vmw_vram_sys_placement = {
141 .num_placement = 1,
142 .placement = &vram_placement_flags,
143 .num_busy_placement = 1,
144 .busy_placement = &sys_placement_flags
145};
146
147struct ttm_placement vmw_vram_ne_placement = {
148 .num_placement = 1,
149 .placement = &vram_ne_placement_flags,
150 .num_busy_placement = 1,
151 .busy_placement = &vram_ne_placement_flags
152};
153
154struct ttm_placement vmw_sys_placement = {
155 .num_placement = 1,
156 .placement = &sys_placement_flags,
157 .num_busy_placement = 1,
158 .busy_placement = &sys_placement_flags
159};
160
161struct ttm_placement vmw_sys_ne_placement = {
162 .num_placement = 1,
163 .placement = &sys_ne_placement_flags,
164 .num_busy_placement = 1,
165 .busy_placement = &sys_ne_placement_flags
166};
167
168static struct ttm_place evictable_placement_flags[] = {
169 {
170 .fpfn = 0,
171 .lpfn = 0,
172 .flags = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED
173 }, {
174 .fpfn = 0,
175 .lpfn = 0,
176 .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED
177 }, {
178 .fpfn = 0,
179 .lpfn = 0,
180 .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED
181 }, {
182 .fpfn = 0,
183 .lpfn = 0,
184 .flags = VMW_PL_FLAG_MOB | TTM_PL_FLAG_CACHED
185 }
186};
187
188struct ttm_placement vmw_evictable_placement = {
189 .num_placement = 4,
190 .placement = evictable_placement_flags,
191 .num_busy_placement = 1,
192 .busy_placement = &sys_placement_flags
193};
194
195struct ttm_placement vmw_srf_placement = {
196 .num_placement = 1,
197 .num_busy_placement = 2,
198 .placement = &gmr_placement_flags,
199 .busy_placement = gmr_vram_placement_flags
200};
201
202struct ttm_placement vmw_mob_placement = {
203 .num_placement = 1,
204 .num_busy_placement = 1,
205 .placement = &mob_placement_flags,
206 .busy_placement = &mob_placement_flags
207};
208
209struct ttm_placement vmw_mob_ne_placement = {
210 .num_placement = 1,
211 .num_busy_placement = 1,
212 .placement = &mob_ne_placement_flags,
213 .busy_placement = &mob_ne_placement_flags
214};
215
216struct vmw_ttm_tt {
217 struct ttm_dma_tt dma_ttm;
218 struct vmw_private *dev_priv;
219 int gmr_id;
220 struct vmw_mob *mob;
221 int mem_type;
222 struct sg_table sgt;
223 struct vmw_sg_table vsgt;
224 uint64_t sg_alloc_size;
225 bool mapped;
226};
227
228const size_t vmw_tt_size = sizeof(struct vmw_ttm_tt);
229
230/**
231 * Helper functions to advance a struct vmw_piter iterator.
232 *
233 * @viter: Pointer to the iterator.
234 *
235 * These functions return false if past the end of the list,
236 * true otherwise. Functions are selected depending on the current
237 * DMA mapping mode.
238 */
239static bool __vmw_piter_non_sg_next(struct vmw_piter *viter)
240{
241 return ++(viter->i) < viter->num_pages;
242}
243
244static bool __vmw_piter_sg_next(struct vmw_piter *viter)
245{
246 return __sg_page_iter_next(&viter->iter);
247}
248
249
250/**
251 * Helper functions to return a pointer to the current page.
252 *
253 * @viter: Pointer to the iterator
254 *
255 * These functions return a pointer to the page currently
256 * pointed to by @viter. Functions are selected depending on the
257 * current mapping mode.
258 */
259static struct page *__vmw_piter_non_sg_page(struct vmw_piter *viter)
260{
261 return viter->pages[viter->i];
262}
263
264static struct page *__vmw_piter_sg_page(struct vmw_piter *viter)
265{
266 return sg_page_iter_page(&viter->iter);
267}
268
269
270/**
271 * Helper functions to return the DMA address of the current page.
272 *
273 * @viter: Pointer to the iterator
274 *
275 * These functions return the DMA address of the page currently
276 * pointed to by @viter. Functions are selected depending on the
277 * current mapping mode.
278 */
279static dma_addr_t __vmw_piter_phys_addr(struct vmw_piter *viter)
280{
281 return page_to_phys(viter->pages[viter->i]);
282}
283
284static dma_addr_t __vmw_piter_dma_addr(struct vmw_piter *viter)
285{
286 return viter->addrs[viter->i];
287}
288
289static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter)
290{
291 return sg_page_iter_dma_address(&viter->iter);
292}
293
294
295/**
296 * vmw_piter_start - Initialize a struct vmw_piter.
297 *
298 * @viter: Pointer to the iterator to initialize
299 * @vsgt: Pointer to a struct vmw_sg_table to initialize from
300 *
301 * Note that we're following the convention of __sg_page_iter_start, so that
302 * the iterator doesn't point to a valid page after initialization; it has
303 * to be advanced one step first.
304 */
305void vmw_piter_start(struct vmw_piter *viter, const struct vmw_sg_table *vsgt,
306 unsigned long p_offset)
307{
308 viter->i = p_offset - 1;
309 viter->num_pages = vsgt->num_pages;
310 switch (vsgt->mode) {
311 case vmw_dma_phys:
312 viter->next = &__vmw_piter_non_sg_next;
313 viter->dma_address = &__vmw_piter_phys_addr;
314 viter->page = &__vmw_piter_non_sg_page;
315 viter->pages = vsgt->pages;
316 break;
317 case vmw_dma_alloc_coherent:
318 viter->next = &__vmw_piter_non_sg_next;
319 viter->dma_address = &__vmw_piter_dma_addr;
320 viter->page = &__vmw_piter_non_sg_page;
321 viter->addrs = vsgt->addrs;
322 viter->pages = vsgt->pages;
323 break;
324 case vmw_dma_map_populate:
325 case vmw_dma_map_bind:
326 viter->next = &__vmw_piter_sg_next;
327 viter->dma_address = &__vmw_piter_sg_addr;
328 viter->page = &__vmw_piter_sg_page;
329 __sg_page_iter_start(&viter->iter, vsgt->sgt->sgl,
330 vsgt->sgt->orig_nents, p_offset);
331 break;
332 default:
333 BUG();
334 }
335}
336
337/**
338 * vmw_ttm_unmap_from_dma - unmap device addresses previsouly mapped for
339 * TTM pages
340 *
341 * @vmw_tt: Pointer to a struct vmw_ttm_backend
342 *
343 * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma.
344 */
345static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt *vmw_tt)
346{
347 struct device *dev = vmw_tt->dev_priv->dev->dev;
348
349 dma_unmap_sg(dev, vmw_tt->sgt.sgl, vmw_tt->sgt.nents,
350 DMA_BIDIRECTIONAL);
351 vmw_tt->sgt.nents = vmw_tt->sgt.orig_nents;
352}
353
354/**
355 * vmw_ttm_map_for_dma - map TTM pages to get device addresses
356 *
357 * @vmw_tt: Pointer to a struct vmw_ttm_backend
358 *
359 * This function is used to get device addresses from the kernel DMA layer.
360 * However, it's violating the DMA API in that when this operation has been
361 * performed, it's illegal for the CPU to write to the pages without first
362 * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is
363 * therefore only legal to call this function if we know that the function
364 * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most
365 * a CPU write buffer flush.
366 */
367static int vmw_ttm_map_for_dma(struct vmw_ttm_tt *vmw_tt)
368{
369 struct device *dev = vmw_tt->dev_priv->dev->dev;
370 int ret;
371
372 ret = dma_map_sg(dev, vmw_tt->sgt.sgl, vmw_tt->sgt.orig_nents,
373 DMA_BIDIRECTIONAL);
374 if (unlikely(ret == 0))
375 return -ENOMEM;
376
377 vmw_tt->sgt.nents = ret;
378
379 return 0;
380}
381
382/**
383 * vmw_ttm_map_dma - Make sure TTM pages are visible to the device
384 *
385 * @vmw_tt: Pointer to a struct vmw_ttm_tt
386 *
387 * Select the correct function for and make sure the TTM pages are
388 * visible to the device. Allocate storage for the device mappings.
389 * If a mapping has already been performed, indicated by the storage
390 * pointer being non NULL, the function returns success.
391 */
392static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
393{
394 struct vmw_private *dev_priv = vmw_tt->dev_priv;
395 struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
396 struct vmw_sg_table *vsgt = &vmw_tt->vsgt;
397 struct vmw_piter iter;
398 dma_addr_t old;
399 int ret = 0;
400 static size_t sgl_size;
401 static size_t sgt_size;
402
403 if (vmw_tt->mapped)
404 return 0;
405
406 vsgt->mode = dev_priv->map_mode;
407 vsgt->pages = vmw_tt->dma_ttm.ttm.pages;
408 vsgt->num_pages = vmw_tt->dma_ttm.ttm.num_pages;
409 vsgt->addrs = vmw_tt->dma_ttm.dma_address;
410 vsgt->sgt = &vmw_tt->sgt;
411
412 switch (dev_priv->map_mode) {
413 case vmw_dma_map_bind:
414 case vmw_dma_map_populate:
415 if (unlikely(!sgl_size)) {
416 sgl_size = ttm_round_pot(sizeof(struct scatterlist));
417 sgt_size = ttm_round_pot(sizeof(struct sg_table));
418 }
419 vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages;
420 ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, false,
421 true);
422 if (unlikely(ret != 0))
423 return ret;
424
425 ret = sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages,
426 vsgt->num_pages, 0,
427 (unsigned long)
428 vsgt->num_pages << PAGE_SHIFT,
429 GFP_KERNEL);
430 if (unlikely(ret != 0))
431 goto out_sg_alloc_fail;
432
433 if (vsgt->num_pages > vmw_tt->sgt.nents) {
434 uint64_t over_alloc =
435 sgl_size * (vsgt->num_pages -
436 vmw_tt->sgt.nents);
437
438 ttm_mem_global_free(glob, over_alloc);
439 vmw_tt->sg_alloc_size -= over_alloc;
440 }
441
442 ret = vmw_ttm_map_for_dma(vmw_tt);
443 if (unlikely(ret != 0))
444 goto out_map_fail;
445
446 break;
447 default:
448 break;
449 }
450
451 old = ~((dma_addr_t) 0);
452 vmw_tt->vsgt.num_regions = 0;
453 for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) {
454 dma_addr_t cur = vmw_piter_dma_addr(&iter);
455
456 if (cur != old + PAGE_SIZE)
457 vmw_tt->vsgt.num_regions++;
458 old = cur;
459 }
460
461 vmw_tt->mapped = true;
462 return 0;
463
464out_map_fail:
465 sg_free_table(vmw_tt->vsgt.sgt);
466 vmw_tt->vsgt.sgt = NULL;
467out_sg_alloc_fail:
468 ttm_mem_global_free(glob, vmw_tt->sg_alloc_size);
469 return ret;
470}
471
472/**
473 * vmw_ttm_unmap_dma - Tear down any TTM page device mappings
474 *
475 * @vmw_tt: Pointer to a struct vmw_ttm_tt
476 *
477 * Tear down any previously set up device DMA mappings and free
478 * any storage space allocated for them. If there are no mappings set up,
479 * this function is a NOP.
480 */
481static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt)
482{
483 struct vmw_private *dev_priv = vmw_tt->dev_priv;
484
485 if (!vmw_tt->vsgt.sgt)
486 return;
487
488 switch (dev_priv->map_mode) {
489 case vmw_dma_map_bind:
490 case vmw_dma_map_populate:
491 vmw_ttm_unmap_from_dma(vmw_tt);
492 sg_free_table(vmw_tt->vsgt.sgt);
493 vmw_tt->vsgt.sgt = NULL;
494 ttm_mem_global_free(vmw_mem_glob(dev_priv),
495 vmw_tt->sg_alloc_size);
496 break;
497 default:
498 break;
499 }
500 vmw_tt->mapped = false;
501}
502
503
504/**
505 * vmw_bo_map_dma - Make sure buffer object pages are visible to the device
506 *
507 * @bo: Pointer to a struct ttm_buffer_object
508 *
509 * Wrapper around vmw_ttm_map_dma, that takes a TTM buffer object pointer
510 * instead of a pointer to a struct vmw_ttm_backend as argument.
511 * Note that the buffer object must be either pinned or reserved before
512 * calling this function.
513 */
514int vmw_bo_map_dma(struct ttm_buffer_object *bo)
515{
516 struct vmw_ttm_tt *vmw_tt =
517 container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
518
519 return vmw_ttm_map_dma(vmw_tt);
520}
521
522
523/**
524 * vmw_bo_unmap_dma - Make sure buffer object pages are visible to the device
525 *
526 * @bo: Pointer to a struct ttm_buffer_object
527 *
528 * Wrapper around vmw_ttm_unmap_dma, that takes a TTM buffer object pointer
529 * instead of a pointer to a struct vmw_ttm_backend as argument.
530 */
531void vmw_bo_unmap_dma(struct ttm_buffer_object *bo)
532{
533 struct vmw_ttm_tt *vmw_tt =
534 container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
535
536 vmw_ttm_unmap_dma(vmw_tt);
537}
538
539
540/**
541 * vmw_bo_sg_table - Return a struct vmw_sg_table object for a
542 * TTM buffer object
543 *
544 * @bo: Pointer to a struct ttm_buffer_object
545 *
546 * Returns a pointer to a struct vmw_sg_table object. The object should
547 * not be freed after use.
548 * Note that for the device addresses to be valid, the buffer object must
549 * either be reserved or pinned.
550 */
551const struct vmw_sg_table *vmw_bo_sg_table(struct ttm_buffer_object *bo)
552{
553 struct vmw_ttm_tt *vmw_tt =
554 container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
555
556 return &vmw_tt->vsgt;
557}
558
559
560static int vmw_ttm_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
561{
562 struct vmw_ttm_tt *vmw_be =
563 container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
564 int ret;
565
566 ret = vmw_ttm_map_dma(vmw_be);
567 if (unlikely(ret != 0))
568 return ret;
569
570 vmw_be->gmr_id = bo_mem->start;
571 vmw_be->mem_type = bo_mem->mem_type;
572
573 switch (bo_mem->mem_type) {
574 case VMW_PL_GMR:
575 return vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt,
576 ttm->num_pages, vmw_be->gmr_id);
577 case VMW_PL_MOB:
578 if (unlikely(vmw_be->mob == NULL)) {
579 vmw_be->mob =
580 vmw_mob_create(ttm->num_pages);
581 if (unlikely(vmw_be->mob == NULL))
582 return -ENOMEM;
583 }
584
585 return vmw_mob_bind(vmw_be->dev_priv, vmw_be->mob,
586 &vmw_be->vsgt, ttm->num_pages,
587 vmw_be->gmr_id);
588 default:
589 BUG();
590 }
591 return 0;
592}
593
594static int vmw_ttm_unbind(struct ttm_tt *ttm)
595{
596 struct vmw_ttm_tt *vmw_be =
597 container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
598
599 switch (vmw_be->mem_type) {
600 case VMW_PL_GMR:
601 vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id);
602 break;
603 case VMW_PL_MOB:
604 vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob);
605 break;
606 default:
607 BUG();
608 }
609
610 if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind)
611 vmw_ttm_unmap_dma(vmw_be);
612
613 return 0;
614}
615
616
617static void vmw_ttm_destroy(struct ttm_tt *ttm)
618{
619 struct vmw_ttm_tt *vmw_be =
620 container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
621
622 vmw_ttm_unmap_dma(vmw_be);
623 if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
624 ttm_dma_tt_fini(&vmw_be->dma_ttm);
625 else
626 ttm_tt_fini(ttm);
627
628 if (vmw_be->mob)
629 vmw_mob_destroy(vmw_be->mob);
630
631 kfree(vmw_be);
632}
633
634
635static int vmw_ttm_populate(struct ttm_tt *ttm)
636{
637 struct vmw_ttm_tt *vmw_tt =
638 container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
639 struct vmw_private *dev_priv = vmw_tt->dev_priv;
640 struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
641 int ret;
642
643 if (ttm->state != tt_unpopulated)
644 return 0;
645
646 if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
647 size_t size =
648 ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
649 ret = ttm_mem_global_alloc(glob, size, false, true);
650 if (unlikely(ret != 0))
651 return ret;
652
653 ret = ttm_dma_populate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
654 if (unlikely(ret != 0))
655 ttm_mem_global_free(glob, size);
656 } else
657 ret = ttm_pool_populate(ttm);
658
659 return ret;
660}
661
662static void vmw_ttm_unpopulate(struct ttm_tt *ttm)
663{
664 struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt,
665 dma_ttm.ttm);
666 struct vmw_private *dev_priv = vmw_tt->dev_priv;
667 struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
668
669
670 if (vmw_tt->mob) {
671 vmw_mob_destroy(vmw_tt->mob);
672 vmw_tt->mob = NULL;
673 }
674
675 vmw_ttm_unmap_dma(vmw_tt);
676 if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
677 size_t size =
678 ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
679
680 ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
681 ttm_mem_global_free(glob, size);
682 } else
683 ttm_pool_unpopulate(ttm);
684}
685
686static struct ttm_backend_func vmw_ttm_func = {
687 .bind = vmw_ttm_bind,
688 .unbind = vmw_ttm_unbind,
689 .destroy = vmw_ttm_destroy,
690};
691
692static struct ttm_tt *vmw_ttm_tt_create(struct ttm_bo_device *bdev,
693 unsigned long size, uint32_t page_flags,
694 struct page *dummy_read_page)
695{
696 struct vmw_ttm_tt *vmw_be;
697 int ret;
698
699 vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL);
700 if (!vmw_be)
701 return NULL;
702
703 vmw_be->dma_ttm.ttm.func = &vmw_ttm_func;
704 vmw_be->dev_priv = container_of(bdev, struct vmw_private, bdev);
705 vmw_be->mob = NULL;
706
707 if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
708 ret = ttm_dma_tt_init(&vmw_be->dma_ttm, bdev, size, page_flags,
709 dummy_read_page);
710 else
711 ret = ttm_tt_init(&vmw_be->dma_ttm.ttm, bdev, size, page_flags,
712 dummy_read_page);
713 if (unlikely(ret != 0))
714 goto out_no_init;
715
716 return &vmw_be->dma_ttm.ttm;
717out_no_init:
718 kfree(vmw_be);
719 return NULL;
720}
721
722static int vmw_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
723{
724 return 0;
725}
726
727static int vmw_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
728 struct ttm_mem_type_manager *man)
729{
730 switch (type) {
731 case TTM_PL_SYSTEM:
732 /* System memory */
733
734 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
735 man->available_caching = TTM_PL_FLAG_CACHED;
736 man->default_caching = TTM_PL_FLAG_CACHED;
737 break;
738 case TTM_PL_VRAM:
739 /* "On-card" video ram */
740 man->func = &ttm_bo_manager_func;
741 man->gpu_offset = 0;
742 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE;
743 man->available_caching = TTM_PL_FLAG_CACHED;
744 man->default_caching = TTM_PL_FLAG_CACHED;
745 break;
746 case VMW_PL_GMR:
747 case VMW_PL_MOB:
748 /*
749 * "Guest Memory Regions" is an aperture like feature with
750 * one slot per bo. There is an upper limit of the number of
751 * slots as well as the bo size.
752 */
753 man->func = &vmw_gmrid_manager_func;
754 man->gpu_offset = 0;
755 man->flags = TTM_MEMTYPE_FLAG_CMA | TTM_MEMTYPE_FLAG_MAPPABLE;
756 man->available_caching = TTM_PL_FLAG_CACHED;
757 man->default_caching = TTM_PL_FLAG_CACHED;
758 break;
759 default:
760 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
761 return -EINVAL;
762 }
763 return 0;
764}
765
766static void vmw_evict_flags(struct ttm_buffer_object *bo,
767 struct ttm_placement *placement)
768{
769 *placement = vmw_sys_placement;
770}
771
772static int vmw_verify_access(struct ttm_buffer_object *bo, struct file *filp)
773{
774 struct ttm_object_file *tfile =
775 vmw_fpriv((struct drm_file *)filp->private_data)->tfile;
776
777 return vmw_user_dmabuf_verify_access(bo, tfile);
778}
779
780static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
781{
782 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
783 struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev);
784
785 mem->bus.addr = NULL;
786 mem->bus.is_iomem = false;
787 mem->bus.offset = 0;
788 mem->bus.size = mem->num_pages << PAGE_SHIFT;
789 mem->bus.base = 0;
790 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
791 return -EINVAL;
792 switch (mem->mem_type) {
793 case TTM_PL_SYSTEM:
794 case VMW_PL_GMR:
795 case VMW_PL_MOB:
796 return 0;
797 case TTM_PL_VRAM:
798 mem->bus.offset = mem->start << PAGE_SHIFT;
799 mem->bus.base = dev_priv->vram_start;
800 mem->bus.is_iomem = true;
801 break;
802 default:
803 return -EINVAL;
804 }
805 return 0;
806}
807
808static void vmw_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
809{
810}
811
812static int vmw_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
813{
814 return 0;
815}
816
817/**
818 * vmw_move_notify - TTM move_notify_callback
819 *
820 * @bo: The TTM buffer object about to move.
821 * @mem: The struct ttm_mem_reg indicating to what memory
822 * region the move is taking place.
823 *
824 * Calls move_notify for all subsystems needing it.
825 * (currently only resources).
826 */
827static void vmw_move_notify(struct ttm_buffer_object *bo,
828 struct ttm_mem_reg *mem)
829{
830 vmw_resource_move_notify(bo, mem);
831 vmw_query_move_notify(bo, mem);
832}
833
834
835/**
836 * vmw_swap_notify - TTM move_notify_callback
837 *
838 * @bo: The TTM buffer object about to be swapped out.
839 */
840static void vmw_swap_notify(struct ttm_buffer_object *bo)
841{
842 ttm_bo_wait(bo, false, false, false);
843}
844
845
846struct ttm_bo_driver vmw_bo_driver = {
847 .ttm_tt_create = &vmw_ttm_tt_create,
848 .ttm_tt_populate = &vmw_ttm_populate,
849 .ttm_tt_unpopulate = &vmw_ttm_unpopulate,
850 .invalidate_caches = vmw_invalidate_caches,
851 .init_mem_type = vmw_init_mem_type,
852 .evict_flags = vmw_evict_flags,
853 .move = NULL,
854 .verify_access = vmw_verify_access,
855 .move_notify = vmw_move_notify,
856 .swap_notify = vmw_swap_notify,
857 .fault_reserve_notify = &vmw_ttm_fault_reserve_notify,
858 .io_mem_reserve = &vmw_ttm_io_mem_reserve,
859 .io_mem_free = &vmw_ttm_io_mem_free,
860};
1/**************************************************************************
2 *
3 * Copyright © 2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28#include "vmwgfx_drv.h"
29#include <drm/ttm/ttm_bo_driver.h>
30#include <drm/ttm/ttm_placement.h>
31#include <drm/ttm/ttm_page_alloc.h>
32
33static uint32_t vram_placement_flags = TTM_PL_FLAG_VRAM |
34 TTM_PL_FLAG_CACHED;
35
36static uint32_t vram_ne_placement_flags = TTM_PL_FLAG_VRAM |
37 TTM_PL_FLAG_CACHED |
38 TTM_PL_FLAG_NO_EVICT;
39
40static uint32_t sys_placement_flags = TTM_PL_FLAG_SYSTEM |
41 TTM_PL_FLAG_CACHED;
42
43static uint32_t sys_ne_placement_flags = TTM_PL_FLAG_SYSTEM |
44 TTM_PL_FLAG_CACHED |
45 TTM_PL_FLAG_NO_EVICT;
46
47static uint32_t gmr_placement_flags = VMW_PL_FLAG_GMR |
48 TTM_PL_FLAG_CACHED;
49
50static uint32_t gmr_ne_placement_flags = VMW_PL_FLAG_GMR |
51 TTM_PL_FLAG_CACHED |
52 TTM_PL_FLAG_NO_EVICT;
53
54static uint32_t mob_placement_flags = VMW_PL_FLAG_MOB |
55 TTM_PL_FLAG_CACHED;
56
57struct ttm_placement vmw_vram_placement = {
58 .fpfn = 0,
59 .lpfn = 0,
60 .num_placement = 1,
61 .placement = &vram_placement_flags,
62 .num_busy_placement = 1,
63 .busy_placement = &vram_placement_flags
64};
65
66static uint32_t vram_gmr_placement_flags[] = {
67 TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED,
68 VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED
69};
70
71static uint32_t gmr_vram_placement_flags[] = {
72 VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED,
73 TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED
74};
75
76struct ttm_placement vmw_vram_gmr_placement = {
77 .fpfn = 0,
78 .lpfn = 0,
79 .num_placement = 2,
80 .placement = vram_gmr_placement_flags,
81 .num_busy_placement = 1,
82 .busy_placement = &gmr_placement_flags
83};
84
85static uint32_t vram_gmr_ne_placement_flags[] = {
86 TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT,
87 VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT
88};
89
90struct ttm_placement vmw_vram_gmr_ne_placement = {
91 .fpfn = 0,
92 .lpfn = 0,
93 .num_placement = 2,
94 .placement = vram_gmr_ne_placement_flags,
95 .num_busy_placement = 1,
96 .busy_placement = &gmr_ne_placement_flags
97};
98
99struct ttm_placement vmw_vram_sys_placement = {
100 .fpfn = 0,
101 .lpfn = 0,
102 .num_placement = 1,
103 .placement = &vram_placement_flags,
104 .num_busy_placement = 1,
105 .busy_placement = &sys_placement_flags
106};
107
108struct ttm_placement vmw_vram_ne_placement = {
109 .fpfn = 0,
110 .lpfn = 0,
111 .num_placement = 1,
112 .placement = &vram_ne_placement_flags,
113 .num_busy_placement = 1,
114 .busy_placement = &vram_ne_placement_flags
115};
116
117struct ttm_placement vmw_sys_placement = {
118 .fpfn = 0,
119 .lpfn = 0,
120 .num_placement = 1,
121 .placement = &sys_placement_flags,
122 .num_busy_placement = 1,
123 .busy_placement = &sys_placement_flags
124};
125
126struct ttm_placement vmw_sys_ne_placement = {
127 .fpfn = 0,
128 .lpfn = 0,
129 .num_placement = 1,
130 .placement = &sys_ne_placement_flags,
131 .num_busy_placement = 1,
132 .busy_placement = &sys_ne_placement_flags
133};
134
135static uint32_t evictable_placement_flags[] = {
136 TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED,
137 TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED,
138 VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED,
139 VMW_PL_FLAG_MOB | TTM_PL_FLAG_CACHED
140};
141
142struct ttm_placement vmw_evictable_placement = {
143 .fpfn = 0,
144 .lpfn = 0,
145 .num_placement = 4,
146 .placement = evictable_placement_flags,
147 .num_busy_placement = 1,
148 .busy_placement = &sys_placement_flags
149};
150
151struct ttm_placement vmw_srf_placement = {
152 .fpfn = 0,
153 .lpfn = 0,
154 .num_placement = 1,
155 .num_busy_placement = 2,
156 .placement = &gmr_placement_flags,
157 .busy_placement = gmr_vram_placement_flags
158};
159
160struct ttm_placement vmw_mob_placement = {
161 .fpfn = 0,
162 .lpfn = 0,
163 .num_placement = 1,
164 .num_busy_placement = 1,
165 .placement = &mob_placement_flags,
166 .busy_placement = &mob_placement_flags
167};
168
169struct vmw_ttm_tt {
170 struct ttm_dma_tt dma_ttm;
171 struct vmw_private *dev_priv;
172 int gmr_id;
173 struct vmw_mob *mob;
174 int mem_type;
175 struct sg_table sgt;
176 struct vmw_sg_table vsgt;
177 uint64_t sg_alloc_size;
178 bool mapped;
179};
180
181const size_t vmw_tt_size = sizeof(struct vmw_ttm_tt);
182
183/**
184 * Helper functions to advance a struct vmw_piter iterator.
185 *
186 * @viter: Pointer to the iterator.
187 *
188 * These functions return false if past the end of the list,
189 * true otherwise. Functions are selected depending on the current
190 * DMA mapping mode.
191 */
192static bool __vmw_piter_non_sg_next(struct vmw_piter *viter)
193{
194 return ++(viter->i) < viter->num_pages;
195}
196
197static bool __vmw_piter_sg_next(struct vmw_piter *viter)
198{
199 return __sg_page_iter_next(&viter->iter);
200}
201
202
203/**
204 * Helper functions to return a pointer to the current page.
205 *
206 * @viter: Pointer to the iterator
207 *
208 * These functions return a pointer to the page currently
209 * pointed to by @viter. Functions are selected depending on the
210 * current mapping mode.
211 */
212static struct page *__vmw_piter_non_sg_page(struct vmw_piter *viter)
213{
214 return viter->pages[viter->i];
215}
216
217static struct page *__vmw_piter_sg_page(struct vmw_piter *viter)
218{
219 return sg_page_iter_page(&viter->iter);
220}
221
222
223/**
224 * Helper functions to return the DMA address of the current page.
225 *
226 * @viter: Pointer to the iterator
227 *
228 * These functions return the DMA address of the page currently
229 * pointed to by @viter. Functions are selected depending on the
230 * current mapping mode.
231 */
232static dma_addr_t __vmw_piter_phys_addr(struct vmw_piter *viter)
233{
234 return page_to_phys(viter->pages[viter->i]);
235}
236
237static dma_addr_t __vmw_piter_dma_addr(struct vmw_piter *viter)
238{
239 return viter->addrs[viter->i];
240}
241
242static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter)
243{
244 return sg_page_iter_dma_address(&viter->iter);
245}
246
247
248/**
249 * vmw_piter_start - Initialize a struct vmw_piter.
250 *
251 * @viter: Pointer to the iterator to initialize
252 * @vsgt: Pointer to a struct vmw_sg_table to initialize from
253 *
254 * Note that we're following the convention of __sg_page_iter_start, so that
255 * the iterator doesn't point to a valid page after initialization; it has
256 * to be advanced one step first.
257 */
258void vmw_piter_start(struct vmw_piter *viter, const struct vmw_sg_table *vsgt,
259 unsigned long p_offset)
260{
261 viter->i = p_offset - 1;
262 viter->num_pages = vsgt->num_pages;
263 switch (vsgt->mode) {
264 case vmw_dma_phys:
265 viter->next = &__vmw_piter_non_sg_next;
266 viter->dma_address = &__vmw_piter_phys_addr;
267 viter->page = &__vmw_piter_non_sg_page;
268 viter->pages = vsgt->pages;
269 break;
270 case vmw_dma_alloc_coherent:
271 viter->next = &__vmw_piter_non_sg_next;
272 viter->dma_address = &__vmw_piter_dma_addr;
273 viter->page = &__vmw_piter_non_sg_page;
274 viter->addrs = vsgt->addrs;
275 viter->pages = vsgt->pages;
276 break;
277 case vmw_dma_map_populate:
278 case vmw_dma_map_bind:
279 viter->next = &__vmw_piter_sg_next;
280 viter->dma_address = &__vmw_piter_sg_addr;
281 viter->page = &__vmw_piter_sg_page;
282 __sg_page_iter_start(&viter->iter, vsgt->sgt->sgl,
283 vsgt->sgt->orig_nents, p_offset);
284 break;
285 default:
286 BUG();
287 }
288}
289
290/**
291 * vmw_ttm_unmap_from_dma - unmap device addresses previsouly mapped for
292 * TTM pages
293 *
294 * @vmw_tt: Pointer to a struct vmw_ttm_backend
295 *
296 * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma.
297 */
298static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt *vmw_tt)
299{
300 struct device *dev = vmw_tt->dev_priv->dev->dev;
301
302 dma_unmap_sg(dev, vmw_tt->sgt.sgl, vmw_tt->sgt.nents,
303 DMA_BIDIRECTIONAL);
304 vmw_tt->sgt.nents = vmw_tt->sgt.orig_nents;
305}
306
307/**
308 * vmw_ttm_map_for_dma - map TTM pages to get device addresses
309 *
310 * @vmw_tt: Pointer to a struct vmw_ttm_backend
311 *
312 * This function is used to get device addresses from the kernel DMA layer.
313 * However, it's violating the DMA API in that when this operation has been
314 * performed, it's illegal for the CPU to write to the pages without first
315 * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is
316 * therefore only legal to call this function if we know that the function
317 * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most
318 * a CPU write buffer flush.
319 */
320static int vmw_ttm_map_for_dma(struct vmw_ttm_tt *vmw_tt)
321{
322 struct device *dev = vmw_tt->dev_priv->dev->dev;
323 int ret;
324
325 ret = dma_map_sg(dev, vmw_tt->sgt.sgl, vmw_tt->sgt.orig_nents,
326 DMA_BIDIRECTIONAL);
327 if (unlikely(ret == 0))
328 return -ENOMEM;
329
330 vmw_tt->sgt.nents = ret;
331
332 return 0;
333}
334
335/**
336 * vmw_ttm_map_dma - Make sure TTM pages are visible to the device
337 *
338 * @vmw_tt: Pointer to a struct vmw_ttm_tt
339 *
340 * Select the correct function for and make sure the TTM pages are
341 * visible to the device. Allocate storage for the device mappings.
342 * If a mapping has already been performed, indicated by the storage
343 * pointer being non NULL, the function returns success.
344 */
345static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
346{
347 struct vmw_private *dev_priv = vmw_tt->dev_priv;
348 struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
349 struct vmw_sg_table *vsgt = &vmw_tt->vsgt;
350 struct vmw_piter iter;
351 dma_addr_t old;
352 int ret = 0;
353 static size_t sgl_size;
354 static size_t sgt_size;
355
356 if (vmw_tt->mapped)
357 return 0;
358
359 vsgt->mode = dev_priv->map_mode;
360 vsgt->pages = vmw_tt->dma_ttm.ttm.pages;
361 vsgt->num_pages = vmw_tt->dma_ttm.ttm.num_pages;
362 vsgt->addrs = vmw_tt->dma_ttm.dma_address;
363 vsgt->sgt = &vmw_tt->sgt;
364
365 switch (dev_priv->map_mode) {
366 case vmw_dma_map_bind:
367 case vmw_dma_map_populate:
368 if (unlikely(!sgl_size)) {
369 sgl_size = ttm_round_pot(sizeof(struct scatterlist));
370 sgt_size = ttm_round_pot(sizeof(struct sg_table));
371 }
372 vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages;
373 ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, false,
374 true);
375 if (unlikely(ret != 0))
376 return ret;
377
378 ret = sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages,
379 vsgt->num_pages, 0,
380 (unsigned long)
381 vsgt->num_pages << PAGE_SHIFT,
382 GFP_KERNEL);
383 if (unlikely(ret != 0))
384 goto out_sg_alloc_fail;
385
386 if (vsgt->num_pages > vmw_tt->sgt.nents) {
387 uint64_t over_alloc =
388 sgl_size * (vsgt->num_pages -
389 vmw_tt->sgt.nents);
390
391 ttm_mem_global_free(glob, over_alloc);
392 vmw_tt->sg_alloc_size -= over_alloc;
393 }
394
395 ret = vmw_ttm_map_for_dma(vmw_tt);
396 if (unlikely(ret != 0))
397 goto out_map_fail;
398
399 break;
400 default:
401 break;
402 }
403
404 old = ~((dma_addr_t) 0);
405 vmw_tt->vsgt.num_regions = 0;
406 for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) {
407 dma_addr_t cur = vmw_piter_dma_addr(&iter);
408
409 if (cur != old + PAGE_SIZE)
410 vmw_tt->vsgt.num_regions++;
411 old = cur;
412 }
413
414 vmw_tt->mapped = true;
415 return 0;
416
417out_map_fail:
418 sg_free_table(vmw_tt->vsgt.sgt);
419 vmw_tt->vsgt.sgt = NULL;
420out_sg_alloc_fail:
421 ttm_mem_global_free(glob, vmw_tt->sg_alloc_size);
422 return ret;
423}
424
425/**
426 * vmw_ttm_unmap_dma - Tear down any TTM page device mappings
427 *
428 * @vmw_tt: Pointer to a struct vmw_ttm_tt
429 *
430 * Tear down any previously set up device DMA mappings and free
431 * any storage space allocated for them. If there are no mappings set up,
432 * this function is a NOP.
433 */
434static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt)
435{
436 struct vmw_private *dev_priv = vmw_tt->dev_priv;
437
438 if (!vmw_tt->vsgt.sgt)
439 return;
440
441 switch (dev_priv->map_mode) {
442 case vmw_dma_map_bind:
443 case vmw_dma_map_populate:
444 vmw_ttm_unmap_from_dma(vmw_tt);
445 sg_free_table(vmw_tt->vsgt.sgt);
446 vmw_tt->vsgt.sgt = NULL;
447 ttm_mem_global_free(vmw_mem_glob(dev_priv),
448 vmw_tt->sg_alloc_size);
449 break;
450 default:
451 break;
452 }
453 vmw_tt->mapped = false;
454}
455
456
457/**
458 * vmw_bo_map_dma - Make sure buffer object pages are visible to the device
459 *
460 * @bo: Pointer to a struct ttm_buffer_object
461 *
462 * Wrapper around vmw_ttm_map_dma, that takes a TTM buffer object pointer
463 * instead of a pointer to a struct vmw_ttm_backend as argument.
464 * Note that the buffer object must be either pinned or reserved before
465 * calling this function.
466 */
467int vmw_bo_map_dma(struct ttm_buffer_object *bo)
468{
469 struct vmw_ttm_tt *vmw_tt =
470 container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
471
472 return vmw_ttm_map_dma(vmw_tt);
473}
474
475
476/**
477 * vmw_bo_unmap_dma - Make sure buffer object pages are visible to the device
478 *
479 * @bo: Pointer to a struct ttm_buffer_object
480 *
481 * Wrapper around vmw_ttm_unmap_dma, that takes a TTM buffer object pointer
482 * instead of a pointer to a struct vmw_ttm_backend as argument.
483 */
484void vmw_bo_unmap_dma(struct ttm_buffer_object *bo)
485{
486 struct vmw_ttm_tt *vmw_tt =
487 container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
488
489 vmw_ttm_unmap_dma(vmw_tt);
490}
491
492
493/**
494 * vmw_bo_sg_table - Return a struct vmw_sg_table object for a
495 * TTM buffer object
496 *
497 * @bo: Pointer to a struct ttm_buffer_object
498 *
499 * Returns a pointer to a struct vmw_sg_table object. The object should
500 * not be freed after use.
501 * Note that for the device addresses to be valid, the buffer object must
502 * either be reserved or pinned.
503 */
504const struct vmw_sg_table *vmw_bo_sg_table(struct ttm_buffer_object *bo)
505{
506 struct vmw_ttm_tt *vmw_tt =
507 container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm);
508
509 return &vmw_tt->vsgt;
510}
511
512
513static int vmw_ttm_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
514{
515 struct vmw_ttm_tt *vmw_be =
516 container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
517 int ret;
518
519 ret = vmw_ttm_map_dma(vmw_be);
520 if (unlikely(ret != 0))
521 return ret;
522
523 vmw_be->gmr_id = bo_mem->start;
524 vmw_be->mem_type = bo_mem->mem_type;
525
526 switch (bo_mem->mem_type) {
527 case VMW_PL_GMR:
528 return vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt,
529 ttm->num_pages, vmw_be->gmr_id);
530 case VMW_PL_MOB:
531 if (unlikely(vmw_be->mob == NULL)) {
532 vmw_be->mob =
533 vmw_mob_create(ttm->num_pages);
534 if (unlikely(vmw_be->mob == NULL))
535 return -ENOMEM;
536 }
537
538 return vmw_mob_bind(vmw_be->dev_priv, vmw_be->mob,
539 &vmw_be->vsgt, ttm->num_pages,
540 vmw_be->gmr_id);
541 default:
542 BUG();
543 }
544 return 0;
545}
546
547static int vmw_ttm_unbind(struct ttm_tt *ttm)
548{
549 struct vmw_ttm_tt *vmw_be =
550 container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
551
552 switch (vmw_be->mem_type) {
553 case VMW_PL_GMR:
554 vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id);
555 break;
556 case VMW_PL_MOB:
557 vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob);
558 break;
559 default:
560 BUG();
561 }
562
563 if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind)
564 vmw_ttm_unmap_dma(vmw_be);
565
566 return 0;
567}
568
569
570static void vmw_ttm_destroy(struct ttm_tt *ttm)
571{
572 struct vmw_ttm_tt *vmw_be =
573 container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
574
575 vmw_ttm_unmap_dma(vmw_be);
576 if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
577 ttm_dma_tt_fini(&vmw_be->dma_ttm);
578 else
579 ttm_tt_fini(ttm);
580
581 if (vmw_be->mob)
582 vmw_mob_destroy(vmw_be->mob);
583
584 kfree(vmw_be);
585}
586
587
588static int vmw_ttm_populate(struct ttm_tt *ttm)
589{
590 struct vmw_ttm_tt *vmw_tt =
591 container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
592 struct vmw_private *dev_priv = vmw_tt->dev_priv;
593 struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
594 int ret;
595
596 if (ttm->state != tt_unpopulated)
597 return 0;
598
599 if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
600 size_t size =
601 ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
602 ret = ttm_mem_global_alloc(glob, size, false, true);
603 if (unlikely(ret != 0))
604 return ret;
605
606 ret = ttm_dma_populate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
607 if (unlikely(ret != 0))
608 ttm_mem_global_free(glob, size);
609 } else
610 ret = ttm_pool_populate(ttm);
611
612 return ret;
613}
614
615static void vmw_ttm_unpopulate(struct ttm_tt *ttm)
616{
617 struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt,
618 dma_ttm.ttm);
619 struct vmw_private *dev_priv = vmw_tt->dev_priv;
620 struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
621
622
623 if (vmw_tt->mob) {
624 vmw_mob_destroy(vmw_tt->mob);
625 vmw_tt->mob = NULL;
626 }
627
628 vmw_ttm_unmap_dma(vmw_tt);
629 if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
630 size_t size =
631 ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
632
633 ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
634 ttm_mem_global_free(glob, size);
635 } else
636 ttm_pool_unpopulate(ttm);
637}
638
639static struct ttm_backend_func vmw_ttm_func = {
640 .bind = vmw_ttm_bind,
641 .unbind = vmw_ttm_unbind,
642 .destroy = vmw_ttm_destroy,
643};
644
645static struct ttm_tt *vmw_ttm_tt_create(struct ttm_bo_device *bdev,
646 unsigned long size, uint32_t page_flags,
647 struct page *dummy_read_page)
648{
649 struct vmw_ttm_tt *vmw_be;
650 int ret;
651
652 vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL);
653 if (!vmw_be)
654 return NULL;
655
656 vmw_be->dma_ttm.ttm.func = &vmw_ttm_func;
657 vmw_be->dev_priv = container_of(bdev, struct vmw_private, bdev);
658 vmw_be->mob = NULL;
659
660 if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
661 ret = ttm_dma_tt_init(&vmw_be->dma_ttm, bdev, size, page_flags,
662 dummy_read_page);
663 else
664 ret = ttm_tt_init(&vmw_be->dma_ttm.ttm, bdev, size, page_flags,
665 dummy_read_page);
666 if (unlikely(ret != 0))
667 goto out_no_init;
668
669 return &vmw_be->dma_ttm.ttm;
670out_no_init:
671 kfree(vmw_be);
672 return NULL;
673}
674
675static int vmw_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
676{
677 return 0;
678}
679
680static int vmw_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
681 struct ttm_mem_type_manager *man)
682{
683 switch (type) {
684 case TTM_PL_SYSTEM:
685 /* System memory */
686
687 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
688 man->available_caching = TTM_PL_FLAG_CACHED;
689 man->default_caching = TTM_PL_FLAG_CACHED;
690 break;
691 case TTM_PL_VRAM:
692 /* "On-card" video ram */
693 man->func = &ttm_bo_manager_func;
694 man->gpu_offset = 0;
695 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE;
696 man->available_caching = TTM_PL_FLAG_CACHED;
697 man->default_caching = TTM_PL_FLAG_CACHED;
698 break;
699 case VMW_PL_GMR:
700 case VMW_PL_MOB:
701 /*
702 * "Guest Memory Regions" is an aperture like feature with
703 * one slot per bo. There is an upper limit of the number of
704 * slots as well as the bo size.
705 */
706 man->func = &vmw_gmrid_manager_func;
707 man->gpu_offset = 0;
708 man->flags = TTM_MEMTYPE_FLAG_CMA | TTM_MEMTYPE_FLAG_MAPPABLE;
709 man->available_caching = TTM_PL_FLAG_CACHED;
710 man->default_caching = TTM_PL_FLAG_CACHED;
711 break;
712 default:
713 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
714 return -EINVAL;
715 }
716 return 0;
717}
718
719static void vmw_evict_flags(struct ttm_buffer_object *bo,
720 struct ttm_placement *placement)
721{
722 *placement = vmw_sys_placement;
723}
724
725static int vmw_verify_access(struct ttm_buffer_object *bo, struct file *filp)
726{
727 struct ttm_object_file *tfile =
728 vmw_fpriv((struct drm_file *)filp->private_data)->tfile;
729
730 return vmw_user_dmabuf_verify_access(bo, tfile);
731}
732
733static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
734{
735 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
736 struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev);
737
738 mem->bus.addr = NULL;
739 mem->bus.is_iomem = false;
740 mem->bus.offset = 0;
741 mem->bus.size = mem->num_pages << PAGE_SHIFT;
742 mem->bus.base = 0;
743 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
744 return -EINVAL;
745 switch (mem->mem_type) {
746 case TTM_PL_SYSTEM:
747 case VMW_PL_GMR:
748 case VMW_PL_MOB:
749 return 0;
750 case TTM_PL_VRAM:
751 mem->bus.offset = mem->start << PAGE_SHIFT;
752 mem->bus.base = dev_priv->vram_start;
753 mem->bus.is_iomem = true;
754 break;
755 default:
756 return -EINVAL;
757 }
758 return 0;
759}
760
761static void vmw_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
762{
763}
764
765static int vmw_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
766{
767 return 0;
768}
769
770/**
771 * FIXME: We're using the old vmware polling method to sync.
772 * Do this with fences instead.
773 */
774
775static void *vmw_sync_obj_ref(void *sync_obj)
776{
777
778 return (void *)
779 vmw_fence_obj_reference((struct vmw_fence_obj *) sync_obj);
780}
781
782static void vmw_sync_obj_unref(void **sync_obj)
783{
784 vmw_fence_obj_unreference((struct vmw_fence_obj **) sync_obj);
785}
786
787static int vmw_sync_obj_flush(void *sync_obj)
788{
789 vmw_fence_obj_flush((struct vmw_fence_obj *) sync_obj);
790 return 0;
791}
792
793static bool vmw_sync_obj_signaled(void *sync_obj)
794{
795 return vmw_fence_obj_signaled((struct vmw_fence_obj *) sync_obj,
796 DRM_VMW_FENCE_FLAG_EXEC);
797
798}
799
800static int vmw_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible)
801{
802 return vmw_fence_obj_wait((struct vmw_fence_obj *) sync_obj,
803 DRM_VMW_FENCE_FLAG_EXEC,
804 lazy, interruptible,
805 VMW_FENCE_WAIT_TIMEOUT);
806}
807
808/**
809 * vmw_move_notify - TTM move_notify_callback
810 *
811 * @bo: The TTM buffer object about to move.
812 * @mem: The truct ttm_mem_reg indicating to what memory
813 * region the move is taking place.
814 *
815 * Calls move_notify for all subsystems needing it.
816 * (currently only resources).
817 */
818static void vmw_move_notify(struct ttm_buffer_object *bo,
819 struct ttm_mem_reg *mem)
820{
821 vmw_resource_move_notify(bo, mem);
822}
823
824
825/**
826 * vmw_swap_notify - TTM move_notify_callback
827 *
828 * @bo: The TTM buffer object about to be swapped out.
829 */
830static void vmw_swap_notify(struct ttm_buffer_object *bo)
831{
832 struct ttm_bo_device *bdev = bo->bdev;
833
834 spin_lock(&bdev->fence_lock);
835 ttm_bo_wait(bo, false, false, false);
836 spin_unlock(&bdev->fence_lock);
837}
838
839
840struct ttm_bo_driver vmw_bo_driver = {
841 .ttm_tt_create = &vmw_ttm_tt_create,
842 .ttm_tt_populate = &vmw_ttm_populate,
843 .ttm_tt_unpopulate = &vmw_ttm_unpopulate,
844 .invalidate_caches = vmw_invalidate_caches,
845 .init_mem_type = vmw_init_mem_type,
846 .evict_flags = vmw_evict_flags,
847 .move = NULL,
848 .verify_access = vmw_verify_access,
849 .sync_obj_signaled = vmw_sync_obj_signaled,
850 .sync_obj_wait = vmw_sync_obj_wait,
851 .sync_obj_flush = vmw_sync_obj_flush,
852 .sync_obj_unref = vmw_sync_obj_unref,
853 .sync_obj_ref = vmw_sync_obj_ref,
854 .move_notify = vmw_move_notify,
855 .swap_notify = vmw_swap_notify,
856 .fault_reserve_notify = &vmw_ttm_fault_reserve_notify,
857 .io_mem_reserve = &vmw_ttm_io_mem_reserve,
858 .io_mem_free = &vmw_ttm_io_mem_free,
859};