Loading...
Note: File does not exist in v5.14.15.
1// SPDX-License-Identifier: MIT
2
3/*
4 * Locking:
5 *
6 * The uvmm mutex protects any operations on the GPU VA space provided by the
7 * DRM GPU VA manager.
8 *
9 * The GEMs dma_resv lock protects the GEMs GPUVA list, hence link/unlink of a
10 * mapping to it's backing GEM must be performed under this lock.
11 *
12 * Actual map/unmap operations within the fence signalling critical path are
13 * protected by installing DMA fences to the corresponding GEMs DMA
14 * reservations, such that concurrent BO moves, which itself walk the GEMs GPUVA
15 * list in order to map/unmap it's entries, can't occur concurrently.
16 *
17 * Accessing the DRM_GPUVA_INVALIDATED flag doesn't need any separate
18 * protection, since there are no accesses other than from BO move callbacks
19 * and from the fence signalling critical path, which are already protected by
20 * the corresponding GEMs DMA reservation fence.
21 */
22
23#include "nouveau_drv.h"
24#include "nouveau_gem.h"
25#include "nouveau_mem.h"
26#include "nouveau_uvmm.h"
27
28#include <nvif/vmm.h>
29#include <nvif/mem.h>
30
31#include <nvif/class.h>
32#include <nvif/if000c.h>
33#include <nvif/if900d.h>
34
35#define NOUVEAU_VA_SPACE_BITS 47 /* FIXME */
36#define NOUVEAU_VA_SPACE_START 0x0
37#define NOUVEAU_VA_SPACE_END (1ULL << NOUVEAU_VA_SPACE_BITS)
38
39#define list_last_op(_ops) list_last_entry(_ops, struct bind_job_op, entry)
40#define list_prev_op(_op) list_prev_entry(_op, entry)
41#define list_for_each_op(_op, _ops) list_for_each_entry(_op, _ops, entry)
42#define list_for_each_op_from_reverse(_op, _ops) \
43 list_for_each_entry_from_reverse(_op, _ops, entry)
44#define list_for_each_op_safe(_op, _n, _ops) list_for_each_entry_safe(_op, _n, _ops, entry)
45
46enum vm_bind_op {
47 OP_MAP = DRM_NOUVEAU_VM_BIND_OP_MAP,
48 OP_UNMAP = DRM_NOUVEAU_VM_BIND_OP_UNMAP,
49 OP_MAP_SPARSE,
50 OP_UNMAP_SPARSE,
51};
52
53struct nouveau_uvma_prealloc {
54 struct nouveau_uvma *map;
55 struct nouveau_uvma *prev;
56 struct nouveau_uvma *next;
57};
58
59struct bind_job_op {
60 struct list_head entry;
61
62 enum vm_bind_op op;
63 u32 flags;
64
65 struct drm_gpuvm_bo *vm_bo;
66
67 struct {
68 u64 addr;
69 u64 range;
70 } va;
71
72 struct {
73 u32 handle;
74 u64 offset;
75 struct drm_gem_object *obj;
76 } gem;
77
78 struct nouveau_uvma_region *reg;
79 struct nouveau_uvma_prealloc new;
80 struct drm_gpuva_ops *ops;
81};
82
83struct uvmm_map_args {
84 struct nouveau_uvma_region *region;
85 u64 addr;
86 u64 range;
87 u8 kind;
88};
89
90static int
91nouveau_uvmm_vmm_sparse_ref(struct nouveau_uvmm *uvmm,
92 u64 addr, u64 range)
93{
94 struct nvif_vmm *vmm = &uvmm->vmm.vmm;
95
96 return nvif_vmm_raw_sparse(vmm, addr, range, true);
97}
98
99static int
100nouveau_uvmm_vmm_sparse_unref(struct nouveau_uvmm *uvmm,
101 u64 addr, u64 range)
102{
103 struct nvif_vmm *vmm = &uvmm->vmm.vmm;
104
105 return nvif_vmm_raw_sparse(vmm, addr, range, false);
106}
107
108static int
109nouveau_uvmm_vmm_get(struct nouveau_uvmm *uvmm,
110 u64 addr, u64 range)
111{
112 struct nvif_vmm *vmm = &uvmm->vmm.vmm;
113
114 return nvif_vmm_raw_get(vmm, addr, range, PAGE_SHIFT);
115}
116
117static int
118nouveau_uvmm_vmm_put(struct nouveau_uvmm *uvmm,
119 u64 addr, u64 range)
120{
121 struct nvif_vmm *vmm = &uvmm->vmm.vmm;
122
123 return nvif_vmm_raw_put(vmm, addr, range, PAGE_SHIFT);
124}
125
126static int
127nouveau_uvmm_vmm_unmap(struct nouveau_uvmm *uvmm,
128 u64 addr, u64 range, bool sparse)
129{
130 struct nvif_vmm *vmm = &uvmm->vmm.vmm;
131
132 return nvif_vmm_raw_unmap(vmm, addr, range, PAGE_SHIFT, sparse);
133}
134
135static int
136nouveau_uvmm_vmm_map(struct nouveau_uvmm *uvmm,
137 u64 addr, u64 range,
138 u64 bo_offset, u8 kind,
139 struct nouveau_mem *mem)
140{
141 struct nvif_vmm *vmm = &uvmm->vmm.vmm;
142 union {
143 struct gf100_vmm_map_v0 gf100;
144 } args;
145 u32 argc = 0;
146
147 switch (vmm->object.oclass) {
148 case NVIF_CLASS_VMM_GF100:
149 case NVIF_CLASS_VMM_GM200:
150 case NVIF_CLASS_VMM_GP100:
151 args.gf100.version = 0;
152 if (mem->mem.type & NVIF_MEM_VRAM)
153 args.gf100.vol = 0;
154 else
155 args.gf100.vol = 1;
156 args.gf100.ro = 0;
157 args.gf100.priv = 0;
158 args.gf100.kind = kind;
159 argc = sizeof(args.gf100);
160 break;
161 default:
162 WARN_ON(1);
163 return -ENOSYS;
164 }
165
166 return nvif_vmm_raw_map(vmm, addr, range, PAGE_SHIFT,
167 &args, argc,
168 &mem->mem, bo_offset);
169}
170
171static int
172nouveau_uvma_region_sparse_unref(struct nouveau_uvma_region *reg)
173{
174 u64 addr = reg->va.addr;
175 u64 range = reg->va.range;
176
177 return nouveau_uvmm_vmm_sparse_unref(reg->uvmm, addr, range);
178}
179
180static int
181nouveau_uvma_vmm_put(struct nouveau_uvma *uvma)
182{
183 u64 addr = uvma->va.va.addr;
184 u64 range = uvma->va.va.range;
185
186 return nouveau_uvmm_vmm_put(to_uvmm(uvma), addr, range);
187}
188
189static int
190nouveau_uvma_map(struct nouveau_uvma *uvma,
191 struct nouveau_mem *mem)
192{
193 u64 addr = uvma->va.va.addr;
194 u64 offset = uvma->va.gem.offset;
195 u64 range = uvma->va.va.range;
196
197 return nouveau_uvmm_vmm_map(to_uvmm(uvma), addr, range,
198 offset, uvma->kind, mem);
199}
200
201static int
202nouveau_uvma_unmap(struct nouveau_uvma *uvma)
203{
204 u64 addr = uvma->va.va.addr;
205 u64 range = uvma->va.va.range;
206 bool sparse = !!uvma->region;
207
208 if (drm_gpuva_invalidated(&uvma->va))
209 return 0;
210
211 return nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse);
212}
213
214static int
215nouveau_uvma_alloc(struct nouveau_uvma **puvma)
216{
217 *puvma = kzalloc(sizeof(**puvma), GFP_KERNEL);
218 if (!*puvma)
219 return -ENOMEM;
220
221 return 0;
222}
223
224static void
225nouveau_uvma_free(struct nouveau_uvma *uvma)
226{
227 kfree(uvma);
228}
229
230static void
231nouveau_uvma_gem_get(struct nouveau_uvma *uvma)
232{
233 drm_gem_object_get(uvma->va.gem.obj);
234}
235
236static void
237nouveau_uvma_gem_put(struct nouveau_uvma *uvma)
238{
239 drm_gem_object_put(uvma->va.gem.obj);
240}
241
242static int
243nouveau_uvma_region_alloc(struct nouveau_uvma_region **preg)
244{
245 *preg = kzalloc(sizeof(**preg), GFP_KERNEL);
246 if (!*preg)
247 return -ENOMEM;
248
249 kref_init(&(*preg)->kref);
250
251 return 0;
252}
253
254static void
255nouveau_uvma_region_free(struct kref *kref)
256{
257 struct nouveau_uvma_region *reg =
258 container_of(kref, struct nouveau_uvma_region, kref);
259
260 kfree(reg);
261}
262
263static void
264nouveau_uvma_region_get(struct nouveau_uvma_region *reg)
265{
266 kref_get(®->kref);
267}
268
269static void
270nouveau_uvma_region_put(struct nouveau_uvma_region *reg)
271{
272 kref_put(®->kref, nouveau_uvma_region_free);
273}
274
275static int
276__nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm,
277 struct nouveau_uvma_region *reg)
278{
279 u64 addr = reg->va.addr;
280 u64 range = reg->va.range;
281 u64 last = addr + range - 1;
282 MA_STATE(mas, &uvmm->region_mt, addr, addr);
283
284 if (unlikely(mas_walk(&mas)))
285 return -EEXIST;
286
287 if (unlikely(mas.last < last))
288 return -EEXIST;
289
290 mas.index = addr;
291 mas.last = last;
292
293 mas_store_gfp(&mas, reg, GFP_KERNEL);
294
295 reg->uvmm = uvmm;
296
297 return 0;
298}
299
300static int
301nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm,
302 struct nouveau_uvma_region *reg,
303 u64 addr, u64 range)
304{
305 int ret;
306
307 reg->uvmm = uvmm;
308 reg->va.addr = addr;
309 reg->va.range = range;
310
311 ret = __nouveau_uvma_region_insert(uvmm, reg);
312 if (ret)
313 return ret;
314
315 return 0;
316}
317
318static void
319nouveau_uvma_region_remove(struct nouveau_uvma_region *reg)
320{
321 struct nouveau_uvmm *uvmm = reg->uvmm;
322 MA_STATE(mas, &uvmm->region_mt, reg->va.addr, 0);
323
324 mas_erase(&mas);
325}
326
327static int
328nouveau_uvma_region_create(struct nouveau_uvmm *uvmm,
329 u64 addr, u64 range)
330{
331 struct nouveau_uvma_region *reg;
332 int ret;
333
334 if (!drm_gpuvm_interval_empty(&uvmm->base, addr, range))
335 return -ENOSPC;
336
337 ret = nouveau_uvma_region_alloc(®);
338 if (ret)
339 return ret;
340
341 ret = nouveau_uvma_region_insert(uvmm, reg, addr, range);
342 if (ret)
343 goto err_free_region;
344
345 ret = nouveau_uvmm_vmm_sparse_ref(uvmm, addr, range);
346 if (ret)
347 goto err_region_remove;
348
349 return 0;
350
351err_region_remove:
352 nouveau_uvma_region_remove(reg);
353err_free_region:
354 nouveau_uvma_region_put(reg);
355 return ret;
356}
357
358static struct nouveau_uvma_region *
359nouveau_uvma_region_find_first(struct nouveau_uvmm *uvmm,
360 u64 addr, u64 range)
361{
362 MA_STATE(mas, &uvmm->region_mt, addr, 0);
363
364 return mas_find(&mas, addr + range - 1);
365}
366
367static struct nouveau_uvma_region *
368nouveau_uvma_region_find(struct nouveau_uvmm *uvmm,
369 u64 addr, u64 range)
370{
371 struct nouveau_uvma_region *reg;
372
373 reg = nouveau_uvma_region_find_first(uvmm, addr, range);
374 if (!reg)
375 return NULL;
376
377 if (reg->va.addr != addr ||
378 reg->va.range != range)
379 return NULL;
380
381 return reg;
382}
383
384static bool
385nouveau_uvma_region_empty(struct nouveau_uvma_region *reg)
386{
387 struct nouveau_uvmm *uvmm = reg->uvmm;
388
389 return drm_gpuvm_interval_empty(&uvmm->base,
390 reg->va.addr,
391 reg->va.range);
392}
393
394static int
395__nouveau_uvma_region_destroy(struct nouveau_uvma_region *reg)
396{
397 struct nouveau_uvmm *uvmm = reg->uvmm;
398 u64 addr = reg->va.addr;
399 u64 range = reg->va.range;
400
401 if (!nouveau_uvma_region_empty(reg))
402 return -EBUSY;
403
404 nouveau_uvma_region_remove(reg);
405 nouveau_uvmm_vmm_sparse_unref(uvmm, addr, range);
406 nouveau_uvma_region_put(reg);
407
408 return 0;
409}
410
411static int
412nouveau_uvma_region_destroy(struct nouveau_uvmm *uvmm,
413 u64 addr, u64 range)
414{
415 struct nouveau_uvma_region *reg;
416
417 reg = nouveau_uvma_region_find(uvmm, addr, range);
418 if (!reg)
419 return -ENOENT;
420
421 return __nouveau_uvma_region_destroy(reg);
422}
423
424static void
425nouveau_uvma_region_dirty(struct nouveau_uvma_region *reg)
426{
427
428 init_completion(®->complete);
429 reg->dirty = true;
430}
431
432static void
433nouveau_uvma_region_complete(struct nouveau_uvma_region *reg)
434{
435 complete_all(®->complete);
436}
437
438static void
439op_map_prepare_unwind(struct nouveau_uvma *uvma)
440{
441 struct drm_gpuva *va = &uvma->va;
442 nouveau_uvma_gem_put(uvma);
443 drm_gpuva_remove(va);
444 nouveau_uvma_free(uvma);
445}
446
447static void
448op_unmap_prepare_unwind(struct drm_gpuva *va)
449{
450 drm_gpuva_insert(va->vm, va);
451}
452
453static void
454nouveau_uvmm_sm_prepare_unwind(struct nouveau_uvmm *uvmm,
455 struct nouveau_uvma_prealloc *new,
456 struct drm_gpuva_ops *ops,
457 struct drm_gpuva_op *last,
458 struct uvmm_map_args *args)
459{
460 struct drm_gpuva_op *op = last;
461 u64 vmm_get_start = args ? args->addr : 0;
462 u64 vmm_get_end = args ? args->addr + args->range : 0;
463
464 /* Unwind GPUVA space. */
465 drm_gpuva_for_each_op_from_reverse(op, ops) {
466 switch (op->op) {
467 case DRM_GPUVA_OP_MAP:
468 op_map_prepare_unwind(new->map);
469 break;
470 case DRM_GPUVA_OP_REMAP: {
471 struct drm_gpuva_op_remap *r = &op->remap;
472 struct drm_gpuva *va = r->unmap->va;
473
474 if (r->next)
475 op_map_prepare_unwind(new->next);
476
477 if (r->prev)
478 op_map_prepare_unwind(new->prev);
479
480 op_unmap_prepare_unwind(va);
481 break;
482 }
483 case DRM_GPUVA_OP_UNMAP:
484 op_unmap_prepare_unwind(op->unmap.va);
485 break;
486 default:
487 break;
488 }
489 }
490
491 /* Unmap operation don't allocate page tables, hence skip the following
492 * page table unwind.
493 */
494 if (!args)
495 return;
496
497 drm_gpuva_for_each_op(op, ops) {
498 switch (op->op) {
499 case DRM_GPUVA_OP_MAP: {
500 u64 vmm_get_range = vmm_get_end - vmm_get_start;
501
502 if (vmm_get_range)
503 nouveau_uvmm_vmm_put(uvmm, vmm_get_start,
504 vmm_get_range);
505 break;
506 }
507 case DRM_GPUVA_OP_REMAP: {
508 struct drm_gpuva_op_remap *r = &op->remap;
509 struct drm_gpuva *va = r->unmap->va;
510 u64 ustart = va->va.addr;
511 u64 urange = va->va.range;
512 u64 uend = ustart + urange;
513
514 if (r->prev)
515 vmm_get_start = uend;
516
517 if (r->next)
518 vmm_get_end = ustart;
519
520 if (r->prev && r->next)
521 vmm_get_start = vmm_get_end = 0;
522
523 break;
524 }
525 case DRM_GPUVA_OP_UNMAP: {
526 struct drm_gpuva_op_unmap *u = &op->unmap;
527 struct drm_gpuva *va = u->va;
528 u64 ustart = va->va.addr;
529 u64 urange = va->va.range;
530 u64 uend = ustart + urange;
531
532 /* Nothing to do for mappings we merge with. */
533 if (uend == vmm_get_start ||
534 ustart == vmm_get_end)
535 break;
536
537 if (ustart > vmm_get_start) {
538 u64 vmm_get_range = ustart - vmm_get_start;
539
540 nouveau_uvmm_vmm_put(uvmm, vmm_get_start,
541 vmm_get_range);
542 }
543 vmm_get_start = uend;
544 break;
545 }
546 default:
547 break;
548 }
549
550 if (op == last)
551 break;
552 }
553}
554
555static void
556nouveau_uvmm_sm_map_prepare_unwind(struct nouveau_uvmm *uvmm,
557 struct nouveau_uvma_prealloc *new,
558 struct drm_gpuva_ops *ops,
559 u64 addr, u64 range)
560{
561 struct drm_gpuva_op *last = drm_gpuva_last_op(ops);
562 struct uvmm_map_args args = {
563 .addr = addr,
564 .range = range,
565 };
566
567 nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, &args);
568}
569
570static void
571nouveau_uvmm_sm_unmap_prepare_unwind(struct nouveau_uvmm *uvmm,
572 struct nouveau_uvma_prealloc *new,
573 struct drm_gpuva_ops *ops)
574{
575 struct drm_gpuva_op *last = drm_gpuva_last_op(ops);
576
577 nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, NULL);
578}
579
580static int
581op_map_prepare(struct nouveau_uvmm *uvmm,
582 struct nouveau_uvma **puvma,
583 struct drm_gpuva_op_map *op,
584 struct uvmm_map_args *args)
585{
586 struct nouveau_uvma *uvma;
587 int ret;
588
589 ret = nouveau_uvma_alloc(&uvma);
590 if (ret)
591 return ret;
592
593 uvma->region = args->region;
594 uvma->kind = args->kind;
595
596 drm_gpuva_map(&uvmm->base, &uvma->va, op);
597
598 /* Keep a reference until this uvma is destroyed. */
599 nouveau_uvma_gem_get(uvma);
600
601 *puvma = uvma;
602 return 0;
603}
604
605static void
606op_unmap_prepare(struct drm_gpuva_op_unmap *u)
607{
608 drm_gpuva_unmap(u);
609}
610
611/*
612 * Note: @args should not be NULL when calling for a map operation.
613 */
614static int
615nouveau_uvmm_sm_prepare(struct nouveau_uvmm *uvmm,
616 struct nouveau_uvma_prealloc *new,
617 struct drm_gpuva_ops *ops,
618 struct uvmm_map_args *args)
619{
620 struct drm_gpuva_op *op;
621 u64 vmm_get_start = args ? args->addr : 0;
622 u64 vmm_get_end = args ? args->addr + args->range : 0;
623 int ret;
624
625 drm_gpuva_for_each_op(op, ops) {
626 switch (op->op) {
627 case DRM_GPUVA_OP_MAP: {
628 u64 vmm_get_range = vmm_get_end - vmm_get_start;
629
630 ret = op_map_prepare(uvmm, &new->map, &op->map, args);
631 if (ret)
632 goto unwind;
633
634 if (vmm_get_range) {
635 ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start,
636 vmm_get_range);
637 if (ret) {
638 op_map_prepare_unwind(new->map);
639 goto unwind;
640 }
641 }
642
643 break;
644 }
645 case DRM_GPUVA_OP_REMAP: {
646 struct drm_gpuva_op_remap *r = &op->remap;
647 struct drm_gpuva *va = r->unmap->va;
648 struct uvmm_map_args remap_args = {
649 .kind = uvma_from_va(va)->kind,
650 .region = uvma_from_va(va)->region,
651 };
652 u64 ustart = va->va.addr;
653 u64 urange = va->va.range;
654 u64 uend = ustart + urange;
655
656 op_unmap_prepare(r->unmap);
657
658 if (r->prev) {
659 ret = op_map_prepare(uvmm, &new->prev, r->prev,
660 &remap_args);
661 if (ret)
662 goto unwind;
663
664 if (args)
665 vmm_get_start = uend;
666 }
667
668 if (r->next) {
669 ret = op_map_prepare(uvmm, &new->next, r->next,
670 &remap_args);
671 if (ret) {
672 if (r->prev)
673 op_map_prepare_unwind(new->prev);
674 goto unwind;
675 }
676
677 if (args)
678 vmm_get_end = ustart;
679 }
680
681 if (args && (r->prev && r->next))
682 vmm_get_start = vmm_get_end = 0;
683
684 break;
685 }
686 case DRM_GPUVA_OP_UNMAP: {
687 struct drm_gpuva_op_unmap *u = &op->unmap;
688 struct drm_gpuva *va = u->va;
689 u64 ustart = va->va.addr;
690 u64 urange = va->va.range;
691 u64 uend = ustart + urange;
692
693 op_unmap_prepare(u);
694
695 if (!args)
696 break;
697
698 /* Nothing to do for mappings we merge with. */
699 if (uend == vmm_get_start ||
700 ustart == vmm_get_end)
701 break;
702
703 if (ustart > vmm_get_start) {
704 u64 vmm_get_range = ustart - vmm_get_start;
705
706 ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start,
707 vmm_get_range);
708 if (ret) {
709 op_unmap_prepare_unwind(va);
710 goto unwind;
711 }
712 }
713 vmm_get_start = uend;
714
715 break;
716 }
717 default:
718 ret = -EINVAL;
719 goto unwind;
720 }
721 }
722
723 return 0;
724
725unwind:
726 if (op != drm_gpuva_first_op(ops))
727 nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops,
728 drm_gpuva_prev_op(op),
729 args);
730 return ret;
731}
732
733static int
734nouveau_uvmm_sm_map_prepare(struct nouveau_uvmm *uvmm,
735 struct nouveau_uvma_prealloc *new,
736 struct nouveau_uvma_region *region,
737 struct drm_gpuva_ops *ops,
738 u64 addr, u64 range, u8 kind)
739{
740 struct uvmm_map_args args = {
741 .region = region,
742 .addr = addr,
743 .range = range,
744 .kind = kind,
745 };
746
747 return nouveau_uvmm_sm_prepare(uvmm, new, ops, &args);
748}
749
750static int
751nouveau_uvmm_sm_unmap_prepare(struct nouveau_uvmm *uvmm,
752 struct nouveau_uvma_prealloc *new,
753 struct drm_gpuva_ops *ops)
754{
755 return nouveau_uvmm_sm_prepare(uvmm, new, ops, NULL);
756}
757
758static struct drm_gem_object *
759op_gem_obj(struct drm_gpuva_op *op)
760{
761 switch (op->op) {
762 case DRM_GPUVA_OP_MAP:
763 return op->map.gem.obj;
764 case DRM_GPUVA_OP_REMAP:
765 /* Actually, we're looking for the GEMs backing remap.prev and
766 * remap.next, but since this is a remap they're identical to
767 * the GEM backing the unmapped GPUVA.
768 */
769 return op->remap.unmap->va->gem.obj;
770 case DRM_GPUVA_OP_UNMAP:
771 return op->unmap.va->gem.obj;
772 default:
773 WARN(1, "Unknown operation.\n");
774 return NULL;
775 }
776}
777
778static void
779op_map(struct nouveau_uvma *uvma)
780{
781 struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj);
782
783 nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource));
784}
785
786static void
787op_unmap(struct drm_gpuva_op_unmap *u)
788{
789 struct drm_gpuva *va = u->va;
790 struct nouveau_uvma *uvma = uvma_from_va(va);
791
792 /* nouveau_uvma_unmap() does not unmap if backing BO is evicted. */
793 if (!u->keep)
794 nouveau_uvma_unmap(uvma);
795}
796
797static void
798op_unmap_range(struct drm_gpuva_op_unmap *u,
799 u64 addr, u64 range)
800{
801 struct nouveau_uvma *uvma = uvma_from_va(u->va);
802 bool sparse = !!uvma->region;
803
804 if (!drm_gpuva_invalidated(u->va))
805 nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse);
806}
807
808static void
809op_remap(struct drm_gpuva_op_remap *r,
810 struct nouveau_uvma_prealloc *new)
811{
812 struct drm_gpuva_op_unmap *u = r->unmap;
813 struct nouveau_uvma *uvma = uvma_from_va(u->va);
814 u64 addr = uvma->va.va.addr;
815 u64 end = uvma->va.va.addr + uvma->va.va.range;
816
817 if (r->prev)
818 addr = r->prev->va.addr + r->prev->va.range;
819
820 if (r->next)
821 end = r->next->va.addr;
822
823 op_unmap_range(u, addr, end - addr);
824}
825
826static int
827nouveau_uvmm_sm(struct nouveau_uvmm *uvmm,
828 struct nouveau_uvma_prealloc *new,
829 struct drm_gpuva_ops *ops)
830{
831 struct drm_gpuva_op *op;
832
833 drm_gpuva_for_each_op(op, ops) {
834 switch (op->op) {
835 case DRM_GPUVA_OP_MAP:
836 op_map(new->map);
837 break;
838 case DRM_GPUVA_OP_REMAP:
839 op_remap(&op->remap, new);
840 break;
841 case DRM_GPUVA_OP_UNMAP:
842 op_unmap(&op->unmap);
843 break;
844 default:
845 break;
846 }
847 }
848
849 return 0;
850}
851
852static int
853nouveau_uvmm_sm_map(struct nouveau_uvmm *uvmm,
854 struct nouveau_uvma_prealloc *new,
855 struct drm_gpuva_ops *ops)
856{
857 return nouveau_uvmm_sm(uvmm, new, ops);
858}
859
860static int
861nouveau_uvmm_sm_unmap(struct nouveau_uvmm *uvmm,
862 struct nouveau_uvma_prealloc *new,
863 struct drm_gpuva_ops *ops)
864{
865 return nouveau_uvmm_sm(uvmm, new, ops);
866}
867
868static void
869nouveau_uvmm_sm_cleanup(struct nouveau_uvmm *uvmm,
870 struct nouveau_uvma_prealloc *new,
871 struct drm_gpuva_ops *ops, bool unmap)
872{
873 struct drm_gpuva_op *op;
874
875 drm_gpuva_for_each_op(op, ops) {
876 switch (op->op) {
877 case DRM_GPUVA_OP_MAP:
878 break;
879 case DRM_GPUVA_OP_REMAP: {
880 struct drm_gpuva_op_remap *r = &op->remap;
881 struct drm_gpuva_op_map *p = r->prev;
882 struct drm_gpuva_op_map *n = r->next;
883 struct drm_gpuva *va = r->unmap->va;
884 struct nouveau_uvma *uvma = uvma_from_va(va);
885
886 if (unmap) {
887 u64 addr = va->va.addr;
888 u64 end = addr + va->va.range;
889
890 if (p)
891 addr = p->va.addr + p->va.range;
892
893 if (n)
894 end = n->va.addr;
895
896 nouveau_uvmm_vmm_put(uvmm, addr, end - addr);
897 }
898
899 nouveau_uvma_gem_put(uvma);
900 nouveau_uvma_free(uvma);
901 break;
902 }
903 case DRM_GPUVA_OP_UNMAP: {
904 struct drm_gpuva_op_unmap *u = &op->unmap;
905 struct drm_gpuva *va = u->va;
906 struct nouveau_uvma *uvma = uvma_from_va(va);
907
908 if (unmap)
909 nouveau_uvma_vmm_put(uvma);
910
911 nouveau_uvma_gem_put(uvma);
912 nouveau_uvma_free(uvma);
913 break;
914 }
915 default:
916 break;
917 }
918 }
919}
920
921static void
922nouveau_uvmm_sm_map_cleanup(struct nouveau_uvmm *uvmm,
923 struct nouveau_uvma_prealloc *new,
924 struct drm_gpuva_ops *ops)
925{
926 nouveau_uvmm_sm_cleanup(uvmm, new, ops, false);
927}
928
929static void
930nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm,
931 struct nouveau_uvma_prealloc *new,
932 struct drm_gpuva_ops *ops)
933{
934 nouveau_uvmm_sm_cleanup(uvmm, new, ops, true);
935}
936
937static int
938nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range)
939{
940 if (addr & ~PAGE_MASK)
941 return -EINVAL;
942
943 if (range & ~PAGE_MASK)
944 return -EINVAL;
945
946 if (!drm_gpuvm_range_valid(&uvmm->base, addr, range))
947 return -EINVAL;
948
949 return 0;
950}
951
952static int
953nouveau_uvmm_bind_job_alloc(struct nouveau_uvmm_bind_job **pjob)
954{
955 *pjob = kzalloc(sizeof(**pjob), GFP_KERNEL);
956 if (!*pjob)
957 return -ENOMEM;
958
959 kref_init(&(*pjob)->kref);
960
961 return 0;
962}
963
964static void
965nouveau_uvmm_bind_job_free(struct kref *kref)
966{
967 struct nouveau_uvmm_bind_job *job =
968 container_of(kref, struct nouveau_uvmm_bind_job, kref);
969 struct bind_job_op *op, *next;
970
971 list_for_each_op_safe(op, next, &job->ops) {
972 list_del(&op->entry);
973 kfree(op);
974 }
975
976 nouveau_job_free(&job->base);
977 kfree(job);
978}
979
980static void
981nouveau_uvmm_bind_job_get(struct nouveau_uvmm_bind_job *job)
982{
983 kref_get(&job->kref);
984}
985
986static void
987nouveau_uvmm_bind_job_put(struct nouveau_uvmm_bind_job *job)
988{
989 kref_put(&job->kref, nouveau_uvmm_bind_job_free);
990}
991
992static int
993bind_validate_op(struct nouveau_job *job,
994 struct bind_job_op *op)
995{
996 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
997 struct drm_gem_object *obj = op->gem.obj;
998
999 if (op->op == OP_MAP) {
1000 if (op->gem.offset & ~PAGE_MASK)
1001 return -EINVAL;
1002
1003 if (obj->size <= op->gem.offset)
1004 return -EINVAL;
1005
1006 if (op->va.range > (obj->size - op->gem.offset))
1007 return -EINVAL;
1008 }
1009
1010 return nouveau_uvmm_validate_range(uvmm, op->va.addr, op->va.range);
1011}
1012
1013static void
1014bind_validate_map_sparse(struct nouveau_job *job, u64 addr, u64 range)
1015{
1016 struct nouveau_sched *sched = job->sched;
1017 struct nouveau_job *__job;
1018 struct bind_job_op *op;
1019 u64 end = addr + range;
1020
1021again:
1022 spin_lock(&sched->job.list.lock);
1023 list_for_each_entry(__job, &sched->job.list.head, entry) {
1024 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(__job);
1025
1026 list_for_each_op(op, &bind_job->ops) {
1027 if (op->op == OP_UNMAP) {
1028 u64 op_addr = op->va.addr;
1029 u64 op_end = op_addr + op->va.range;
1030
1031 if (!(end <= op_addr || addr >= op_end)) {
1032 nouveau_uvmm_bind_job_get(bind_job);
1033 spin_unlock(&sched->job.list.lock);
1034 wait_for_completion(&bind_job->complete);
1035 nouveau_uvmm_bind_job_put(bind_job);
1036 goto again;
1037 }
1038 }
1039 }
1040 }
1041 spin_unlock(&sched->job.list.lock);
1042}
1043
1044static int
1045bind_validate_map_common(struct nouveau_job *job, u64 addr, u64 range,
1046 bool sparse)
1047{
1048 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1049 struct nouveau_uvma_region *reg;
1050 u64 reg_addr, reg_end;
1051 u64 end = addr + range;
1052
1053again:
1054 nouveau_uvmm_lock(uvmm);
1055 reg = nouveau_uvma_region_find_first(uvmm, addr, range);
1056 if (!reg) {
1057 nouveau_uvmm_unlock(uvmm);
1058 return 0;
1059 }
1060
1061 /* Generally, job submits are serialized, hence only
1062 * dirty regions can be modified concurrently.
1063 */
1064 if (reg->dirty) {
1065 nouveau_uvma_region_get(reg);
1066 nouveau_uvmm_unlock(uvmm);
1067 wait_for_completion(®->complete);
1068 nouveau_uvma_region_put(reg);
1069 goto again;
1070 }
1071 nouveau_uvmm_unlock(uvmm);
1072
1073 if (sparse)
1074 return -ENOSPC;
1075
1076 reg_addr = reg->va.addr;
1077 reg_end = reg_addr + reg->va.range;
1078
1079 /* Make sure the mapping is either outside of a
1080 * region or fully enclosed by a region.
1081 */
1082 if (reg_addr > addr || reg_end < end)
1083 return -ENOSPC;
1084
1085 return 0;
1086}
1087
1088static int
1089bind_validate_region(struct nouveau_job *job)
1090{
1091 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1092 struct bind_job_op *op;
1093 int ret;
1094
1095 list_for_each_op(op, &bind_job->ops) {
1096 u64 op_addr = op->va.addr;
1097 u64 op_range = op->va.range;
1098 bool sparse = false;
1099
1100 switch (op->op) {
1101 case OP_MAP_SPARSE:
1102 sparse = true;
1103 bind_validate_map_sparse(job, op_addr, op_range);
1104 fallthrough;
1105 case OP_MAP:
1106 ret = bind_validate_map_common(job, op_addr, op_range,
1107 sparse);
1108 if (ret)
1109 return ret;
1110 break;
1111 default:
1112 break;
1113 }
1114 }
1115
1116 return 0;
1117}
1118
1119static void
1120bind_link_gpuvas(struct bind_job_op *bop)
1121{
1122 struct nouveau_uvma_prealloc *new = &bop->new;
1123 struct drm_gpuvm_bo *vm_bo = bop->vm_bo;
1124 struct drm_gpuva_ops *ops = bop->ops;
1125 struct drm_gpuva_op *op;
1126
1127 drm_gpuva_for_each_op(op, ops) {
1128 switch (op->op) {
1129 case DRM_GPUVA_OP_MAP:
1130 drm_gpuva_link(&new->map->va, vm_bo);
1131 break;
1132 case DRM_GPUVA_OP_REMAP: {
1133 struct drm_gpuva *va = op->remap.unmap->va;
1134
1135 if (op->remap.prev)
1136 drm_gpuva_link(&new->prev->va, va->vm_bo);
1137 if (op->remap.next)
1138 drm_gpuva_link(&new->next->va, va->vm_bo);
1139 drm_gpuva_unlink(va);
1140 break;
1141 }
1142 case DRM_GPUVA_OP_UNMAP:
1143 drm_gpuva_unlink(op->unmap.va);
1144 break;
1145 default:
1146 break;
1147 }
1148 }
1149}
1150
1151static int
1152bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec,
1153 unsigned int num_fences)
1154{
1155 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1156 struct bind_job_op *op;
1157 int ret;
1158
1159 list_for_each_op(op, &bind_job->ops) {
1160 struct drm_gpuva_op *va_op;
1161
1162 if (!op->ops)
1163 continue;
1164
1165 drm_gpuva_for_each_op(va_op, op->ops) {
1166 struct drm_gem_object *obj = op_gem_obj(va_op);
1167
1168 if (unlikely(!obj))
1169 continue;
1170
1171 ret = drm_exec_prepare_obj(exec, obj, num_fences);
1172 if (ret)
1173 return ret;
1174
1175 /* Don't validate GEMs backing mappings we're about to
1176 * unmap, it's not worth the effort.
1177 */
1178 if (va_op->op == DRM_GPUVA_OP_UNMAP)
1179 continue;
1180
1181 ret = nouveau_bo_validate(nouveau_gem_object(obj),
1182 true, false);
1183 if (ret)
1184 return ret;
1185 }
1186 }
1187
1188 return 0;
1189}
1190
1191static int
1192nouveau_uvmm_bind_job_submit(struct nouveau_job *job,
1193 struct drm_gpuvm_exec *vme)
1194{
1195 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1196 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1197 struct drm_exec *exec = &vme->exec;
1198 struct bind_job_op *op;
1199 int ret;
1200
1201 list_for_each_op(op, &bind_job->ops) {
1202 if (op->op == OP_MAP) {
1203 struct drm_gem_object *obj = op->gem.obj =
1204 drm_gem_object_lookup(job->file_priv,
1205 op->gem.handle);
1206 if (!obj)
1207 return -ENOENT;
1208
1209 dma_resv_lock(obj->resv, NULL);
1210 op->vm_bo = drm_gpuvm_bo_obtain(&uvmm->base, obj);
1211 dma_resv_unlock(obj->resv);
1212 if (IS_ERR(op->vm_bo))
1213 return PTR_ERR(op->vm_bo);
1214
1215 drm_gpuvm_bo_extobj_add(op->vm_bo);
1216 }
1217
1218 ret = bind_validate_op(job, op);
1219 if (ret)
1220 return ret;
1221 }
1222
1223 /* If a sparse region or mapping overlaps a dirty region, we need to
1224 * wait for the region to complete the unbind process. This is due to
1225 * how page table management is currently implemented. A future
1226 * implementation might change this.
1227 */
1228 ret = bind_validate_region(job);
1229 if (ret)
1230 return ret;
1231
1232 /* Once we start modifying the GPU VA space we need to keep holding the
1233 * uvmm lock until we can't fail anymore. This is due to the set of GPU
1234 * VA space changes must appear atomically and we need to be able to
1235 * unwind all GPU VA space changes on failure.
1236 */
1237 nouveau_uvmm_lock(uvmm);
1238
1239 list_for_each_op(op, &bind_job->ops) {
1240 switch (op->op) {
1241 case OP_MAP_SPARSE:
1242 ret = nouveau_uvma_region_create(uvmm,
1243 op->va.addr,
1244 op->va.range);
1245 if (ret)
1246 goto unwind_continue;
1247
1248 break;
1249 case OP_UNMAP_SPARSE:
1250 op->reg = nouveau_uvma_region_find(uvmm, op->va.addr,
1251 op->va.range);
1252 if (!op->reg || op->reg->dirty) {
1253 ret = -ENOENT;
1254 goto unwind_continue;
1255 }
1256
1257 op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base,
1258 op->va.addr,
1259 op->va.range);
1260 if (IS_ERR(op->ops)) {
1261 ret = PTR_ERR(op->ops);
1262 goto unwind_continue;
1263 }
1264
1265 ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new,
1266 op->ops);
1267 if (ret) {
1268 drm_gpuva_ops_free(&uvmm->base, op->ops);
1269 op->ops = NULL;
1270 op->reg = NULL;
1271 goto unwind_continue;
1272 }
1273
1274 nouveau_uvma_region_dirty(op->reg);
1275
1276 break;
1277 case OP_MAP: {
1278 struct nouveau_uvma_region *reg;
1279
1280 reg = nouveau_uvma_region_find_first(uvmm,
1281 op->va.addr,
1282 op->va.range);
1283 if (reg) {
1284 u64 reg_addr = reg->va.addr;
1285 u64 reg_end = reg_addr + reg->va.range;
1286 u64 op_addr = op->va.addr;
1287 u64 op_end = op_addr + op->va.range;
1288
1289 if (unlikely(reg->dirty)) {
1290 ret = -EINVAL;
1291 goto unwind_continue;
1292 }
1293
1294 /* Make sure the mapping is either outside of a
1295 * region or fully enclosed by a region.
1296 */
1297 if (reg_addr > op_addr || reg_end < op_end) {
1298 ret = -ENOSPC;
1299 goto unwind_continue;
1300 }
1301 }
1302
1303 op->ops = drm_gpuvm_sm_map_ops_create(&uvmm->base,
1304 op->va.addr,
1305 op->va.range,
1306 op->gem.obj,
1307 op->gem.offset);
1308 if (IS_ERR(op->ops)) {
1309 ret = PTR_ERR(op->ops);
1310 goto unwind_continue;
1311 }
1312
1313 ret = nouveau_uvmm_sm_map_prepare(uvmm, &op->new,
1314 reg, op->ops,
1315 op->va.addr,
1316 op->va.range,
1317 op->flags & 0xff);
1318 if (ret) {
1319 drm_gpuva_ops_free(&uvmm->base, op->ops);
1320 op->ops = NULL;
1321 goto unwind_continue;
1322 }
1323
1324 break;
1325 }
1326 case OP_UNMAP:
1327 op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base,
1328 op->va.addr,
1329 op->va.range);
1330 if (IS_ERR(op->ops)) {
1331 ret = PTR_ERR(op->ops);
1332 goto unwind_continue;
1333 }
1334
1335 ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new,
1336 op->ops);
1337 if (ret) {
1338 drm_gpuva_ops_free(&uvmm->base, op->ops);
1339 op->ops = NULL;
1340 goto unwind_continue;
1341 }
1342
1343 break;
1344 default:
1345 ret = -EINVAL;
1346 goto unwind_continue;
1347 }
1348 }
1349
1350 drm_exec_init(exec, vme->flags, 0);
1351 drm_exec_until_all_locked(exec) {
1352 ret = bind_lock_validate(job, exec, vme->num_fences);
1353 drm_exec_retry_on_contention(exec);
1354 if (ret) {
1355 op = list_last_op(&bind_job->ops);
1356 goto unwind;
1357 }
1358 }
1359
1360 /* Link and unlink GPUVAs while holding the dma_resv lock.
1361 *
1362 * As long as we validate() all GEMs and add fences to all GEMs DMA
1363 * reservations backing map and remap operations we can be sure there
1364 * won't be any concurrent (in)validations during job execution, hence
1365 * we're safe to check drm_gpuva_invalidated() within the fence
1366 * signalling critical path without holding a separate lock.
1367 *
1368 * GPUVAs about to be unmapped are safe as well, since they're unlinked
1369 * already.
1370 *
1371 * GEMs from map and remap operations must be validated before linking
1372 * their corresponding mappings to prevent the actual PT update to
1373 * happen right away in validate() rather than asynchronously as
1374 * intended.
1375 *
1376 * Note that after linking and unlinking the GPUVAs in this loop this
1377 * function cannot fail anymore, hence there is no need for an unwind
1378 * path.
1379 */
1380 list_for_each_op(op, &bind_job->ops) {
1381 switch (op->op) {
1382 case OP_UNMAP_SPARSE:
1383 case OP_MAP:
1384 case OP_UNMAP:
1385 bind_link_gpuvas(op);
1386 break;
1387 default:
1388 break;
1389 }
1390 }
1391 nouveau_uvmm_unlock(uvmm);
1392
1393 return 0;
1394
1395unwind_continue:
1396 op = list_prev_op(op);
1397unwind:
1398 list_for_each_op_from_reverse(op, &bind_job->ops) {
1399 switch (op->op) {
1400 case OP_MAP_SPARSE:
1401 nouveau_uvma_region_destroy(uvmm, op->va.addr,
1402 op->va.range);
1403 break;
1404 case OP_UNMAP_SPARSE:
1405 __nouveau_uvma_region_insert(uvmm, op->reg);
1406 nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new,
1407 op->ops);
1408 break;
1409 case OP_MAP:
1410 nouveau_uvmm_sm_map_prepare_unwind(uvmm, &op->new,
1411 op->ops,
1412 op->va.addr,
1413 op->va.range);
1414 break;
1415 case OP_UNMAP:
1416 nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new,
1417 op->ops);
1418 break;
1419 }
1420
1421 drm_gpuva_ops_free(&uvmm->base, op->ops);
1422 op->ops = NULL;
1423 op->reg = NULL;
1424 }
1425
1426 nouveau_uvmm_unlock(uvmm);
1427 drm_gpuvm_exec_unlock(vme);
1428 return ret;
1429}
1430
1431static void
1432nouveau_uvmm_bind_job_armed_submit(struct nouveau_job *job,
1433 struct drm_gpuvm_exec *vme)
1434{
1435 drm_gpuvm_exec_resv_add_fence(vme, job->done_fence,
1436 job->resv_usage, job->resv_usage);
1437 drm_gpuvm_exec_unlock(vme);
1438}
1439
1440static struct dma_fence *
1441nouveau_uvmm_bind_job_run(struct nouveau_job *job)
1442{
1443 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1444 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1445 struct bind_job_op *op;
1446 int ret = 0;
1447
1448 list_for_each_op(op, &bind_job->ops) {
1449 switch (op->op) {
1450 case OP_MAP_SPARSE:
1451 /* noop */
1452 break;
1453 case OP_MAP:
1454 ret = nouveau_uvmm_sm_map(uvmm, &op->new, op->ops);
1455 if (ret)
1456 goto out;
1457 break;
1458 case OP_UNMAP_SPARSE:
1459 fallthrough;
1460 case OP_UNMAP:
1461 ret = nouveau_uvmm_sm_unmap(uvmm, &op->new, op->ops);
1462 if (ret)
1463 goto out;
1464 break;
1465 }
1466 }
1467
1468out:
1469 if (ret)
1470 NV_PRINTK(err, job->cli, "bind job failed: %d\n", ret);
1471 return ERR_PTR(ret);
1472}
1473
1474static void
1475nouveau_uvmm_bind_job_cleanup(struct nouveau_job *job)
1476{
1477 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1478 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1479 struct bind_job_op *op;
1480
1481 list_for_each_op(op, &bind_job->ops) {
1482 struct drm_gem_object *obj = op->gem.obj;
1483
1484 /* When nouveau_uvmm_bind_job_submit() fails op->ops and op->reg
1485 * will be NULL, hence skip the cleanup.
1486 */
1487 switch (op->op) {
1488 case OP_MAP_SPARSE:
1489 /* noop */
1490 break;
1491 case OP_UNMAP_SPARSE:
1492 if (!IS_ERR_OR_NULL(op->ops))
1493 nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new,
1494 op->ops);
1495
1496 if (op->reg) {
1497 nouveau_uvma_region_sparse_unref(op->reg);
1498 nouveau_uvmm_lock(uvmm);
1499 nouveau_uvma_region_remove(op->reg);
1500 nouveau_uvmm_unlock(uvmm);
1501 nouveau_uvma_region_complete(op->reg);
1502 nouveau_uvma_region_put(op->reg);
1503 }
1504
1505 break;
1506 case OP_MAP:
1507 if (!IS_ERR_OR_NULL(op->ops))
1508 nouveau_uvmm_sm_map_cleanup(uvmm, &op->new,
1509 op->ops);
1510 break;
1511 case OP_UNMAP:
1512 if (!IS_ERR_OR_NULL(op->ops))
1513 nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new,
1514 op->ops);
1515 break;
1516 }
1517
1518 if (!IS_ERR_OR_NULL(op->ops))
1519 drm_gpuva_ops_free(&uvmm->base, op->ops);
1520
1521 if (!IS_ERR_OR_NULL(op->vm_bo)) {
1522 dma_resv_lock(obj->resv, NULL);
1523 drm_gpuvm_bo_put(op->vm_bo);
1524 dma_resv_unlock(obj->resv);
1525 }
1526
1527 if (obj)
1528 drm_gem_object_put(obj);
1529 }
1530
1531 nouveau_job_done(job);
1532 complete_all(&bind_job->complete);
1533
1534 nouveau_uvmm_bind_job_put(bind_job);
1535}
1536
1537static const struct nouveau_job_ops nouveau_bind_job_ops = {
1538 .submit = nouveau_uvmm_bind_job_submit,
1539 .armed_submit = nouveau_uvmm_bind_job_armed_submit,
1540 .run = nouveau_uvmm_bind_job_run,
1541 .free = nouveau_uvmm_bind_job_cleanup,
1542};
1543
1544static int
1545bind_job_op_from_uop(struct bind_job_op **pop,
1546 struct drm_nouveau_vm_bind_op *uop)
1547{
1548 struct bind_job_op *op;
1549
1550 op = *pop = kzalloc(sizeof(*op), GFP_KERNEL);
1551 if (!op)
1552 return -ENOMEM;
1553
1554 switch (uop->op) {
1555 case OP_MAP:
1556 op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ?
1557 OP_MAP_SPARSE : OP_MAP;
1558 break;
1559 case OP_UNMAP:
1560 op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ?
1561 OP_UNMAP_SPARSE : OP_UNMAP;
1562 break;
1563 default:
1564 op->op = uop->op;
1565 break;
1566 }
1567
1568 op->flags = uop->flags;
1569 op->va.addr = uop->addr;
1570 op->va.range = uop->range;
1571 op->gem.handle = uop->handle;
1572 op->gem.offset = uop->bo_offset;
1573
1574 return 0;
1575}
1576
1577static void
1578bind_job_ops_free(struct list_head *ops)
1579{
1580 struct bind_job_op *op, *next;
1581
1582 list_for_each_op_safe(op, next, ops) {
1583 list_del(&op->entry);
1584 kfree(op);
1585 }
1586}
1587
1588static int
1589nouveau_uvmm_bind_job_init(struct nouveau_uvmm_bind_job **pjob,
1590 struct nouveau_uvmm_bind_job_args *__args)
1591{
1592 struct nouveau_uvmm_bind_job *job;
1593 struct nouveau_job_args args = {};
1594 struct bind_job_op *op;
1595 int i, ret;
1596
1597 ret = nouveau_uvmm_bind_job_alloc(&job);
1598 if (ret)
1599 return ret;
1600
1601 INIT_LIST_HEAD(&job->ops);
1602
1603 for (i = 0; i < __args->op.count; i++) {
1604 ret = bind_job_op_from_uop(&op, &__args->op.s[i]);
1605 if (ret)
1606 goto err_free;
1607
1608 list_add_tail(&op->entry, &job->ops);
1609 }
1610
1611 init_completion(&job->complete);
1612
1613 args.file_priv = __args->file_priv;
1614
1615 args.sched = __args->sched;
1616 args.credits = 1;
1617
1618 args.in_sync.count = __args->in_sync.count;
1619 args.in_sync.s = __args->in_sync.s;
1620
1621 args.out_sync.count = __args->out_sync.count;
1622 args.out_sync.s = __args->out_sync.s;
1623
1624 args.sync = !(__args->flags & DRM_NOUVEAU_VM_BIND_RUN_ASYNC);
1625 args.ops = &nouveau_bind_job_ops;
1626 args.resv_usage = DMA_RESV_USAGE_BOOKKEEP;
1627
1628 ret = nouveau_job_init(&job->base, &args);
1629 if (ret)
1630 goto err_free;
1631
1632 *pjob = job;
1633 return 0;
1634
1635err_free:
1636 bind_job_ops_free(&job->ops);
1637 kfree(job);
1638 *pjob = NULL;
1639
1640 return ret;
1641}
1642
1643static int
1644nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args)
1645{
1646 struct nouveau_uvmm_bind_job *job;
1647 int ret;
1648
1649 ret = nouveau_uvmm_bind_job_init(&job, args);
1650 if (ret)
1651 return ret;
1652
1653 ret = nouveau_job_submit(&job->base);
1654 if (ret)
1655 goto err_job_fini;
1656
1657 return 0;
1658
1659err_job_fini:
1660 nouveau_job_fini(&job->base);
1661 return ret;
1662}
1663
1664static int
1665nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args,
1666 struct drm_nouveau_vm_bind *req)
1667{
1668 struct drm_nouveau_sync **s;
1669 u32 inc = req->wait_count;
1670 u64 ins = req->wait_ptr;
1671 u32 outc = req->sig_count;
1672 u64 outs = req->sig_ptr;
1673 u32 opc = req->op_count;
1674 u64 ops = req->op_ptr;
1675 int ret;
1676
1677 args->flags = req->flags;
1678
1679 if (opc) {
1680 args->op.count = opc;
1681 args->op.s = u_memcpya(ops, opc,
1682 sizeof(*args->op.s));
1683 if (IS_ERR(args->op.s))
1684 return PTR_ERR(args->op.s);
1685 }
1686
1687 if (inc) {
1688 s = &args->in_sync.s;
1689
1690 args->in_sync.count = inc;
1691 *s = u_memcpya(ins, inc, sizeof(**s));
1692 if (IS_ERR(*s)) {
1693 ret = PTR_ERR(*s);
1694 goto err_free_ops;
1695 }
1696 }
1697
1698 if (outc) {
1699 s = &args->out_sync.s;
1700
1701 args->out_sync.count = outc;
1702 *s = u_memcpya(outs, outc, sizeof(**s));
1703 if (IS_ERR(*s)) {
1704 ret = PTR_ERR(*s);
1705 goto err_free_ins;
1706 }
1707 }
1708
1709 return 0;
1710
1711err_free_ops:
1712 u_free(args->op.s);
1713err_free_ins:
1714 u_free(args->in_sync.s);
1715 return ret;
1716}
1717
1718static void
1719nouveau_uvmm_vm_bind_ufree(struct nouveau_uvmm_bind_job_args *args)
1720{
1721 u_free(args->op.s);
1722 u_free(args->in_sync.s);
1723 u_free(args->out_sync.s);
1724}
1725
1726int
1727nouveau_uvmm_ioctl_vm_bind(struct drm_device *dev,
1728 void *data,
1729 struct drm_file *file_priv)
1730{
1731 struct nouveau_cli *cli = nouveau_cli(file_priv);
1732 struct nouveau_uvmm_bind_job_args args = {};
1733 struct drm_nouveau_vm_bind *req = data;
1734 int ret = 0;
1735
1736 if (unlikely(!nouveau_cli_uvmm_locked(cli)))
1737 return -ENOSYS;
1738
1739 ret = nouveau_uvmm_vm_bind_ucopy(&args, req);
1740 if (ret)
1741 return ret;
1742
1743 args.sched = cli->sched;
1744 args.file_priv = file_priv;
1745
1746 ret = nouveau_uvmm_vm_bind(&args);
1747 if (ret)
1748 goto out_free_args;
1749
1750out_free_args:
1751 nouveau_uvmm_vm_bind_ufree(&args);
1752 return ret;
1753}
1754
1755void
1756nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem)
1757{
1758 struct drm_gem_object *obj = &nvbo->bo.base;
1759 struct drm_gpuvm_bo *vm_bo;
1760 struct drm_gpuva *va;
1761
1762 dma_resv_assert_held(obj->resv);
1763
1764 drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
1765 drm_gpuvm_bo_for_each_va(va, vm_bo) {
1766 struct nouveau_uvma *uvma = uvma_from_va(va);
1767
1768 nouveau_uvma_map(uvma, mem);
1769 drm_gpuva_invalidate(va, false);
1770 }
1771 }
1772}
1773
1774void
1775nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo)
1776{
1777 struct drm_gem_object *obj = &nvbo->bo.base;
1778 struct drm_gpuvm_bo *vm_bo;
1779 struct drm_gpuva *va;
1780
1781 dma_resv_assert_held(obj->resv);
1782
1783 drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
1784 drm_gpuvm_bo_for_each_va(va, vm_bo) {
1785 struct nouveau_uvma *uvma = uvma_from_va(va);
1786
1787 nouveau_uvma_unmap(uvma);
1788 drm_gpuva_invalidate(va, true);
1789 }
1790 }
1791}
1792
1793static void
1794nouveau_uvmm_free(struct drm_gpuvm *gpuvm)
1795{
1796 struct nouveau_uvmm *uvmm = uvmm_from_gpuvm(gpuvm);
1797
1798 kfree(uvmm);
1799}
1800
1801static int
1802nouveau_uvmm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
1803{
1804 struct nouveau_bo *nvbo = nouveau_gem_object(vm_bo->obj);
1805
1806 nouveau_bo_placement_set(nvbo, nvbo->valid_domains, 0);
1807 return nouveau_bo_validate(nvbo, true, false);
1808}
1809
1810static const struct drm_gpuvm_ops gpuvm_ops = {
1811 .vm_free = nouveau_uvmm_free,
1812 .vm_bo_validate = nouveau_uvmm_bo_validate,
1813};
1814
1815int
1816nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,
1817 void *data,
1818 struct drm_file *file_priv)
1819{
1820 struct nouveau_uvmm *uvmm;
1821 struct nouveau_cli *cli = nouveau_cli(file_priv);
1822 struct drm_device *drm = cli->drm->dev;
1823 struct drm_gem_object *r_obj;
1824 struct drm_nouveau_vm_init *init = data;
1825 u64 kernel_managed_end;
1826 int ret;
1827
1828 if (check_add_overflow(init->kernel_managed_addr,
1829 init->kernel_managed_size,
1830 &kernel_managed_end))
1831 return -EINVAL;
1832
1833 if (kernel_managed_end > NOUVEAU_VA_SPACE_END)
1834 return -EINVAL;
1835
1836 mutex_lock(&cli->mutex);
1837
1838 if (unlikely(cli->uvmm.disabled)) {
1839 ret = -ENOSYS;
1840 goto out_unlock;
1841 }
1842
1843 uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL);
1844 if (!uvmm) {
1845 ret = -ENOMEM;
1846 goto out_unlock;
1847 }
1848
1849 r_obj = drm_gpuvm_resv_object_alloc(drm);
1850 if (!r_obj) {
1851 kfree(uvmm);
1852 ret = -ENOMEM;
1853 goto out_unlock;
1854 }
1855
1856 mutex_init(&uvmm->mutex);
1857 mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN);
1858 mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex);
1859
1860 drm_gpuvm_init(&uvmm->base, cli->name, 0, drm, r_obj,
1861 NOUVEAU_VA_SPACE_START,
1862 NOUVEAU_VA_SPACE_END,
1863 init->kernel_managed_addr,
1864 init->kernel_managed_size,
1865 &gpuvm_ops);
1866 /* GPUVM takes care from here on. */
1867 drm_gem_object_put(r_obj);
1868
1869 ret = nvif_vmm_ctor(&cli->mmu, "uvmm",
1870 cli->vmm.vmm.object.oclass, RAW,
1871 init->kernel_managed_addr,
1872 init->kernel_managed_size,
1873 NULL, 0, &uvmm->vmm.vmm);
1874 if (ret)
1875 goto out_gpuvm_fini;
1876
1877 uvmm->vmm.cli = cli;
1878 cli->uvmm.ptr = uvmm;
1879 mutex_unlock(&cli->mutex);
1880
1881 return 0;
1882
1883out_gpuvm_fini:
1884 drm_gpuvm_put(&uvmm->base);
1885out_unlock:
1886 mutex_unlock(&cli->mutex);
1887 return ret;
1888}
1889
1890void
1891nouveau_uvmm_fini(struct nouveau_uvmm *uvmm)
1892{
1893 MA_STATE(mas, &uvmm->region_mt, 0, 0);
1894 struct nouveau_uvma_region *reg;
1895 struct nouveau_cli *cli = uvmm->vmm.cli;
1896 struct drm_gpuva *va, *next;
1897
1898 nouveau_uvmm_lock(uvmm);
1899 drm_gpuvm_for_each_va_safe(va, next, &uvmm->base) {
1900 struct nouveau_uvma *uvma = uvma_from_va(va);
1901 struct drm_gem_object *obj = va->gem.obj;
1902
1903 if (unlikely(va == &uvmm->base.kernel_alloc_node))
1904 continue;
1905
1906 drm_gpuva_remove(va);
1907
1908 dma_resv_lock(obj->resv, NULL);
1909 drm_gpuva_unlink(va);
1910 dma_resv_unlock(obj->resv);
1911
1912 nouveau_uvma_unmap(uvma);
1913 nouveau_uvma_vmm_put(uvma);
1914
1915 nouveau_uvma_gem_put(uvma);
1916 nouveau_uvma_free(uvma);
1917 }
1918
1919 mas_for_each(&mas, reg, ULONG_MAX) {
1920 mas_erase(&mas);
1921 nouveau_uvma_region_sparse_unref(reg);
1922 nouveau_uvma_region_put(reg);
1923 }
1924
1925 WARN(!mtree_empty(&uvmm->region_mt),
1926 "nouveau_uvma_region tree not empty, potentially leaking memory.");
1927 __mt_destroy(&uvmm->region_mt);
1928 nouveau_uvmm_unlock(uvmm);
1929
1930 mutex_lock(&cli->mutex);
1931 nouveau_vmm_fini(&uvmm->vmm);
1932 drm_gpuvm_put(&uvmm->base);
1933 mutex_unlock(&cli->mutex);
1934}