Loading...
1/*
2 * Copyright 2014-2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#define pr_fmt(fmt) "kfd2kgd: " fmt
24
25#include <linux/list.h>
26#include <drm/drmP.h>
27#include "amdgpu_object.h"
28#include "amdgpu_vm.h"
29#include "amdgpu_amdkfd.h"
30
31/* Special VM and GART address alignment needed for VI pre-Fiji due to
32 * a HW bug.
33 */
34#define VI_BO_SIZE_ALIGN (0x8000)
35
36/* Impose limit on how much memory KFD can use */
37static struct {
38 uint64_t max_system_mem_limit;
39 int64_t system_mem_used;
40 spinlock_t mem_limit_lock;
41} kfd_mem_limit;
42
43/* Struct used for amdgpu_amdkfd_bo_validate */
44struct amdgpu_vm_parser {
45 uint32_t domain;
46 bool wait;
47};
48
49static const char * const domain_bit_to_string[] = {
50 "CPU",
51 "GTT",
52 "VRAM",
53 "GDS",
54 "GWS",
55 "OA"
56};
57
58#define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
59
60
61
62static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
63{
64 return (struct amdgpu_device *)kgd;
65}
66
67static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm,
68 struct kgd_mem *mem)
69{
70 struct kfd_bo_va_list *entry;
71
72 list_for_each_entry(entry, &mem->bo_va_list, bo_list)
73 if (entry->bo_va->base.vm == avm)
74 return false;
75
76 return true;
77}
78
79/* Set memory usage limits. Current, limits are
80 * System (kernel) memory - 3/8th System RAM
81 */
82void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
83{
84 struct sysinfo si;
85 uint64_t mem;
86
87 si_meminfo(&si);
88 mem = si.totalram - si.totalhigh;
89 mem *= si.mem_unit;
90
91 spin_lock_init(&kfd_mem_limit.mem_limit_lock);
92 kfd_mem_limit.max_system_mem_limit = (mem >> 1) - (mem >> 3);
93 pr_debug("Kernel memory limit %lluM\n",
94 (kfd_mem_limit.max_system_mem_limit >> 20));
95}
96
97static int amdgpu_amdkfd_reserve_system_mem_limit(struct amdgpu_device *adev,
98 uint64_t size, u32 domain)
99{
100 size_t acc_size;
101 int ret = 0;
102
103 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
104 sizeof(struct amdgpu_bo));
105
106 spin_lock(&kfd_mem_limit.mem_limit_lock);
107 if (domain == AMDGPU_GEM_DOMAIN_GTT) {
108 if (kfd_mem_limit.system_mem_used + (acc_size + size) >
109 kfd_mem_limit.max_system_mem_limit) {
110 ret = -ENOMEM;
111 goto err_no_mem;
112 }
113 kfd_mem_limit.system_mem_used += (acc_size + size);
114 }
115err_no_mem:
116 spin_unlock(&kfd_mem_limit.mem_limit_lock);
117 return ret;
118}
119
120static void unreserve_system_mem_limit(struct amdgpu_device *adev,
121 uint64_t size, u32 domain)
122{
123 size_t acc_size;
124
125 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
126 sizeof(struct amdgpu_bo));
127
128 spin_lock(&kfd_mem_limit.mem_limit_lock);
129 if (domain == AMDGPU_GEM_DOMAIN_GTT)
130 kfd_mem_limit.system_mem_used -= (acc_size + size);
131 WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
132 "kfd system memory accounting unbalanced");
133
134 spin_unlock(&kfd_mem_limit.mem_limit_lock);
135}
136
137void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo)
138{
139 spin_lock(&kfd_mem_limit.mem_limit_lock);
140
141 if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT) {
142 kfd_mem_limit.system_mem_used -=
143 (bo->tbo.acc_size + amdgpu_bo_size(bo));
144 }
145 WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
146 "kfd system memory accounting unbalanced");
147
148 spin_unlock(&kfd_mem_limit.mem_limit_lock);
149}
150
151
152/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence(s) from BO's
153 * reservation object.
154 *
155 * @bo: [IN] Remove eviction fence(s) from this BO
156 * @ef: [IN] If ef is specified, then this eviction fence is removed if it
157 * is present in the shared list.
158 * @ef_list: [OUT] Returns list of eviction fences. These fences are removed
159 * from BO's reservation object shared list.
160 * @ef_count: [OUT] Number of fences in ef_list.
161 *
162 * NOTE: If called with ef_list, then amdgpu_amdkfd_add_eviction_fence must be
163 * called to restore the eviction fences and to avoid memory leak. This is
164 * useful for shared BOs.
165 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
166 */
167static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
168 struct amdgpu_amdkfd_fence *ef,
169 struct amdgpu_amdkfd_fence ***ef_list,
170 unsigned int *ef_count)
171{
172 struct reservation_object_list *fobj;
173 struct reservation_object *resv;
174 unsigned int i = 0, j = 0, k = 0, shared_count;
175 unsigned int count = 0;
176 struct amdgpu_amdkfd_fence **fence_list;
177
178 if (!ef && !ef_list)
179 return -EINVAL;
180
181 if (ef_list) {
182 *ef_list = NULL;
183 *ef_count = 0;
184 }
185
186 resv = bo->tbo.resv;
187 fobj = reservation_object_get_list(resv);
188
189 if (!fobj)
190 return 0;
191
192 preempt_disable();
193 write_seqcount_begin(&resv->seq);
194
195 /* Go through all the shared fences in the resevation object. If
196 * ef is specified and it exists in the list, remove it and reduce the
197 * count. If ef is not specified, then get the count of eviction fences
198 * present.
199 */
200 shared_count = fobj->shared_count;
201 for (i = 0; i < shared_count; ++i) {
202 struct dma_fence *f;
203
204 f = rcu_dereference_protected(fobj->shared[i],
205 reservation_object_held(resv));
206
207 if (ef) {
208 if (f->context == ef->base.context) {
209 dma_fence_put(f);
210 fobj->shared_count--;
211 } else {
212 RCU_INIT_POINTER(fobj->shared[j++], f);
213 }
214 } else if (to_amdgpu_amdkfd_fence(f))
215 count++;
216 }
217 write_seqcount_end(&resv->seq);
218 preempt_enable();
219
220 if (ef || !count)
221 return 0;
222
223 /* Alloc memory for count number of eviction fence pointers. Fill the
224 * ef_list array and ef_count
225 */
226 fence_list = kcalloc(count, sizeof(struct amdgpu_amdkfd_fence *),
227 GFP_KERNEL);
228 if (!fence_list)
229 return -ENOMEM;
230
231 preempt_disable();
232 write_seqcount_begin(&resv->seq);
233
234 j = 0;
235 for (i = 0; i < shared_count; ++i) {
236 struct dma_fence *f;
237 struct amdgpu_amdkfd_fence *efence;
238
239 f = rcu_dereference_protected(fobj->shared[i],
240 reservation_object_held(resv));
241
242 efence = to_amdgpu_amdkfd_fence(f);
243 if (efence) {
244 fence_list[k++] = efence;
245 fobj->shared_count--;
246 } else {
247 RCU_INIT_POINTER(fobj->shared[j++], f);
248 }
249 }
250
251 write_seqcount_end(&resv->seq);
252 preempt_enable();
253
254 *ef_list = fence_list;
255 *ef_count = k;
256
257 return 0;
258}
259
260/* amdgpu_amdkfd_add_eviction_fence - Adds eviction fence(s) back into BO's
261 * reservation object.
262 *
263 * @bo: [IN] Add eviction fences to this BO
264 * @ef_list: [IN] List of eviction fences to be added
265 * @ef_count: [IN] Number of fences in ef_list.
266 *
267 * NOTE: Must call amdgpu_amdkfd_remove_eviction_fence before calling this
268 * function.
269 */
270static void amdgpu_amdkfd_add_eviction_fence(struct amdgpu_bo *bo,
271 struct amdgpu_amdkfd_fence **ef_list,
272 unsigned int ef_count)
273{
274 int i;
275
276 if (!ef_list || !ef_count)
277 return;
278
279 for (i = 0; i < ef_count; i++) {
280 amdgpu_bo_fence(bo, &ef_list[i]->base, true);
281 /* Re-adding the fence takes an additional reference. Drop that
282 * reference.
283 */
284 dma_fence_put(&ef_list[i]->base);
285 }
286
287 kfree(ef_list);
288}
289
290static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
291 bool wait)
292{
293 struct ttm_operation_ctx ctx = { false, false };
294 int ret;
295
296 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
297 "Called with userptr BO"))
298 return -EINVAL;
299
300 amdgpu_ttm_placement_from_domain(bo, domain);
301
302 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
303 if (ret)
304 goto validate_fail;
305 if (wait) {
306 struct amdgpu_amdkfd_fence **ef_list;
307 unsigned int ef_count;
308
309 ret = amdgpu_amdkfd_remove_eviction_fence(bo, NULL, &ef_list,
310 &ef_count);
311 if (ret)
312 goto validate_fail;
313
314 ttm_bo_wait(&bo->tbo, false, false);
315 amdgpu_amdkfd_add_eviction_fence(bo, ef_list, ef_count);
316 }
317
318validate_fail:
319 return ret;
320}
321
322static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo)
323{
324 struct amdgpu_vm_parser *p = param;
325
326 return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait);
327}
328
329/* vm_validate_pt_pd_bos - Validate page table and directory BOs
330 *
331 * Page directories are not updated here because huge page handling
332 * during page table updates can invalidate page directory entries
333 * again. Page directories are only updated after updating page
334 * tables.
335 */
336static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
337{
338 struct amdgpu_bo *pd = vm->root.base.bo;
339 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
340 struct amdgpu_vm_parser param;
341 uint64_t addr, flags = AMDGPU_PTE_VALID;
342 int ret;
343
344 param.domain = AMDGPU_GEM_DOMAIN_VRAM;
345 param.wait = false;
346
347 ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate,
348 ¶m);
349 if (ret) {
350 pr_err("amdgpu: failed to validate PT BOs\n");
351 return ret;
352 }
353
354 ret = amdgpu_amdkfd_validate(¶m, pd);
355 if (ret) {
356 pr_err("amdgpu: failed to validate PD\n");
357 return ret;
358 }
359
360 addr = amdgpu_bo_gpu_offset(vm->root.base.bo);
361 amdgpu_gmc_get_vm_pde(adev, -1, &addr, &flags);
362 vm->pd_phys_addr = addr;
363
364 if (vm->use_cpu_for_update) {
365 ret = amdgpu_bo_kmap(pd, NULL);
366 if (ret) {
367 pr_err("amdgpu: failed to kmap PD, ret=%d\n", ret);
368 return ret;
369 }
370 }
371
372 return 0;
373}
374
375static int sync_vm_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
376 struct dma_fence *f)
377{
378 int ret = amdgpu_sync_fence(adev, sync, f, false);
379
380 /* Sync objects can't handle multiple GPUs (contexts) updating
381 * sync->last_vm_update. Fortunately we don't need it for
382 * KFD's purposes, so we can just drop that fence.
383 */
384 if (sync->last_vm_update) {
385 dma_fence_put(sync->last_vm_update);
386 sync->last_vm_update = NULL;
387 }
388
389 return ret;
390}
391
392static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
393{
394 struct amdgpu_bo *pd = vm->root.base.bo;
395 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
396 int ret;
397
398 ret = amdgpu_vm_update_directories(adev, vm);
399 if (ret)
400 return ret;
401
402 return sync_vm_fence(adev, sync, vm->last_update);
403}
404
405/* add_bo_to_vm - Add a BO to a VM
406 *
407 * Everything that needs to bo done only once when a BO is first added
408 * to a VM. It can later be mapped and unmapped many times without
409 * repeating these steps.
410 *
411 * 1. Allocate and initialize BO VA entry data structure
412 * 2. Add BO to the VM
413 * 3. Determine ASIC-specific PTE flags
414 * 4. Alloc page tables and directories if needed
415 * 4a. Validate new page tables and directories
416 */
417static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem,
418 struct amdgpu_vm *vm, bool is_aql,
419 struct kfd_bo_va_list **p_bo_va_entry)
420{
421 int ret;
422 struct kfd_bo_va_list *bo_va_entry;
423 struct amdgpu_bo *pd = vm->root.base.bo;
424 struct amdgpu_bo *bo = mem->bo;
425 uint64_t va = mem->va;
426 struct list_head *list_bo_va = &mem->bo_va_list;
427 unsigned long bo_size = bo->tbo.mem.size;
428
429 if (!va) {
430 pr_err("Invalid VA when adding BO to VM\n");
431 return -EINVAL;
432 }
433
434 if (is_aql)
435 va += bo_size;
436
437 bo_va_entry = kzalloc(sizeof(*bo_va_entry), GFP_KERNEL);
438 if (!bo_va_entry)
439 return -ENOMEM;
440
441 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
442 va + bo_size, vm);
443
444 /* Add BO to VM internal data structures*/
445 bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, vm, bo);
446 if (!bo_va_entry->bo_va) {
447 ret = -EINVAL;
448 pr_err("Failed to add BO object to VM. ret == %d\n",
449 ret);
450 goto err_vmadd;
451 }
452
453 bo_va_entry->va = va;
454 bo_va_entry->pte_flags = amdgpu_gmc_get_pte_flags(adev,
455 mem->mapping_flags);
456 bo_va_entry->kgd_dev = (void *)adev;
457 list_add(&bo_va_entry->bo_list, list_bo_va);
458
459 if (p_bo_va_entry)
460 *p_bo_va_entry = bo_va_entry;
461
462 /* Allocate new page tables if needed and validate
463 * them. Clearing of new page tables and validate need to wait
464 * on move fences. We don't want that to trigger the eviction
465 * fence, so remove it temporarily.
466 */
467 amdgpu_amdkfd_remove_eviction_fence(pd,
468 vm->process_info->eviction_fence,
469 NULL, NULL);
470
471 ret = amdgpu_vm_alloc_pts(adev, vm, va, amdgpu_bo_size(bo));
472 if (ret) {
473 pr_err("Failed to allocate pts, err=%d\n", ret);
474 goto err_alloc_pts;
475 }
476
477 ret = vm_validate_pt_pd_bos(vm);
478 if (ret) {
479 pr_err("validate_pt_pd_bos() failed\n");
480 goto err_alloc_pts;
481 }
482
483 /* Add the eviction fence back */
484 amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true);
485
486 return 0;
487
488err_alloc_pts:
489 amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true);
490 amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va);
491 list_del(&bo_va_entry->bo_list);
492err_vmadd:
493 kfree(bo_va_entry);
494 return ret;
495}
496
497static void remove_bo_from_vm(struct amdgpu_device *adev,
498 struct kfd_bo_va_list *entry, unsigned long size)
499{
500 pr_debug("\t remove VA 0x%llx - 0x%llx in entry %p\n",
501 entry->va,
502 entry->va + size, entry);
503 amdgpu_vm_bo_rmv(adev, entry->bo_va);
504 list_del(&entry->bo_list);
505 kfree(entry);
506}
507
508static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
509 struct amdkfd_process_info *process_info)
510{
511 struct ttm_validate_buffer *entry = &mem->validate_list;
512 struct amdgpu_bo *bo = mem->bo;
513
514 INIT_LIST_HEAD(&entry->head);
515 entry->shared = true;
516 entry->bo = &bo->tbo;
517 mutex_lock(&process_info->lock);
518 list_add_tail(&entry->head, &process_info->kfd_bo_list);
519 mutex_unlock(&process_info->lock);
520}
521
522/* Reserving a BO and its page table BOs must happen atomically to
523 * avoid deadlocks. Some operations update multiple VMs at once. Track
524 * all the reservation info in a context structure. Optionally a sync
525 * object can track VM updates.
526 */
527struct bo_vm_reservation_context {
528 struct amdgpu_bo_list_entry kfd_bo; /* BO list entry for the KFD BO */
529 unsigned int n_vms; /* Number of VMs reserved */
530 struct amdgpu_bo_list_entry *vm_pd; /* Array of VM BO list entries */
531 struct ww_acquire_ctx ticket; /* Reservation ticket */
532 struct list_head list, duplicates; /* BO lists */
533 struct amdgpu_sync *sync; /* Pointer to sync object */
534 bool reserved; /* Whether BOs are reserved */
535};
536
537enum bo_vm_match {
538 BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */
539 BO_VM_MAPPED, /* Match VMs where a BO is mapped */
540 BO_VM_ALL, /* Match all VMs a BO was added to */
541};
542
543/**
544 * reserve_bo_and_vm - reserve a BO and a VM unconditionally.
545 * @mem: KFD BO structure.
546 * @vm: the VM to reserve.
547 * @ctx: the struct that will be used in unreserve_bo_and_vms().
548 */
549static int reserve_bo_and_vm(struct kgd_mem *mem,
550 struct amdgpu_vm *vm,
551 struct bo_vm_reservation_context *ctx)
552{
553 struct amdgpu_bo *bo = mem->bo;
554 int ret;
555
556 WARN_ON(!vm);
557
558 ctx->reserved = false;
559 ctx->n_vms = 1;
560 ctx->sync = &mem->sync;
561
562 INIT_LIST_HEAD(&ctx->list);
563 INIT_LIST_HEAD(&ctx->duplicates);
564
565 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), GFP_KERNEL);
566 if (!ctx->vm_pd)
567 return -ENOMEM;
568
569 ctx->kfd_bo.robj = bo;
570 ctx->kfd_bo.priority = 0;
571 ctx->kfd_bo.tv.bo = &bo->tbo;
572 ctx->kfd_bo.tv.shared = true;
573 ctx->kfd_bo.user_pages = NULL;
574 list_add(&ctx->kfd_bo.tv.head, &ctx->list);
575
576 amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);
577
578 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
579 false, &ctx->duplicates);
580 if (!ret)
581 ctx->reserved = true;
582 else {
583 pr_err("Failed to reserve buffers in ttm\n");
584 kfree(ctx->vm_pd);
585 ctx->vm_pd = NULL;
586 }
587
588 return ret;
589}
590
591/**
592 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
593 * @mem: KFD BO structure.
594 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO
595 * is used. Otherwise, a single VM associated with the BO.
596 * @map_type: the mapping status that will be used to filter the VMs.
597 * @ctx: the struct that will be used in unreserve_bo_and_vms().
598 *
599 * Returns 0 for success, negative for failure.
600 */
601static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
602 struct amdgpu_vm *vm, enum bo_vm_match map_type,
603 struct bo_vm_reservation_context *ctx)
604{
605 struct amdgpu_bo *bo = mem->bo;
606 struct kfd_bo_va_list *entry;
607 unsigned int i;
608 int ret;
609
610 ctx->reserved = false;
611 ctx->n_vms = 0;
612 ctx->vm_pd = NULL;
613 ctx->sync = &mem->sync;
614
615 INIT_LIST_HEAD(&ctx->list);
616 INIT_LIST_HEAD(&ctx->duplicates);
617
618 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
619 if ((vm && vm != entry->bo_va->base.vm) ||
620 (entry->is_mapped != map_type
621 && map_type != BO_VM_ALL))
622 continue;
623
624 ctx->n_vms++;
625 }
626
627 if (ctx->n_vms != 0) {
628 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd),
629 GFP_KERNEL);
630 if (!ctx->vm_pd)
631 return -ENOMEM;
632 }
633
634 ctx->kfd_bo.robj = bo;
635 ctx->kfd_bo.priority = 0;
636 ctx->kfd_bo.tv.bo = &bo->tbo;
637 ctx->kfd_bo.tv.shared = true;
638 ctx->kfd_bo.user_pages = NULL;
639 list_add(&ctx->kfd_bo.tv.head, &ctx->list);
640
641 i = 0;
642 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
643 if ((vm && vm != entry->bo_va->base.vm) ||
644 (entry->is_mapped != map_type
645 && map_type != BO_VM_ALL))
646 continue;
647
648 amdgpu_vm_get_pd_bo(entry->bo_va->base.vm, &ctx->list,
649 &ctx->vm_pd[i]);
650 i++;
651 }
652
653 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
654 false, &ctx->duplicates);
655 if (!ret)
656 ctx->reserved = true;
657 else
658 pr_err("Failed to reserve buffers in ttm.\n");
659
660 if (ret) {
661 kfree(ctx->vm_pd);
662 ctx->vm_pd = NULL;
663 }
664
665 return ret;
666}
667
668/**
669 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
670 * @ctx: Reservation context to unreserve
671 * @wait: Optionally wait for a sync object representing pending VM updates
672 * @intr: Whether the wait is interruptible
673 *
674 * Also frees any resources allocated in
675 * reserve_bo_and_(cond_)vm(s). Returns the status from
676 * amdgpu_sync_wait.
677 */
678static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
679 bool wait, bool intr)
680{
681 int ret = 0;
682
683 if (wait)
684 ret = amdgpu_sync_wait(ctx->sync, intr);
685
686 if (ctx->reserved)
687 ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list);
688 kfree(ctx->vm_pd);
689
690 ctx->sync = NULL;
691
692 ctx->reserved = false;
693 ctx->vm_pd = NULL;
694
695 return ret;
696}
697
698static int unmap_bo_from_gpuvm(struct amdgpu_device *adev,
699 struct kfd_bo_va_list *entry,
700 struct amdgpu_sync *sync)
701{
702 struct amdgpu_bo_va *bo_va = entry->bo_va;
703 struct amdgpu_vm *vm = bo_va->base.vm;
704 struct amdgpu_bo *pd = vm->root.base.bo;
705
706 /* Remove eviction fence from PD (and thereby from PTs too as
707 * they share the resv. object). Otherwise during PT update
708 * job (see amdgpu_vm_bo_update_mapping), eviction fence would
709 * get added to job->sync object and job execution would
710 * trigger the eviction fence.
711 */
712 amdgpu_amdkfd_remove_eviction_fence(pd,
713 vm->process_info->eviction_fence,
714 NULL, NULL);
715 amdgpu_vm_bo_unmap(adev, bo_va, entry->va);
716
717 amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
718
719 /* Add the eviction fence back */
720 amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true);
721
722 sync_vm_fence(adev, sync, bo_va->last_pt_update);
723
724 return 0;
725}
726
727static int update_gpuvm_pte(struct amdgpu_device *adev,
728 struct kfd_bo_va_list *entry,
729 struct amdgpu_sync *sync)
730{
731 int ret;
732 struct amdgpu_vm *vm;
733 struct amdgpu_bo_va *bo_va;
734 struct amdgpu_bo *bo;
735
736 bo_va = entry->bo_va;
737 vm = bo_va->base.vm;
738 bo = bo_va->base.bo;
739
740 /* Update the page tables */
741 ret = amdgpu_vm_bo_update(adev, bo_va, false);
742 if (ret) {
743 pr_err("amdgpu_vm_bo_update failed\n");
744 return ret;
745 }
746
747 return sync_vm_fence(adev, sync, bo_va->last_pt_update);
748}
749
750static int map_bo_to_gpuvm(struct amdgpu_device *adev,
751 struct kfd_bo_va_list *entry, struct amdgpu_sync *sync)
752{
753 int ret;
754
755 /* Set virtual address for the allocation */
756 ret = amdgpu_vm_bo_map(adev, entry->bo_va, entry->va, 0,
757 amdgpu_bo_size(entry->bo_va->base.bo),
758 entry->pte_flags);
759 if (ret) {
760 pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
761 entry->va, ret);
762 return ret;
763 }
764
765 ret = update_gpuvm_pte(adev, entry, sync);
766 if (ret) {
767 pr_err("update_gpuvm_pte() failed\n");
768 goto update_gpuvm_pte_failed;
769 }
770
771 return 0;
772
773update_gpuvm_pte_failed:
774 unmap_bo_from_gpuvm(adev, entry, sync);
775 return ret;
776}
777
778static int process_validate_vms(struct amdkfd_process_info *process_info)
779{
780 struct amdgpu_vm *peer_vm;
781 int ret;
782
783 list_for_each_entry(peer_vm, &process_info->vm_list_head,
784 vm_list_node) {
785 ret = vm_validate_pt_pd_bos(peer_vm);
786 if (ret)
787 return ret;
788 }
789
790 return 0;
791}
792
793static int process_update_pds(struct amdkfd_process_info *process_info,
794 struct amdgpu_sync *sync)
795{
796 struct amdgpu_vm *peer_vm;
797 int ret;
798
799 list_for_each_entry(peer_vm, &process_info->vm_list_head,
800 vm_list_node) {
801 ret = vm_update_pds(peer_vm, sync);
802 if (ret)
803 return ret;
804 }
805
806 return 0;
807}
808
809static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
810 struct dma_fence **ef)
811{
812 struct amdkfd_process_info *info = NULL;
813 int ret;
814
815 if (!*process_info) {
816 info = kzalloc(sizeof(*info), GFP_KERNEL);
817 if (!info)
818 return -ENOMEM;
819
820 mutex_init(&info->lock);
821 INIT_LIST_HEAD(&info->vm_list_head);
822 INIT_LIST_HEAD(&info->kfd_bo_list);
823
824 info->eviction_fence =
825 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
826 current->mm);
827 if (!info->eviction_fence) {
828 pr_err("Failed to create eviction fence\n");
829 ret = -ENOMEM;
830 goto create_evict_fence_fail;
831 }
832
833 *process_info = info;
834 *ef = dma_fence_get(&info->eviction_fence->base);
835 }
836
837 vm->process_info = *process_info;
838
839 /* Validate page directory and attach eviction fence */
840 ret = amdgpu_bo_reserve(vm->root.base.bo, true);
841 if (ret)
842 goto reserve_pd_fail;
843 ret = vm_validate_pt_pd_bos(vm);
844 if (ret) {
845 pr_err("validate_pt_pd_bos() failed\n");
846 goto validate_pd_fail;
847 }
848 ret = ttm_bo_wait(&vm->root.base.bo->tbo, false, false);
849 if (ret)
850 goto wait_pd_fail;
851 amdgpu_bo_fence(vm->root.base.bo,
852 &vm->process_info->eviction_fence->base, true);
853 amdgpu_bo_unreserve(vm->root.base.bo);
854
855 /* Update process info */
856 mutex_lock(&vm->process_info->lock);
857 list_add_tail(&vm->vm_list_node,
858 &(vm->process_info->vm_list_head));
859 vm->process_info->n_vms++;
860 mutex_unlock(&vm->process_info->lock);
861
862 return 0;
863
864wait_pd_fail:
865validate_pd_fail:
866 amdgpu_bo_unreserve(vm->root.base.bo);
867reserve_pd_fail:
868 vm->process_info = NULL;
869 if (info) {
870 /* Two fence references: one in info and one in *ef */
871 dma_fence_put(&info->eviction_fence->base);
872 dma_fence_put(*ef);
873 *ef = NULL;
874 *process_info = NULL;
875create_evict_fence_fail:
876 mutex_destroy(&info->lock);
877 kfree(info);
878 }
879 return ret;
880}
881
882int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
883 void **process_info,
884 struct dma_fence **ef)
885{
886 struct amdgpu_device *adev = get_amdgpu_device(kgd);
887 struct amdgpu_vm *new_vm;
888 int ret;
889
890 new_vm = kzalloc(sizeof(*new_vm), GFP_KERNEL);
891 if (!new_vm)
892 return -ENOMEM;
893
894 /* Initialize AMDGPU part of the VM */
895 ret = amdgpu_vm_init(adev, new_vm, AMDGPU_VM_CONTEXT_COMPUTE, 0);
896 if (ret) {
897 pr_err("Failed init vm ret %d\n", ret);
898 goto amdgpu_vm_init_fail;
899 }
900
901 /* Initialize KFD part of the VM and process info */
902 ret = init_kfd_vm(new_vm, process_info, ef);
903 if (ret)
904 goto init_kfd_vm_fail;
905
906 *vm = (void *) new_vm;
907
908 return 0;
909
910init_kfd_vm_fail:
911 amdgpu_vm_fini(adev, new_vm);
912amdgpu_vm_init_fail:
913 kfree(new_vm);
914 return ret;
915}
916
917int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
918 struct file *filp,
919 void **vm, void **process_info,
920 struct dma_fence **ef)
921{
922 struct amdgpu_device *adev = get_amdgpu_device(kgd);
923 struct drm_file *drm_priv = filp->private_data;
924 struct amdgpu_fpriv *drv_priv = drm_priv->driver_priv;
925 struct amdgpu_vm *avm = &drv_priv->vm;
926 int ret;
927
928 /* Already a compute VM? */
929 if (avm->process_info)
930 return -EINVAL;
931
932 /* Convert VM into a compute VM */
933 ret = amdgpu_vm_make_compute(adev, avm);
934 if (ret)
935 return ret;
936
937 /* Initialize KFD part of the VM and process info */
938 ret = init_kfd_vm(avm, process_info, ef);
939 if (ret)
940 return ret;
941
942 *vm = (void *)avm;
943
944 return 0;
945}
946
947void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
948 struct amdgpu_vm *vm)
949{
950 struct amdkfd_process_info *process_info = vm->process_info;
951 struct amdgpu_bo *pd = vm->root.base.bo;
952
953 if (!process_info)
954 return;
955
956 /* Release eviction fence from PD */
957 amdgpu_bo_reserve(pd, false);
958 amdgpu_bo_fence(pd, NULL, false);
959 amdgpu_bo_unreserve(pd);
960
961 /* Update process info */
962 mutex_lock(&process_info->lock);
963 process_info->n_vms--;
964 list_del(&vm->vm_list_node);
965 mutex_unlock(&process_info->lock);
966
967 /* Release per-process resources when last compute VM is destroyed */
968 if (!process_info->n_vms) {
969 WARN_ON(!list_empty(&process_info->kfd_bo_list));
970
971 dma_fence_put(&process_info->eviction_fence->base);
972 mutex_destroy(&process_info->lock);
973 kfree(process_info);
974 }
975}
976
977void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm)
978{
979 struct amdgpu_device *adev = get_amdgpu_device(kgd);
980 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
981
982 if (WARN_ON(!kgd || !vm))
983 return;
984
985 pr_debug("Destroying process vm %p\n", vm);
986
987 /* Release the VM context */
988 amdgpu_vm_fini(adev, avm);
989 kfree(vm);
990}
991
992uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm)
993{
994 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
995
996 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
997}
998
999int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1000 struct kgd_dev *kgd, uint64_t va, uint64_t size,
1001 void *vm, struct kgd_mem **mem,
1002 uint64_t *offset, uint32_t flags)
1003{
1004 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1005 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1006 struct amdgpu_bo *bo;
1007 int byte_align;
1008 u32 alloc_domain;
1009 u64 alloc_flags;
1010 uint32_t mapping_flags;
1011 int ret;
1012
1013 /*
1014 * Check on which domain to allocate BO
1015 */
1016 if (flags & ALLOC_MEM_FLAGS_VRAM) {
1017 alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1018 alloc_flags = AMDGPU_GEM_CREATE_VRAM_CLEARED;
1019 alloc_flags |= (flags & ALLOC_MEM_FLAGS_PUBLIC) ?
1020 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED :
1021 AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
1022 } else if (flags & ALLOC_MEM_FLAGS_GTT) {
1023 alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1024 alloc_flags = 0;
1025 } else {
1026 return -EINVAL;
1027 }
1028
1029 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1030 if (!*mem)
1031 return -ENOMEM;
1032 INIT_LIST_HEAD(&(*mem)->bo_va_list);
1033 mutex_init(&(*mem)->lock);
1034 (*mem)->aql_queue = !!(flags & ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1035
1036 /* Workaround for AQL queue wraparound bug. Map the same
1037 * memory twice. That means we only actually allocate half
1038 * the memory.
1039 */
1040 if ((*mem)->aql_queue)
1041 size = size >> 1;
1042
1043 /* Workaround for TLB bug on older VI chips */
1044 byte_align = (adev->family == AMDGPU_FAMILY_VI &&
1045 adev->asic_type != CHIP_FIJI &&
1046 adev->asic_type != CHIP_POLARIS10 &&
1047 adev->asic_type != CHIP_POLARIS11) ?
1048 VI_BO_SIZE_ALIGN : 1;
1049
1050 mapping_flags = AMDGPU_VM_PAGE_READABLE;
1051 if (flags & ALLOC_MEM_FLAGS_WRITABLE)
1052 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
1053 if (flags & ALLOC_MEM_FLAGS_EXECUTABLE)
1054 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
1055 if (flags & ALLOC_MEM_FLAGS_COHERENT)
1056 mapping_flags |= AMDGPU_VM_MTYPE_UC;
1057 else
1058 mapping_flags |= AMDGPU_VM_MTYPE_NC;
1059 (*mem)->mapping_flags = mapping_flags;
1060
1061 amdgpu_sync_create(&(*mem)->sync);
1062
1063 ret = amdgpu_amdkfd_reserve_system_mem_limit(adev, size, alloc_domain);
1064 if (ret) {
1065 pr_debug("Insufficient system memory\n");
1066 goto err_reserve_system_mem;
1067 }
1068
1069 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n",
1070 va, size, domain_string(alloc_domain));
1071
1072 ret = amdgpu_bo_create(adev, size, byte_align,
1073 alloc_domain, alloc_flags, ttm_bo_type_device, NULL, &bo);
1074 if (ret) {
1075 pr_debug("Failed to create BO on domain %s. ret %d\n",
1076 domain_string(alloc_domain), ret);
1077 goto err_bo_create;
1078 }
1079 bo->kfd_bo = *mem;
1080 (*mem)->bo = bo;
1081
1082 (*mem)->va = va;
1083 (*mem)->domain = alloc_domain;
1084 (*mem)->mapped_to_gpu_memory = 0;
1085 (*mem)->process_info = avm->process_info;
1086 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info);
1087
1088 if (offset)
1089 *offset = amdgpu_bo_mmap_offset(bo);
1090
1091 return 0;
1092
1093err_bo_create:
1094 unreserve_system_mem_limit(adev, size, alloc_domain);
1095err_reserve_system_mem:
1096 mutex_destroy(&(*mem)->lock);
1097 kfree(*mem);
1098 return ret;
1099}
1100
1101int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1102 struct kgd_dev *kgd, struct kgd_mem *mem)
1103{
1104 struct amdkfd_process_info *process_info = mem->process_info;
1105 unsigned long bo_size = mem->bo->tbo.mem.size;
1106 struct kfd_bo_va_list *entry, *tmp;
1107 struct bo_vm_reservation_context ctx;
1108 struct ttm_validate_buffer *bo_list_entry;
1109 int ret;
1110
1111 mutex_lock(&mem->lock);
1112
1113 if (mem->mapped_to_gpu_memory > 0) {
1114 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1115 mem->va, bo_size);
1116 mutex_unlock(&mem->lock);
1117 return -EBUSY;
1118 }
1119
1120 mutex_unlock(&mem->lock);
1121 /* lock is not needed after this, since mem is unused and will
1122 * be freed anyway
1123 */
1124
1125 /* Make sure restore workers don't access the BO any more */
1126 bo_list_entry = &mem->validate_list;
1127 mutex_lock(&process_info->lock);
1128 list_del(&bo_list_entry->head);
1129 mutex_unlock(&process_info->lock);
1130
1131 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
1132 if (unlikely(ret))
1133 return ret;
1134
1135 /* The eviction fence should be removed by the last unmap.
1136 * TODO: Log an error condition if the bo still has the eviction fence
1137 * attached
1138 */
1139 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1140 process_info->eviction_fence,
1141 NULL, NULL);
1142 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1143 mem->va + bo_size * (1 + mem->aql_queue));
1144
1145 /* Remove from VM internal data structures */
1146 list_for_each_entry_safe(entry, tmp, &mem->bo_va_list, bo_list)
1147 remove_bo_from_vm((struct amdgpu_device *)entry->kgd_dev,
1148 entry, bo_size);
1149
1150 ret = unreserve_bo_and_vms(&ctx, false, false);
1151
1152 /* Free the sync object */
1153 amdgpu_sync_free(&mem->sync);
1154
1155 /* Free the BO*/
1156 amdgpu_bo_unref(&mem->bo);
1157 mutex_destroy(&mem->lock);
1158 kfree(mem);
1159
1160 return ret;
1161}
1162
1163int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1164 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
1165{
1166 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1167 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1168 int ret;
1169 struct amdgpu_bo *bo;
1170 uint32_t domain;
1171 struct kfd_bo_va_list *entry;
1172 struct bo_vm_reservation_context ctx;
1173 struct kfd_bo_va_list *bo_va_entry = NULL;
1174 struct kfd_bo_va_list *bo_va_entry_aql = NULL;
1175 unsigned long bo_size;
1176
1177 /* Make sure restore is not running concurrently.
1178 */
1179 mutex_lock(&mem->process_info->lock);
1180
1181 mutex_lock(&mem->lock);
1182
1183 bo = mem->bo;
1184
1185 if (!bo) {
1186 pr_err("Invalid BO when mapping memory to GPU\n");
1187 ret = -EINVAL;
1188 goto out;
1189 }
1190
1191 domain = mem->domain;
1192 bo_size = bo->tbo.mem.size;
1193
1194 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
1195 mem->va,
1196 mem->va + bo_size * (1 + mem->aql_queue),
1197 vm, domain_string(domain));
1198
1199 ret = reserve_bo_and_vm(mem, vm, &ctx);
1200 if (unlikely(ret))
1201 goto out;
1202
1203 if (check_if_add_bo_to_vm(avm, mem)) {
1204 ret = add_bo_to_vm(adev, mem, avm, false,
1205 &bo_va_entry);
1206 if (ret)
1207 goto add_bo_to_vm_failed;
1208 if (mem->aql_queue) {
1209 ret = add_bo_to_vm(adev, mem, avm,
1210 true, &bo_va_entry_aql);
1211 if (ret)
1212 goto add_bo_to_vm_failed_aql;
1213 }
1214 } else {
1215 ret = vm_validate_pt_pd_bos(avm);
1216 if (unlikely(ret))
1217 goto add_bo_to_vm_failed;
1218 }
1219
1220 if (mem->mapped_to_gpu_memory == 0) {
1221 /* Validate BO only once. The eviction fence gets added to BO
1222 * the first time it is mapped. Validate will wait for all
1223 * background evictions to complete.
1224 */
1225 ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
1226 if (ret) {
1227 pr_debug("Validate failed\n");
1228 goto map_bo_to_gpuvm_failed;
1229 }
1230 }
1231
1232 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
1233 if (entry->bo_va->base.vm == vm && !entry->is_mapped) {
1234 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
1235 entry->va, entry->va + bo_size,
1236 entry);
1237
1238 ret = map_bo_to_gpuvm(adev, entry, ctx.sync);
1239 if (ret) {
1240 pr_err("Failed to map radeon bo to gpuvm\n");
1241 goto map_bo_to_gpuvm_failed;
1242 }
1243
1244 ret = vm_update_pds(vm, ctx.sync);
1245 if (ret) {
1246 pr_err("Failed to update page directories\n");
1247 goto map_bo_to_gpuvm_failed;
1248 }
1249
1250 entry->is_mapped = true;
1251 mem->mapped_to_gpu_memory++;
1252 pr_debug("\t INC mapping count %d\n",
1253 mem->mapped_to_gpu_memory);
1254 }
1255 }
1256
1257 if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->pin_count)
1258 amdgpu_bo_fence(bo,
1259 &avm->process_info->eviction_fence->base,
1260 true);
1261 ret = unreserve_bo_and_vms(&ctx, false, false);
1262
1263 goto out;
1264
1265map_bo_to_gpuvm_failed:
1266 if (bo_va_entry_aql)
1267 remove_bo_from_vm(adev, bo_va_entry_aql, bo_size);
1268add_bo_to_vm_failed_aql:
1269 if (bo_va_entry)
1270 remove_bo_from_vm(adev, bo_va_entry, bo_size);
1271add_bo_to_vm_failed:
1272 unreserve_bo_and_vms(&ctx, false, false);
1273out:
1274 mutex_unlock(&mem->process_info->lock);
1275 mutex_unlock(&mem->lock);
1276 return ret;
1277}
1278
1279int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1280 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
1281{
1282 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1283 struct amdkfd_process_info *process_info =
1284 ((struct amdgpu_vm *)vm)->process_info;
1285 unsigned long bo_size = mem->bo->tbo.mem.size;
1286 struct kfd_bo_va_list *entry;
1287 struct bo_vm_reservation_context ctx;
1288 int ret;
1289
1290 mutex_lock(&mem->lock);
1291
1292 ret = reserve_bo_and_cond_vms(mem, vm, BO_VM_MAPPED, &ctx);
1293 if (unlikely(ret))
1294 goto out;
1295 /* If no VMs were reserved, it means the BO wasn't actually mapped */
1296 if (ctx.n_vms == 0) {
1297 ret = -EINVAL;
1298 goto unreserve_out;
1299 }
1300
1301 ret = vm_validate_pt_pd_bos((struct amdgpu_vm *)vm);
1302 if (unlikely(ret))
1303 goto unreserve_out;
1304
1305 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
1306 mem->va,
1307 mem->va + bo_size * (1 + mem->aql_queue),
1308 vm);
1309
1310 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
1311 if (entry->bo_va->base.vm == vm && entry->is_mapped) {
1312 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
1313 entry->va,
1314 entry->va + bo_size,
1315 entry);
1316
1317 ret = unmap_bo_from_gpuvm(adev, entry, ctx.sync);
1318 if (ret == 0) {
1319 entry->is_mapped = false;
1320 } else {
1321 pr_err("failed to unmap VA 0x%llx\n",
1322 mem->va);
1323 goto unreserve_out;
1324 }
1325
1326 mem->mapped_to_gpu_memory--;
1327 pr_debug("\t DEC mapping count %d\n",
1328 mem->mapped_to_gpu_memory);
1329 }
1330 }
1331
1332 /* If BO is unmapped from all VMs, unfence it. It can be evicted if
1333 * required.
1334 */
1335 if (mem->mapped_to_gpu_memory == 0 &&
1336 !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count)
1337 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1338 process_info->eviction_fence,
1339 NULL, NULL);
1340
1341unreserve_out:
1342 unreserve_bo_and_vms(&ctx, false, false);
1343out:
1344 mutex_unlock(&mem->lock);
1345 return ret;
1346}
1347
1348int amdgpu_amdkfd_gpuvm_sync_memory(
1349 struct kgd_dev *kgd, struct kgd_mem *mem, bool intr)
1350{
1351 struct amdgpu_sync sync;
1352 int ret;
1353
1354 amdgpu_sync_create(&sync);
1355
1356 mutex_lock(&mem->lock);
1357 amdgpu_sync_clone(&mem->sync, &sync);
1358 mutex_unlock(&mem->lock);
1359
1360 ret = amdgpu_sync_wait(&sync, intr);
1361 amdgpu_sync_free(&sync);
1362 return ret;
1363}
1364
1365int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
1366 struct kgd_mem *mem, void **kptr, uint64_t *size)
1367{
1368 int ret;
1369 struct amdgpu_bo *bo = mem->bo;
1370
1371 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1372 pr_err("userptr can't be mapped to kernel\n");
1373 return -EINVAL;
1374 }
1375
1376 /* delete kgd_mem from kfd_bo_list to avoid re-validating
1377 * this BO in BO's restoring after eviction.
1378 */
1379 mutex_lock(&mem->process_info->lock);
1380
1381 ret = amdgpu_bo_reserve(bo, true);
1382 if (ret) {
1383 pr_err("Failed to reserve bo. ret %d\n", ret);
1384 goto bo_reserve_failed;
1385 }
1386
1387 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT, NULL);
1388 if (ret) {
1389 pr_err("Failed to pin bo. ret %d\n", ret);
1390 goto pin_failed;
1391 }
1392
1393 ret = amdgpu_bo_kmap(bo, kptr);
1394 if (ret) {
1395 pr_err("Failed to map bo to kernel. ret %d\n", ret);
1396 goto kmap_failed;
1397 }
1398
1399 amdgpu_amdkfd_remove_eviction_fence(
1400 bo, mem->process_info->eviction_fence, NULL, NULL);
1401 list_del_init(&mem->validate_list.head);
1402
1403 if (size)
1404 *size = amdgpu_bo_size(bo);
1405
1406 amdgpu_bo_unreserve(bo);
1407
1408 mutex_unlock(&mem->process_info->lock);
1409 return 0;
1410
1411kmap_failed:
1412 amdgpu_bo_unpin(bo);
1413pin_failed:
1414 amdgpu_bo_unreserve(bo);
1415bo_reserve_failed:
1416 mutex_unlock(&mem->process_info->lock);
1417
1418 return ret;
1419}
1420
1421/** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
1422 * KFD process identified by process_info
1423 *
1424 * @process_info: amdkfd_process_info of the KFD process
1425 *
1426 * After memory eviction, restore thread calls this function. The function
1427 * should be called when the Process is still valid. BO restore involves -
1428 *
1429 * 1. Release old eviction fence and create new one
1430 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
1431 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of
1432 * BOs that need to be reserved.
1433 * 4. Reserve all the BOs
1434 * 5. Validate of PD and PT BOs.
1435 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence
1436 * 7. Add fence to all PD and PT BOs.
1437 * 8. Unreserve all BOs
1438 */
1439int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
1440{
1441 struct amdgpu_bo_list_entry *pd_bo_list;
1442 struct amdkfd_process_info *process_info = info;
1443 struct amdgpu_vm *peer_vm;
1444 struct kgd_mem *mem;
1445 struct bo_vm_reservation_context ctx;
1446 struct amdgpu_amdkfd_fence *new_fence;
1447 int ret = 0, i;
1448 struct list_head duplicate_save;
1449 struct amdgpu_sync sync_obj;
1450
1451 INIT_LIST_HEAD(&duplicate_save);
1452 INIT_LIST_HEAD(&ctx.list);
1453 INIT_LIST_HEAD(&ctx.duplicates);
1454
1455 pd_bo_list = kcalloc(process_info->n_vms,
1456 sizeof(struct amdgpu_bo_list_entry),
1457 GFP_KERNEL);
1458 if (!pd_bo_list)
1459 return -ENOMEM;
1460
1461 i = 0;
1462 mutex_lock(&process_info->lock);
1463 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1464 vm_list_node)
1465 amdgpu_vm_get_pd_bo(peer_vm, &ctx.list, &pd_bo_list[i++]);
1466
1467 /* Reserve all BOs and page tables/directory. Add all BOs from
1468 * kfd_bo_list to ctx.list
1469 */
1470 list_for_each_entry(mem, &process_info->kfd_bo_list,
1471 validate_list.head) {
1472
1473 list_add_tail(&mem->resv_list.head, &ctx.list);
1474 mem->resv_list.bo = mem->validate_list.bo;
1475 mem->resv_list.shared = mem->validate_list.shared;
1476 }
1477
1478 ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list,
1479 false, &duplicate_save);
1480 if (ret) {
1481 pr_debug("Memory eviction: TTM Reserve Failed. Try again\n");
1482 goto ttm_reserve_fail;
1483 }
1484
1485 amdgpu_sync_create(&sync_obj);
1486
1487 /* Validate PDs and PTs */
1488 ret = process_validate_vms(process_info);
1489 if (ret)
1490 goto validate_map_fail;
1491
1492 /* Wait for PD/PTs validate to finish */
1493 /* FIXME: I think this isn't needed */
1494 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1495 vm_list_node) {
1496 struct amdgpu_bo *bo = peer_vm->root.base.bo;
1497
1498 ttm_bo_wait(&bo->tbo, false, false);
1499 }
1500
1501 /* Validate BOs and map them to GPUVM (update VM page tables). */
1502 list_for_each_entry(mem, &process_info->kfd_bo_list,
1503 validate_list.head) {
1504
1505 struct amdgpu_bo *bo = mem->bo;
1506 uint32_t domain = mem->domain;
1507 struct kfd_bo_va_list *bo_va_entry;
1508
1509 ret = amdgpu_amdkfd_bo_validate(bo, domain, false);
1510 if (ret) {
1511 pr_debug("Memory eviction: Validate BOs failed. Try again\n");
1512 goto validate_map_fail;
1513 }
1514
1515 list_for_each_entry(bo_va_entry, &mem->bo_va_list,
1516 bo_list) {
1517 ret = update_gpuvm_pte((struct amdgpu_device *)
1518 bo_va_entry->kgd_dev,
1519 bo_va_entry,
1520 &sync_obj);
1521 if (ret) {
1522 pr_debug("Memory eviction: update PTE failed. Try again\n");
1523 goto validate_map_fail;
1524 }
1525 }
1526 }
1527
1528 /* Update page directories */
1529 ret = process_update_pds(process_info, &sync_obj);
1530 if (ret) {
1531 pr_debug("Memory eviction: update PDs failed. Try again\n");
1532 goto validate_map_fail;
1533 }
1534
1535 amdgpu_sync_wait(&sync_obj, false);
1536
1537 /* Release old eviction fence and create new one, because fence only
1538 * goes from unsignaled to signaled, fence cannot be reused.
1539 * Use context and mm from the old fence.
1540 */
1541 new_fence = amdgpu_amdkfd_fence_create(
1542 process_info->eviction_fence->base.context,
1543 process_info->eviction_fence->mm);
1544 if (!new_fence) {
1545 pr_err("Failed to create eviction fence\n");
1546 ret = -ENOMEM;
1547 goto validate_map_fail;
1548 }
1549 dma_fence_put(&process_info->eviction_fence->base);
1550 process_info->eviction_fence = new_fence;
1551 *ef = dma_fence_get(&new_fence->base);
1552
1553 /* Wait for validate to finish and attach new eviction fence */
1554 list_for_each_entry(mem, &process_info->kfd_bo_list,
1555 validate_list.head)
1556 ttm_bo_wait(&mem->bo->tbo, false, false);
1557 list_for_each_entry(mem, &process_info->kfd_bo_list,
1558 validate_list.head)
1559 amdgpu_bo_fence(mem->bo,
1560 &process_info->eviction_fence->base, true);
1561
1562 /* Attach eviction fence to PD / PT BOs */
1563 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1564 vm_list_node) {
1565 struct amdgpu_bo *bo = peer_vm->root.base.bo;
1566
1567 amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);
1568 }
1569
1570validate_map_fail:
1571 ttm_eu_backoff_reservation(&ctx.ticket, &ctx.list);
1572 amdgpu_sync_free(&sync_obj);
1573ttm_reserve_fail:
1574 mutex_unlock(&process_info->lock);
1575 kfree(pd_bo_list);
1576 return ret;
1577}
1/*
2 * Copyright 2014-2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#define pr_fmt(fmt) "kfd2kgd: " fmt
24
25#include <linux/dma-buf.h>
26#include <linux/list.h>
27#include <linux/pagemap.h>
28#include <linux/sched/mm.h>
29#include <linux/sched/task.h>
30
31#include "amdgpu_object.h"
32#include "amdgpu_vm.h"
33#include "amdgpu_amdkfd.h"
34#include "amdgpu_dma_buf.h"
35
36/* Special VM and GART address alignment needed for VI pre-Fiji due to
37 * a HW bug.
38 */
39#define VI_BO_SIZE_ALIGN (0x8000)
40
41/* BO flag to indicate a KFD userptr BO */
42#define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63)
43
44/* Userptr restore delay, just long enough to allow consecutive VM
45 * changes to accumulate
46 */
47#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
48
49/* Impose limit on how much memory KFD can use */
50static struct {
51 uint64_t max_system_mem_limit;
52 uint64_t max_ttm_mem_limit;
53 int64_t system_mem_used;
54 int64_t ttm_mem_used;
55 spinlock_t mem_limit_lock;
56} kfd_mem_limit;
57
58/* Struct used for amdgpu_amdkfd_bo_validate */
59struct amdgpu_vm_parser {
60 uint32_t domain;
61 bool wait;
62};
63
64static const char * const domain_bit_to_string[] = {
65 "CPU",
66 "GTT",
67 "VRAM",
68 "GDS",
69 "GWS",
70 "OA"
71};
72
73#define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
74
75static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
76
77
78static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
79{
80 return (struct amdgpu_device *)kgd;
81}
82
83static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm,
84 struct kgd_mem *mem)
85{
86 struct kfd_bo_va_list *entry;
87
88 list_for_each_entry(entry, &mem->bo_va_list, bo_list)
89 if (entry->bo_va->base.vm == avm)
90 return false;
91
92 return true;
93}
94
95/* Set memory usage limits. Current, limits are
96 * System (TTM + userptr) memory - 3/4th System RAM
97 * TTM memory - 3/8th System RAM
98 */
99void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
100{
101 struct sysinfo si;
102 uint64_t mem;
103
104 si_meminfo(&si);
105 mem = si.totalram - si.totalhigh;
106 mem *= si.mem_unit;
107
108 spin_lock_init(&kfd_mem_limit.mem_limit_lock);
109 kfd_mem_limit.max_system_mem_limit = (mem >> 1) + (mem >> 2);
110 kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3);
111 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
112 (kfd_mem_limit.max_system_mem_limit >> 20),
113 (kfd_mem_limit.max_ttm_mem_limit >> 20));
114}
115
116static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
117 uint64_t size, u32 domain, bool sg)
118{
119 size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed;
120 uint64_t reserved_for_pt = amdgpu_amdkfd_total_mem_size >> 9;
121 int ret = 0;
122
123 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
124 sizeof(struct amdgpu_bo));
125
126 vram_needed = 0;
127 if (domain == AMDGPU_GEM_DOMAIN_GTT) {
128 /* TTM GTT memory */
129 system_mem_needed = acc_size + size;
130 ttm_mem_needed = acc_size + size;
131 } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
132 /* Userptr */
133 system_mem_needed = acc_size + size;
134 ttm_mem_needed = acc_size;
135 } else {
136 /* VRAM and SG */
137 system_mem_needed = acc_size;
138 ttm_mem_needed = acc_size;
139 if (domain == AMDGPU_GEM_DOMAIN_VRAM)
140 vram_needed = size;
141 }
142
143 spin_lock(&kfd_mem_limit.mem_limit_lock);
144
145 if ((kfd_mem_limit.system_mem_used + system_mem_needed >
146 kfd_mem_limit.max_system_mem_limit) ||
147 (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
148 kfd_mem_limit.max_ttm_mem_limit) ||
149 (adev->kfd.vram_used + vram_needed >
150 adev->gmc.real_vram_size - reserved_for_pt)) {
151 ret = -ENOMEM;
152 } else {
153 kfd_mem_limit.system_mem_used += system_mem_needed;
154 kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
155 adev->kfd.vram_used += vram_needed;
156 }
157
158 spin_unlock(&kfd_mem_limit.mem_limit_lock);
159 return ret;
160}
161
162static void unreserve_mem_limit(struct amdgpu_device *adev,
163 uint64_t size, u32 domain, bool sg)
164{
165 size_t acc_size;
166
167 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
168 sizeof(struct amdgpu_bo));
169
170 spin_lock(&kfd_mem_limit.mem_limit_lock);
171 if (domain == AMDGPU_GEM_DOMAIN_GTT) {
172 kfd_mem_limit.system_mem_used -= (acc_size + size);
173 kfd_mem_limit.ttm_mem_used -= (acc_size + size);
174 } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
175 kfd_mem_limit.system_mem_used -= (acc_size + size);
176 kfd_mem_limit.ttm_mem_used -= acc_size;
177 } else {
178 kfd_mem_limit.system_mem_used -= acc_size;
179 kfd_mem_limit.ttm_mem_used -= acc_size;
180 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
181 adev->kfd.vram_used -= size;
182 WARN_ONCE(adev->kfd.vram_used < 0,
183 "kfd VRAM memory accounting unbalanced");
184 }
185 }
186 WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
187 "kfd system memory accounting unbalanced");
188 WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
189 "kfd TTM memory accounting unbalanced");
190
191 spin_unlock(&kfd_mem_limit.mem_limit_lock);
192}
193
194void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)
195{
196 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
197 u32 domain = bo->preferred_domains;
198 bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU);
199
200 if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) {
201 domain = AMDGPU_GEM_DOMAIN_CPU;
202 sg = false;
203 }
204
205 unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg);
206}
207
208
209/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
210 * reservation object.
211 *
212 * @bo: [IN] Remove eviction fence(s) from this BO
213 * @ef: [IN] This eviction fence is removed if it
214 * is present in the shared list.
215 *
216 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
217 */
218static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
219 struct amdgpu_amdkfd_fence *ef)
220{
221 struct dma_resv *resv = bo->tbo.base.resv;
222 struct dma_resv_list *old, *new;
223 unsigned int i, j, k;
224
225 if (!ef)
226 return -EINVAL;
227
228 old = dma_resv_get_list(resv);
229 if (!old)
230 return 0;
231
232 new = kmalloc(offsetof(typeof(*new), shared[old->shared_max]),
233 GFP_KERNEL);
234 if (!new)
235 return -ENOMEM;
236
237 /* Go through all the shared fences in the resevation object and sort
238 * the interesting ones to the end of the list.
239 */
240 for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) {
241 struct dma_fence *f;
242
243 f = rcu_dereference_protected(old->shared[i],
244 dma_resv_held(resv));
245
246 if (f->context == ef->base.context)
247 RCU_INIT_POINTER(new->shared[--j], f);
248 else
249 RCU_INIT_POINTER(new->shared[k++], f);
250 }
251 new->shared_max = old->shared_max;
252 new->shared_count = k;
253
254 /* Install the new fence list, seqcount provides the barriers */
255 preempt_disable();
256 write_seqcount_begin(&resv->seq);
257 RCU_INIT_POINTER(resv->fence, new);
258 write_seqcount_end(&resv->seq);
259 preempt_enable();
260
261 /* Drop the references to the removed fences or move them to ef_list */
262 for (i = j, k = 0; i < old->shared_count; ++i) {
263 struct dma_fence *f;
264
265 f = rcu_dereference_protected(new->shared[i],
266 dma_resv_held(resv));
267 dma_fence_put(f);
268 }
269 kfree_rcu(old, rcu);
270
271 return 0;
272}
273
274static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
275 bool wait)
276{
277 struct ttm_operation_ctx ctx = { false, false };
278 int ret;
279
280 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
281 "Called with userptr BO"))
282 return -EINVAL;
283
284 amdgpu_bo_placement_from_domain(bo, domain);
285
286 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
287 if (ret)
288 goto validate_fail;
289 if (wait)
290 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
291
292validate_fail:
293 return ret;
294}
295
296static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo)
297{
298 struct amdgpu_vm_parser *p = param;
299
300 return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait);
301}
302
303/* vm_validate_pt_pd_bos - Validate page table and directory BOs
304 *
305 * Page directories are not updated here because huge page handling
306 * during page table updates can invalidate page directory entries
307 * again. Page directories are only updated after updating page
308 * tables.
309 */
310static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
311{
312 struct amdgpu_bo *pd = vm->root.base.bo;
313 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
314 struct amdgpu_vm_parser param;
315 int ret;
316
317 param.domain = AMDGPU_GEM_DOMAIN_VRAM;
318 param.wait = false;
319
320 ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate,
321 ¶m);
322 if (ret) {
323 pr_err("amdgpu: failed to validate PT BOs\n");
324 return ret;
325 }
326
327 ret = amdgpu_amdkfd_validate(¶m, pd);
328 if (ret) {
329 pr_err("amdgpu: failed to validate PD\n");
330 return ret;
331 }
332
333 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo);
334
335 if (vm->use_cpu_for_update) {
336 ret = amdgpu_bo_kmap(pd, NULL);
337 if (ret) {
338 pr_err("amdgpu: failed to kmap PD, ret=%d\n", ret);
339 return ret;
340 }
341 }
342
343 return 0;
344}
345
346static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
347{
348 struct amdgpu_bo *pd = vm->root.base.bo;
349 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
350 int ret;
351
352 ret = amdgpu_vm_update_directories(adev, vm);
353 if (ret)
354 return ret;
355
356 return amdgpu_sync_fence(NULL, sync, vm->last_update, false);
357}
358
359/* add_bo_to_vm - Add a BO to a VM
360 *
361 * Everything that needs to bo done only once when a BO is first added
362 * to a VM. It can later be mapped and unmapped many times without
363 * repeating these steps.
364 *
365 * 1. Allocate and initialize BO VA entry data structure
366 * 2. Add BO to the VM
367 * 3. Determine ASIC-specific PTE flags
368 * 4. Alloc page tables and directories if needed
369 * 4a. Validate new page tables and directories
370 */
371static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem,
372 struct amdgpu_vm *vm, bool is_aql,
373 struct kfd_bo_va_list **p_bo_va_entry)
374{
375 int ret;
376 struct kfd_bo_va_list *bo_va_entry;
377 struct amdgpu_bo *bo = mem->bo;
378 uint64_t va = mem->va;
379 struct list_head *list_bo_va = &mem->bo_va_list;
380 unsigned long bo_size = bo->tbo.mem.size;
381
382 if (!va) {
383 pr_err("Invalid VA when adding BO to VM\n");
384 return -EINVAL;
385 }
386
387 if (is_aql)
388 va += bo_size;
389
390 bo_va_entry = kzalloc(sizeof(*bo_va_entry), GFP_KERNEL);
391 if (!bo_va_entry)
392 return -ENOMEM;
393
394 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
395 va + bo_size, vm);
396
397 /* Add BO to VM internal data structures*/
398 bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, vm, bo);
399 if (!bo_va_entry->bo_va) {
400 ret = -EINVAL;
401 pr_err("Failed to add BO object to VM. ret == %d\n",
402 ret);
403 goto err_vmadd;
404 }
405
406 bo_va_entry->va = va;
407 bo_va_entry->pte_flags = amdgpu_gmc_get_pte_flags(adev,
408 mem->mapping_flags);
409 bo_va_entry->kgd_dev = (void *)adev;
410 list_add(&bo_va_entry->bo_list, list_bo_va);
411
412 if (p_bo_va_entry)
413 *p_bo_va_entry = bo_va_entry;
414
415 /* Allocate validate page tables if needed */
416 ret = vm_validate_pt_pd_bos(vm);
417 if (ret) {
418 pr_err("validate_pt_pd_bos() failed\n");
419 goto err_alloc_pts;
420 }
421
422 return 0;
423
424err_alloc_pts:
425 amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va);
426 list_del(&bo_va_entry->bo_list);
427err_vmadd:
428 kfree(bo_va_entry);
429 return ret;
430}
431
432static void remove_bo_from_vm(struct amdgpu_device *adev,
433 struct kfd_bo_va_list *entry, unsigned long size)
434{
435 pr_debug("\t remove VA 0x%llx - 0x%llx in entry %p\n",
436 entry->va,
437 entry->va + size, entry);
438 amdgpu_vm_bo_rmv(adev, entry->bo_va);
439 list_del(&entry->bo_list);
440 kfree(entry);
441}
442
443static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
444 struct amdkfd_process_info *process_info,
445 bool userptr)
446{
447 struct ttm_validate_buffer *entry = &mem->validate_list;
448 struct amdgpu_bo *bo = mem->bo;
449
450 INIT_LIST_HEAD(&entry->head);
451 entry->num_shared = 1;
452 entry->bo = &bo->tbo;
453 mutex_lock(&process_info->lock);
454 if (userptr)
455 list_add_tail(&entry->head, &process_info->userptr_valid_list);
456 else
457 list_add_tail(&entry->head, &process_info->kfd_bo_list);
458 mutex_unlock(&process_info->lock);
459}
460
461static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
462 struct amdkfd_process_info *process_info)
463{
464 struct ttm_validate_buffer *bo_list_entry;
465
466 bo_list_entry = &mem->validate_list;
467 mutex_lock(&process_info->lock);
468 list_del(&bo_list_entry->head);
469 mutex_unlock(&process_info->lock);
470}
471
472/* Initializes user pages. It registers the MMU notifier and validates
473 * the userptr BO in the GTT domain.
474 *
475 * The BO must already be on the userptr_valid_list. Otherwise an
476 * eviction and restore may happen that leaves the new BO unmapped
477 * with the user mode queues running.
478 *
479 * Takes the process_info->lock to protect against concurrent restore
480 * workers.
481 *
482 * Returns 0 for success, negative errno for errors.
483 */
484static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
485 uint64_t user_addr)
486{
487 struct amdkfd_process_info *process_info = mem->process_info;
488 struct amdgpu_bo *bo = mem->bo;
489 struct ttm_operation_ctx ctx = { true, false };
490 int ret = 0;
491
492 mutex_lock(&process_info->lock);
493
494 ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0);
495 if (ret) {
496 pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
497 goto out;
498 }
499
500 ret = amdgpu_mn_register(bo, user_addr);
501 if (ret) {
502 pr_err("%s: Failed to register MMU notifier: %d\n",
503 __func__, ret);
504 goto out;
505 }
506
507 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
508 if (ret) {
509 pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
510 goto unregister_out;
511 }
512
513 ret = amdgpu_bo_reserve(bo, true);
514 if (ret) {
515 pr_err("%s: Failed to reserve BO\n", __func__);
516 goto release_out;
517 }
518 amdgpu_bo_placement_from_domain(bo, mem->domain);
519 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
520 if (ret)
521 pr_err("%s: failed to validate BO\n", __func__);
522 amdgpu_bo_unreserve(bo);
523
524release_out:
525 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
526unregister_out:
527 if (ret)
528 amdgpu_mn_unregister(bo);
529out:
530 mutex_unlock(&process_info->lock);
531 return ret;
532}
533
534/* Reserving a BO and its page table BOs must happen atomically to
535 * avoid deadlocks. Some operations update multiple VMs at once. Track
536 * all the reservation info in a context structure. Optionally a sync
537 * object can track VM updates.
538 */
539struct bo_vm_reservation_context {
540 struct amdgpu_bo_list_entry kfd_bo; /* BO list entry for the KFD BO */
541 unsigned int n_vms; /* Number of VMs reserved */
542 struct amdgpu_bo_list_entry *vm_pd; /* Array of VM BO list entries */
543 struct ww_acquire_ctx ticket; /* Reservation ticket */
544 struct list_head list, duplicates; /* BO lists */
545 struct amdgpu_sync *sync; /* Pointer to sync object */
546 bool reserved; /* Whether BOs are reserved */
547};
548
549enum bo_vm_match {
550 BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */
551 BO_VM_MAPPED, /* Match VMs where a BO is mapped */
552 BO_VM_ALL, /* Match all VMs a BO was added to */
553};
554
555/**
556 * reserve_bo_and_vm - reserve a BO and a VM unconditionally.
557 * @mem: KFD BO structure.
558 * @vm: the VM to reserve.
559 * @ctx: the struct that will be used in unreserve_bo_and_vms().
560 */
561static int reserve_bo_and_vm(struct kgd_mem *mem,
562 struct amdgpu_vm *vm,
563 struct bo_vm_reservation_context *ctx)
564{
565 struct amdgpu_bo *bo = mem->bo;
566 int ret;
567
568 WARN_ON(!vm);
569
570 ctx->reserved = false;
571 ctx->n_vms = 1;
572 ctx->sync = &mem->sync;
573
574 INIT_LIST_HEAD(&ctx->list);
575 INIT_LIST_HEAD(&ctx->duplicates);
576
577 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), GFP_KERNEL);
578 if (!ctx->vm_pd)
579 return -ENOMEM;
580
581 ctx->kfd_bo.priority = 0;
582 ctx->kfd_bo.tv.bo = &bo->tbo;
583 ctx->kfd_bo.tv.num_shared = 1;
584 list_add(&ctx->kfd_bo.tv.head, &ctx->list);
585
586 amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);
587
588 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
589 false, &ctx->duplicates, true);
590 if (!ret)
591 ctx->reserved = true;
592 else {
593 pr_err("Failed to reserve buffers in ttm\n");
594 kfree(ctx->vm_pd);
595 ctx->vm_pd = NULL;
596 }
597
598 return ret;
599}
600
601/**
602 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
603 * @mem: KFD BO structure.
604 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO
605 * is used. Otherwise, a single VM associated with the BO.
606 * @map_type: the mapping status that will be used to filter the VMs.
607 * @ctx: the struct that will be used in unreserve_bo_and_vms().
608 *
609 * Returns 0 for success, negative for failure.
610 */
611static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
612 struct amdgpu_vm *vm, enum bo_vm_match map_type,
613 struct bo_vm_reservation_context *ctx)
614{
615 struct amdgpu_bo *bo = mem->bo;
616 struct kfd_bo_va_list *entry;
617 unsigned int i;
618 int ret;
619
620 ctx->reserved = false;
621 ctx->n_vms = 0;
622 ctx->vm_pd = NULL;
623 ctx->sync = &mem->sync;
624
625 INIT_LIST_HEAD(&ctx->list);
626 INIT_LIST_HEAD(&ctx->duplicates);
627
628 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
629 if ((vm && vm != entry->bo_va->base.vm) ||
630 (entry->is_mapped != map_type
631 && map_type != BO_VM_ALL))
632 continue;
633
634 ctx->n_vms++;
635 }
636
637 if (ctx->n_vms != 0) {
638 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd),
639 GFP_KERNEL);
640 if (!ctx->vm_pd)
641 return -ENOMEM;
642 }
643
644 ctx->kfd_bo.priority = 0;
645 ctx->kfd_bo.tv.bo = &bo->tbo;
646 ctx->kfd_bo.tv.num_shared = 1;
647 list_add(&ctx->kfd_bo.tv.head, &ctx->list);
648
649 i = 0;
650 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
651 if ((vm && vm != entry->bo_va->base.vm) ||
652 (entry->is_mapped != map_type
653 && map_type != BO_VM_ALL))
654 continue;
655
656 amdgpu_vm_get_pd_bo(entry->bo_va->base.vm, &ctx->list,
657 &ctx->vm_pd[i]);
658 i++;
659 }
660
661 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
662 false, &ctx->duplicates, true);
663 if (!ret)
664 ctx->reserved = true;
665 else
666 pr_err("Failed to reserve buffers in ttm.\n");
667
668 if (ret) {
669 kfree(ctx->vm_pd);
670 ctx->vm_pd = NULL;
671 }
672
673 return ret;
674}
675
676/**
677 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
678 * @ctx: Reservation context to unreserve
679 * @wait: Optionally wait for a sync object representing pending VM updates
680 * @intr: Whether the wait is interruptible
681 *
682 * Also frees any resources allocated in
683 * reserve_bo_and_(cond_)vm(s). Returns the status from
684 * amdgpu_sync_wait.
685 */
686static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
687 bool wait, bool intr)
688{
689 int ret = 0;
690
691 if (wait)
692 ret = amdgpu_sync_wait(ctx->sync, intr);
693
694 if (ctx->reserved)
695 ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list);
696 kfree(ctx->vm_pd);
697
698 ctx->sync = NULL;
699
700 ctx->reserved = false;
701 ctx->vm_pd = NULL;
702
703 return ret;
704}
705
706static int unmap_bo_from_gpuvm(struct amdgpu_device *adev,
707 struct kfd_bo_va_list *entry,
708 struct amdgpu_sync *sync)
709{
710 struct amdgpu_bo_va *bo_va = entry->bo_va;
711 struct amdgpu_vm *vm = bo_va->base.vm;
712
713 amdgpu_vm_bo_unmap(adev, bo_va, entry->va);
714
715 amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
716
717 amdgpu_sync_fence(NULL, sync, bo_va->last_pt_update, false);
718
719 return 0;
720}
721
722static int update_gpuvm_pte(struct amdgpu_device *adev,
723 struct kfd_bo_va_list *entry,
724 struct amdgpu_sync *sync)
725{
726 int ret;
727 struct amdgpu_bo_va *bo_va = entry->bo_va;
728
729 /* Update the page tables */
730 ret = amdgpu_vm_bo_update(adev, bo_va, false);
731 if (ret) {
732 pr_err("amdgpu_vm_bo_update failed\n");
733 return ret;
734 }
735
736 return amdgpu_sync_fence(NULL, sync, bo_va->last_pt_update, false);
737}
738
739static int map_bo_to_gpuvm(struct amdgpu_device *adev,
740 struct kfd_bo_va_list *entry, struct amdgpu_sync *sync,
741 bool no_update_pte)
742{
743 int ret;
744
745 /* Set virtual address for the allocation */
746 ret = amdgpu_vm_bo_map(adev, entry->bo_va, entry->va, 0,
747 amdgpu_bo_size(entry->bo_va->base.bo),
748 entry->pte_flags);
749 if (ret) {
750 pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
751 entry->va, ret);
752 return ret;
753 }
754
755 if (no_update_pte)
756 return 0;
757
758 ret = update_gpuvm_pte(adev, entry, sync);
759 if (ret) {
760 pr_err("update_gpuvm_pte() failed\n");
761 goto update_gpuvm_pte_failed;
762 }
763
764 return 0;
765
766update_gpuvm_pte_failed:
767 unmap_bo_from_gpuvm(adev, entry, sync);
768 return ret;
769}
770
771static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
772{
773 struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
774
775 if (!sg)
776 return NULL;
777 if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
778 kfree(sg);
779 return NULL;
780 }
781 sg->sgl->dma_address = addr;
782 sg->sgl->length = size;
783#ifdef CONFIG_NEED_SG_DMA_LENGTH
784 sg->sgl->dma_length = size;
785#endif
786 return sg;
787}
788
789static int process_validate_vms(struct amdkfd_process_info *process_info)
790{
791 struct amdgpu_vm *peer_vm;
792 int ret;
793
794 list_for_each_entry(peer_vm, &process_info->vm_list_head,
795 vm_list_node) {
796 ret = vm_validate_pt_pd_bos(peer_vm);
797 if (ret)
798 return ret;
799 }
800
801 return 0;
802}
803
804static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
805 struct amdgpu_sync *sync)
806{
807 struct amdgpu_vm *peer_vm;
808 int ret;
809
810 list_for_each_entry(peer_vm, &process_info->vm_list_head,
811 vm_list_node) {
812 struct amdgpu_bo *pd = peer_vm->root.base.bo;
813
814 ret = amdgpu_sync_resv(NULL,
815 sync, pd->tbo.base.resv,
816 AMDGPU_FENCE_OWNER_KFD, false);
817 if (ret)
818 return ret;
819 }
820
821 return 0;
822}
823
824static int process_update_pds(struct amdkfd_process_info *process_info,
825 struct amdgpu_sync *sync)
826{
827 struct amdgpu_vm *peer_vm;
828 int ret;
829
830 list_for_each_entry(peer_vm, &process_info->vm_list_head,
831 vm_list_node) {
832 ret = vm_update_pds(peer_vm, sync);
833 if (ret)
834 return ret;
835 }
836
837 return 0;
838}
839
840static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
841 struct dma_fence **ef)
842{
843 struct amdkfd_process_info *info = NULL;
844 int ret;
845
846 if (!*process_info) {
847 info = kzalloc(sizeof(*info), GFP_KERNEL);
848 if (!info)
849 return -ENOMEM;
850
851 mutex_init(&info->lock);
852 INIT_LIST_HEAD(&info->vm_list_head);
853 INIT_LIST_HEAD(&info->kfd_bo_list);
854 INIT_LIST_HEAD(&info->userptr_valid_list);
855 INIT_LIST_HEAD(&info->userptr_inval_list);
856
857 info->eviction_fence =
858 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
859 current->mm);
860 if (!info->eviction_fence) {
861 pr_err("Failed to create eviction fence\n");
862 ret = -ENOMEM;
863 goto create_evict_fence_fail;
864 }
865
866 info->pid = get_task_pid(current->group_leader, PIDTYPE_PID);
867 atomic_set(&info->evicted_bos, 0);
868 INIT_DELAYED_WORK(&info->restore_userptr_work,
869 amdgpu_amdkfd_restore_userptr_worker);
870
871 *process_info = info;
872 *ef = dma_fence_get(&info->eviction_fence->base);
873 }
874
875 vm->process_info = *process_info;
876
877 /* Validate page directory and attach eviction fence */
878 ret = amdgpu_bo_reserve(vm->root.base.bo, true);
879 if (ret)
880 goto reserve_pd_fail;
881 ret = vm_validate_pt_pd_bos(vm);
882 if (ret) {
883 pr_err("validate_pt_pd_bos() failed\n");
884 goto validate_pd_fail;
885 }
886 ret = amdgpu_bo_sync_wait(vm->root.base.bo,
887 AMDGPU_FENCE_OWNER_KFD, false);
888 if (ret)
889 goto wait_pd_fail;
890 ret = dma_resv_reserve_shared(vm->root.base.bo->tbo.base.resv, 1);
891 if (ret)
892 goto reserve_shared_fail;
893 amdgpu_bo_fence(vm->root.base.bo,
894 &vm->process_info->eviction_fence->base, true);
895 amdgpu_bo_unreserve(vm->root.base.bo);
896
897 /* Update process info */
898 mutex_lock(&vm->process_info->lock);
899 list_add_tail(&vm->vm_list_node,
900 &(vm->process_info->vm_list_head));
901 vm->process_info->n_vms++;
902 mutex_unlock(&vm->process_info->lock);
903
904 return 0;
905
906reserve_shared_fail:
907wait_pd_fail:
908validate_pd_fail:
909 amdgpu_bo_unreserve(vm->root.base.bo);
910reserve_pd_fail:
911 vm->process_info = NULL;
912 if (info) {
913 /* Two fence references: one in info and one in *ef */
914 dma_fence_put(&info->eviction_fence->base);
915 dma_fence_put(*ef);
916 *ef = NULL;
917 *process_info = NULL;
918 put_pid(info->pid);
919create_evict_fence_fail:
920 mutex_destroy(&info->lock);
921 kfree(info);
922 }
923 return ret;
924}
925
926int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid,
927 void **vm, void **process_info,
928 struct dma_fence **ef)
929{
930 struct amdgpu_device *adev = get_amdgpu_device(kgd);
931 struct amdgpu_vm *new_vm;
932 int ret;
933
934 new_vm = kzalloc(sizeof(*new_vm), GFP_KERNEL);
935 if (!new_vm)
936 return -ENOMEM;
937
938 /* Initialize AMDGPU part of the VM */
939 ret = amdgpu_vm_init(adev, new_vm, AMDGPU_VM_CONTEXT_COMPUTE, pasid);
940 if (ret) {
941 pr_err("Failed init vm ret %d\n", ret);
942 goto amdgpu_vm_init_fail;
943 }
944
945 /* Initialize KFD part of the VM and process info */
946 ret = init_kfd_vm(new_vm, process_info, ef);
947 if (ret)
948 goto init_kfd_vm_fail;
949
950 *vm = (void *) new_vm;
951
952 return 0;
953
954init_kfd_vm_fail:
955 amdgpu_vm_fini(adev, new_vm);
956amdgpu_vm_init_fail:
957 kfree(new_vm);
958 return ret;
959}
960
961int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
962 struct file *filp, unsigned int pasid,
963 void **vm, void **process_info,
964 struct dma_fence **ef)
965{
966 struct amdgpu_device *adev = get_amdgpu_device(kgd);
967 struct drm_file *drm_priv = filp->private_data;
968 struct amdgpu_fpriv *drv_priv = drm_priv->driver_priv;
969 struct amdgpu_vm *avm = &drv_priv->vm;
970 int ret;
971
972 /* Already a compute VM? */
973 if (avm->process_info)
974 return -EINVAL;
975
976 /* Convert VM into a compute VM */
977 ret = amdgpu_vm_make_compute(adev, avm, pasid);
978 if (ret)
979 return ret;
980
981 /* Initialize KFD part of the VM and process info */
982 ret = init_kfd_vm(avm, process_info, ef);
983 if (ret)
984 return ret;
985
986 *vm = (void *)avm;
987
988 return 0;
989}
990
991void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
992 struct amdgpu_vm *vm)
993{
994 struct amdkfd_process_info *process_info = vm->process_info;
995 struct amdgpu_bo *pd = vm->root.base.bo;
996
997 if (!process_info)
998 return;
999
1000 /* Release eviction fence from PD */
1001 amdgpu_bo_reserve(pd, false);
1002 amdgpu_bo_fence(pd, NULL, false);
1003 amdgpu_bo_unreserve(pd);
1004
1005 /* Update process info */
1006 mutex_lock(&process_info->lock);
1007 process_info->n_vms--;
1008 list_del(&vm->vm_list_node);
1009 mutex_unlock(&process_info->lock);
1010
1011 /* Release per-process resources when last compute VM is destroyed */
1012 if (!process_info->n_vms) {
1013 WARN_ON(!list_empty(&process_info->kfd_bo_list));
1014 WARN_ON(!list_empty(&process_info->userptr_valid_list));
1015 WARN_ON(!list_empty(&process_info->userptr_inval_list));
1016
1017 dma_fence_put(&process_info->eviction_fence->base);
1018 cancel_delayed_work_sync(&process_info->restore_userptr_work);
1019 put_pid(process_info->pid);
1020 mutex_destroy(&process_info->lock);
1021 kfree(process_info);
1022 }
1023}
1024
1025void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm)
1026{
1027 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1028 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1029
1030 if (WARN_ON(!kgd || !vm))
1031 return;
1032
1033 pr_debug("Destroying process vm %p\n", vm);
1034
1035 /* Release the VM context */
1036 amdgpu_vm_fini(adev, avm);
1037 kfree(vm);
1038}
1039
1040void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm)
1041{
1042 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1043 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1044
1045 if (WARN_ON(!kgd || !vm))
1046 return;
1047
1048 pr_debug("Releasing process vm %p\n", vm);
1049
1050 /* The original pasid of amdgpu vm has already been
1051 * released during making a amdgpu vm to a compute vm
1052 * The current pasid is managed by kfd and will be
1053 * released on kfd process destroy. Set amdgpu pasid
1054 * to 0 to avoid duplicate release.
1055 */
1056 amdgpu_vm_release_compute(adev, avm);
1057}
1058
1059uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm)
1060{
1061 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1062 struct amdgpu_bo *pd = avm->root.base.bo;
1063 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
1064
1065 if (adev->asic_type < CHIP_VEGA10)
1066 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
1067 return avm->pd_phys_addr;
1068}
1069
1070int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1071 struct kgd_dev *kgd, uint64_t va, uint64_t size,
1072 void *vm, struct kgd_mem **mem,
1073 uint64_t *offset, uint32_t flags)
1074{
1075 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1076 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1077 enum ttm_bo_type bo_type = ttm_bo_type_device;
1078 struct sg_table *sg = NULL;
1079 uint64_t user_addr = 0;
1080 struct amdgpu_bo *bo;
1081 struct amdgpu_bo_param bp;
1082 int byte_align;
1083 u32 domain, alloc_domain;
1084 u64 alloc_flags;
1085 uint32_t mapping_flags;
1086 int ret;
1087
1088 /*
1089 * Check on which domain to allocate BO
1090 */
1091 if (flags & ALLOC_MEM_FLAGS_VRAM) {
1092 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1093 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
1094 alloc_flags |= (flags & ALLOC_MEM_FLAGS_PUBLIC) ?
1095 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED :
1096 AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
1097 } else if (flags & ALLOC_MEM_FLAGS_GTT) {
1098 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1099 alloc_flags = 0;
1100 } else if (flags & ALLOC_MEM_FLAGS_USERPTR) {
1101 domain = AMDGPU_GEM_DOMAIN_GTT;
1102 alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1103 alloc_flags = 0;
1104 if (!offset || !*offset)
1105 return -EINVAL;
1106 user_addr = untagged_addr(*offset);
1107 } else if (flags & (ALLOC_MEM_FLAGS_DOORBELL |
1108 ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1109 domain = AMDGPU_GEM_DOMAIN_GTT;
1110 alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1111 bo_type = ttm_bo_type_sg;
1112 alloc_flags = 0;
1113 if (size > UINT_MAX)
1114 return -EINVAL;
1115 sg = create_doorbell_sg(*offset, size);
1116 if (!sg)
1117 return -ENOMEM;
1118 } else {
1119 return -EINVAL;
1120 }
1121
1122 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1123 if (!*mem) {
1124 ret = -ENOMEM;
1125 goto err;
1126 }
1127 INIT_LIST_HEAD(&(*mem)->bo_va_list);
1128 mutex_init(&(*mem)->lock);
1129 (*mem)->aql_queue = !!(flags & ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1130
1131 /* Workaround for AQL queue wraparound bug. Map the same
1132 * memory twice. That means we only actually allocate half
1133 * the memory.
1134 */
1135 if ((*mem)->aql_queue)
1136 size = size >> 1;
1137
1138 /* Workaround for TLB bug on older VI chips */
1139 byte_align = (adev->family == AMDGPU_FAMILY_VI &&
1140 adev->asic_type != CHIP_FIJI &&
1141 adev->asic_type != CHIP_POLARIS10 &&
1142 adev->asic_type != CHIP_POLARIS11 &&
1143 adev->asic_type != CHIP_POLARIS12 &&
1144 adev->asic_type != CHIP_VEGAM) ?
1145 VI_BO_SIZE_ALIGN : 1;
1146
1147 mapping_flags = AMDGPU_VM_PAGE_READABLE;
1148 if (flags & ALLOC_MEM_FLAGS_WRITABLE)
1149 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
1150 if (flags & ALLOC_MEM_FLAGS_EXECUTABLE)
1151 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
1152 if (flags & ALLOC_MEM_FLAGS_COHERENT)
1153 mapping_flags |= AMDGPU_VM_MTYPE_UC;
1154 else
1155 mapping_flags |= AMDGPU_VM_MTYPE_NC;
1156 (*mem)->mapping_flags = mapping_flags;
1157
1158 amdgpu_sync_create(&(*mem)->sync);
1159
1160 ret = amdgpu_amdkfd_reserve_mem_limit(adev, size, alloc_domain, !!sg);
1161 if (ret) {
1162 pr_debug("Insufficient system memory\n");
1163 goto err_reserve_limit;
1164 }
1165
1166 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n",
1167 va, size, domain_string(alloc_domain));
1168
1169 memset(&bp, 0, sizeof(bp));
1170 bp.size = size;
1171 bp.byte_align = byte_align;
1172 bp.domain = alloc_domain;
1173 bp.flags = alloc_flags;
1174 bp.type = bo_type;
1175 bp.resv = NULL;
1176 ret = amdgpu_bo_create(adev, &bp, &bo);
1177 if (ret) {
1178 pr_debug("Failed to create BO on domain %s. ret %d\n",
1179 domain_string(alloc_domain), ret);
1180 goto err_bo_create;
1181 }
1182 if (bo_type == ttm_bo_type_sg) {
1183 bo->tbo.sg = sg;
1184 bo->tbo.ttm->sg = sg;
1185 }
1186 bo->kfd_bo = *mem;
1187 (*mem)->bo = bo;
1188 if (user_addr)
1189 bo->flags |= AMDGPU_AMDKFD_USERPTR_BO;
1190
1191 (*mem)->va = va;
1192 (*mem)->domain = domain;
1193 (*mem)->mapped_to_gpu_memory = 0;
1194 (*mem)->process_info = avm->process_info;
1195 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);
1196
1197 if (user_addr) {
1198 ret = init_user_pages(*mem, current->mm, user_addr);
1199 if (ret)
1200 goto allocate_init_user_pages_failed;
1201 }
1202
1203 if (offset)
1204 *offset = amdgpu_bo_mmap_offset(bo);
1205
1206 return 0;
1207
1208allocate_init_user_pages_failed:
1209 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
1210 amdgpu_bo_unref(&bo);
1211 /* Don't unreserve system mem limit twice */
1212 goto err_reserve_limit;
1213err_bo_create:
1214 unreserve_mem_limit(adev, size, alloc_domain, !!sg);
1215err_reserve_limit:
1216 mutex_destroy(&(*mem)->lock);
1217 kfree(*mem);
1218err:
1219 if (sg) {
1220 sg_free_table(sg);
1221 kfree(sg);
1222 }
1223 return ret;
1224}
1225
1226int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1227 struct kgd_dev *kgd, struct kgd_mem *mem)
1228{
1229 struct amdkfd_process_info *process_info = mem->process_info;
1230 unsigned long bo_size = mem->bo->tbo.mem.size;
1231 struct kfd_bo_va_list *entry, *tmp;
1232 struct bo_vm_reservation_context ctx;
1233 struct ttm_validate_buffer *bo_list_entry;
1234 int ret;
1235
1236 mutex_lock(&mem->lock);
1237
1238 if (mem->mapped_to_gpu_memory > 0) {
1239 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1240 mem->va, bo_size);
1241 mutex_unlock(&mem->lock);
1242 return -EBUSY;
1243 }
1244
1245 mutex_unlock(&mem->lock);
1246 /* lock is not needed after this, since mem is unused and will
1247 * be freed anyway
1248 */
1249
1250 /* No more MMU notifiers */
1251 amdgpu_mn_unregister(mem->bo);
1252
1253 /* Make sure restore workers don't access the BO any more */
1254 bo_list_entry = &mem->validate_list;
1255 mutex_lock(&process_info->lock);
1256 list_del(&bo_list_entry->head);
1257 mutex_unlock(&process_info->lock);
1258
1259 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
1260 if (unlikely(ret))
1261 return ret;
1262
1263 /* The eviction fence should be removed by the last unmap.
1264 * TODO: Log an error condition if the bo still has the eviction fence
1265 * attached
1266 */
1267 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1268 process_info->eviction_fence);
1269 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1270 mem->va + bo_size * (1 + mem->aql_queue));
1271
1272 /* Remove from VM internal data structures */
1273 list_for_each_entry_safe(entry, tmp, &mem->bo_va_list, bo_list)
1274 remove_bo_from_vm((struct amdgpu_device *)entry->kgd_dev,
1275 entry, bo_size);
1276
1277 ret = unreserve_bo_and_vms(&ctx, false, false);
1278
1279 /* Free the sync object */
1280 amdgpu_sync_free(&mem->sync);
1281
1282 /* If the SG is not NULL, it's one we created for a doorbell or mmio
1283 * remap BO. We need to free it.
1284 */
1285 if (mem->bo->tbo.sg) {
1286 sg_free_table(mem->bo->tbo.sg);
1287 kfree(mem->bo->tbo.sg);
1288 }
1289
1290 /* Free the BO*/
1291 amdgpu_bo_unref(&mem->bo);
1292 mutex_destroy(&mem->lock);
1293 kfree(mem);
1294
1295 return ret;
1296}
1297
1298int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1299 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
1300{
1301 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1302 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1303 int ret;
1304 struct amdgpu_bo *bo;
1305 uint32_t domain;
1306 struct kfd_bo_va_list *entry;
1307 struct bo_vm_reservation_context ctx;
1308 struct kfd_bo_va_list *bo_va_entry = NULL;
1309 struct kfd_bo_va_list *bo_va_entry_aql = NULL;
1310 unsigned long bo_size;
1311 bool is_invalid_userptr = false;
1312
1313 bo = mem->bo;
1314 if (!bo) {
1315 pr_err("Invalid BO when mapping memory to GPU\n");
1316 return -EINVAL;
1317 }
1318
1319 /* Make sure restore is not running concurrently. Since we
1320 * don't map invalid userptr BOs, we rely on the next restore
1321 * worker to do the mapping
1322 */
1323 mutex_lock(&mem->process_info->lock);
1324
1325 /* Lock mmap-sem. If we find an invalid userptr BO, we can be
1326 * sure that the MMU notifier is no longer running
1327 * concurrently and the queues are actually stopped
1328 */
1329 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1330 down_write(¤t->mm->mmap_sem);
1331 is_invalid_userptr = atomic_read(&mem->invalid);
1332 up_write(¤t->mm->mmap_sem);
1333 }
1334
1335 mutex_lock(&mem->lock);
1336
1337 domain = mem->domain;
1338 bo_size = bo->tbo.mem.size;
1339
1340 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
1341 mem->va,
1342 mem->va + bo_size * (1 + mem->aql_queue),
1343 vm, domain_string(domain));
1344
1345 ret = reserve_bo_and_vm(mem, vm, &ctx);
1346 if (unlikely(ret))
1347 goto out;
1348
1349 /* Userptr can be marked as "not invalid", but not actually be
1350 * validated yet (still in the system domain). In that case
1351 * the queues are still stopped and we can leave mapping for
1352 * the next restore worker
1353 */
1354 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&
1355 bo->tbo.mem.mem_type == TTM_PL_SYSTEM)
1356 is_invalid_userptr = true;
1357
1358 if (check_if_add_bo_to_vm(avm, mem)) {
1359 ret = add_bo_to_vm(adev, mem, avm, false,
1360 &bo_va_entry);
1361 if (ret)
1362 goto add_bo_to_vm_failed;
1363 if (mem->aql_queue) {
1364 ret = add_bo_to_vm(adev, mem, avm,
1365 true, &bo_va_entry_aql);
1366 if (ret)
1367 goto add_bo_to_vm_failed_aql;
1368 }
1369 } else {
1370 ret = vm_validate_pt_pd_bos(avm);
1371 if (unlikely(ret))
1372 goto add_bo_to_vm_failed;
1373 }
1374
1375 if (mem->mapped_to_gpu_memory == 0 &&
1376 !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1377 /* Validate BO only once. The eviction fence gets added to BO
1378 * the first time it is mapped. Validate will wait for all
1379 * background evictions to complete.
1380 */
1381 ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
1382 if (ret) {
1383 pr_debug("Validate failed\n");
1384 goto map_bo_to_gpuvm_failed;
1385 }
1386 }
1387
1388 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
1389 if (entry->bo_va->base.vm == vm && !entry->is_mapped) {
1390 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
1391 entry->va, entry->va + bo_size,
1392 entry);
1393
1394 ret = map_bo_to_gpuvm(adev, entry, ctx.sync,
1395 is_invalid_userptr);
1396 if (ret) {
1397 pr_err("Failed to map bo to gpuvm\n");
1398 goto map_bo_to_gpuvm_failed;
1399 }
1400
1401 ret = vm_update_pds(vm, ctx.sync);
1402 if (ret) {
1403 pr_err("Failed to update page directories\n");
1404 goto map_bo_to_gpuvm_failed;
1405 }
1406
1407 entry->is_mapped = true;
1408 mem->mapped_to_gpu_memory++;
1409 pr_debug("\t INC mapping count %d\n",
1410 mem->mapped_to_gpu_memory);
1411 }
1412 }
1413
1414 if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->pin_count)
1415 amdgpu_bo_fence(bo,
1416 &avm->process_info->eviction_fence->base,
1417 true);
1418 ret = unreserve_bo_and_vms(&ctx, false, false);
1419
1420 goto out;
1421
1422map_bo_to_gpuvm_failed:
1423 if (bo_va_entry_aql)
1424 remove_bo_from_vm(adev, bo_va_entry_aql, bo_size);
1425add_bo_to_vm_failed_aql:
1426 if (bo_va_entry)
1427 remove_bo_from_vm(adev, bo_va_entry, bo_size);
1428add_bo_to_vm_failed:
1429 unreserve_bo_and_vms(&ctx, false, false);
1430out:
1431 mutex_unlock(&mem->process_info->lock);
1432 mutex_unlock(&mem->lock);
1433 return ret;
1434}
1435
1436int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1437 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
1438{
1439 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1440 struct amdkfd_process_info *process_info =
1441 ((struct amdgpu_vm *)vm)->process_info;
1442 unsigned long bo_size = mem->bo->tbo.mem.size;
1443 struct kfd_bo_va_list *entry;
1444 struct bo_vm_reservation_context ctx;
1445 int ret;
1446
1447 mutex_lock(&mem->lock);
1448
1449 ret = reserve_bo_and_cond_vms(mem, vm, BO_VM_MAPPED, &ctx);
1450 if (unlikely(ret))
1451 goto out;
1452 /* If no VMs were reserved, it means the BO wasn't actually mapped */
1453 if (ctx.n_vms == 0) {
1454 ret = -EINVAL;
1455 goto unreserve_out;
1456 }
1457
1458 ret = vm_validate_pt_pd_bos((struct amdgpu_vm *)vm);
1459 if (unlikely(ret))
1460 goto unreserve_out;
1461
1462 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
1463 mem->va,
1464 mem->va + bo_size * (1 + mem->aql_queue),
1465 vm);
1466
1467 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
1468 if (entry->bo_va->base.vm == vm && entry->is_mapped) {
1469 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
1470 entry->va,
1471 entry->va + bo_size,
1472 entry);
1473
1474 ret = unmap_bo_from_gpuvm(adev, entry, ctx.sync);
1475 if (ret == 0) {
1476 entry->is_mapped = false;
1477 } else {
1478 pr_err("failed to unmap VA 0x%llx\n",
1479 mem->va);
1480 goto unreserve_out;
1481 }
1482
1483 mem->mapped_to_gpu_memory--;
1484 pr_debug("\t DEC mapping count %d\n",
1485 mem->mapped_to_gpu_memory);
1486 }
1487 }
1488
1489 /* If BO is unmapped from all VMs, unfence it. It can be evicted if
1490 * required.
1491 */
1492 if (mem->mapped_to_gpu_memory == 0 &&
1493 !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count)
1494 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1495 process_info->eviction_fence);
1496
1497unreserve_out:
1498 unreserve_bo_and_vms(&ctx, false, false);
1499out:
1500 mutex_unlock(&mem->lock);
1501 return ret;
1502}
1503
1504int amdgpu_amdkfd_gpuvm_sync_memory(
1505 struct kgd_dev *kgd, struct kgd_mem *mem, bool intr)
1506{
1507 struct amdgpu_sync sync;
1508 int ret;
1509
1510 amdgpu_sync_create(&sync);
1511
1512 mutex_lock(&mem->lock);
1513 amdgpu_sync_clone(&mem->sync, &sync);
1514 mutex_unlock(&mem->lock);
1515
1516 ret = amdgpu_sync_wait(&sync, intr);
1517 amdgpu_sync_free(&sync);
1518 return ret;
1519}
1520
1521int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
1522 struct kgd_mem *mem, void **kptr, uint64_t *size)
1523{
1524 int ret;
1525 struct amdgpu_bo *bo = mem->bo;
1526
1527 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1528 pr_err("userptr can't be mapped to kernel\n");
1529 return -EINVAL;
1530 }
1531
1532 /* delete kgd_mem from kfd_bo_list to avoid re-validating
1533 * this BO in BO's restoring after eviction.
1534 */
1535 mutex_lock(&mem->process_info->lock);
1536
1537 ret = amdgpu_bo_reserve(bo, true);
1538 if (ret) {
1539 pr_err("Failed to reserve bo. ret %d\n", ret);
1540 goto bo_reserve_failed;
1541 }
1542
1543 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
1544 if (ret) {
1545 pr_err("Failed to pin bo. ret %d\n", ret);
1546 goto pin_failed;
1547 }
1548
1549 ret = amdgpu_bo_kmap(bo, kptr);
1550 if (ret) {
1551 pr_err("Failed to map bo to kernel. ret %d\n", ret);
1552 goto kmap_failed;
1553 }
1554
1555 amdgpu_amdkfd_remove_eviction_fence(
1556 bo, mem->process_info->eviction_fence);
1557 list_del_init(&mem->validate_list.head);
1558
1559 if (size)
1560 *size = amdgpu_bo_size(bo);
1561
1562 amdgpu_bo_unreserve(bo);
1563
1564 mutex_unlock(&mem->process_info->lock);
1565 return 0;
1566
1567kmap_failed:
1568 amdgpu_bo_unpin(bo);
1569pin_failed:
1570 amdgpu_bo_unreserve(bo);
1571bo_reserve_failed:
1572 mutex_unlock(&mem->process_info->lock);
1573
1574 return ret;
1575}
1576
1577int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
1578 struct kfd_vm_fault_info *mem)
1579{
1580 struct amdgpu_device *adev;
1581
1582 adev = (struct amdgpu_device *)kgd;
1583 if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
1584 *mem = *adev->gmc.vm_fault_info;
1585 mb();
1586 atomic_set(&adev->gmc.vm_fault_info_updated, 0);
1587 }
1588 return 0;
1589}
1590
1591int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
1592 struct dma_buf *dma_buf,
1593 uint64_t va, void *vm,
1594 struct kgd_mem **mem, uint64_t *size,
1595 uint64_t *mmap_offset)
1596{
1597 struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
1598 struct drm_gem_object *obj;
1599 struct amdgpu_bo *bo;
1600 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1601
1602 if (dma_buf->ops != &amdgpu_dmabuf_ops)
1603 /* Can't handle non-graphics buffers */
1604 return -EINVAL;
1605
1606 obj = dma_buf->priv;
1607 if (obj->dev->dev_private != adev)
1608 /* Can't handle buffers from other devices */
1609 return -EINVAL;
1610
1611 bo = gem_to_amdgpu_bo(obj);
1612 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
1613 AMDGPU_GEM_DOMAIN_GTT)))
1614 /* Only VRAM and GTT BOs are supported */
1615 return -EINVAL;
1616
1617 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1618 if (!*mem)
1619 return -ENOMEM;
1620
1621 if (size)
1622 *size = amdgpu_bo_size(bo);
1623
1624 if (mmap_offset)
1625 *mmap_offset = amdgpu_bo_mmap_offset(bo);
1626
1627 INIT_LIST_HEAD(&(*mem)->bo_va_list);
1628 mutex_init(&(*mem)->lock);
1629 (*mem)->mapping_flags =
1630 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
1631 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_NC;
1632
1633 (*mem)->bo = amdgpu_bo_ref(bo);
1634 (*mem)->va = va;
1635 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
1636 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
1637 (*mem)->mapped_to_gpu_memory = 0;
1638 (*mem)->process_info = avm->process_info;
1639 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);
1640 amdgpu_sync_create(&(*mem)->sync);
1641
1642 return 0;
1643}
1644
1645/* Evict a userptr BO by stopping the queues if necessary
1646 *
1647 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
1648 * cannot do any memory allocations, and cannot take any locks that
1649 * are held elsewhere while allocating memory. Therefore this is as
1650 * simple as possible, using atomic counters.
1651 *
1652 * It doesn't do anything to the BO itself. The real work happens in
1653 * restore, where we get updated page addresses. This function only
1654 * ensures that GPU access to the BO is stopped.
1655 */
1656int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem,
1657 struct mm_struct *mm)
1658{
1659 struct amdkfd_process_info *process_info = mem->process_info;
1660 int invalid, evicted_bos;
1661 int r = 0;
1662
1663 invalid = atomic_inc_return(&mem->invalid);
1664 evicted_bos = atomic_inc_return(&process_info->evicted_bos);
1665 if (evicted_bos == 1) {
1666 /* First eviction, stop the queues */
1667 r = kgd2kfd_quiesce_mm(mm);
1668 if (r)
1669 pr_err("Failed to quiesce KFD\n");
1670 schedule_delayed_work(&process_info->restore_userptr_work,
1671 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
1672 }
1673
1674 return r;
1675}
1676
1677/* Update invalid userptr BOs
1678 *
1679 * Moves invalidated (evicted) userptr BOs from userptr_valid_list to
1680 * userptr_inval_list and updates user pages for all BOs that have
1681 * been invalidated since their last update.
1682 */
1683static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
1684 struct mm_struct *mm)
1685{
1686 struct kgd_mem *mem, *tmp_mem;
1687 struct amdgpu_bo *bo;
1688 struct ttm_operation_ctx ctx = { false, false };
1689 int invalid, ret;
1690
1691 /* Move all invalidated BOs to the userptr_inval_list and
1692 * release their user pages by migration to the CPU domain
1693 */
1694 list_for_each_entry_safe(mem, tmp_mem,
1695 &process_info->userptr_valid_list,
1696 validate_list.head) {
1697 if (!atomic_read(&mem->invalid))
1698 continue; /* BO is still valid */
1699
1700 bo = mem->bo;
1701
1702 if (amdgpu_bo_reserve(bo, true))
1703 return -EAGAIN;
1704 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
1705 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1706 amdgpu_bo_unreserve(bo);
1707 if (ret) {
1708 pr_err("%s: Failed to invalidate userptr BO\n",
1709 __func__);
1710 return -EAGAIN;
1711 }
1712
1713 list_move_tail(&mem->validate_list.head,
1714 &process_info->userptr_inval_list);
1715 }
1716
1717 if (list_empty(&process_info->userptr_inval_list))
1718 return 0; /* All evicted userptr BOs were freed */
1719
1720 /* Go through userptr_inval_list and update any invalid user_pages */
1721 list_for_each_entry(mem, &process_info->userptr_inval_list,
1722 validate_list.head) {
1723 invalid = atomic_read(&mem->invalid);
1724 if (!invalid)
1725 /* BO hasn't been invalidated since the last
1726 * revalidation attempt. Keep its BO list.
1727 */
1728 continue;
1729
1730 bo = mem->bo;
1731
1732 /* Get updated user pages */
1733 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
1734 if (ret) {
1735 pr_debug("%s: Failed to get user pages: %d\n",
1736 __func__, ret);
1737
1738 /* Return error -EBUSY or -ENOMEM, retry restore */
1739 return ret;
1740 }
1741
1742 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
1743
1744 /* Mark the BO as valid unless it was invalidated
1745 * again concurrently.
1746 */
1747 if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)
1748 return -EAGAIN;
1749 }
1750
1751 return 0;
1752}
1753
1754/* Validate invalid userptr BOs
1755 *
1756 * Validates BOs on the userptr_inval_list, and moves them back to the
1757 * userptr_valid_list. Also updates GPUVM page tables with new page
1758 * addresses and waits for the page table updates to complete.
1759 */
1760static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
1761{
1762 struct amdgpu_bo_list_entry *pd_bo_list_entries;
1763 struct list_head resv_list, duplicates;
1764 struct ww_acquire_ctx ticket;
1765 struct amdgpu_sync sync;
1766
1767 struct amdgpu_vm *peer_vm;
1768 struct kgd_mem *mem, *tmp_mem;
1769 struct amdgpu_bo *bo;
1770 struct ttm_operation_ctx ctx = { false, false };
1771 int i, ret;
1772
1773 pd_bo_list_entries = kcalloc(process_info->n_vms,
1774 sizeof(struct amdgpu_bo_list_entry),
1775 GFP_KERNEL);
1776 if (!pd_bo_list_entries) {
1777 pr_err("%s: Failed to allocate PD BO list entries\n", __func__);
1778 ret = -ENOMEM;
1779 goto out_no_mem;
1780 }
1781
1782 INIT_LIST_HEAD(&resv_list);
1783 INIT_LIST_HEAD(&duplicates);
1784
1785 /* Get all the page directory BOs that need to be reserved */
1786 i = 0;
1787 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1788 vm_list_node)
1789 amdgpu_vm_get_pd_bo(peer_vm, &resv_list,
1790 &pd_bo_list_entries[i++]);
1791 /* Add the userptr_inval_list entries to resv_list */
1792 list_for_each_entry(mem, &process_info->userptr_inval_list,
1793 validate_list.head) {
1794 list_add_tail(&mem->resv_list.head, &resv_list);
1795 mem->resv_list.bo = mem->validate_list.bo;
1796 mem->resv_list.num_shared = mem->validate_list.num_shared;
1797 }
1798
1799 /* Reserve all BOs and page tables for validation */
1800 ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates,
1801 true);
1802 WARN(!list_empty(&duplicates), "Duplicates should be empty");
1803 if (ret)
1804 goto out_free;
1805
1806 amdgpu_sync_create(&sync);
1807
1808 ret = process_validate_vms(process_info);
1809 if (ret)
1810 goto unreserve_out;
1811
1812 /* Validate BOs and update GPUVM page tables */
1813 list_for_each_entry_safe(mem, tmp_mem,
1814 &process_info->userptr_inval_list,
1815 validate_list.head) {
1816 struct kfd_bo_va_list *bo_va_entry;
1817
1818 bo = mem->bo;
1819
1820 /* Validate the BO if we got user pages */
1821 if (bo->tbo.ttm->pages[0]) {
1822 amdgpu_bo_placement_from_domain(bo, mem->domain);
1823 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1824 if (ret) {
1825 pr_err("%s: failed to validate BO\n", __func__);
1826 goto unreserve_out;
1827 }
1828 }
1829
1830 list_move_tail(&mem->validate_list.head,
1831 &process_info->userptr_valid_list);
1832
1833 /* Update mapping. If the BO was not validated
1834 * (because we couldn't get user pages), this will
1835 * clear the page table entries, which will result in
1836 * VM faults if the GPU tries to access the invalid
1837 * memory.
1838 */
1839 list_for_each_entry(bo_va_entry, &mem->bo_va_list, bo_list) {
1840 if (!bo_va_entry->is_mapped)
1841 continue;
1842
1843 ret = update_gpuvm_pte((struct amdgpu_device *)
1844 bo_va_entry->kgd_dev,
1845 bo_va_entry, &sync);
1846 if (ret) {
1847 pr_err("%s: update PTE failed\n", __func__);
1848 /* make sure this gets validated again */
1849 atomic_inc(&mem->invalid);
1850 goto unreserve_out;
1851 }
1852 }
1853 }
1854
1855 /* Update page directories */
1856 ret = process_update_pds(process_info, &sync);
1857
1858unreserve_out:
1859 ttm_eu_backoff_reservation(&ticket, &resv_list);
1860 amdgpu_sync_wait(&sync, false);
1861 amdgpu_sync_free(&sync);
1862out_free:
1863 kfree(pd_bo_list_entries);
1864out_no_mem:
1865
1866 return ret;
1867}
1868
1869/* Worker callback to restore evicted userptr BOs
1870 *
1871 * Tries to update and validate all userptr BOs. If successful and no
1872 * concurrent evictions happened, the queues are restarted. Otherwise,
1873 * reschedule for another attempt later.
1874 */
1875static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
1876{
1877 struct delayed_work *dwork = to_delayed_work(work);
1878 struct amdkfd_process_info *process_info =
1879 container_of(dwork, struct amdkfd_process_info,
1880 restore_userptr_work);
1881 struct task_struct *usertask;
1882 struct mm_struct *mm;
1883 int evicted_bos;
1884
1885 evicted_bos = atomic_read(&process_info->evicted_bos);
1886 if (!evicted_bos)
1887 return;
1888
1889 /* Reference task and mm in case of concurrent process termination */
1890 usertask = get_pid_task(process_info->pid, PIDTYPE_PID);
1891 if (!usertask)
1892 return;
1893 mm = get_task_mm(usertask);
1894 if (!mm) {
1895 put_task_struct(usertask);
1896 return;
1897 }
1898
1899 mutex_lock(&process_info->lock);
1900
1901 if (update_invalid_user_pages(process_info, mm))
1902 goto unlock_out;
1903 /* userptr_inval_list can be empty if all evicted userptr BOs
1904 * have been freed. In that case there is nothing to validate
1905 * and we can just restart the queues.
1906 */
1907 if (!list_empty(&process_info->userptr_inval_list)) {
1908 if (atomic_read(&process_info->evicted_bos) != evicted_bos)
1909 goto unlock_out; /* Concurrent eviction, try again */
1910
1911 if (validate_invalid_user_pages(process_info))
1912 goto unlock_out;
1913 }
1914 /* Final check for concurrent evicton and atomic update. If
1915 * another eviction happens after successful update, it will
1916 * be a first eviction that calls quiesce_mm. The eviction
1917 * reference counting inside KFD will handle this case.
1918 */
1919 if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) !=
1920 evicted_bos)
1921 goto unlock_out;
1922 evicted_bos = 0;
1923 if (kgd2kfd_resume_mm(mm)) {
1924 pr_err("%s: Failed to resume KFD\n", __func__);
1925 /* No recovery from this failure. Probably the CP is
1926 * hanging. No point trying again.
1927 */
1928 }
1929
1930unlock_out:
1931 mutex_unlock(&process_info->lock);
1932 mmput(mm);
1933 put_task_struct(usertask);
1934
1935 /* If validation failed, reschedule another attempt */
1936 if (evicted_bos)
1937 schedule_delayed_work(&process_info->restore_userptr_work,
1938 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
1939}
1940
1941/** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
1942 * KFD process identified by process_info
1943 *
1944 * @process_info: amdkfd_process_info of the KFD process
1945 *
1946 * After memory eviction, restore thread calls this function. The function
1947 * should be called when the Process is still valid. BO restore involves -
1948 *
1949 * 1. Release old eviction fence and create new one
1950 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
1951 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of
1952 * BOs that need to be reserved.
1953 * 4. Reserve all the BOs
1954 * 5. Validate of PD and PT BOs.
1955 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence
1956 * 7. Add fence to all PD and PT BOs.
1957 * 8. Unreserve all BOs
1958 */
1959int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
1960{
1961 struct amdgpu_bo_list_entry *pd_bo_list;
1962 struct amdkfd_process_info *process_info = info;
1963 struct amdgpu_vm *peer_vm;
1964 struct kgd_mem *mem;
1965 struct bo_vm_reservation_context ctx;
1966 struct amdgpu_amdkfd_fence *new_fence;
1967 int ret = 0, i;
1968 struct list_head duplicate_save;
1969 struct amdgpu_sync sync_obj;
1970
1971 INIT_LIST_HEAD(&duplicate_save);
1972 INIT_LIST_HEAD(&ctx.list);
1973 INIT_LIST_HEAD(&ctx.duplicates);
1974
1975 pd_bo_list = kcalloc(process_info->n_vms,
1976 sizeof(struct amdgpu_bo_list_entry),
1977 GFP_KERNEL);
1978 if (!pd_bo_list)
1979 return -ENOMEM;
1980
1981 i = 0;
1982 mutex_lock(&process_info->lock);
1983 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1984 vm_list_node)
1985 amdgpu_vm_get_pd_bo(peer_vm, &ctx.list, &pd_bo_list[i++]);
1986
1987 /* Reserve all BOs and page tables/directory. Add all BOs from
1988 * kfd_bo_list to ctx.list
1989 */
1990 list_for_each_entry(mem, &process_info->kfd_bo_list,
1991 validate_list.head) {
1992
1993 list_add_tail(&mem->resv_list.head, &ctx.list);
1994 mem->resv_list.bo = mem->validate_list.bo;
1995 mem->resv_list.num_shared = mem->validate_list.num_shared;
1996 }
1997
1998 ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list,
1999 false, &duplicate_save, true);
2000 if (ret) {
2001 pr_debug("Memory eviction: TTM Reserve Failed. Try again\n");
2002 goto ttm_reserve_fail;
2003 }
2004
2005 amdgpu_sync_create(&sync_obj);
2006
2007 /* Validate PDs and PTs */
2008 ret = process_validate_vms(process_info);
2009 if (ret)
2010 goto validate_map_fail;
2011
2012 ret = process_sync_pds_resv(process_info, &sync_obj);
2013 if (ret) {
2014 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
2015 goto validate_map_fail;
2016 }
2017
2018 /* Validate BOs and map them to GPUVM (update VM page tables). */
2019 list_for_each_entry(mem, &process_info->kfd_bo_list,
2020 validate_list.head) {
2021
2022 struct amdgpu_bo *bo = mem->bo;
2023 uint32_t domain = mem->domain;
2024 struct kfd_bo_va_list *bo_va_entry;
2025
2026 ret = amdgpu_amdkfd_bo_validate(bo, domain, false);
2027 if (ret) {
2028 pr_debug("Memory eviction: Validate BOs failed. Try again\n");
2029 goto validate_map_fail;
2030 }
2031 ret = amdgpu_sync_fence(NULL, &sync_obj, bo->tbo.moving, false);
2032 if (ret) {
2033 pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2034 goto validate_map_fail;
2035 }
2036 list_for_each_entry(bo_va_entry, &mem->bo_va_list,
2037 bo_list) {
2038 ret = update_gpuvm_pte((struct amdgpu_device *)
2039 bo_va_entry->kgd_dev,
2040 bo_va_entry,
2041 &sync_obj);
2042 if (ret) {
2043 pr_debug("Memory eviction: update PTE failed. Try again\n");
2044 goto validate_map_fail;
2045 }
2046 }
2047 }
2048
2049 /* Update page directories */
2050 ret = process_update_pds(process_info, &sync_obj);
2051 if (ret) {
2052 pr_debug("Memory eviction: update PDs failed. Try again\n");
2053 goto validate_map_fail;
2054 }
2055
2056 /* Wait for validate and PT updates to finish */
2057 amdgpu_sync_wait(&sync_obj, false);
2058
2059 /* Release old eviction fence and create new one, because fence only
2060 * goes from unsignaled to signaled, fence cannot be reused.
2061 * Use context and mm from the old fence.
2062 */
2063 new_fence = amdgpu_amdkfd_fence_create(
2064 process_info->eviction_fence->base.context,
2065 process_info->eviction_fence->mm);
2066 if (!new_fence) {
2067 pr_err("Failed to create eviction fence\n");
2068 ret = -ENOMEM;
2069 goto validate_map_fail;
2070 }
2071 dma_fence_put(&process_info->eviction_fence->base);
2072 process_info->eviction_fence = new_fence;
2073 *ef = dma_fence_get(&new_fence->base);
2074
2075 /* Attach new eviction fence to all BOs */
2076 list_for_each_entry(mem, &process_info->kfd_bo_list,
2077 validate_list.head)
2078 amdgpu_bo_fence(mem->bo,
2079 &process_info->eviction_fence->base, true);
2080
2081 /* Attach eviction fence to PD / PT BOs */
2082 list_for_each_entry(peer_vm, &process_info->vm_list_head,
2083 vm_list_node) {
2084 struct amdgpu_bo *bo = peer_vm->root.base.bo;
2085
2086 amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);
2087 }
2088
2089validate_map_fail:
2090 ttm_eu_backoff_reservation(&ctx.ticket, &ctx.list);
2091 amdgpu_sync_free(&sync_obj);
2092ttm_reserve_fail:
2093 mutex_unlock(&process_info->lock);
2094 kfree(pd_bo_list);
2095 return ret;
2096}
2097
2098int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
2099{
2100 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
2101 struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
2102 int ret;
2103
2104 if (!info || !gws)
2105 return -EINVAL;
2106
2107 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2108 if (!*mem)
2109 return -ENOMEM;
2110
2111 mutex_init(&(*mem)->lock);
2112 (*mem)->bo = amdgpu_bo_ref(gws_bo);
2113 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
2114 (*mem)->process_info = process_info;
2115 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);
2116 amdgpu_sync_create(&(*mem)->sync);
2117
2118
2119 /* Validate gws bo the first time it is added to process */
2120 mutex_lock(&(*mem)->process_info->lock);
2121 ret = amdgpu_bo_reserve(gws_bo, false);
2122 if (unlikely(ret)) {
2123 pr_err("Reserve gws bo failed %d\n", ret);
2124 goto bo_reservation_failure;
2125 }
2126
2127 ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true);
2128 if (ret) {
2129 pr_err("GWS BO validate failed %d\n", ret);
2130 goto bo_validation_failure;
2131 }
2132 /* GWS resource is shared b/t amdgpu and amdkfd
2133 * Add process eviction fence to bo so they can
2134 * evict each other.
2135 */
2136 ret = dma_resv_reserve_shared(gws_bo->tbo.base.resv, 1);
2137 if (ret)
2138 goto reserve_shared_fail;
2139 amdgpu_bo_fence(gws_bo, &process_info->eviction_fence->base, true);
2140 amdgpu_bo_unreserve(gws_bo);
2141 mutex_unlock(&(*mem)->process_info->lock);
2142
2143 return ret;
2144
2145reserve_shared_fail:
2146bo_validation_failure:
2147 amdgpu_bo_unreserve(gws_bo);
2148bo_reservation_failure:
2149 mutex_unlock(&(*mem)->process_info->lock);
2150 amdgpu_sync_free(&(*mem)->sync);
2151 remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
2152 amdgpu_bo_unref(&gws_bo);
2153 mutex_destroy(&(*mem)->lock);
2154 kfree(*mem);
2155 *mem = NULL;
2156 return ret;
2157}
2158
2159int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
2160{
2161 int ret;
2162 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
2163 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
2164 struct amdgpu_bo *gws_bo = kgd_mem->bo;
2165
2166 /* Remove BO from process's validate list so restore worker won't touch
2167 * it anymore
2168 */
2169 remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);
2170
2171 ret = amdgpu_bo_reserve(gws_bo, false);
2172 if (unlikely(ret)) {
2173 pr_err("Reserve gws bo failed %d\n", ret);
2174 //TODO add BO back to validate_list?
2175 return ret;
2176 }
2177 amdgpu_amdkfd_remove_eviction_fence(gws_bo,
2178 process_info->eviction_fence);
2179 amdgpu_bo_unreserve(gws_bo);
2180 amdgpu_sync_free(&kgd_mem->sync);
2181 amdgpu_bo_unref(&gws_bo);
2182 mutex_destroy(&kgd_mem->lock);
2183 kfree(mem);
2184 return 0;
2185}