Linux Audio

Check our new training course

Loading...
v4.17
   1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
 
  28#include <linux/dma-fence-array.h>
  29#include <linux/interval_tree_generic.h>
  30#include <linux/idr.h>
  31#include <drm/drmP.h>
 
  32#include <drm/amdgpu_drm.h>
 
  33#include "amdgpu.h"
  34#include "amdgpu_trace.h"
  35#include "amdgpu_amdkfd.h"
 
 
 
 
 
  36
  37/*
  38 * GPUVM
  39 * GPUVM is similar to the legacy gart on older asics, however
  40 * rather than there being a single global gart table
  41 * for the entire GPU, there are multiple VM page tables active
  42 * at any given time.  The VM page tables can contain a mix
  43 * vram pages and system memory pages and system memory pages
 
 
  44 * can be mapped as snooped (cached system pages) or unsnooped
  45 * (uncached system pages).
  46 * Each VM has an ID associated with it and there is a page table
  47 * associated with each VMID.  When execting a command buffer,
  48 * the kernel tells the the ring what VMID to use for that command
 
  49 * buffer.  VMIDs are allocated dynamically as commands are submitted.
  50 * The userspace drivers maintain their own address space and the kernel
  51 * sets up their pages tables accordingly when they submit their
  52 * command buffers and a VMID is assigned.
  53 * Cayman/Trinity support up to 8 active VMs at any given time;
  54 * SI supports 16.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  55 */
  56
  57#define START(node) ((node)->start)
  58#define LAST(node) ((node)->last)
  59
  60INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  61		     START, LAST, static, amdgpu_vm_it)
  62
  63#undef START
  64#undef LAST
  65
  66/* Local structure. Encapsulate some VM table update parameters to reduce
  67 * the number of function parameters
  68 */
  69struct amdgpu_pte_update_params {
  70	/* amdgpu device we do this update for */
 
 
 
  71	struct amdgpu_device *adev;
  72	/* optional amdgpu_vm we do this update for */
  73	struct amdgpu_vm *vm;
  74	/* address where to copy page table entries from */
  75	uint64_t src;
  76	/* indirect buffer to fill with commands */
  77	struct amdgpu_ib *ib;
  78	/* Function which actually does the update */
  79	void (*func)(struct amdgpu_pte_update_params *params,
  80		     struct amdgpu_bo *bo, uint64_t pe,
  81		     uint64_t addr, unsigned count, uint32_t incr,
  82		     uint64_t flags);
  83	/* The next two are used during VM update by CPU
  84	 *  DMA addresses to use for mapping
  85	 *  Kernel pointer of PD/PT BO that needs to be updated
  86	 */
  87	dma_addr_t *pages_addr;
  88	void *kptr;
  89};
  90
  91/* Helper to disable partial resident texture feature from a fence callback */
  92struct amdgpu_prt_cb {
  93	struct amdgpu_device *adev;
 
 
 
 
 
 
 
 
 
  94	struct dma_fence_cb cb;
  95};
  96
  97/**
  98 * amdgpu_vm_level_shift - return the addr shift for each level
  99 *
 100 * @adev: amdgpu_device pointer
 
 
 
 
 
 101 *
 102 * Returns the number of bits the pfn needs to be right shifted for a level.
 103 */
 104static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
 105				      unsigned level)
 106{
 107	unsigned shift = 0xff;
 108
 109	switch (level) {
 110	case AMDGPU_VM_PDB2:
 111	case AMDGPU_VM_PDB1:
 112	case AMDGPU_VM_PDB0:
 113		shift = 9 * (AMDGPU_VM_PDB0 - level) +
 114			adev->vm_manager.block_size;
 115		break;
 116	case AMDGPU_VM_PTB:
 117		shift = 0;
 118		break;
 119	default:
 120		dev_err(adev->dev, "the level%d isn't supported.\n", level);
 
 
 
 
 
 
 121	}
 122
 123	return shift;
 
 124}
 125
 126/**
 127 * amdgpu_vm_num_entries - return the number of entries in a PD/PT
 128 *
 129 * @adev: amdgpu_device pointer
 130 *
 131 * Calculate the number of entries in a page directory or page table.
 
 132 */
 133static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
 134				      unsigned level)
 135{
 136	unsigned shift = amdgpu_vm_level_shift(adev,
 137					       adev->vm_manager.root_level);
 138
 139	if (level == adev->vm_manager.root_level)
 140		/* For the root directory */
 141		return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift;
 142	else if (level != AMDGPU_VM_PTB)
 143		/* Everything in between */
 144		return 512;
 145	else
 146		/* For the page tables on the leaves */
 147		return AMDGPU_VM_PTE_COUNT(adev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 148}
 149
 150/**
 151 * amdgpu_vm_bo_size - returns the size of the BOs in bytes
 152 *
 153 * @adev: amdgpu_device pointer
 154 *
 155 * Calculate the size of the BO for a page directory or page table in bytes.
 
 156 */
 157static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
 158{
 159	return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 160}
 161
 162/**
 163 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
 164 *
 165 * @vm: vm providing the BOs
 166 * @validated: head of validation list
 167 * @entry: entry to add
 168 *
 169 * Add the page directory to the list of BOs to
 170 * validate for command submission.
 171 */
 172void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 173			 struct list_head *validated,
 174			 struct amdgpu_bo_list_entry *entry)
 175{
 176	entry->robj = vm->root.base.bo;
 177	entry->priority = 0;
 178	entry->tv.bo = &entry->robj->tbo;
 179	entry->tv.shared = true;
 
 180	entry->user_pages = NULL;
 181	list_add(&entry->tv.head, validated);
 182}
 183
 184/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 185 * amdgpu_vm_validate_pt_bos - validate the page table BOs
 186 *
 187 * @adev: amdgpu device pointer
 188 * @vm: vm providing the BOs
 189 * @validate: callback to do the validation
 190 * @param: parameter for the validation callback
 191 *
 192 * Validate the page table BOs on command submission if neccessary.
 
 
 
 193 */
 194int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 195			      int (*validate)(void *p, struct amdgpu_bo *bo),
 196			      void *param)
 197{
 198	struct ttm_bo_global *glob = adev->mman.bdev.glob;
 
 
 199	int r;
 200
 201	spin_lock(&vm->status_lock);
 202	while (!list_empty(&vm->evicted)) {
 203		struct amdgpu_vm_bo_base *bo_base;
 204		struct amdgpu_bo *bo;
 205
 206		bo_base = list_first_entry(&vm->evicted,
 207					   struct amdgpu_vm_bo_base,
 208					   vm_status);
 209		spin_unlock(&vm->status_lock);
 210
 211		bo = bo_base->bo;
 212		BUG_ON(!bo);
 213		if (bo->parent) {
 214			r = validate(param, bo);
 215			if (r)
 216				return r;
 217
 218			spin_lock(&glob->lru_lock);
 219			ttm_bo_move_to_lru_tail(&bo->tbo);
 220			if (bo->shadow)
 221				ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
 222			spin_unlock(&glob->lru_lock);
 223		}
 224
 225		if (bo->tbo.type == ttm_bo_type_kernel &&
 226		    vm->use_cpu_for_update) {
 227			r = amdgpu_bo_kmap(bo, NULL);
 
 
 228			if (r)
 229				return r;
 230		}
 231
 
 
 
 
 
 
 232		spin_lock(&vm->status_lock);
 233		if (bo->tbo.type != ttm_bo_type_kernel)
 234			list_move(&bo_base->vm_status, &vm->moved);
 235		else
 236			list_move(&bo_base->vm_status, &vm->relocated);
 237	}
 238	spin_unlock(&vm->status_lock);
 239
 
 
 
 
 240	return 0;
 241}
 242
 243/**
 244 * amdgpu_vm_ready - check VM is ready for updates
 245 *
 246 * @vm: VM to check
 247 *
 248 * Check if all VM PDs/PTs are ready for updates
 
 
 
 249 */
 250bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 251{
 252	bool ready;
 
 
 
 
 
 253
 254	spin_lock(&vm->status_lock);
 255	ready = list_empty(&vm->evicted);
 256	spin_unlock(&vm->status_lock);
 257
 258	return ready;
 259}
 260
 261/**
 262 * amdgpu_vm_clear_bo - initially clear the PDs/PTs
 263 *
 264 * @adev: amdgpu_device pointer
 265 * @bo: BO to clear
 266 * @level: level this BO is at
 267 *
 268 * Root PD needs to be reserved when calling this.
 269 */
 270static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 271			      struct amdgpu_vm *vm, struct amdgpu_bo *bo,
 272			      unsigned level, bool pte_support_ats)
 273{
 274	struct ttm_operation_ctx ctx = { true, false };
 275	struct dma_fence *fence = NULL;
 276	unsigned entries, ats_entries;
 277	struct amdgpu_ring *ring;
 278	struct amdgpu_job *job;
 279	uint64_t addr;
 280	int r;
 281
 282	addr = amdgpu_bo_gpu_offset(bo);
 283	entries = amdgpu_bo_size(bo) / 8;
 284
 285	if (pte_support_ats) {
 286		if (level == adev->vm_manager.root_level) {
 287			ats_entries = amdgpu_vm_level_shift(adev, level);
 288			ats_entries += AMDGPU_GPU_PAGE_SHIFT;
 289			ats_entries = AMDGPU_VA_HOLE_START >> ats_entries;
 290			ats_entries = min(ats_entries, entries);
 291			entries -= ats_entries;
 292		} else {
 293			ats_entries = entries;
 294			entries = 0;
 295		}
 296	} else {
 297		ats_entries = 0;
 298	}
 299
 300	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 301
 302	r = reservation_object_reserve_shared(bo->tbo.resv);
 303	if (r)
 304		return r;
 305
 306	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 307	if (r)
 308		goto error;
 309
 310	r = amdgpu_job_alloc_with_ib(adev, 64, &job);
 311	if (r)
 312		goto error;
 313
 314	if (ats_entries) {
 315		uint64_t ats_value;
 316
 317		ats_value = AMDGPU_PTE_DEFAULT_ATC;
 318		if (level != AMDGPU_VM_PTB)
 319			ats_value |= AMDGPU_PDE_PTE;
 320
 321		amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
 322				      ats_entries, 0, ats_value);
 323		addr += ats_entries * 8;
 324	}
 325
 326	if (entries)
 327		amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
 328				      entries, 0, 0);
 329
 330	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 331
 332	WARN_ON(job->ibs[0].length_dw > 64);
 333	r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv,
 334			     AMDGPU_FENCE_OWNER_UNDEFINED, false);
 335	if (r)
 336		goto error_free;
 337
 338	r = amdgpu_job_submit(job, ring, &vm->entity,
 339			      AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
 340	if (r)
 341		goto error_free;
 342
 343	amdgpu_bo_fence(bo, fence, true);
 344	dma_fence_put(fence);
 345
 346	if (bo->shadow)
 347		return amdgpu_vm_clear_bo(adev, vm, bo->shadow,
 348					  level, pte_support_ats);
 349
 350	return 0;
 351
 352error_free:
 353	amdgpu_job_free(job);
 354
 355error:
 356	return r;
 357}
 358
 359/**
 360 * amdgpu_vm_alloc_levels - allocate the PD/PT levels
 361 *
 362 * @adev: amdgpu_device pointer
 363 * @vm: requested vm
 364 * @saddr: start of the address range
 365 * @eaddr: end of the address range
 366 *
 367 * Make sure the page directories and page tables are allocated
 368 */
 369static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 370				  struct amdgpu_vm *vm,
 371				  struct amdgpu_vm_pt *parent,
 372				  uint64_t saddr, uint64_t eaddr,
 373				  unsigned level, bool ats)
 374{
 375	unsigned shift = amdgpu_vm_level_shift(adev, level);
 376	unsigned pt_idx, from, to;
 377	u64 flags;
 378	int r;
 379
 380	if (!parent->entries) {
 381		unsigned num_entries = amdgpu_vm_num_entries(adev, level);
 382
 383		parent->entries = kvmalloc_array(num_entries,
 384						   sizeof(struct amdgpu_vm_pt),
 385						   GFP_KERNEL | __GFP_ZERO);
 386		if (!parent->entries)
 387			return -ENOMEM;
 388		memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt));
 389	}
 390
 391	from = saddr >> shift;
 392	to = eaddr >> shift;
 393	if (from >= amdgpu_vm_num_entries(adev, level) ||
 394	    to >= amdgpu_vm_num_entries(adev, level))
 395		return -EINVAL;
 396
 397	++level;
 398	saddr = saddr & ((1 << shift) - 1);
 399	eaddr = eaddr & ((1 << shift) - 1);
 400
 401	flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
 402	if (vm->use_cpu_for_update)
 403		flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 404	else
 405		flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
 406				AMDGPU_GEM_CREATE_SHADOW);
 407
 408	/* walk over the address space and allocate the page tables */
 409	for (pt_idx = from; pt_idx <= to; ++pt_idx) {
 410		struct reservation_object *resv = vm->root.base.bo->tbo.resv;
 411		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 412		struct amdgpu_bo *pt;
 413
 414		if (!entry->base.bo) {
 415			r = amdgpu_bo_create(adev,
 416					     amdgpu_vm_bo_size(adev, level),
 417					     AMDGPU_GPU_PAGE_SIZE,
 418					     AMDGPU_GEM_DOMAIN_VRAM, flags,
 419					     ttm_bo_type_kernel, resv, &pt);
 420			if (r)
 421				return r;
 422
 423			r = amdgpu_vm_clear_bo(adev, vm, pt, level, ats);
 424			if (r) {
 425				amdgpu_bo_unref(&pt->shadow);
 426				amdgpu_bo_unref(&pt);
 427				return r;
 428			}
 429
 430			if (vm->use_cpu_for_update) {
 431				r = amdgpu_bo_kmap(pt, NULL);
 432				if (r) {
 433					amdgpu_bo_unref(&pt->shadow);
 434					amdgpu_bo_unref(&pt);
 435					return r;
 436				}
 437			}
 438
 439			/* Keep a reference to the root directory to avoid
 440			* freeing them up in the wrong order.
 441			*/
 442			pt->parent = amdgpu_bo_ref(parent->base.bo);
 443
 444			entry->base.vm = vm;
 445			entry->base.bo = pt;
 446			list_add_tail(&entry->base.bo_list, &pt->va);
 447			spin_lock(&vm->status_lock);
 448			list_add(&entry->base.vm_status, &vm->relocated);
 449			spin_unlock(&vm->status_lock);
 450		}
 451
 452		if (level < AMDGPU_VM_PTB) {
 453			uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
 454			uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
 455				((1 << shift) - 1);
 456			r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
 457						   sub_eaddr, level, ats);
 458			if (r)
 459				return r;
 460		}
 461	}
 462
 463	return 0;
 464}
 465
 466/**
 467 * amdgpu_vm_alloc_pts - Allocate page tables.
 468 *
 469 * @adev: amdgpu_device pointer
 470 * @vm: VM to allocate page tables for
 471 * @saddr: Start address which needs to be allocated
 472 * @size: Size from start address we need.
 473 *
 474 * Make sure the page tables are allocated.
 475 */
 476int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 477			struct amdgpu_vm *vm,
 478			uint64_t saddr, uint64_t size)
 479{
 480	uint64_t eaddr;
 481	bool ats = false;
 482
 483	/* validate the parameters */
 484	if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
 485		return -EINVAL;
 486
 487	eaddr = saddr + size - 1;
 488
 489	if (vm->pte_support_ats)
 490		ats = saddr < AMDGPU_VA_HOLE_START;
 491
 492	saddr /= AMDGPU_GPU_PAGE_SIZE;
 493	eaddr /= AMDGPU_GPU_PAGE_SIZE;
 494
 495	if (eaddr >= adev->vm_manager.max_pfn) {
 496		dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
 497			eaddr, adev->vm_manager.max_pfn);
 498		return -EINVAL;
 499	}
 500
 501	return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr,
 502				      adev->vm_manager.root_level, ats);
 503}
 504
 505/**
 506 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
 507 *
 508 * @adev: amdgpu_device pointer
 509 */
 510void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
 511{
 512	const struct amdgpu_ip_block *ip_block;
 513	bool has_compute_vm_bug;
 514	struct amdgpu_ring *ring;
 515	int i;
 516
 517	has_compute_vm_bug = false;
 518
 519	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 520	if (ip_block) {
 521		/* Compute has a VM bug for GFX version < 7.
 522		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
 523		if (ip_block->version->major <= 7)
 524			has_compute_vm_bug = true;
 525		else if (ip_block->version->major == 8)
 526			if (adev->gfx.mec_fw_version < 673)
 527				has_compute_vm_bug = true;
 528	}
 529
 530	for (i = 0; i < adev->num_rings; i++) {
 531		ring = adev->rings[i];
 532		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
 533			/* only compute rings */
 534			ring->has_compute_vm_bug = has_compute_vm_bug;
 535		else
 536			ring->has_compute_vm_bug = false;
 537	}
 538}
 539
 
 
 
 
 
 
 
 
 
 540bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 541				  struct amdgpu_job *job)
 542{
 543	struct amdgpu_device *adev = ring->adev;
 544	unsigned vmhub = ring->funcs->vmhub;
 545	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 546	struct amdgpu_vmid *id;
 547	bool gds_switch_needed;
 548	bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
 549
 550	if (job->vmid == 0)
 551		return false;
 552	id = &id_mgr->ids[job->vmid];
 553	gds_switch_needed = ring->funcs->emit_gds_switch && (
 554		id->gds_base != job->gds_base ||
 555		id->gds_size != job->gds_size ||
 556		id->gws_base != job->gws_base ||
 557		id->gws_size != job->gws_size ||
 558		id->oa_base != job->oa_base ||
 559		id->oa_size != job->oa_size);
 560
 561	if (amdgpu_vmid_had_gpu_reset(adev, id))
 562		return true;
 563
 564	return vm_flush_needed || gds_switch_needed;
 565}
 566
 567static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
 568{
 569	return (adev->gmc.real_vram_size == adev->gmc.visible_vram_size);
 
 570}
 571
 572/**
 573 * amdgpu_vm_flush - hardware flush the vm
 574 *
 575 * @ring: ring to use for flush
 576 * @vmid: vmid number to use
 577 * @pd_addr: address of the page directory
 578 *
 579 * Emit a VM flush when it is necessary.
 
 
 
 580 */
 581int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
 
 582{
 583	struct amdgpu_device *adev = ring->adev;
 584	unsigned vmhub = ring->funcs->vmhub;
 585	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 586	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
 587	bool gds_switch_needed = ring->funcs->emit_gds_switch && (
 588		id->gds_base != job->gds_base ||
 589		id->gds_size != job->gds_size ||
 590		id->gws_base != job->gws_base ||
 591		id->gws_size != job->gws_size ||
 592		id->oa_base != job->oa_base ||
 593		id->oa_size != job->oa_size);
 594	bool vm_flush_needed = job->vm_needs_flush;
 595	bool pasid_mapping_needed = id->pasid != job->pasid ||
 596		!id->pasid_mapping ||
 597		!dma_fence_is_signaled(id->pasid_mapping);
 598	struct dma_fence *fence = NULL;
 
 599	unsigned patch_offset = 0;
 600	int r;
 601
 602	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
 603		gds_switch_needed = true;
 604		vm_flush_needed = true;
 605		pasid_mapping_needed = true;
 
 606	}
 607
 
 
 
 
 
 
 608	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
 609	vm_flush_needed &= !!ring->funcs->emit_vm_flush;
 
 610	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
 611		ring->funcs->emit_wreg;
 612
 613	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
 614		return 0;
 615
 
 616	if (ring->funcs->init_cond_exec)
 617		patch_offset = amdgpu_ring_init_cond_exec(ring);
 618
 619	if (need_pipe_sync)
 620		amdgpu_ring_emit_pipeline_sync(ring);
 621
 622	if (vm_flush_needed) {
 623		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
 624		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
 625	}
 626
 627	if (pasid_mapping_needed)
 628		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
 629
 
 
 
 
 
 
 
 
 
 
 
 630	if (vm_flush_needed || pasid_mapping_needed) {
 631		r = amdgpu_fence_emit(ring, &fence);
 632		if (r)
 633			return r;
 634	}
 635
 636	if (vm_flush_needed) {
 637		mutex_lock(&id_mgr->lock);
 638		dma_fence_put(id->last_flush);
 639		id->last_flush = dma_fence_get(fence);
 640		id->current_gpu_reset_count =
 641			atomic_read(&adev->gpu_reset_counter);
 642		mutex_unlock(&id_mgr->lock);
 643	}
 644
 645	if (pasid_mapping_needed) {
 
 646		id->pasid = job->pasid;
 647		dma_fence_put(id->pasid_mapping);
 648		id->pasid_mapping = dma_fence_get(fence);
 
 649	}
 650	dma_fence_put(fence);
 651
 652	if (ring->funcs->emit_gds_switch && gds_switch_needed) {
 653		id->gds_base = job->gds_base;
 654		id->gds_size = job->gds_size;
 655		id->gws_base = job->gws_base;
 656		id->gws_size = job->gws_size;
 657		id->oa_base = job->oa_base;
 658		id->oa_size = job->oa_size;
 659		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
 660					    job->gds_size, job->gws_base,
 661					    job->gws_size, job->oa_base,
 662					    job->oa_size);
 663	}
 664
 665	if (ring->funcs->patch_cond_exec)
 666		amdgpu_ring_patch_cond_exec(ring, patch_offset);
 667
 668	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
 669	if (ring->funcs->emit_switch_buffer) {
 670		amdgpu_ring_emit_switch_buffer(ring);
 671		amdgpu_ring_emit_switch_buffer(ring);
 672	}
 
 673	return 0;
 674}
 675
 676/**
 677 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 678 *
 679 * @vm: requested vm
 680 * @bo: requested buffer object
 681 *
 682 * Find @bo inside the requested vm.
 683 * Search inside the @bos vm list for the requested vm
 684 * Returns the found bo_va or NULL if none is found
 685 *
 686 * Object has to be reserved!
 
 
 
 687 */
 688struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 689				       struct amdgpu_bo *bo)
 690{
 691	struct amdgpu_bo_va *bo_va;
 692
 693	list_for_each_entry(bo_va, &bo->va, base.bo_list) {
 694		if (bo_va->base.vm == vm) {
 695			return bo_va;
 696		}
 697	}
 698	return NULL;
 699}
 700
 701/**
 702 * amdgpu_vm_do_set_ptes - helper to call the right asic function
 703 *
 704 * @params: see amdgpu_pte_update_params definition
 705 * @bo: PD/PT to update
 706 * @pe: addr of the page entry
 707 * @addr: dst addr to write into pe
 708 * @count: number of page entries to update
 709 * @incr: increase next addr by incr bytes
 710 * @flags: hw access flags
 711 *
 712 * Traces the parameters and calls the right asic functions
 713 * to setup the page table using the DMA.
 714 */
 715static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
 716				  struct amdgpu_bo *bo,
 717				  uint64_t pe, uint64_t addr,
 718				  unsigned count, uint32_t incr,
 719				  uint64_t flags)
 720{
 721	pe += amdgpu_bo_gpu_offset(bo);
 722	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
 723
 724	if (count < 3) {
 725		amdgpu_vm_write_pte(params->adev, params->ib, pe,
 726				    addr | flags, count, incr);
 727
 728	} else {
 729		amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
 730				      count, incr, flags);
 731	}
 732}
 733
 734/**
 735 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
 736 *
 737 * @params: see amdgpu_pte_update_params definition
 738 * @bo: PD/PT to update
 739 * @pe: addr of the page entry
 740 * @addr: dst addr to write into pe
 741 * @count: number of page entries to update
 742 * @incr: increase next addr by incr bytes
 743 * @flags: hw access flags
 744 *
 745 * Traces the parameters and calls the DMA function to copy the PTEs.
 746 */
 747static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
 748				   struct amdgpu_bo *bo,
 749				   uint64_t pe, uint64_t addr,
 750				   unsigned count, uint32_t incr,
 751				   uint64_t flags)
 752{
 753	uint64_t src = (params->src + (addr >> 12) * 8);
 754
 755	pe += amdgpu_bo_gpu_offset(bo);
 756	trace_amdgpu_vm_copy_ptes(pe, src, count);
 757
 758	amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
 759}
 760
 761/**
 762 * amdgpu_vm_map_gart - Resolve gart mapping of addr
 763 *
 764 * @pages_addr: optional DMA address to use for lookup
 765 * @addr: the unmapped addr
 766 *
 767 * Look up the physical address of the page that the pte resolves
 768 * to and return the pointer for the page table entry.
 
 
 
 769 */
 770static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 771{
 772	uint64_t result;
 773
 774	/* page table offset */
 775	result = pages_addr[addr >> PAGE_SHIFT];
 776
 777	/* in case cpu page size != gpu page size*/
 778	result |= addr & (~PAGE_MASK);
 779
 780	result &= 0xFFFFFFFFFFFFF000ULL;
 781
 782	return result;
 783}
 784
 785/**
 786 * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
 787 *
 788 * @params: see amdgpu_pte_update_params definition
 789 * @bo: PD/PT to update
 790 * @pe: kmap addr of the page entry
 791 * @addr: dst addr to write into pe
 792 * @count: number of page entries to update
 793 * @incr: increase next addr by incr bytes
 794 * @flags: hw access flags
 795 *
 796 * Write count number of PT/PD entries directly.
 797 */
 798static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
 799				   struct amdgpu_bo *bo,
 800				   uint64_t pe, uint64_t addr,
 801				   unsigned count, uint32_t incr,
 802				   uint64_t flags)
 803{
 804	unsigned int i;
 805	uint64_t value;
 806
 807	pe += (unsigned long)amdgpu_bo_kptr(bo);
 808
 809	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
 810
 811	for (i = 0; i < count; i++) {
 812		value = params->pages_addr ?
 813			amdgpu_vm_map_gart(params->pages_addr, addr) :
 814			addr;
 815		amdgpu_gmc_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
 816				       i, value, flags);
 817		addr += incr;
 818	}
 819}
 820
 821static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 822			     void *owner)
 823{
 824	struct amdgpu_sync sync;
 825	int r;
 826
 827	amdgpu_sync_create(&sync);
 828	amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);
 829	r = amdgpu_sync_wait(&sync, true);
 830	amdgpu_sync_free(&sync);
 831
 832	return r;
 833}
 834
 835/*
 836 * amdgpu_vm_update_pde - update a single level in the hierarchy
 837 *
 838 * @param: parameters for the update
 839 * @vm: requested vm
 840 * @parent: parent directory
 841 * @entry: entry to update
 842 *
 843 * Makes sure the requested entry in parent is up to date.
 844 */
 845static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params *params,
 846				 struct amdgpu_vm *vm,
 847				 struct amdgpu_vm_pt *parent,
 848				 struct amdgpu_vm_pt *entry)
 849{
 850	struct amdgpu_bo *bo = parent->base.bo, *pbo;
 851	uint64_t pde, pt, flags;
 852	unsigned level;
 853
 854	/* Don't update huge pages here */
 855	if (entry->huge)
 856		return;
 857
 858	for (level = 0, pbo = bo->parent; pbo; ++level)
 859		pbo = pbo->parent;
 860
 861	level += params->adev->vm_manager.root_level;
 862	pt = amdgpu_bo_gpu_offset(entry->base.bo);
 863	flags = AMDGPU_PTE_VALID;
 864	amdgpu_gmc_get_vm_pde(params->adev, level, &pt, &flags);
 865	pde = (entry - parent->entries) * 8;
 866	if (bo->shadow)
 867		params->func(params, bo->shadow, pde, pt, 1, 0, flags);
 868	params->func(params, bo, pde, pt, 1, 0, flags);
 869}
 870
 871/*
 872 * amdgpu_vm_invalidate_level - mark all PD levels as invalid
 873 *
 874 * @parent: parent PD
 875 *
 876 * Mark all PD level as invalid after an error.
 877 */
 878static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev,
 879				       struct amdgpu_vm *vm,
 880				       struct amdgpu_vm_pt *parent,
 881				       unsigned level)
 882{
 883	unsigned pt_idx, num_entries;
 884
 885	/*
 886	 * Recurse into the subdirectories. This recursion is harmless because
 887	 * we only have a maximum of 5 layers.
 888	 */
 889	num_entries = amdgpu_vm_num_entries(adev, level);
 890	for (pt_idx = 0; pt_idx < num_entries; ++pt_idx) {
 891		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 892
 893		if (!entry->base.bo)
 894			continue;
 895
 896		spin_lock(&vm->status_lock);
 897		if (list_empty(&entry->base.vm_status))
 898			list_add(&entry->base.vm_status, &vm->relocated);
 899		spin_unlock(&vm->status_lock);
 900		amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);
 901	}
 902}
 903
 904/*
 905 * amdgpu_vm_update_directories - make sure that all directories are valid
 906 *
 907 * @adev: amdgpu_device pointer
 908 * @vm: requested vm
 
 909 *
 910 * Makes sure all directories are up to date.
 911 * Returns 0 for success, error for failure.
 
 
 912 */
 913int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 914				 struct amdgpu_vm *vm)
 915{
 916	struct amdgpu_pte_update_params params;
 917	struct amdgpu_job *job;
 918	unsigned ndw = 0;
 919	int r = 0;
 920
 921	if (list_empty(&vm->relocated))
 922		return 0;
 923
 924restart:
 925	memset(&params, 0, sizeof(params));
 926	params.adev = adev;
 927
 928	if (vm->use_cpu_for_update) {
 929		r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
 930		if (unlikely(r))
 931			return r;
 932
 933		params.func = amdgpu_vm_cpu_set_ptes;
 934	} else {
 935		ndw = 512 * 8;
 936		r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 937		if (r)
 938			return r;
 939
 940		params.ib = &job->ibs[0];
 941		params.func = amdgpu_vm_do_set_ptes;
 942	}
 943
 944	spin_lock(&vm->status_lock);
 945	while (!list_empty(&vm->relocated)) {
 946		struct amdgpu_vm_bo_base *bo_base, *parent;
 947		struct amdgpu_vm_pt *pt, *entry;
 948		struct amdgpu_bo *bo;
 949
 950		bo_base = list_first_entry(&vm->relocated,
 951					   struct amdgpu_vm_bo_base,
 952					   vm_status);
 953		list_del_init(&bo_base->vm_status);
 954		spin_unlock(&vm->status_lock);
 955
 956		bo = bo_base->bo->parent;
 957		if (!bo) {
 958			spin_lock(&vm->status_lock);
 959			continue;
 960		}
 961
 962		parent = list_first_entry(&bo->va, struct amdgpu_vm_bo_base,
 963					  bo_list);
 964		pt = container_of(parent, struct amdgpu_vm_pt, base);
 965		entry = container_of(bo_base, struct amdgpu_vm_pt, base);
 966
 967		amdgpu_vm_update_pde(&params, vm, pt, entry);
 
 
 968
 969		spin_lock(&vm->status_lock);
 970		if (!vm->use_cpu_for_update &&
 971		    (ndw - params.ib->length_dw) < 32)
 972			break;
 973	}
 974	spin_unlock(&vm->status_lock);
 975
 976	if (vm->use_cpu_for_update) {
 977		/* Flush HDP */
 978		mb();
 979		amdgpu_asic_flush_hdp(adev, NULL);
 980	} else if (params.ib->length_dw == 0) {
 981		amdgpu_job_free(job);
 982	} else {
 983		struct amdgpu_bo *root = vm->root.base.bo;
 984		struct amdgpu_ring *ring;
 985		struct dma_fence *fence;
 986
 987		ring = container_of(vm->entity.sched, struct amdgpu_ring,
 988				    sched);
 989
 990		amdgpu_ring_pad_ib(ring, params.ib);
 991		amdgpu_sync_resv(adev, &job->sync, root->tbo.resv,
 992				 AMDGPU_FENCE_OWNER_VM, false);
 993		WARN_ON(params.ib->length_dw > ndw);
 994		r = amdgpu_job_submit(job, ring, &vm->entity,
 995				      AMDGPU_FENCE_OWNER_VM, &fence);
 996		if (r)
 997			goto error;
 998
 999		amdgpu_bo_fence(root, fence, true);
1000		dma_fence_put(vm->last_update);
1001		vm->last_update = fence;
1002	}
1003
1004	if (!list_empty(&vm->relocated))
1005		goto restart;
1006
1007	return 0;
1008
1009error:
1010	amdgpu_vm_invalidate_level(adev, vm, &vm->root,
1011				   adev->vm_manager.root_level);
1012	amdgpu_job_free(job);
1013	return r;
1014}
1015
1016/**
1017 * amdgpu_vm_find_entry - find the entry for an address
1018 *
1019 * @p: see amdgpu_pte_update_params definition
1020 * @addr: virtual address in question
1021 * @entry: resulting entry or NULL
1022 * @parent: parent entry
1023 *
1024 * Find the vm_pt entry and it's parent for the given address.
1025 */
1026void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
1027			 struct amdgpu_vm_pt **entry,
1028			 struct amdgpu_vm_pt **parent)
1029{
1030	unsigned level = p->adev->vm_manager.root_level;
1031
1032	*parent = NULL;
1033	*entry = &p->vm->root;
1034	while ((*entry)->entries) {
1035		unsigned shift = amdgpu_vm_level_shift(p->adev, level++);
1036
1037		*parent = *entry;
1038		*entry = &(*entry)->entries[addr >> shift];
1039		addr &= (1ULL << shift) - 1;
 
1040	}
1041
1042	if (level != AMDGPU_VM_PTB)
1043		*entry = NULL;
 
1044}
1045
1046/**
1047 * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages
1048 *
1049 * @p: see amdgpu_pte_update_params definition
1050 * @entry: vm_pt entry to check
1051 * @parent: parent entry
1052 * @nptes: number of PTEs updated with this operation
1053 * @dst: destination address where the PTEs should point to
1054 * @flags: access flags fro the PTEs
1055 *
1056 * Check if we can update the PD with a huge page.
1057 */
1058static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
1059					struct amdgpu_vm_pt *entry,
1060					struct amdgpu_vm_pt *parent,
1061					unsigned nptes, uint64_t dst,
1062					uint64_t flags)
1063{
1064	uint64_t pde;
1065
1066	/* In the case of a mixed PT the PDE must point to it*/
1067	if (p->adev->asic_type >= CHIP_VEGA10 && !p->src &&
1068	    nptes == AMDGPU_VM_PTE_COUNT(p->adev)) {
1069		/* Set the huge page flag to stop scanning at this PDE */
1070		flags |= AMDGPU_PDE_PTE;
1071	}
1072
1073	if (!(flags & AMDGPU_PDE_PTE)) {
1074		if (entry->huge) {
1075			/* Add the entry to the relocated list to update it. */
1076			entry->huge = false;
1077			spin_lock(&p->vm->status_lock);
1078			list_move(&entry->base.vm_status, &p->vm->relocated);
1079			spin_unlock(&p->vm->status_lock);
1080		}
1081		return;
1082	}
1083
1084	entry->huge = true;
1085	amdgpu_gmc_get_vm_pde(p->adev, AMDGPU_VM_PDB0, &dst, &flags);
1086
1087	pde = (entry - parent->entries) * 8;
1088	if (parent->base.bo->shadow)
1089		p->func(p, parent->base.bo->shadow, pde, dst, 1, 0, flags);
1090	p->func(p, parent->base.bo, pde, dst, 1, 0, flags);
1091}
1092
1093/**
1094 * amdgpu_vm_update_ptes - make sure that page tables are valid
1095 *
1096 * @params: see amdgpu_pte_update_params definition
1097 * @vm: requested vm
1098 * @start: start of GPU address range
1099 * @end: end of GPU address range
1100 * @dst: destination address to map to, the next dst inside the function
1101 * @flags: mapping flags
1102 *
1103 * Update the page tables in the range @start - @end.
1104 * Returns 0 for success, -EINVAL for failure.
1105 */
1106static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
1107				  uint64_t start, uint64_t end,
1108				  uint64_t dst, uint64_t flags)
1109{
1110	struct amdgpu_device *adev = params->adev;
1111	const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
1112
1113	uint64_t addr, pe_start;
1114	struct amdgpu_bo *pt;
1115	unsigned nptes;
1116
1117	/* walk over the address space and update the page tables */
1118	for (addr = start; addr < end; addr += nptes,
1119	     dst += nptes * AMDGPU_GPU_PAGE_SIZE) {
1120		struct amdgpu_vm_pt *entry, *parent;
1121
1122		amdgpu_vm_get_entry(params, addr, &entry, &parent);
1123		if (!entry)
1124			return -ENOENT;
1125
1126		if ((addr & ~mask) == (end & ~mask))
1127			nptes = end - addr;
1128		else
1129			nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
1130
1131		amdgpu_vm_handle_huge_pages(params, entry, parent,
1132					    nptes, dst, flags);
1133		/* We don't need to update PTEs for huge pages */
1134		if (entry->huge)
1135			continue;
1136
1137		pt = entry->base.bo;
1138		pe_start = (addr & mask) * 8;
1139		if (pt->shadow)
1140			params->func(params, pt->shadow, pe_start, dst, nptes,
1141				     AMDGPU_GPU_PAGE_SIZE, flags);
1142		params->func(params, pt, pe_start, dst, nptes,
1143			     AMDGPU_GPU_PAGE_SIZE, flags);
1144	}
1145
1146	return 0;
1147}
1148
1149/*
1150 * amdgpu_vm_frag_ptes - add fragment information to PTEs
1151 *
1152 * @params: see amdgpu_pte_update_params definition
1153 * @vm: requested vm
1154 * @start: first PTE to handle
1155 * @end: last PTE to handle
1156 * @dst: addr those PTEs should point to
1157 * @flags: hw mapping flags
1158 * Returns 0 for success, -EINVAL for failure.
1159 */
1160static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
1161				uint64_t start, uint64_t end,
1162				uint64_t dst, uint64_t flags)
1163{
1164	/**
1165	 * The MC L1 TLB supports variable sized pages, based on a fragment
1166	 * field in the PTE. When this field is set to a non-zero value, page
1167	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1168	 * flags are considered valid for all PTEs within the fragment range
1169	 * and corresponding mappings are assumed to be physically contiguous.
1170	 *
1171	 * The L1 TLB can store a single PTE for the whole fragment,
1172	 * significantly increasing the space available for translation
1173	 * caching. This leads to large improvements in throughput when the
1174	 * TLB is under pressure.
1175	 *
1176	 * The L2 TLB distributes small and large fragments into two
1177	 * asymmetric partitions. The large fragment cache is significantly
1178	 * larger. Thus, we try to use large fragments wherever possible.
1179	 * Userspace can support this by aligning virtual base address and
1180	 * allocation size to the fragment size.
1181	 */
1182	unsigned max_frag = params->adev->vm_manager.fragment_size;
1183	int r;
1184
1185	/* system pages are non continuously */
1186	if (params->src || !(flags & AMDGPU_PTE_VALID))
1187		return amdgpu_vm_update_ptes(params, start, end, dst, flags);
1188
1189	while (start != end) {
1190		uint64_t frag_flags, frag_end;
1191		unsigned frag;
1192
1193		/* This intentionally wraps around if no bit is set */
1194		frag = min((unsigned)ffs(start) - 1,
1195			   (unsigned)fls64(end - start) - 1);
1196		if (frag >= max_frag) {
1197			frag_flags = AMDGPU_PTE_FRAG(max_frag);
1198			frag_end = end & ~((1ULL << max_frag) - 1);
1199		} else {
1200			frag_flags = AMDGPU_PTE_FRAG(frag);
1201			frag_end = start + (1 << frag);
1202		}
1203
1204		r = amdgpu_vm_update_ptes(params, start, frag_end, dst,
1205					  flags | frag_flags);
1206		if (r)
1207			return r;
1208
1209		dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE;
1210		start = frag_end;
1211	}
1212
1213	return 0;
 
 
1214}
1215
1216/**
1217 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1218 *
1219 * @adev: amdgpu_device pointer
1220 * @exclusive: fence we need to sync to
1221 * @pages_addr: DMA addresses to use for mapping
1222 * @vm: requested vm
 
 
1223 * @start: start of mapped range
1224 * @last: last mapped entry
1225 * @flags: flags for the entries
1226 * @addr: addr to set the area to
 
 
 
1227 * @fence: optional resulting fence
1228 *
1229 * Fill in the page table entries between @start and @last.
1230 * Returns 0 for success, -EINVAL for failure.
 
 
1231 */
1232static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1233				       struct dma_fence *exclusive,
1234				       dma_addr_t *pages_addr,
1235				       struct amdgpu_vm *vm,
1236				       uint64_t start, uint64_t last,
1237				       uint64_t flags, uint64_t addr,
1238				       struct dma_fence **fence)
1239{
1240	struct amdgpu_ring *ring;
1241	void *owner = AMDGPU_FENCE_OWNER_VM;
1242	unsigned nptes, ncmds, ndw;
1243	struct amdgpu_job *job;
1244	struct amdgpu_pte_update_params params;
1245	struct dma_fence *f = NULL;
1246	int r;
1247
1248	memset(&params, 0, sizeof(params));
1249	params.adev = adev;
1250	params.vm = vm;
1251
1252	/* sync to everything on unmapping */
1253	if (!(flags & AMDGPU_PTE_VALID))
1254		owner = AMDGPU_FENCE_OWNER_UNDEFINED;
1255
1256	if (vm->use_cpu_for_update) {
1257		/* params.src is used as flag to indicate system Memory */
1258		if (pages_addr)
1259			params.src = ~0;
1260
1261		/* Wait for PT BOs to be free. PTs share the same resv. object
1262		 * as the root PD BO
1263		 */
1264		r = amdgpu_vm_wait_pd(adev, vm, owner);
1265		if (unlikely(r))
1266			return r;
1267
1268		params.func = amdgpu_vm_cpu_set_ptes;
1269		params.pages_addr = pages_addr;
1270		return amdgpu_vm_frag_ptes(&params, start, last + 1,
1271					   addr, flags);
1272	}
1273
1274	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
1275
1276	nptes = last - start + 1;
 
 
1277
1278	/*
1279	 * reserve space for two commands every (1 << BLOCK_SIZE)
1280	 *  entries or 2k dwords (whatever is smaller)
1281         *
1282         * The second command is for the shadow pagetables.
1283	 */
1284	if (vm->root.base.bo->shadow)
1285		ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
1286	else
1287		ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1);
1288
1289	/* padding, etc. */
1290	ndw = 64;
1291
1292	if (pages_addr) {
1293		/* copy commands needed */
1294		ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
1295
1296		/* and also PTEs */
1297		ndw += nptes * 2;
1298
1299		params.func = amdgpu_vm_do_copy_ptes;
1300
1301	} else {
1302		/* set page commands needed */
1303		ndw += ncmds * 10;
1304
1305		/* extra commands for begin/end fragments */
1306		ndw += 2 * 10 * adev->vm_manager.fragment_size;
 
 
 
 
 
1307
1308		params.func = amdgpu_vm_do_set_ptes;
 
 
 
1309	}
1310
1311	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1312	if (r)
1313		return r;
1314
1315	params.ib = &job->ibs[0];
1316
1317	if (pages_addr) {
1318		uint64_t *pte;
1319		unsigned i;
1320
1321		/* Put the PTEs at the end of the IB. */
1322		i = ndw - nptes * 2;
1323		pte= (uint64_t *)&(job->ibs->ptr[i]);
1324		params.src = job->ibs->gpu_addr + i * 4;
1325
1326		for (i = 0; i < nptes; ++i) {
1327			pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
1328						    AMDGPU_GPU_PAGE_SIZE);
1329			pte[i] |= flags;
1330		}
1331		addr = 0;
1332	}
1333
1334	r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
1335	if (r)
1336		goto error_free;
1337
1338	r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
1339			     owner, false);
1340	if (r)
1341		goto error_free;
1342
1343	r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
1344	if (r)
1345		goto error_free;
1346
1347	r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
1348	if (r)
1349		goto error_free;
1350
1351	amdgpu_ring_pad_ib(ring, params.ib);
1352	WARN_ON(params.ib->length_dw > ndw);
1353	r = amdgpu_job_submit(job, ring, &vm->entity,
1354			      AMDGPU_FENCE_OWNER_VM, &f);
1355	if (r)
1356		goto error_free;
1357
1358	amdgpu_bo_fence(vm->root.base.bo, f, true);
1359	dma_fence_put(*fence);
1360	*fence = f;
1361	return 0;
1362
1363error_free:
1364	amdgpu_job_free(job);
1365	return r;
1366}
1367
1368/**
1369 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1370 *
1371 * @adev: amdgpu_device pointer
1372 * @exclusive: fence we need to sync to
1373 * @pages_addr: DMA addresses to use for mapping
1374 * @vm: requested vm
1375 * @mapping: mapped range and flags to use for the update
1376 * @flags: HW flags for the mapping
1377 * @nodes: array of drm_mm_nodes with the MC addresses
1378 * @fence: optional resulting fence
1379 *
1380 * Split the mapping into smaller chunks so that each update fits
1381 * into a SDMA IB.
1382 * Returns 0 for success, -EINVAL for failure.
1383 */
1384static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1385				      struct dma_fence *exclusive,
1386				      dma_addr_t *pages_addr,
1387				      struct amdgpu_vm *vm,
1388				      struct amdgpu_bo_va_mapping *mapping,
1389				      uint64_t flags,
1390				      struct drm_mm_node *nodes,
1391				      struct dma_fence **fence)
1392{
1393	unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
1394	uint64_t pfn, start = mapping->start;
1395	int r;
1396
1397	/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1398	 * but in case of something, we filter the flags in first place
1399	 */
1400	if (!(mapping->flags & AMDGPU_PTE_READABLE))
1401		flags &= ~AMDGPU_PTE_READABLE;
1402	if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1403		flags &= ~AMDGPU_PTE_WRITEABLE;
1404
1405	flags &= ~AMDGPU_PTE_EXECUTABLE;
1406	flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1407
1408	flags &= ~AMDGPU_PTE_MTYPE_MASK;
1409	flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
1410
1411	if ((mapping->flags & AMDGPU_PTE_PRT) &&
1412	    (adev->asic_type >= CHIP_VEGA10)) {
1413		flags |= AMDGPU_PTE_PRT;
1414		flags &= ~AMDGPU_PTE_VALID;
1415	}
1416
1417	trace_amdgpu_vm_bo_update(mapping);
1418
1419	pfn = mapping->offset >> PAGE_SHIFT;
1420	if (nodes) {
1421		while (pfn >= nodes->size) {
1422			pfn -= nodes->size;
1423			++nodes;
1424		}
1425	}
1426
1427	do {
1428		dma_addr_t *dma_addr = NULL;
1429		uint64_t max_entries;
1430		uint64_t addr, last;
1431
1432		if (nodes) {
1433			addr = nodes->start << PAGE_SHIFT;
1434			max_entries = (nodes->size - pfn) *
1435				(PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
1436		} else {
1437			addr = 0;
1438			max_entries = S64_MAX;
1439		}
1440
 
1441		if (pages_addr) {
1442			uint64_t count;
1443
1444			max_entries = min(max_entries, 16ull * 1024ull);
1445			for (count = 1; count < max_entries; ++count) {
1446				uint64_t idx = pfn + count;
1447
1448				if (pages_addr[idx] !=
1449				    (pages_addr[idx - 1] + PAGE_SIZE))
1450					break;
 
 
 
 
 
 
 
 
 
 
 
1451			}
1452
1453			if (count < min_linear_pages) {
1454				addr = pfn << PAGE_SHIFT;
1455				dma_addr = pages_addr;
1456			} else {
1457				addr = pages_addr[pfn];
1458				max_entries = count;
1459			}
1460
1461		} else if (flags & AMDGPU_PTE_VALID) {
1462			addr += adev->vm_manager.vram_base_offset;
1463			addr += pfn << PAGE_SHIFT;
 
1464		}
1465
1466		last = min((uint64_t)mapping->last, start + max_entries - 1);
1467		r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
1468						start, last, flags, addr,
1469						fence);
1470		if (r)
1471			return r;
1472
1473		pfn += last - start + 1;
1474		if (nodes && nodes->size == pfn) {
1475			pfn = 0;
1476			++nodes;
 
 
 
 
 
 
 
 
 
 
 
1477		}
1478		start = last + 1;
 
1479
1480	} while (unlikely(start != mapping->last + 1));
 
1481
1482	return 0;
 
 
 
1483}
1484
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1485/**
1486 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1487 *
1488 * @adev: amdgpu_device pointer
1489 * @bo_va: requested BO and VM object
1490 * @clear: if true clear the entries
1491 *
1492 * Fill in the page table entries for @bo_va.
1493 * Returns 0 for success, -EINVAL for failure.
 
 
1494 */
1495int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1496			struct amdgpu_bo_va *bo_va,
1497			bool clear)
1498{
1499	struct amdgpu_bo *bo = bo_va->base.bo;
1500	struct amdgpu_vm *vm = bo_va->base.vm;
1501	struct amdgpu_bo_va_mapping *mapping;
1502	dma_addr_t *pages_addr = NULL;
1503	struct ttm_mem_reg *mem;
1504	struct drm_mm_node *nodes;
1505	struct dma_fence *exclusive, **last_update;
 
 
1506	uint64_t flags;
1507	int r;
1508
1509	if (clear || !bo_va->base.bo) {
1510		mem = NULL;
1511		nodes = NULL;
1512		exclusive = NULL;
1513	} else {
1514		struct ttm_dma_tt *ttm;
 
 
 
 
 
 
1515
1516		mem = &bo_va->base.bo->tbo.mem;
1517		nodes = mem->mm_node;
1518		if (mem->mem_type == TTM_PL_TT) {
1519			ttm = container_of(bo_va->base.bo->tbo.ttm,
1520					   struct ttm_dma_tt, ttm);
1521			pages_addr = ttm->dma_address;
1522		}
1523		exclusive = reservation_object_get_excl(bo->tbo.resv);
 
 
 
1524	}
1525
1526	if (bo)
 
 
1527		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1528	else
 
 
 
 
 
 
1529		flags = 0x0;
 
 
1530
1531	if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
 
1532		last_update = &vm->last_update;
1533	else
1534		last_update = &bo_va->last_pt_update;
1535
1536	if (!clear && bo_va->base.moved) {
1537		bo_va->base.moved = false;
1538		list_splice_init(&bo_va->valids, &bo_va->invalids);
1539
1540	} else if (bo_va->cleared != clear) {
1541		list_splice_init(&bo_va->valids, &bo_va->invalids);
1542	}
1543
1544	list_for_each_entry(mapping, &bo_va->invalids, list) {
1545		r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
1546					       mapping, flags, nodes,
1547					       last_update);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1548		if (r)
1549			return r;
1550	}
1551
1552	if (vm->use_cpu_for_update) {
1553		/* Flush HDP */
1554		mb();
1555		amdgpu_asic_flush_hdp(adev, NULL);
1556	}
 
1557
1558	spin_lock(&vm->status_lock);
1559	list_del_init(&bo_va->base.vm_status);
1560	spin_unlock(&vm->status_lock);
 
 
 
 
 
1561
1562	list_splice_init(&bo_va->invalids, &bo_va->valids);
1563	bo_va->cleared = clear;
 
1564
1565	if (trace_amdgpu_vm_bo_mapping_enabled()) {
1566		list_for_each_entry(mapping, &bo_va->valids, list)
1567			trace_amdgpu_vm_bo_mapping(mapping);
1568	}
1569
1570	return 0;
1571}
1572
1573/**
1574 * amdgpu_vm_update_prt_state - update the global PRT state
 
 
1575 */
1576static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1577{
1578	unsigned long flags;
1579	bool enable;
1580
1581	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1582	enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1583	adev->gmc.gmc_funcs->set_prt(adev, enable);
1584	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1585}
1586
1587/**
1588 * amdgpu_vm_prt_get - add a PRT user
 
 
1589 */
1590static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1591{
1592	if (!adev->gmc.gmc_funcs->set_prt)
1593		return;
1594
1595	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1596		amdgpu_vm_update_prt_state(adev);
1597}
1598
1599/**
1600 * amdgpu_vm_prt_put - drop a PRT user
 
 
1601 */
1602static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1603{
1604	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1605		amdgpu_vm_update_prt_state(adev);
1606}
1607
1608/**
1609 * amdgpu_vm_prt_cb - callback for updating the PRT status
 
 
 
1610 */
1611static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1612{
1613	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1614
1615	amdgpu_vm_prt_put(cb->adev);
1616	kfree(cb);
1617}
1618
1619/**
1620 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
 
 
 
1621 */
1622static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1623				 struct dma_fence *fence)
1624{
1625	struct amdgpu_prt_cb *cb;
1626
1627	if (!adev->gmc.gmc_funcs->set_prt)
1628		return;
1629
1630	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1631	if (!cb) {
1632		/* Last resort when we are OOM */
1633		if (fence)
1634			dma_fence_wait(fence, false);
1635
1636		amdgpu_vm_prt_put(adev);
1637	} else {
1638		cb->adev = adev;
1639		if (!fence || dma_fence_add_callback(fence, &cb->cb,
1640						     amdgpu_vm_prt_cb))
1641			amdgpu_vm_prt_cb(fence, &cb->cb);
1642	}
1643}
1644
1645/**
1646 * amdgpu_vm_free_mapping - free a mapping
1647 *
1648 * @adev: amdgpu_device pointer
1649 * @vm: requested vm
1650 * @mapping: mapping to be freed
1651 * @fence: fence of the unmap operation
1652 *
1653 * Free a mapping and make sure we decrease the PRT usage count if applicable.
1654 */
1655static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1656				   struct amdgpu_vm *vm,
1657				   struct amdgpu_bo_va_mapping *mapping,
1658				   struct dma_fence *fence)
1659{
1660	if (mapping->flags & AMDGPU_PTE_PRT)
1661		amdgpu_vm_add_prt_cb(adev, fence);
1662	kfree(mapping);
1663}
1664
1665/**
1666 * amdgpu_vm_prt_fini - finish all prt mappings
1667 *
1668 * @adev: amdgpu_device pointer
1669 * @vm: requested vm
1670 *
1671 * Register a cleanup callback to disable PRT support after VM dies.
1672 */
1673static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1674{
1675	struct reservation_object *resv = vm->root.base.bo->tbo.resv;
1676	struct dma_fence *excl, **shared;
1677	unsigned i, shared_count;
1678	int r;
1679
1680	r = reservation_object_get_fences_rcu(resv, &excl,
1681					      &shared_count, &shared);
1682	if (r) {
1683		/* Not enough memory to grab the fence list, as last resort
1684		 * block for all the fences to complete.
1685		 */
1686		reservation_object_wait_timeout_rcu(resv, true, false,
1687						    MAX_SCHEDULE_TIMEOUT);
1688		return;
1689	}
1690
1691	/* Add a callback for each fence in the reservation object */
1692	amdgpu_vm_prt_get(adev);
1693	amdgpu_vm_add_prt_cb(adev, excl);
1694
1695	for (i = 0; i < shared_count; ++i) {
1696		amdgpu_vm_prt_get(adev);
1697		amdgpu_vm_add_prt_cb(adev, shared[i]);
1698	}
1699
1700	kfree(shared);
1701}
1702
1703/**
1704 * amdgpu_vm_clear_freed - clear freed BOs in the PT
1705 *
1706 * @adev: amdgpu_device pointer
1707 * @vm: requested vm
1708 * @fence: optional resulting fence (unchanged if no work needed to be done
1709 * or if an error occurred)
1710 *
1711 * Make sure all freed BOs are cleared in the PT.
1712 * Returns 0 for success.
1713 *
1714 * PTs have to be reserved and mutex must be locked!
 
 
 
 
1715 */
1716int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1717			  struct amdgpu_vm *vm,
1718			  struct dma_fence **fence)
1719{
 
1720	struct amdgpu_bo_va_mapping *mapping;
1721	uint64_t init_pte_value = 0;
1722	struct dma_fence *f = NULL;
1723	int r;
1724
1725	while (!list_empty(&vm->freed)) {
1726		mapping = list_first_entry(&vm->freed,
1727			struct amdgpu_bo_va_mapping, list);
1728		list_del(&mapping->list);
1729
1730		if (vm->pte_support_ats && mapping->start < AMDGPU_VA_HOLE_START)
 
1731			init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
1732
1733		r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
1734						mapping->start, mapping->last,
1735						init_pte_value, 0, &f);
 
1736		amdgpu_vm_free_mapping(adev, vm, mapping, f);
1737		if (r) {
1738			dma_fence_put(f);
1739			return r;
1740		}
1741	}
1742
1743	if (fence && f) {
1744		dma_fence_put(*fence);
1745		*fence = f;
1746	} else {
1747		dma_fence_put(f);
1748	}
1749
1750	return 0;
1751
1752}
1753
1754/**
1755 * amdgpu_vm_handle_moved - handle moved BOs in the PT
1756 *
1757 * @adev: amdgpu_device pointer
1758 * @vm: requested vm
1759 * @sync: sync object to add fences to
1760 *
1761 * Make sure all BOs which are moved are updated in the PTs.
1762 * Returns 0 for success.
 
 
1763 *
1764 * PTs have to be reserved!
1765 */
1766int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1767			   struct amdgpu_vm *vm)
1768{
 
 
1769	bool clear;
1770	int r = 0;
1771
1772	spin_lock(&vm->status_lock);
1773	while (!list_empty(&vm->moved)) {
1774		struct amdgpu_bo_va *bo_va;
1775		struct reservation_object *resv;
1776
1777		bo_va = list_first_entry(&vm->moved,
1778			struct amdgpu_bo_va, base.vm_status);
1779		spin_unlock(&vm->status_lock);
1780
1781		resv = bo_va->base.bo->tbo.resv;
1782
1783		/* Per VM BOs never need to bo cleared in the page tables */
1784		if (resv == vm->root.base.bo->tbo.resv)
1785			clear = false;
 
 
 
 
 
 
 
 
 
 
1786		/* Try to reserve the BO to avoid clearing its ptes */
1787		else if (!amdgpu_vm_debug && reservation_object_trylock(resv))
1788			clear = false;
1789		/* Somebody else is using the BO right now */
1790		else
1791			clear = true;
1792
1793		r = amdgpu_vm_bo_update(adev, bo_va, clear);
1794		if (r)
1795			return r;
1796
1797		if (!clear && resv != vm->root.base.bo->tbo.resv)
1798			reservation_object_unlock(resv);
1799
1800		spin_lock(&vm->status_lock);
1801	}
1802	spin_unlock(&vm->status_lock);
1803
1804	return r;
1805}
1806
1807/**
1808 * amdgpu_vm_bo_add - add a bo to a specific vm
1809 *
1810 * @adev: amdgpu_device pointer
1811 * @vm: requested vm
1812 * @bo: amdgpu buffer object
1813 *
1814 * Add @bo into the requested vm.
1815 * Add @bo to the list of bos associated with the vm
1816 * Returns newly added bo_va or NULL for failure
 
 
1817 *
1818 * Object has to be reserved!
1819 */
1820struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1821				      struct amdgpu_vm *vm,
1822				      struct amdgpu_bo *bo)
1823{
1824	struct amdgpu_bo_va *bo_va;
1825
1826	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1827	if (bo_va == NULL) {
1828		return NULL;
1829	}
1830	bo_va->base.vm = vm;
1831	bo_va->base.bo = bo;
1832	INIT_LIST_HEAD(&bo_va->base.bo_list);
1833	INIT_LIST_HEAD(&bo_va->base.vm_status);
1834
1835	bo_va->ref_count = 1;
1836	INIT_LIST_HEAD(&bo_va->valids);
1837	INIT_LIST_HEAD(&bo_va->invalids);
1838
1839	if (!bo)
1840		return bo_va;
1841
1842	list_add_tail(&bo_va->base.bo_list, &bo->va);
1843
1844	if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
1845		return bo_va;
1846
1847	if (bo->preferred_domains &
1848	    amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
1849		return bo_va;
1850
1851	/*
1852	 * We checked all the prerequisites, but it looks like this per VM BO
1853	 * is currently evicted. add the BO to the evicted list to make sure it
1854	 * is validated on next VM use to avoid fault.
1855	 * */
1856	spin_lock(&vm->status_lock);
1857	list_move_tail(&bo_va->base.vm_status, &vm->evicted);
1858	spin_unlock(&vm->status_lock);
1859
1860	return bo_va;
1861}
1862
1863
1864/**
1865 * amdgpu_vm_bo_insert_mapping - insert a new mapping
1866 *
1867 * @adev: amdgpu_device pointer
1868 * @bo_va: bo_va to store the address
1869 * @mapping: the mapping to insert
1870 *
1871 * Insert a new mapping into all structures.
1872 */
1873static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
1874				    struct amdgpu_bo_va *bo_va,
1875				    struct amdgpu_bo_va_mapping *mapping)
1876{
1877	struct amdgpu_vm *vm = bo_va->base.vm;
1878	struct amdgpu_bo *bo = bo_va->base.bo;
1879
1880	mapping->bo_va = bo_va;
1881	list_add(&mapping->list, &bo_va->invalids);
1882	amdgpu_vm_it_insert(mapping, &vm->va);
1883
1884	if (mapping->flags & AMDGPU_PTE_PRT)
1885		amdgpu_vm_prt_get(adev);
1886
1887	if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
1888		spin_lock(&vm->status_lock);
1889		if (list_empty(&bo_va->base.vm_status))
1890			list_add(&bo_va->base.vm_status, &vm->moved);
1891		spin_unlock(&vm->status_lock);
1892	}
1893	trace_amdgpu_vm_bo_map(bo_va, mapping);
1894}
1895
1896/**
1897 * amdgpu_vm_bo_map - map bo inside a vm
1898 *
1899 * @adev: amdgpu_device pointer
1900 * @bo_va: bo_va to store the address
1901 * @saddr: where to map the BO
1902 * @offset: requested offset in the BO
 
1903 * @flags: attributes of pages (read/write/valid/etc.)
1904 *
1905 * Add a mapping of the BO at the specefied addr into the VM.
1906 * Returns 0 for success, error for failure.
 
 
1907 *
1908 * Object has to be reserved and unreserved outside!
1909 */
1910int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1911		     struct amdgpu_bo_va *bo_va,
1912		     uint64_t saddr, uint64_t offset,
1913		     uint64_t size, uint64_t flags)
1914{
1915	struct amdgpu_bo_va_mapping *mapping, *tmp;
1916	struct amdgpu_bo *bo = bo_va->base.bo;
1917	struct amdgpu_vm *vm = bo_va->base.vm;
1918	uint64_t eaddr;
1919
1920	/* validate the parameters */
1921	if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1922	    size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1923		return -EINVAL;
1924
1925	/* make sure object fit at this offset */
1926	eaddr = saddr + size - 1;
1927	if (saddr >= eaddr ||
1928	    (bo && offset + size > amdgpu_bo_size(bo)))
 
1929		return -EINVAL;
1930
1931	saddr /= AMDGPU_GPU_PAGE_SIZE;
1932	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1933
1934	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1935	if (tmp) {
1936		/* bo and tmp overlap, invalid addr */
1937		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1938			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1939			tmp->start, tmp->last + 1);
1940		return -EINVAL;
1941	}
1942
1943	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1944	if (!mapping)
1945		return -ENOMEM;
1946
1947	mapping->start = saddr;
1948	mapping->last = eaddr;
1949	mapping->offset = offset;
1950	mapping->flags = flags;
1951
1952	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1953
1954	return 0;
1955}
1956
1957/**
1958 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1959 *
1960 * @adev: amdgpu_device pointer
1961 * @bo_va: bo_va to store the address
1962 * @saddr: where to map the BO
1963 * @offset: requested offset in the BO
 
1964 * @flags: attributes of pages (read/write/valid/etc.)
1965 *
1966 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
1967 * mappings as we do so.
1968 * Returns 0 for success, error for failure.
 
 
1969 *
1970 * Object has to be reserved and unreserved outside!
1971 */
1972int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1973			     struct amdgpu_bo_va *bo_va,
1974			     uint64_t saddr, uint64_t offset,
1975			     uint64_t size, uint64_t flags)
1976{
1977	struct amdgpu_bo_va_mapping *mapping;
1978	struct amdgpu_bo *bo = bo_va->base.bo;
1979	uint64_t eaddr;
1980	int r;
1981
1982	/* validate the parameters */
1983	if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1984	    size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1985		return -EINVAL;
1986
1987	/* make sure object fit at this offset */
1988	eaddr = saddr + size - 1;
1989	if (saddr >= eaddr ||
1990	    (bo && offset + size > amdgpu_bo_size(bo)))
 
1991		return -EINVAL;
1992
1993	/* Allocate all the needed memory */
1994	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1995	if (!mapping)
1996		return -ENOMEM;
1997
1998	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
1999	if (r) {
2000		kfree(mapping);
2001		return r;
2002	}
2003
2004	saddr /= AMDGPU_GPU_PAGE_SIZE;
2005	eaddr /= AMDGPU_GPU_PAGE_SIZE;
2006
2007	mapping->start = saddr;
2008	mapping->last = eaddr;
2009	mapping->offset = offset;
2010	mapping->flags = flags;
2011
2012	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2013
2014	return 0;
2015}
2016
2017/**
2018 * amdgpu_vm_bo_unmap - remove bo mapping from vm
2019 *
2020 * @adev: amdgpu_device pointer
2021 * @bo_va: bo_va to remove the address from
2022 * @saddr: where to the BO is mapped
2023 *
2024 * Remove a mapping of the BO at the specefied addr from the VM.
2025 * Returns 0 for success, error for failure.
 
 
2026 *
2027 * Object has to be reserved and unreserved outside!
2028 */
2029int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
2030		       struct amdgpu_bo_va *bo_va,
2031		       uint64_t saddr)
2032{
2033	struct amdgpu_bo_va_mapping *mapping;
2034	struct amdgpu_vm *vm = bo_va->base.vm;
2035	bool valid = true;
2036
2037	saddr /= AMDGPU_GPU_PAGE_SIZE;
2038
2039	list_for_each_entry(mapping, &bo_va->valids, list) {
2040		if (mapping->start == saddr)
2041			break;
2042	}
2043
2044	if (&mapping->list == &bo_va->valids) {
2045		valid = false;
2046
2047		list_for_each_entry(mapping, &bo_va->invalids, list) {
2048			if (mapping->start == saddr)
2049				break;
2050		}
2051
2052		if (&mapping->list == &bo_va->invalids)
2053			return -ENOENT;
2054	}
2055
2056	list_del(&mapping->list);
2057	amdgpu_vm_it_remove(mapping, &vm->va);
2058	mapping->bo_va = NULL;
2059	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2060
2061	if (valid)
2062		list_add(&mapping->list, &vm->freed);
2063	else
2064		amdgpu_vm_free_mapping(adev, vm, mapping,
2065				       bo_va->last_pt_update);
2066
2067	return 0;
2068}
2069
2070/**
2071 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2072 *
2073 * @adev: amdgpu_device pointer
2074 * @vm: VM structure to use
2075 * @saddr: start of the range
2076 * @size: size of the range
2077 *
2078 * Remove all mappings in a range, split them as appropriate.
2079 * Returns 0 for success, error for failure.
 
 
2080 */
2081int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2082				struct amdgpu_vm *vm,
2083				uint64_t saddr, uint64_t size)
2084{
2085	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2086	LIST_HEAD(removed);
2087	uint64_t eaddr;
2088
2089	eaddr = saddr + size - 1;
2090	saddr /= AMDGPU_GPU_PAGE_SIZE;
2091	eaddr /= AMDGPU_GPU_PAGE_SIZE;
2092
2093	/* Allocate all the needed memory */
2094	before = kzalloc(sizeof(*before), GFP_KERNEL);
2095	if (!before)
2096		return -ENOMEM;
2097	INIT_LIST_HEAD(&before->list);
2098
2099	after = kzalloc(sizeof(*after), GFP_KERNEL);
2100	if (!after) {
2101		kfree(before);
2102		return -ENOMEM;
2103	}
2104	INIT_LIST_HEAD(&after->list);
2105
2106	/* Now gather all removed mappings */
2107	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2108	while (tmp) {
2109		/* Remember mapping split at the start */
2110		if (tmp->start < saddr) {
2111			before->start = tmp->start;
2112			before->last = saddr - 1;
2113			before->offset = tmp->offset;
2114			before->flags = tmp->flags;
2115			list_add(&before->list, &tmp->list);
 
2116		}
2117
2118		/* Remember mapping split at the end */
2119		if (tmp->last > eaddr) {
2120			after->start = eaddr + 1;
2121			after->last = tmp->last;
2122			after->offset = tmp->offset;
2123			after->offset += after->start - tmp->start;
2124			after->flags = tmp->flags;
2125			list_add(&after->list, &tmp->list);
 
2126		}
2127
2128		list_del(&tmp->list);
2129		list_add(&tmp->list, &removed);
2130
2131		tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2132	}
2133
2134	/* And free them up */
2135	list_for_each_entry_safe(tmp, next, &removed, list) {
2136		amdgpu_vm_it_remove(tmp, &vm->va);
2137		list_del(&tmp->list);
2138
2139		if (tmp->start < saddr)
2140		    tmp->start = saddr;
2141		if (tmp->last > eaddr)
2142		    tmp->last = eaddr;
2143
2144		tmp->bo_va = NULL;
2145		list_add(&tmp->list, &vm->freed);
2146		trace_amdgpu_vm_bo_unmap(NULL, tmp);
2147	}
2148
2149	/* Insert partial mapping before the range */
2150	if (!list_empty(&before->list)) {
2151		amdgpu_vm_it_insert(before, &vm->va);
2152		if (before->flags & AMDGPU_PTE_PRT)
2153			amdgpu_vm_prt_get(adev);
2154	} else {
2155		kfree(before);
2156	}
2157
2158	/* Insert partial mapping after the range */
2159	if (!list_empty(&after->list)) {
2160		amdgpu_vm_it_insert(after, &vm->va);
2161		if (after->flags & AMDGPU_PTE_PRT)
2162			amdgpu_vm_prt_get(adev);
2163	} else {
2164		kfree(after);
2165	}
2166
2167	return 0;
2168}
2169
2170/**
2171 * amdgpu_vm_bo_lookup_mapping - find mapping by address
2172 *
2173 * @vm: the requested VM
 
2174 *
2175 * Find a mapping by it's address.
 
 
 
 
2176 */
2177struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2178							 uint64_t addr)
2179{
2180	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2181}
2182
2183/**
2184 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2185 *
2186 * @adev: amdgpu_device pointer
2187 * @bo_va: requested bo_va
2188 *
2189 * Remove @bo_va->bo from the requested vm.
2190 *
2191 * Object have to be reserved!
2192 */
2193void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2194		      struct amdgpu_bo_va *bo_va)
2195{
2196	struct amdgpu_bo_va_mapping *mapping, *next;
 
2197	struct amdgpu_vm *vm = bo_va->base.vm;
 
2198
2199	list_del(&bo_va->base.bo_list);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2200
2201	spin_lock(&vm->status_lock);
2202	list_del(&bo_va->base.vm_status);
2203	spin_unlock(&vm->status_lock);
2204
2205	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2206		list_del(&mapping->list);
2207		amdgpu_vm_it_remove(mapping, &vm->va);
2208		mapping->bo_va = NULL;
2209		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2210		list_add(&mapping->list, &vm->freed);
2211	}
2212	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2213		list_del(&mapping->list);
2214		amdgpu_vm_it_remove(mapping, &vm->va);
2215		amdgpu_vm_free_mapping(adev, vm, mapping,
2216				       bo_va->last_pt_update);
2217	}
2218
2219	dma_fence_put(bo_va->last_pt_update);
 
 
 
 
2220	kfree(bo_va);
2221}
2222
2223/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2224 * amdgpu_vm_bo_invalidate - mark the bo as invalid
2225 *
2226 * @adev: amdgpu_device pointer
2227 * @vm: requested vm
2228 * @bo: amdgpu buffer object
 
2229 *
2230 * Mark @bo as invalid.
2231 */
2232void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2233			     struct amdgpu_bo *bo, bool evicted)
2234{
2235	struct amdgpu_vm_bo_base *bo_base;
2236
2237	list_for_each_entry(bo_base, &bo->va, bo_list) {
 
 
 
 
2238		struct amdgpu_vm *vm = bo_base->vm;
2239
2240		bo_base->moved = true;
2241		if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
2242			spin_lock(&bo_base->vm->status_lock);
2243			if (bo->tbo.type == ttm_bo_type_kernel)
2244				list_move(&bo_base->vm_status, &vm->evicted);
2245			else
2246				list_move_tail(&bo_base->vm_status,
2247					       &vm->evicted);
2248			spin_unlock(&bo_base->vm->status_lock);
2249			continue;
2250		}
2251
2252		if (bo->tbo.type == ttm_bo_type_kernel) {
2253			spin_lock(&bo_base->vm->status_lock);
2254			if (list_empty(&bo_base->vm_status))
2255				list_add(&bo_base->vm_status, &vm->relocated);
2256			spin_unlock(&bo_base->vm->status_lock);
2257			continue;
2258		}
2259
2260		spin_lock(&bo_base->vm->status_lock);
2261		if (list_empty(&bo_base->vm_status))
2262			list_add(&bo_base->vm_status, &vm->moved);
2263		spin_unlock(&bo_base->vm->status_lock);
 
 
2264	}
2265}
2266
 
 
 
 
 
 
 
 
2267static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2268{
2269	/* Total bits covered by PD + PTs */
2270	unsigned bits = ilog2(vm_size) + 18;
2271
2272	/* Make sure the PD is 4K in size up to 8GB address space.
2273	   Above that split equal between PD and PTs */
2274	if (vm_size <= 8)
2275		return (bits - 9);
2276	else
2277		return ((bits + 3) / 2);
2278}
2279
2280/**
2281 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2282 *
2283 * @adev: amdgpu_device pointer
2284 * @vm_size: the default vm size if it's set auto
 
 
 
 
2285 */
2286void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
2287			   uint32_t fragment_size_default, unsigned max_level,
2288			   unsigned max_bits)
2289{
 
 
2290	uint64_t tmp;
2291
2292	/* adjust vm size first */
2293	if (amdgpu_vm_size != -1) {
2294		unsigned max_size = 1 << (max_bits - 30);
2295
2296		vm_size = amdgpu_vm_size;
2297		if (vm_size > max_size) {
2298			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2299				 amdgpu_vm_size, max_size);
2300			vm_size = max_size;
2301		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2302	}
2303
2304	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2305
2306	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2307	if (amdgpu_vm_block_size != -1)
2308		tmp >>= amdgpu_vm_block_size - 9;
2309	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2310	adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2311	switch (adev->vm_manager.num_level) {
2312	case 3:
2313		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2314		break;
2315	case 2:
2316		adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2317		break;
2318	case 1:
2319		adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2320		break;
2321	default:
2322		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2323	}
2324	/* block size depends on vm size and hw setup*/
2325	if (amdgpu_vm_block_size != -1)
2326		adev->vm_manager.block_size =
2327			min((unsigned)amdgpu_vm_block_size, max_bits
2328			    - AMDGPU_GPU_PAGE_SHIFT
2329			    - 9 * adev->vm_manager.num_level);
2330	else if (adev->vm_manager.num_level > 1)
2331		adev->vm_manager.block_size = 9;
2332	else
2333		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2334
2335	if (amdgpu_vm_fragment_size == -1)
2336		adev->vm_manager.fragment_size = fragment_size_default;
2337	else
2338		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2339
2340	DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2341		 vm_size, adev->vm_manager.num_level + 1,
2342		 adev->vm_manager.block_size,
2343		 adev->vm_manager.fragment_size);
2344}
2345
2346/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2347 * amdgpu_vm_init - initialize a vm instance
2348 *
2349 * @adev: amdgpu_device pointer
2350 * @vm: requested vm
2351 * @vm_context: Indicates if it GFX or Compute context
2352 *
2353 * Init @vm fields.
 
 
 
2354 */
2355int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2356		   int vm_context, unsigned int pasid)
2357{
2358	const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
2359		AMDGPU_VM_PTE_COUNT(adev) * 8);
2360	unsigned ring_instance;
2361	struct amdgpu_ring *ring;
2362	struct drm_sched_rq *rq;
2363	unsigned long size;
2364	uint64_t flags;
2365	int r, i;
2366
2367	vm->va = RB_ROOT_CACHED;
2368	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2369		vm->reserved_vmid[i] = NULL;
2370	spin_lock_init(&vm->status_lock);
2371	INIT_LIST_HEAD(&vm->evicted);
2372	INIT_LIST_HEAD(&vm->relocated);
2373	INIT_LIST_HEAD(&vm->moved);
 
 
 
2374	INIT_LIST_HEAD(&vm->freed);
2375
2376	/* create scheduler entity for page table updates */
2377
2378	ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
2379	ring_instance %= adev->vm_manager.vm_pte_num_rings;
2380	ring = adev->vm_manager.vm_pte_rings[ring_instance];
2381	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
2382	r = drm_sched_entity_init(&ring->sched, &vm->entity,
2383				  rq, amdgpu_sched_jobs, NULL);
2384	if (r)
2385		return r;
2386
 
 
 
 
 
 
2387	vm->pte_support_ats = false;
 
2388
2389	if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
2390		vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2391						AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2392
2393		if (adev->asic_type == CHIP_RAVEN)
2394			vm->pte_support_ats = true;
2395	} else {
2396		vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2397						AMDGPU_VM_USE_CPU_FOR_GFX);
2398	}
2399	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2400			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2401	WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
 
2402		  "CPU update of VM recommended only for large BAR system\n");
2403	vm->last_update = NULL;
2404
2405	flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
2406	if (vm->use_cpu_for_update)
2407		flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
2408	else
2409		flags |= AMDGPU_GEM_CREATE_SHADOW;
 
 
 
2410
2411	size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level);
2412	r = amdgpu_bo_create(adev, size, align, AMDGPU_GEM_DOMAIN_VRAM, flags,
2413			     ttm_bo_type_kernel, NULL, &vm->root.base.bo);
2414	if (r)
2415		goto error_free_sched_entity;
2416
2417	r = amdgpu_bo_reserve(vm->root.base.bo, true);
 
 
 
 
 
2418	if (r)
2419		goto error_free_root;
2420
2421	r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo,
2422			       adev->vm_manager.root_level,
2423			       vm->pte_support_ats);
2424	if (r)
2425		goto error_unreserve;
2426
2427	vm->root.base.vm = vm;
2428	list_add_tail(&vm->root.base.bo_list, &vm->root.base.bo->va);
2429	list_add_tail(&vm->root.base.vm_status, &vm->evicted);
2430	amdgpu_bo_unreserve(vm->root.base.bo);
2431
2432	if (pasid) {
2433		unsigned long flags;
 
2434
2435		spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2436		r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2437			      GFP_ATOMIC);
2438		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2439		if (r < 0)
2440			goto error_free_root;
2441
2442		vm->pasid = pasid;
2443	}
2444
2445	INIT_KFIFO(vm->faults);
2446	vm->fault_credit = 16;
2447
2448	return 0;
2449
2450error_unreserve:
2451	amdgpu_bo_unreserve(vm->root.base.bo);
2452
2453error_free_root:
2454	amdgpu_bo_unref(&vm->root.base.bo->shadow);
2455	amdgpu_bo_unref(&vm->root.base.bo);
2456	vm->root.base.bo = NULL;
 
 
 
 
 
2457
2458error_free_sched_entity:
2459	drm_sched_entity_fini(&ring->sched, &vm->entity);
2460
2461	return r;
2462}
2463
2464/**
2465 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2466 *
 
 
 
2467 * This only works on GFX VMs that don't have any BOs added and no
2468 * page tables allocated yet.
2469 *
2470 * Changes the following VM parameters:
2471 * - use_cpu_for_update
2472 * - pte_supports_ats
2473 * - pasid (old PASID is released, because compute manages its own PASIDs)
2474 *
2475 * Reinitializes the page directory to reflect the changed ATS
2476 * setting. May leave behind an unused shadow BO for the page
2477 * directory when switching from SDMA updates to CPU updates.
2478 *
2479 * Returns 0 for success, -errno for errors.
 
2480 */
2481int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2482{
2483	bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2484	int r;
2485
2486	r = amdgpu_bo_reserve(vm->root.base.bo, true);
2487	if (r)
2488		return r;
2489
2490	/* Sanity checks */
2491	if (!RB_EMPTY_ROOT(&vm->va.rb_root) || vm->root.entries) {
2492		r = -EINVAL;
2493		goto error;
2494	}
2495
2496	/* Check if PD needs to be reinitialized and do it before
2497	 * changing any other state, in case it fails.
2498	 */
2499	if (pte_support_ats != vm->pte_support_ats) {
2500		r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo,
2501			       adev->vm_manager.root_level,
2502			       pte_support_ats);
2503		if (r)
2504			goto error;
2505	}
2506
2507	/* Update VM state */
2508	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2509				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2510	vm->pte_support_ats = pte_support_ats;
2511	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2512			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2513	WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
 
2514		  "CPU update of VM recommended only for large BAR system\n");
2515
2516	if (vm->pasid) {
2517		unsigned long flags;
2518
2519		spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2520		idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2521		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2522
2523		vm->pasid = 0;
 
 
2524	}
 
 
 
 
 
 
 
2525
2526error:
2527	amdgpu_bo_unreserve(vm->root.base.bo);
 
 
 
 
 
 
 
 
 
2528	return r;
2529}
2530
2531/**
2532 * amdgpu_vm_free_levels - free PD/PT levels
2533 *
2534 * @adev: amdgpu device structure
2535 * @parent: PD/PT starting level to free
2536 * @level: level of parent structure
2537 *
2538 * Free the page directory or page table level and all sub levels.
 
2539 */
2540static void amdgpu_vm_free_levels(struct amdgpu_device *adev,
2541				  struct amdgpu_vm_pt *parent,
2542				  unsigned level)
2543{
2544	unsigned i, num_entries = amdgpu_vm_num_entries(adev, level);
2545
2546	if (parent->base.bo) {
2547		list_del(&parent->base.bo_list);
2548		list_del(&parent->base.vm_status);
2549		amdgpu_bo_unref(&parent->base.bo->shadow);
2550		amdgpu_bo_unref(&parent->base.bo);
2551	}
2552
2553	if (parent->entries)
2554		for (i = 0; i < num_entries; i++)
2555			amdgpu_vm_free_levels(adev, &parent->entries[i],
2556					      level + 1);
2557
2558	kvfree(parent->entries);
2559}
2560
2561/**
2562 * amdgpu_vm_fini - tear down a vm instance
2563 *
2564 * @adev: amdgpu_device pointer
2565 * @vm: requested vm
2566 *
2567 * Tear down @vm.
2568 * Unbind the VM and remove all bos from the vm bo list
2569 */
2570void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2571{
2572	struct amdgpu_bo_va_mapping *mapping, *tmp;
2573	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2574	struct amdgpu_bo *root;
2575	u64 fault;
2576	int i, r;
2577
2578	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2579
2580	/* Clear pending page faults from IH when the VM is destroyed */
2581	while (kfifo_get(&vm->faults, &fault))
2582		amdgpu_ih_clear_fault(adev, fault);
2583
2584	if (vm->pasid) {
2585		unsigned long flags;
2586
2587		spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2588		idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2589		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2590	}
2591
2592	drm_sched_entity_fini(vm->entity.sched, &vm->entity);
 
 
 
 
 
 
 
 
 
2593
2594	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2595		dev_err(adev->dev, "still active bo inside vm\n");
2596	}
2597	rbtree_postorder_for_each_entry_safe(mapping, tmp,
2598					     &vm->va.rb_root, rb) {
2599		list_del(&mapping->list);
2600		amdgpu_vm_it_remove(mapping, &vm->va);
2601		kfree(mapping);
2602	}
2603	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2604		if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2605			amdgpu_vm_prt_fini(adev, vm);
2606			prt_fini_needed = false;
2607		}
2608
2609		list_del(&mapping->list);
2610		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2611	}
2612
2613	root = amdgpu_bo_ref(vm->root.base.bo);
2614	r = amdgpu_bo_reserve(root, true);
2615	if (r) {
2616		dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
2617	} else {
2618		amdgpu_vm_free_levels(adev, &vm->root,
2619				      adev->vm_manager.root_level);
2620		amdgpu_bo_unreserve(root);
2621	}
2622	amdgpu_bo_unref(&root);
2623	dma_fence_put(vm->last_update);
2624	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2625		amdgpu_vmid_free_reserved(adev, vm, i);
2626}
2627
2628/**
2629 * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
2630 *
2631 * @adev: amdgpu_device pointer
2632 * @pasid: PASID do identify the VM
2633 *
2634 * This function is expected to be called in interrupt context. Returns
2635 * true if there was fault credit, false otherwise
2636 */
2637bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
2638				  unsigned int pasid)
2639{
2640	struct amdgpu_vm *vm;
2641
2642	spin_lock(&adev->vm_manager.pasid_lock);
2643	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
2644	if (!vm) {
2645		/* VM not found, can't track fault credit */
2646		spin_unlock(&adev->vm_manager.pasid_lock);
2647		return true;
2648	}
2649
2650	/* No lock needed. only accessed by IRQ handler */
2651	if (!vm->fault_credit) {
2652		/* Too many faults in this VM */
2653		spin_unlock(&adev->vm_manager.pasid_lock);
2654		return false;
 
2655	}
2656
2657	vm->fault_credit--;
2658	spin_unlock(&adev->vm_manager.pasid_lock);
2659	return true;
2660}
2661
2662/**
2663 * amdgpu_vm_manager_init - init the VM manager
2664 *
2665 * @adev: amdgpu_device pointer
2666 *
2667 * Initialize the VM manager structures
2668 */
2669void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2670{
2671	unsigned i;
2672
 
 
 
 
 
 
2673	amdgpu_vmid_mgr_init(adev);
2674
2675	adev->vm_manager.fence_context =
2676		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2677	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2678		adev->vm_manager.seqno[i] = 0;
2679
2680	atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
2681	spin_lock_init(&adev->vm_manager.prt_lock);
2682	atomic_set(&adev->vm_manager.num_prt_users, 0);
2683
2684	/* If not overridden by the user, by default, only in large BAR systems
2685	 * Compute VM tables will be updated by CPU
2686	 */
2687#ifdef CONFIG_X86_64
2688	if (amdgpu_vm_update_mode == -1) {
2689		if (amdgpu_vm_is_large_bar(adev))
 
 
 
 
2690			adev->vm_manager.vm_update_mode =
2691				AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2692		else
2693			adev->vm_manager.vm_update_mode = 0;
2694	} else
2695		adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2696#else
2697	adev->vm_manager.vm_update_mode = 0;
2698#endif
2699
2700	idr_init(&adev->vm_manager.pasid_idr);
2701	spin_lock_init(&adev->vm_manager.pasid_lock);
2702}
2703
2704/**
2705 * amdgpu_vm_manager_fini - cleanup VM manager
2706 *
2707 * @adev: amdgpu_device pointer
2708 *
2709 * Cleanup the VM manager and free resources.
2710 */
2711void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2712{
2713	WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
2714	idr_destroy(&adev->vm_manager.pasid_idr);
2715
2716	amdgpu_vmid_mgr_fini(adev);
2717}
2718
 
 
 
 
 
 
 
 
 
 
2719int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2720{
2721	union drm_amdgpu_vm *args = data;
2722	struct amdgpu_device *adev = dev->dev_private;
2723	struct amdgpu_fpriv *fpriv = filp->driver_priv;
2724	int r;
2725
2726	switch (args->in.op) {
2727	case AMDGPU_VM_OP_RESERVE_VMID:
2728		/* current, we only have requirement to reserve vmid from gfxhub */
2729		r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
 
2730		if (r)
2731			return r;
2732		break;
2733	case AMDGPU_VM_OP_UNRESERVE_VMID:
2734		amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
2735		break;
2736	default:
2737		return -EINVAL;
2738	}
2739
2740	return 0;
2741}
v6.2
   1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28
  29#include <linux/dma-fence-array.h>
  30#include <linux/interval_tree_generic.h>
  31#include <linux/idr.h>
  32#include <linux/dma-buf.h>
  33
  34#include <drm/amdgpu_drm.h>
  35#include <drm/drm_drv.h>
  36#include "amdgpu.h"
  37#include "amdgpu_trace.h"
  38#include "amdgpu_amdkfd.h"
  39#include "amdgpu_gmc.h"
  40#include "amdgpu_xgmi.h"
  41#include "amdgpu_dma_buf.h"
  42#include "amdgpu_res_cursor.h"
  43#include "kfd_svm.h"
  44
  45/**
  46 * DOC: GPUVM
  47 *
  48 * GPUVM is the MMU functionality provided on the GPU.
  49 * GPUVM is similar to the legacy GART on older asics, however
  50 * rather than there being a single global GART table
  51 * for the entire GPU, there can be multiple GPUVM page tables active
  52 * at any given time.  The GPUVM page tables can contain a mix
  53 * VRAM pages and system pages (both memory and MMIO) and system pages
  54 * can be mapped as snooped (cached system pages) or unsnooped
  55 * (uncached system pages).
  56 *
  57 * Each active GPUVM has an ID associated with it and there is a page table
  58 * linked with each VMID.  When executing a command buffer,
  59 * the kernel tells the engine what VMID to use for that command
  60 * buffer.  VMIDs are allocated dynamically as commands are submitted.
  61 * The userspace drivers maintain their own address space and the kernel
  62 * sets up their pages tables accordingly when they submit their
  63 * command buffers and a VMID is assigned.
  64 * The hardware supports up to 16 active GPUVMs at any given time.
  65 *
  66 * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending
  67 * on the ASIC family.  GPUVM supports RWX attributes on each page as well
  68 * as other features such as encryption and caching attributes.
  69 *
  70 * VMID 0 is special.  It is the GPUVM used for the kernel driver.  In
  71 * addition to an aperture managed by a page table, VMID 0 also has
  72 * several other apertures.  There is an aperture for direct access to VRAM
  73 * and there is a legacy AGP aperture which just forwards accesses directly
  74 * to the matching system physical addresses (or IOVAs when an IOMMU is
  75 * present).  These apertures provide direct access to these memories without
  76 * incurring the overhead of a page table.  VMID 0 is used by the kernel
  77 * driver for tasks like memory management.
  78 *
  79 * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory.
  80 * For user applications, each application can have their own unique GPUVM
  81 * address space.  The application manages the address space and the kernel
  82 * driver manages the GPUVM page tables for each process.  If an GPU client
  83 * accesses an invalid page, it will generate a GPU page fault, similar to
  84 * accessing an invalid page on a CPU.
  85 */
  86
  87#define START(node) ((node)->start)
  88#define LAST(node) ((node)->last)
  89
  90INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  91		     START, LAST, static, amdgpu_vm_it)
  92
  93#undef START
  94#undef LAST
  95
  96/**
  97 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
  98 */
  99struct amdgpu_prt_cb {
 100
 101	/**
 102	 * @adev: amdgpu device
 103	 */
 104	struct amdgpu_device *adev;
 105
 106	/**
 107	 * @cb: callback
 
 
 
 
 
 
 
 
 
 
 
 108	 */
 109	struct dma_fence_cb cb;
 
 110};
 111
 112/**
 113 * struct amdgpu_vm_tlb_seq_cb - Helper to increment the TLB flush sequence
 114 */
 115struct amdgpu_vm_tlb_seq_cb {
 116	/**
 117	 * @vm: pointer to the amdgpu_vm structure to set the fence sequence on
 118	 */
 119	struct amdgpu_vm *vm;
 120
 121	/**
 122	 * @cb: callback
 123	 */
 124	struct dma_fence_cb cb;
 125};
 126
 127/**
 128 * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping
 129 *
 130 * @adev: amdgpu_device pointer
 131 * @vm: amdgpu_vm pointer
 132 * @pasid: the pasid the VM is using on this GPU
 133 *
 134 * Set the pasid this VM is using on this GPU, can also be used to remove the
 135 * pasid by passing in zero.
 136 *
 
 137 */
 138int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 139			u32 pasid)
 140{
 141	int r;
 142
 143	if (vm->pasid == pasid)
 144		return 0;
 145
 146	if (vm->pasid) {
 147		r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid));
 148		if (r < 0)
 149			return r;
 150
 151		vm->pasid = 0;
 152	}
 153
 154	if (pasid) {
 155		r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm,
 156					GFP_KERNEL));
 157		if (r < 0)
 158			return r;
 159
 160		vm->pasid = pasid;
 161	}
 162
 163
 164	return 0;
 165}
 166
 167/**
 168 * amdgpu_vm_bo_evicted - vm_bo is evicted
 169 *
 170 * @vm_bo: vm_bo which is evicted
 171 *
 172 * State for PDs/PTs and per VM BOs which are not at the location they should
 173 * be.
 174 */
 175static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 
 176{
 177	struct amdgpu_vm *vm = vm_bo->vm;
 178	struct amdgpu_bo *bo = vm_bo->bo;
 179
 180	vm_bo->moved = true;
 181	spin_lock(&vm_bo->vm->status_lock);
 182	if (bo->tbo.type == ttm_bo_type_kernel)
 183		list_move(&vm_bo->vm_status, &vm->evicted);
 
 
 184	else
 185		list_move_tail(&vm_bo->vm_status, &vm->evicted);
 186	spin_unlock(&vm_bo->vm->status_lock);
 187}
 188/**
 189 * amdgpu_vm_bo_moved - vm_bo is moved
 190 *
 191 * @vm_bo: vm_bo which is moved
 192 *
 193 * State for per VM BOs which are moved, but that change is not yet reflected
 194 * in the page tables.
 195 */
 196static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 197{
 198	spin_lock(&vm_bo->vm->status_lock);
 199	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
 200	spin_unlock(&vm_bo->vm->status_lock);
 201}
 202
 203/**
 204 * amdgpu_vm_bo_idle - vm_bo is idle
 205 *
 206 * @vm_bo: vm_bo which is now idle
 207 *
 208 * State for PDs/PTs and per VM BOs which have gone through the state machine
 209 * and are now idle.
 210 */
 211static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 212{
 213	spin_lock(&vm_bo->vm->status_lock);
 214	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
 215	spin_unlock(&vm_bo->vm->status_lock);
 216	vm_bo->moved = false;
 217}
 218
 219/**
 220 * amdgpu_vm_bo_invalidated - vm_bo is invalidated
 221 *
 222 * @vm_bo: vm_bo which is now invalidated
 223 *
 224 * State for normal BOs which are invalidated and that change not yet reflected
 225 * in the PTs.
 226 */
 227static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
 228{
 229	spin_lock(&vm_bo->vm->status_lock);
 230	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
 231	spin_unlock(&vm_bo->vm->status_lock);
 232}
 233
 234/**
 235 * amdgpu_vm_bo_relocated - vm_bo is reloacted
 236 *
 237 * @vm_bo: vm_bo which is relocated
 238 *
 239 * State for PDs/PTs which needs to update their parent PD.
 240 * For the root PD, just move to idle state.
 241 */
 242static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 243{
 244	if (vm_bo->bo->parent) {
 245		spin_lock(&vm_bo->vm->status_lock);
 246		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
 247		spin_unlock(&vm_bo->vm->status_lock);
 248	} else {
 249		amdgpu_vm_bo_idle(vm_bo);
 250	}
 251}
 252
 253/**
 254 * amdgpu_vm_bo_done - vm_bo is done
 255 *
 256 * @vm_bo: vm_bo which is now done
 257 *
 258 * State for normal BOs which are invalidated and that change has been updated
 259 * in the PTs.
 260 */
 261static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
 262{
 263	spin_lock(&vm_bo->vm->status_lock);
 264	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
 265	spin_unlock(&vm_bo->vm->status_lock);
 266}
 267
 268/**
 269 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
 270 *
 271 * @base: base structure for tracking BO usage in a VM
 272 * @vm: vm to which bo is to be added
 273 * @bo: amdgpu buffer object
 274 *
 275 * Initialize a bo_va_base structure and add it to the appropriate lists
 276 *
 277 */
 278void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 279			    struct amdgpu_vm *vm, struct amdgpu_bo *bo)
 280{
 281	base->vm = vm;
 282	base->bo = bo;
 283	base->next = NULL;
 284	INIT_LIST_HEAD(&base->vm_status);
 285
 286	if (!bo)
 287		return;
 288	base->next = bo->vm_bo;
 289	bo->vm_bo = base;
 290
 291	if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv)
 292		return;
 293
 294	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 295
 296	ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
 297	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
 298		amdgpu_vm_bo_relocated(base);
 299	else
 300		amdgpu_vm_bo_idle(base);
 301
 302	if (bo->preferred_domains &
 303	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
 304		return;
 305
 306	/*
 307	 * we checked all the prerequisites, but it looks like this per vm bo
 308	 * is currently evicted. add the bo to the evicted list to make sure it
 309	 * is validated on next vm use to avoid fault.
 310	 * */
 311	amdgpu_vm_bo_evicted(base);
 312}
 313
 314/**
 315 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
 316 *
 317 * @vm: vm providing the BOs
 318 * @validated: head of validation list
 319 * @entry: entry to add
 320 *
 321 * Add the page directory to the list of BOs to
 322 * validate for command submission.
 323 */
 324void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 325			 struct list_head *validated,
 326			 struct amdgpu_bo_list_entry *entry)
 327{
 
 328	entry->priority = 0;
 329	entry->tv.bo = &vm->root.bo->tbo;
 330	/* Two for VM updates, one for TTM and one for the CS job */
 331	entry->tv.num_shared = 4;
 332	entry->user_pages = NULL;
 333	list_add(&entry->tv.head, validated);
 334}
 335
 336/**
 337 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
 338 *
 339 * @adev: amdgpu device pointer
 340 * @vm: vm providing the BOs
 341 *
 342 * Move all BOs to the end of LRU and remember their positions to put them
 343 * together.
 344 */
 345void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 346				struct amdgpu_vm *vm)
 347{
 348	spin_lock(&adev->mman.bdev.lru_lock);
 349	ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
 350	spin_unlock(&adev->mman.bdev.lru_lock);
 351}
 352
 353/**
 354 * amdgpu_vm_validate_pt_bos - validate the page table BOs
 355 *
 356 * @adev: amdgpu device pointer
 357 * @vm: vm providing the BOs
 358 * @validate: callback to do the validation
 359 * @param: parameter for the validation callback
 360 *
 361 * Validate the page table BOs on command submission if neccessary.
 362 *
 363 * Returns:
 364 * Validation result.
 365 */
 366int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 367			      int (*validate)(void *p, struct amdgpu_bo *bo),
 368			      void *param)
 369{
 370	struct amdgpu_vm_bo_base *bo_base;
 371	struct amdgpu_bo *shadow;
 372	struct amdgpu_bo *bo;
 373	int r;
 374
 375	spin_lock(&vm->status_lock);
 376	while (!list_empty(&vm->evicted)) {
 
 
 
 377		bo_base = list_first_entry(&vm->evicted,
 378					   struct amdgpu_vm_bo_base,
 379					   vm_status);
 380		spin_unlock(&vm->status_lock);
 381
 382		bo = bo_base->bo;
 383		shadow = amdgpu_bo_shadowed(bo);
 
 
 
 
 
 
 
 
 
 
 
 384
 385		r = validate(param, bo);
 386		if (r)
 387			return r;
 388		if (shadow) {
 389			r = validate(param, shadow);
 390			if (r)
 391				return r;
 392		}
 393
 394		if (bo->tbo.type != ttm_bo_type_kernel) {
 395			amdgpu_vm_bo_moved(bo_base);
 396		} else {
 397			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
 398			amdgpu_vm_bo_relocated(bo_base);
 399		}
 400		spin_lock(&vm->status_lock);
 
 
 
 
 401	}
 402	spin_unlock(&vm->status_lock);
 403
 404	amdgpu_vm_eviction_lock(vm);
 405	vm->evicting = false;
 406	amdgpu_vm_eviction_unlock(vm);
 407
 408	return 0;
 409}
 410
 411/**
 412 * amdgpu_vm_ready - check VM is ready for updates
 413 *
 414 * @vm: VM to check
 415 *
 416 * Check if all VM PDs/PTs are ready for updates
 417 *
 418 * Returns:
 419 * True if VM is not evicting.
 420 */
 421bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 422{
 423	bool empty;
 424	bool ret;
 425
 426	amdgpu_vm_eviction_lock(vm);
 427	ret = !vm->evicting;
 428	amdgpu_vm_eviction_unlock(vm);
 429
 430	spin_lock(&vm->status_lock);
 431	empty = list_empty(&vm->evicted);
 432	spin_unlock(&vm->status_lock);
 433
 434	return ret && empty;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 435}
 436
 437/**
 438 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
 439 *
 440 * @adev: amdgpu_device pointer
 441 */
 442void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
 443{
 444	const struct amdgpu_ip_block *ip_block;
 445	bool has_compute_vm_bug;
 446	struct amdgpu_ring *ring;
 447	int i;
 448
 449	has_compute_vm_bug = false;
 450
 451	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 452	if (ip_block) {
 453		/* Compute has a VM bug for GFX version < 7.
 454		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
 455		if (ip_block->version->major <= 7)
 456			has_compute_vm_bug = true;
 457		else if (ip_block->version->major == 8)
 458			if (adev->gfx.mec_fw_version < 673)
 459				has_compute_vm_bug = true;
 460	}
 461
 462	for (i = 0; i < adev->num_rings; i++) {
 463		ring = adev->rings[i];
 464		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
 465			/* only compute rings */
 466			ring->has_compute_vm_bug = has_compute_vm_bug;
 467		else
 468			ring->has_compute_vm_bug = false;
 469	}
 470}
 471
 472/**
 473 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
 474 *
 475 * @ring: ring on which the job will be submitted
 476 * @job: job to submit
 477 *
 478 * Returns:
 479 * True if sync is needed.
 480 */
 481bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 482				  struct amdgpu_job *job)
 483{
 484	struct amdgpu_device *adev = ring->adev;
 485	unsigned vmhub = ring->funcs->vmhub;
 486	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 
 
 
 487
 488	if (job->vmid == 0)
 489		return false;
 
 
 
 
 
 
 
 
 490
 491	if (job->vm_needs_flush || ring->has_compute_vm_bug)
 492		return true;
 493
 494	if (ring->funcs->emit_gds_switch && job->gds_switch_needed)
 495		return true;
 496
 497	if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid]))
 498		return true;
 499
 500	return false;
 501}
 502
 503/**
 504 * amdgpu_vm_flush - hardware flush the vm
 505 *
 506 * @ring: ring to use for flush
 507 * @job:  related job
 508 * @need_pipe_sync: is pipe sync needed
 509 *
 510 * Emit a VM flush when it is necessary.
 511 *
 512 * Returns:
 513 * 0 on success, errno otherwise.
 514 */
 515int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 516		    bool need_pipe_sync)
 517{
 518	struct amdgpu_device *adev = ring->adev;
 519	unsigned vmhub = ring->funcs->vmhub;
 520	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 521	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
 522	bool spm_update_needed = job->spm_update_needed;
 523	bool gds_switch_needed = ring->funcs->emit_gds_switch &&
 524		job->gds_switch_needed;
 
 
 
 
 525	bool vm_flush_needed = job->vm_needs_flush;
 
 
 
 526	struct dma_fence *fence = NULL;
 527	bool pasid_mapping_needed = false;
 528	unsigned patch_offset = 0;
 529	int r;
 530
 531	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
 532		gds_switch_needed = true;
 533		vm_flush_needed = true;
 534		pasid_mapping_needed = true;
 535		spm_update_needed = true;
 536	}
 537
 538	mutex_lock(&id_mgr->lock);
 539	if (id->pasid != job->pasid || !id->pasid_mapping ||
 540	    !dma_fence_is_signaled(id->pasid_mapping))
 541		pasid_mapping_needed = true;
 542	mutex_unlock(&id_mgr->lock);
 543
 544	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
 545	vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&
 546			job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
 547	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
 548		ring->funcs->emit_wreg;
 549
 550	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
 551		return 0;
 552
 553	amdgpu_ring_ib_begin(ring);
 554	if (ring->funcs->init_cond_exec)
 555		patch_offset = amdgpu_ring_init_cond_exec(ring);
 556
 557	if (need_pipe_sync)
 558		amdgpu_ring_emit_pipeline_sync(ring);
 559
 560	if (vm_flush_needed) {
 561		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
 562		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
 563	}
 564
 565	if (pasid_mapping_needed)
 566		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
 567
 568	if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid)
 569		adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid);
 570
 571	if (!ring->is_mes_queue && ring->funcs->emit_gds_switch &&
 572	    gds_switch_needed) {
 573		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
 574					    job->gds_size, job->gws_base,
 575					    job->gws_size, job->oa_base,
 576					    job->oa_size);
 577	}
 578
 579	if (vm_flush_needed || pasid_mapping_needed) {
 580		r = amdgpu_fence_emit(ring, &fence, NULL, 0);
 581		if (r)
 582			return r;
 583	}
 584
 585	if (vm_flush_needed) {
 586		mutex_lock(&id_mgr->lock);
 587		dma_fence_put(id->last_flush);
 588		id->last_flush = dma_fence_get(fence);
 589		id->current_gpu_reset_count =
 590			atomic_read(&adev->gpu_reset_counter);
 591		mutex_unlock(&id_mgr->lock);
 592	}
 593
 594	if (pasid_mapping_needed) {
 595		mutex_lock(&id_mgr->lock);
 596		id->pasid = job->pasid;
 597		dma_fence_put(id->pasid_mapping);
 598		id->pasid_mapping = dma_fence_get(fence);
 599		mutex_unlock(&id_mgr->lock);
 600	}
 601	dma_fence_put(fence);
 602
 
 
 
 
 
 
 
 
 
 
 
 
 
 603	if (ring->funcs->patch_cond_exec)
 604		amdgpu_ring_patch_cond_exec(ring, patch_offset);
 605
 606	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
 607	if (ring->funcs->emit_switch_buffer) {
 608		amdgpu_ring_emit_switch_buffer(ring);
 609		amdgpu_ring_emit_switch_buffer(ring);
 610	}
 611	amdgpu_ring_ib_end(ring);
 612	return 0;
 613}
 614
 615/**
 616 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 617 *
 618 * @vm: requested vm
 619 * @bo: requested buffer object
 620 *
 621 * Find @bo inside the requested vm.
 622 * Search inside the @bos vm list for the requested vm
 623 * Returns the found bo_va or NULL if none is found
 624 *
 625 * Object has to be reserved!
 626 *
 627 * Returns:
 628 * Found bo_va or NULL.
 629 */
 630struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 631				       struct amdgpu_bo *bo)
 632{
 633	struct amdgpu_vm_bo_base *base;
 634
 635	for (base = bo->vm_bo; base; base = base->next) {
 636		if (base->vm != vm)
 637			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 638
 639		return container_of(base, struct amdgpu_bo_va, base);
 
 
 640	}
 641	return NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 642}
 643
 644/**
 645 * amdgpu_vm_map_gart - Resolve gart mapping of addr
 646 *
 647 * @pages_addr: optional DMA address to use for lookup
 648 * @addr: the unmapped addr
 649 *
 650 * Look up the physical address of the page that the pte resolves
 651 * to.
 652 *
 653 * Returns:
 654 * The pointer for the page table entry.
 655 */
 656uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 657{
 658	uint64_t result;
 659
 660	/* page table offset */
 661	result = pages_addr[addr >> PAGE_SHIFT];
 662
 663	/* in case cpu page size != gpu page size*/
 664	result |= addr & (~PAGE_MASK);
 665
 666	result &= 0xFFFFFFFFFFFFF000ULL;
 667
 668	return result;
 669}
 670
 671/**
 672 * amdgpu_vm_update_pdes - make sure that all directories are valid
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 673 *
 674 * @adev: amdgpu_device pointer
 675 * @vm: requested vm
 676 * @immediate: submit immediately to the paging queue
 677 *
 678 * Makes sure all directories are up to date.
 679 *
 680 * Returns:
 681 * 0 for success, error for failure.
 682 */
 683int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 684			  struct amdgpu_vm *vm, bool immediate)
 685{
 686	struct amdgpu_vm_update_params params;
 687	struct amdgpu_vm_bo_base *entry;
 688	bool flush_tlb_needed = false;
 689	LIST_HEAD(relocated);
 690	int r, idx;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 691
 692	spin_lock(&vm->status_lock);
 693	list_splice_init(&vm->relocated, &relocated);
 694	spin_unlock(&vm->status_lock);
 
 
 695
 696	if (list_empty(&relocated))
 697		return 0;
 
 
 
 698
 699	if (!drm_dev_enter(adev_to_drm(adev), &idx))
 700		return -ENODEV;
 
 
 
 701
 702	memset(&params, 0, sizeof(params));
 703	params.adev = adev;
 704	params.vm = vm;
 705	params.immediate = immediate;
 706
 707	r = vm->update_funcs->prepare(&params, NULL, AMDGPU_SYNC_EXPLICIT);
 708	if (r)
 709		goto error;
 710
 711	list_for_each_entry(entry, &relocated, vm_status) {
 712		/* vm_flush_needed after updating moved PDEs */
 713		flush_tlb_needed |= entry->moved;
 
 
 
 714
 715		r = amdgpu_vm_pde_update(&params, entry);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 716		if (r)
 717			goto error;
 
 
 
 
 718	}
 719
 720	r = vm->update_funcs->commit(&params, &vm->last_update);
 721	if (r)
 722		goto error;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 723
 724	if (flush_tlb_needed)
 725		atomic64_inc(&vm->tlb_seq);
 
 
 726
 727	while (!list_empty(&relocated)) {
 728		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
 729					 vm_status);
 730		amdgpu_vm_bo_idle(entry);
 731	}
 732
 733error:
 734	drm_dev_exit(idx);
 735	return r;
 736}
 737
 738/**
 739 * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence
 740 * @fence: unused
 741 * @cb: the callback structure
 
 
 
 
 
 742 *
 743 * Increments the tlb sequence to make sure that future CS execute a VM flush.
 744 */
 745static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,
 746				 struct dma_fence_cb *cb)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 747{
 748	struct amdgpu_vm_tlb_seq_cb *tlb_cb;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 749
 750	tlb_cb = container_of(cb, typeof(*tlb_cb), cb);
 751	atomic64_inc(&tlb_cb->vm->tlb_seq);
 752	kfree(tlb_cb);
 753}
 754
 755/**
 756 * amdgpu_vm_update_range - update a range in the vm page table
 757 *
 758 * @adev: amdgpu_device pointer to use for commands
 759 * @vm: the VM to update the range
 760 * @immediate: immediate submission in a page fault
 761 * @unlocked: unlocked invalidation during MM callback
 762 * @flush_tlb: trigger tlb invalidation after update completed
 763 * @resv: fences we need to sync to
 764 * @start: start of mapped range
 765 * @last: last mapped entry
 766 * @flags: flags for the entries
 767 * @offset: offset into nodes and pages_addr
 768 * @vram_base: base for vram mappings
 769 * @res: ttm_resource to map
 770 * @pages_addr: DMA addresses to use for mapping
 771 * @fence: optional resulting fence
 772 *
 773 * Fill in the page table entries between @start and @last.
 774 *
 775 * Returns:
 776 * 0 for success, negative erro code for failure.
 777 */
 778int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 779			   bool immediate, bool unlocked, bool flush_tlb,
 780			   struct dma_resv *resv, uint64_t start, uint64_t last,
 781			   uint64_t flags, uint64_t offset, uint64_t vram_base,
 782			   struct ttm_resource *res, dma_addr_t *pages_addr,
 783			   struct dma_fence **fence)
 784{
 785	struct amdgpu_vm_update_params params;
 786	struct amdgpu_vm_tlb_seq_cb *tlb_cb;
 787	struct amdgpu_res_cursor cursor;
 788	enum amdgpu_sync_mode sync_mode;
 789	int r, idx;
 790
 791	if (!drm_dev_enter(adev_to_drm(adev), &idx))
 792		return -ENODEV;
 793
 794	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
 795	if (!tlb_cb) {
 796		r = -ENOMEM;
 797		goto error_unlock;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 798	}
 799
 800	/* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,
 801	 * heavy-weight flush TLB unconditionally.
 802	 */
 803	flush_tlb |= adev->gmc.xgmi.num_physical_nodes &&
 804		     adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0);
 805
 806	/*
 807	 * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB
 
 
 
 808	 */
 809	flush_tlb |= adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 0, 0);
 
 
 
 
 
 
 
 
 
 
 810
 811	memset(&params, 0, sizeof(params));
 812	params.adev = adev;
 813	params.vm = vm;
 814	params.immediate = immediate;
 815	params.pages_addr = pages_addr;
 816	params.unlocked = unlocked;
 
 
 817
 818	/* Implicitly sync to command submissions in the same VM before
 819	 * unmapping. Sync to moving fences before mapping.
 820	 */
 821	if (!(flags & AMDGPU_PTE_VALID))
 822		sync_mode = AMDGPU_SYNC_EQ_OWNER;
 823	else
 824		sync_mode = AMDGPU_SYNC_EXPLICIT;
 825
 826	amdgpu_vm_eviction_lock(vm);
 827	if (vm->evicting) {
 828		r = -EBUSY;
 829		goto error_free;
 830	}
 831
 832	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
 833		struct dma_fence *tmp = dma_fence_get_stub();
 
 
 
 834
 835		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
 836		swap(vm->last_unlocked, tmp);
 837		dma_fence_put(tmp);
 
 
 
 
 
 
 
 
 
 
 
 
 838	}
 839
 840	r = vm->update_funcs->prepare(&params, resv, sync_mode);
 
 
 
 
 
 
 
 
 
 
 
 
 
 841	if (r)
 842		goto error_free;
 843
 844	amdgpu_res_first(pages_addr ? NULL : res, offset,
 845			 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);
 846	while (cursor.remaining) {
 847		uint64_t tmp, num_entries, addr;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 848
 849		num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
 850		if (pages_addr) {
 851			bool contiguous = true;
 852
 853			if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
 854				uint64_t pfn = cursor.start >> PAGE_SHIFT;
 855				uint64_t count;
 856
 857				contiguous = pages_addr[pfn + 1] ==
 858					pages_addr[pfn] + PAGE_SIZE;
 859
 860				tmp = num_entries /
 861					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 862				for (count = 2; count < tmp; ++count) {
 863					uint64_t idx = pfn + count;
 864
 865					if (contiguous != (pages_addr[idx] ==
 866					    pages_addr[idx - 1] + PAGE_SIZE))
 867						break;
 868				}
 869				num_entries = count *
 870					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 871			}
 872
 873			if (!contiguous) {
 874				addr = cursor.start;
 875				params.pages_addr = pages_addr;
 876			} else {
 877				addr = pages_addr[cursor.start >> PAGE_SHIFT];
 878				params.pages_addr = NULL;
 879			}
 880
 881		} else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
 882			addr = vram_base + cursor.start;
 883		} else {
 884			addr = 0;
 885		}
 886
 887		tmp = start + num_entries;
 888		r = amdgpu_vm_ptes_update(&params, start, tmp, addr, flags);
 
 
 889		if (r)
 890			goto error_free;
 891
 892		amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);
 893		start = tmp;
 894	}
 895
 896	r = vm->update_funcs->commit(&params, fence);
 897
 898	if (flush_tlb || params.table_freed) {
 899		tlb_cb->vm = vm;
 900		if (fence && *fence &&
 901		    !dma_fence_add_callback(*fence, &tlb_cb->cb,
 902					   amdgpu_vm_tlb_seq_cb)) {
 903			dma_fence_put(vm->last_tlb_flush);
 904			vm->last_tlb_flush = dma_fence_get(*fence);
 905		} else {
 906			amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
 907		}
 908		tlb_cb = NULL;
 909	}
 910
 911error_free:
 912	kfree(tlb_cb);
 913
 914error_unlock:
 915	amdgpu_vm_eviction_unlock(vm);
 916	drm_dev_exit(idx);
 917	return r;
 918}
 919
 920void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
 921				uint64_t *gtt_mem, uint64_t *cpu_mem)
 922{
 923	struct amdgpu_bo_va *bo_va, *tmp;
 924
 925	spin_lock(&vm->status_lock);
 926	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
 927		if (!bo_va->base.bo)
 928			continue;
 929		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
 930				gtt_mem, cpu_mem);
 931	}
 932	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
 933		if (!bo_va->base.bo)
 934			continue;
 935		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
 936				gtt_mem, cpu_mem);
 937	}
 938	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
 939		if (!bo_va->base.bo)
 940			continue;
 941		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
 942				gtt_mem, cpu_mem);
 943	}
 944	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
 945		if (!bo_va->base.bo)
 946			continue;
 947		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
 948				gtt_mem, cpu_mem);
 949	}
 950	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
 951		if (!bo_va->base.bo)
 952			continue;
 953		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
 954				gtt_mem, cpu_mem);
 955	}
 956	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
 957		if (!bo_va->base.bo)
 958			continue;
 959		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
 960				gtt_mem, cpu_mem);
 961	}
 962	spin_unlock(&vm->status_lock);
 963}
 964/**
 965 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
 966 *
 967 * @adev: amdgpu_device pointer
 968 * @bo_va: requested BO and VM object
 969 * @clear: if true clear the entries
 970 *
 971 * Fill in the page table entries for @bo_va.
 972 *
 973 * Returns:
 974 * 0 for success, -EINVAL for failure.
 975 */
 976int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 
 977			bool clear)
 978{
 979	struct amdgpu_bo *bo = bo_va->base.bo;
 980	struct amdgpu_vm *vm = bo_va->base.vm;
 981	struct amdgpu_bo_va_mapping *mapping;
 982	dma_addr_t *pages_addr = NULL;
 983	struct ttm_resource *mem;
 984	struct dma_fence **last_update;
 985	bool flush_tlb = clear;
 986	struct dma_resv *resv;
 987	uint64_t vram_base;
 988	uint64_t flags;
 989	int r;
 990
 991	if (clear || !bo) {
 992		mem = NULL;
 993		resv = vm->root.bo->tbo.base.resv;
 
 994	} else {
 995		struct drm_gem_object *obj = &bo->tbo.base;
 996
 997		resv = bo->tbo.base.resv;
 998		if (obj->import_attach && bo_va->is_xgmi) {
 999			struct dma_buf *dma_buf = obj->import_attach->dmabuf;
1000			struct drm_gem_object *gobj = dma_buf->priv;
1001			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
1002
1003			if (abo->tbo.resource->mem_type == TTM_PL_VRAM)
1004				bo = gem_to_amdgpu_bo(gobj);
 
 
 
 
1005		}
1006		mem = bo->tbo.resource;
1007		if (mem->mem_type == TTM_PL_TT ||
1008		    mem->mem_type == AMDGPU_PL_PREEMPT)
1009			pages_addr = bo->tbo.ttm->dma_address;
1010	}
1011
1012	if (bo) {
1013		struct amdgpu_device *bo_adev;
1014
1015		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1016
1017		if (amdgpu_bo_encrypted(bo))
1018			flags |= AMDGPU_PTE_TMZ;
1019
1020		bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1021		vram_base = bo_adev->vm_manager.vram_base_offset;
1022	} else {
1023		flags = 0x0;
1024		vram_base = 0;
1025	}
1026
1027	if (clear || (bo && bo->tbo.base.resv ==
1028		      vm->root.bo->tbo.base.resv))
1029		last_update = &vm->last_update;
1030	else
1031		last_update = &bo_va->last_pt_update;
1032
1033	if (!clear && bo_va->base.moved) {
1034		flush_tlb = true;
1035		list_splice_init(&bo_va->valids, &bo_va->invalids);
1036
1037	} else if (bo_va->cleared != clear) {
1038		list_splice_init(&bo_va->valids, &bo_va->invalids);
1039	}
1040
1041	list_for_each_entry(mapping, &bo_va->invalids, list) {
1042		uint64_t update_flags = flags;
1043
1044		/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1045		 * but in case of something, we filter the flags in first place
1046		 */
1047		if (!(mapping->flags & AMDGPU_PTE_READABLE))
1048			update_flags &= ~AMDGPU_PTE_READABLE;
1049		if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1050			update_flags &= ~AMDGPU_PTE_WRITEABLE;
1051
1052		/* Apply ASIC specific mapping flags */
1053		amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags);
1054
1055		trace_amdgpu_vm_bo_update(mapping);
1056
1057		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,
1058					   resv, mapping->start, mapping->last,
1059					   update_flags, mapping->offset,
1060					   vram_base, mem, pages_addr,
1061					   last_update);
1062		if (r)
1063			return r;
1064	}
1065
1066	/* If the BO is not in its preferred location add it back to
1067	 * the evicted list so that it gets validated again on the
1068	 * next command submission.
1069	 */
1070	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1071		uint32_t mem_type = bo->tbo.resource->mem_type;
1072
1073		if (!(bo->preferred_domains &
1074		      amdgpu_mem_type_to_domain(mem_type)))
1075			amdgpu_vm_bo_evicted(&bo_va->base);
1076		else
1077			amdgpu_vm_bo_idle(&bo_va->base);
1078	} else {
1079		amdgpu_vm_bo_done(&bo_va->base);
1080	}
1081
1082	list_splice_init(&bo_va->invalids, &bo_va->valids);
1083	bo_va->cleared = clear;
1084	bo_va->base.moved = false;
1085
1086	if (trace_amdgpu_vm_bo_mapping_enabled()) {
1087		list_for_each_entry(mapping, &bo_va->valids, list)
1088			trace_amdgpu_vm_bo_mapping(mapping);
1089	}
1090
1091	return 0;
1092}
1093
1094/**
1095 * amdgpu_vm_update_prt_state - update the global PRT state
1096 *
1097 * @adev: amdgpu_device pointer
1098 */
1099static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1100{
1101	unsigned long flags;
1102	bool enable;
1103
1104	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1105	enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1106	adev->gmc.gmc_funcs->set_prt(adev, enable);
1107	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1108}
1109
1110/**
1111 * amdgpu_vm_prt_get - add a PRT user
1112 *
1113 * @adev: amdgpu_device pointer
1114 */
1115static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1116{
1117	if (!adev->gmc.gmc_funcs->set_prt)
1118		return;
1119
1120	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1121		amdgpu_vm_update_prt_state(adev);
1122}
1123
1124/**
1125 * amdgpu_vm_prt_put - drop a PRT user
1126 *
1127 * @adev: amdgpu_device pointer
1128 */
1129static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1130{
1131	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1132		amdgpu_vm_update_prt_state(adev);
1133}
1134
1135/**
1136 * amdgpu_vm_prt_cb - callback for updating the PRT status
1137 *
1138 * @fence: fence for the callback
1139 * @_cb: the callback function
1140 */
1141static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1142{
1143	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1144
1145	amdgpu_vm_prt_put(cb->adev);
1146	kfree(cb);
1147}
1148
1149/**
1150 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1151 *
1152 * @adev: amdgpu_device pointer
1153 * @fence: fence for the callback
1154 */
1155static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1156				 struct dma_fence *fence)
1157{
1158	struct amdgpu_prt_cb *cb;
1159
1160	if (!adev->gmc.gmc_funcs->set_prt)
1161		return;
1162
1163	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1164	if (!cb) {
1165		/* Last resort when we are OOM */
1166		if (fence)
1167			dma_fence_wait(fence, false);
1168
1169		amdgpu_vm_prt_put(adev);
1170	} else {
1171		cb->adev = adev;
1172		if (!fence || dma_fence_add_callback(fence, &cb->cb,
1173						     amdgpu_vm_prt_cb))
1174			amdgpu_vm_prt_cb(fence, &cb->cb);
1175	}
1176}
1177
1178/**
1179 * amdgpu_vm_free_mapping - free a mapping
1180 *
1181 * @adev: amdgpu_device pointer
1182 * @vm: requested vm
1183 * @mapping: mapping to be freed
1184 * @fence: fence of the unmap operation
1185 *
1186 * Free a mapping and make sure we decrease the PRT usage count if applicable.
1187 */
1188static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1189				   struct amdgpu_vm *vm,
1190				   struct amdgpu_bo_va_mapping *mapping,
1191				   struct dma_fence *fence)
1192{
1193	if (mapping->flags & AMDGPU_PTE_PRT)
1194		amdgpu_vm_add_prt_cb(adev, fence);
1195	kfree(mapping);
1196}
1197
1198/**
1199 * amdgpu_vm_prt_fini - finish all prt mappings
1200 *
1201 * @adev: amdgpu_device pointer
1202 * @vm: requested vm
1203 *
1204 * Register a cleanup callback to disable PRT support after VM dies.
1205 */
1206static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1207{
1208	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1209	struct dma_resv_iter cursor;
1210	struct dma_fence *fence;
 
1211
1212	dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
1213		/* Add a callback for each fence in the reservation object */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1214		amdgpu_vm_prt_get(adev);
1215		amdgpu_vm_add_prt_cb(adev, fence);
1216	}
 
 
1217}
1218
1219/**
1220 * amdgpu_vm_clear_freed - clear freed BOs in the PT
1221 *
1222 * @adev: amdgpu_device pointer
1223 * @vm: requested vm
1224 * @fence: optional resulting fence (unchanged if no work needed to be done
1225 * or if an error occurred)
1226 *
1227 * Make sure all freed BOs are cleared in the PT.
 
 
1228 * PTs have to be reserved and mutex must be locked!
1229 *
1230 * Returns:
1231 * 0 for success.
1232 *
1233 */
1234int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1235			  struct amdgpu_vm *vm,
1236			  struct dma_fence **fence)
1237{
1238	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1239	struct amdgpu_bo_va_mapping *mapping;
1240	uint64_t init_pte_value = 0;
1241	struct dma_fence *f = NULL;
1242	int r;
1243
1244	while (!list_empty(&vm->freed)) {
1245		mapping = list_first_entry(&vm->freed,
1246			struct amdgpu_bo_va_mapping, list);
1247		list_del(&mapping->list);
1248
1249		if (vm->pte_support_ats &&
1250		    mapping->start < AMDGPU_GMC_HOLE_START)
1251			init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
1252
1253		r = amdgpu_vm_update_range(adev, vm, false, false, true, resv,
1254					   mapping->start, mapping->last,
1255					   init_pte_value, 0, 0, NULL, NULL,
1256					   &f);
1257		amdgpu_vm_free_mapping(adev, vm, mapping, f);
1258		if (r) {
1259			dma_fence_put(f);
1260			return r;
1261		}
1262	}
1263
1264	if (fence && f) {
1265		dma_fence_put(*fence);
1266		*fence = f;
1267	} else {
1268		dma_fence_put(f);
1269	}
1270
1271	return 0;
1272
1273}
1274
1275/**
1276 * amdgpu_vm_handle_moved - handle moved BOs in the PT
1277 *
1278 * @adev: amdgpu_device pointer
1279 * @vm: requested vm
 
1280 *
1281 * Make sure all BOs which are moved are updated in the PTs.
1282 *
1283 * Returns:
1284 * 0 for success.
1285 *
1286 * PTs have to be reserved!
1287 */
1288int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1289			   struct amdgpu_vm *vm)
1290{
1291	struct amdgpu_bo_va *bo_va;
1292	struct dma_resv *resv;
1293	bool clear;
1294	int r;
1295
1296	spin_lock(&vm->status_lock);
1297	while (!list_empty(&vm->moved)) {
1298		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
1299					 base.vm_status);
 
 
 
1300		spin_unlock(&vm->status_lock);
1301
 
 
1302		/* Per VM BOs never need to bo cleared in the page tables */
1303		r = amdgpu_vm_bo_update(adev, bo_va, false);
1304		if (r)
1305			return r;
1306		spin_lock(&vm->status_lock);
1307	}
1308
1309	while (!list_empty(&vm->invalidated)) {
1310		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1311					 base.vm_status);
1312		resv = bo_va->base.bo->tbo.base.resv;
1313		spin_unlock(&vm->status_lock);
1314
1315		/* Try to reserve the BO to avoid clearing its ptes */
1316		if (!amdgpu_vm_debug && dma_resv_trylock(resv))
1317			clear = false;
1318		/* Somebody else is using the BO right now */
1319		else
1320			clear = true;
1321
1322		r = amdgpu_vm_bo_update(adev, bo_va, clear);
1323		if (r)
1324			return r;
1325
1326		if (!clear)
1327			dma_resv_unlock(resv);
 
1328		spin_lock(&vm->status_lock);
1329	}
1330	spin_unlock(&vm->status_lock);
1331
1332	return 0;
1333}
1334
1335/**
1336 * amdgpu_vm_bo_add - add a bo to a specific vm
1337 *
1338 * @adev: amdgpu_device pointer
1339 * @vm: requested vm
1340 * @bo: amdgpu buffer object
1341 *
1342 * Add @bo into the requested vm.
1343 * Add @bo to the list of bos associated with the vm
1344 *
1345 * Returns:
1346 * Newly added bo_va or NULL for failure
1347 *
1348 * Object has to be reserved!
1349 */
1350struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1351				      struct amdgpu_vm *vm,
1352				      struct amdgpu_bo *bo)
1353{
1354	struct amdgpu_bo_va *bo_va;
1355
1356	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1357	if (bo_va == NULL) {
1358		return NULL;
1359	}
1360	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
 
 
 
1361
1362	bo_va->ref_count = 1;
1363	INIT_LIST_HEAD(&bo_va->valids);
1364	INIT_LIST_HEAD(&bo_va->invalids);
1365
1366	if (!bo)
1367		return bo_va;
1368
1369	dma_resv_assert_held(bo->tbo.base.resv);
1370	if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {
1371		bo_va->is_xgmi = true;
1372		/* Power up XGMI if it can be potentially used */
1373		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);
1374	}
 
 
 
 
 
 
 
 
 
 
 
1375
1376	return bo_va;
1377}
1378
1379
1380/**
1381 * amdgpu_vm_bo_insert_map - insert a new mapping
1382 *
1383 * @adev: amdgpu_device pointer
1384 * @bo_va: bo_va to store the address
1385 * @mapping: the mapping to insert
1386 *
1387 * Insert a new mapping into all structures.
1388 */
1389static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
1390				    struct amdgpu_bo_va *bo_va,
1391				    struct amdgpu_bo_va_mapping *mapping)
1392{
1393	struct amdgpu_vm *vm = bo_va->base.vm;
1394	struct amdgpu_bo *bo = bo_va->base.bo;
1395
1396	mapping->bo_va = bo_va;
1397	list_add(&mapping->list, &bo_va->invalids);
1398	amdgpu_vm_it_insert(mapping, &vm->va);
1399
1400	if (mapping->flags & AMDGPU_PTE_PRT)
1401		amdgpu_vm_prt_get(adev);
1402
1403	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
1404	    !bo_va->base.moved) {
1405		amdgpu_vm_bo_moved(&bo_va->base);
 
 
1406	}
1407	trace_amdgpu_vm_bo_map(bo_va, mapping);
1408}
1409
1410/**
1411 * amdgpu_vm_bo_map - map bo inside a vm
1412 *
1413 * @adev: amdgpu_device pointer
1414 * @bo_va: bo_va to store the address
1415 * @saddr: where to map the BO
1416 * @offset: requested offset in the BO
1417 * @size: BO size in bytes
1418 * @flags: attributes of pages (read/write/valid/etc.)
1419 *
1420 * Add a mapping of the BO at the specefied addr into the VM.
1421 *
1422 * Returns:
1423 * 0 for success, error for failure.
1424 *
1425 * Object has to be reserved and unreserved outside!
1426 */
1427int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1428		     struct amdgpu_bo_va *bo_va,
1429		     uint64_t saddr, uint64_t offset,
1430		     uint64_t size, uint64_t flags)
1431{
1432	struct amdgpu_bo_va_mapping *mapping, *tmp;
1433	struct amdgpu_bo *bo = bo_va->base.bo;
1434	struct amdgpu_vm *vm = bo_va->base.vm;
1435	uint64_t eaddr;
1436
1437	/* validate the parameters */
1438	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
1439	    size == 0 || size & ~PAGE_MASK)
1440		return -EINVAL;
1441
1442	/* make sure object fit at this offset */
1443	eaddr = saddr + size - 1;
1444	if (saddr >= eaddr ||
1445	    (bo && offset + size > amdgpu_bo_size(bo)) ||
1446	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
1447		return -EINVAL;
1448
1449	saddr /= AMDGPU_GPU_PAGE_SIZE;
1450	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1451
1452	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1453	if (tmp) {
1454		/* bo and tmp overlap, invalid addr */
1455		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1456			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1457			tmp->start, tmp->last + 1);
1458		return -EINVAL;
1459	}
1460
1461	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1462	if (!mapping)
1463		return -ENOMEM;
1464
1465	mapping->start = saddr;
1466	mapping->last = eaddr;
1467	mapping->offset = offset;
1468	mapping->flags = flags;
1469
1470	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1471
1472	return 0;
1473}
1474
1475/**
1476 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1477 *
1478 * @adev: amdgpu_device pointer
1479 * @bo_va: bo_va to store the address
1480 * @saddr: where to map the BO
1481 * @offset: requested offset in the BO
1482 * @size: BO size in bytes
1483 * @flags: attributes of pages (read/write/valid/etc.)
1484 *
1485 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
1486 * mappings as we do so.
1487 *
1488 * Returns:
1489 * 0 for success, error for failure.
1490 *
1491 * Object has to be reserved and unreserved outside!
1492 */
1493int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1494			     struct amdgpu_bo_va *bo_va,
1495			     uint64_t saddr, uint64_t offset,
1496			     uint64_t size, uint64_t flags)
1497{
1498	struct amdgpu_bo_va_mapping *mapping;
1499	struct amdgpu_bo *bo = bo_va->base.bo;
1500	uint64_t eaddr;
1501	int r;
1502
1503	/* validate the parameters */
1504	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
1505	    size == 0 || size & ~PAGE_MASK)
1506		return -EINVAL;
1507
1508	/* make sure object fit at this offset */
1509	eaddr = saddr + size - 1;
1510	if (saddr >= eaddr ||
1511	    (bo && offset + size > amdgpu_bo_size(bo)) ||
1512	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
1513		return -EINVAL;
1514
1515	/* Allocate all the needed memory */
1516	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1517	if (!mapping)
1518		return -ENOMEM;
1519
1520	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
1521	if (r) {
1522		kfree(mapping);
1523		return r;
1524	}
1525
1526	saddr /= AMDGPU_GPU_PAGE_SIZE;
1527	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1528
1529	mapping->start = saddr;
1530	mapping->last = eaddr;
1531	mapping->offset = offset;
1532	mapping->flags = flags;
1533
1534	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1535
1536	return 0;
1537}
1538
1539/**
1540 * amdgpu_vm_bo_unmap - remove bo mapping from vm
1541 *
1542 * @adev: amdgpu_device pointer
1543 * @bo_va: bo_va to remove the address from
1544 * @saddr: where to the BO is mapped
1545 *
1546 * Remove a mapping of the BO at the specefied addr from the VM.
1547 *
1548 * Returns:
1549 * 0 for success, error for failure.
1550 *
1551 * Object has to be reserved and unreserved outside!
1552 */
1553int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1554		       struct amdgpu_bo_va *bo_va,
1555		       uint64_t saddr)
1556{
1557	struct amdgpu_bo_va_mapping *mapping;
1558	struct amdgpu_vm *vm = bo_va->base.vm;
1559	bool valid = true;
1560
1561	saddr /= AMDGPU_GPU_PAGE_SIZE;
1562
1563	list_for_each_entry(mapping, &bo_va->valids, list) {
1564		if (mapping->start == saddr)
1565			break;
1566	}
1567
1568	if (&mapping->list == &bo_va->valids) {
1569		valid = false;
1570
1571		list_for_each_entry(mapping, &bo_va->invalids, list) {
1572			if (mapping->start == saddr)
1573				break;
1574		}
1575
1576		if (&mapping->list == &bo_va->invalids)
1577			return -ENOENT;
1578	}
1579
1580	list_del(&mapping->list);
1581	amdgpu_vm_it_remove(mapping, &vm->va);
1582	mapping->bo_va = NULL;
1583	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1584
1585	if (valid)
1586		list_add(&mapping->list, &vm->freed);
1587	else
1588		amdgpu_vm_free_mapping(adev, vm, mapping,
1589				       bo_va->last_pt_update);
1590
1591	return 0;
1592}
1593
1594/**
1595 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
1596 *
1597 * @adev: amdgpu_device pointer
1598 * @vm: VM structure to use
1599 * @saddr: start of the range
1600 * @size: size of the range
1601 *
1602 * Remove all mappings in a range, split them as appropriate.
1603 *
1604 * Returns:
1605 * 0 for success, error for failure.
1606 */
1607int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1608				struct amdgpu_vm *vm,
1609				uint64_t saddr, uint64_t size)
1610{
1611	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
1612	LIST_HEAD(removed);
1613	uint64_t eaddr;
1614
1615	eaddr = saddr + size - 1;
1616	saddr /= AMDGPU_GPU_PAGE_SIZE;
1617	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1618
1619	/* Allocate all the needed memory */
1620	before = kzalloc(sizeof(*before), GFP_KERNEL);
1621	if (!before)
1622		return -ENOMEM;
1623	INIT_LIST_HEAD(&before->list);
1624
1625	after = kzalloc(sizeof(*after), GFP_KERNEL);
1626	if (!after) {
1627		kfree(before);
1628		return -ENOMEM;
1629	}
1630	INIT_LIST_HEAD(&after->list);
1631
1632	/* Now gather all removed mappings */
1633	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1634	while (tmp) {
1635		/* Remember mapping split at the start */
1636		if (tmp->start < saddr) {
1637			before->start = tmp->start;
1638			before->last = saddr - 1;
1639			before->offset = tmp->offset;
1640			before->flags = tmp->flags;
1641			before->bo_va = tmp->bo_va;
1642			list_add(&before->list, &tmp->bo_va->invalids);
1643		}
1644
1645		/* Remember mapping split at the end */
1646		if (tmp->last > eaddr) {
1647			after->start = eaddr + 1;
1648			after->last = tmp->last;
1649			after->offset = tmp->offset;
1650			after->offset += (after->start - tmp->start) << PAGE_SHIFT;
1651			after->flags = tmp->flags;
1652			after->bo_va = tmp->bo_va;
1653			list_add(&after->list, &tmp->bo_va->invalids);
1654		}
1655
1656		list_del(&tmp->list);
1657		list_add(&tmp->list, &removed);
1658
1659		tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
1660	}
1661
1662	/* And free them up */
1663	list_for_each_entry_safe(tmp, next, &removed, list) {
1664		amdgpu_vm_it_remove(tmp, &vm->va);
1665		list_del(&tmp->list);
1666
1667		if (tmp->start < saddr)
1668		    tmp->start = saddr;
1669		if (tmp->last > eaddr)
1670		    tmp->last = eaddr;
1671
1672		tmp->bo_va = NULL;
1673		list_add(&tmp->list, &vm->freed);
1674		trace_amdgpu_vm_bo_unmap(NULL, tmp);
1675	}
1676
1677	/* Insert partial mapping before the range */
1678	if (!list_empty(&before->list)) {
1679		amdgpu_vm_it_insert(before, &vm->va);
1680		if (before->flags & AMDGPU_PTE_PRT)
1681			amdgpu_vm_prt_get(adev);
1682	} else {
1683		kfree(before);
1684	}
1685
1686	/* Insert partial mapping after the range */
1687	if (!list_empty(&after->list)) {
1688		amdgpu_vm_it_insert(after, &vm->va);
1689		if (after->flags & AMDGPU_PTE_PRT)
1690			amdgpu_vm_prt_get(adev);
1691	} else {
1692		kfree(after);
1693	}
1694
1695	return 0;
1696}
1697
1698/**
1699 * amdgpu_vm_bo_lookup_mapping - find mapping by address
1700 *
1701 * @vm: the requested VM
1702 * @addr: the address
1703 *
1704 * Find a mapping by it's address.
1705 *
1706 * Returns:
1707 * The amdgpu_bo_va_mapping matching for addr or NULL
1708 *
1709 */
1710struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
1711							 uint64_t addr)
1712{
1713	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
1714}
1715
1716/**
1717 * amdgpu_vm_bo_trace_cs - trace all reserved mappings
1718 *
1719 * @vm: the requested vm
1720 * @ticket: CS ticket
1721 *
1722 * Trace all mappings of BOs reserved during a command submission.
1723 */
1724void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
1725{
1726	struct amdgpu_bo_va_mapping *mapping;
1727
1728	if (!trace_amdgpu_vm_bo_cs_enabled())
1729		return;
1730
1731	for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
1732	     mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
1733		if (mapping->bo_va && mapping->bo_va->base.bo) {
1734			struct amdgpu_bo *bo;
1735
1736			bo = mapping->bo_va->base.bo;
1737			if (dma_resv_locking_ctx(bo->tbo.base.resv) !=
1738			    ticket)
1739				continue;
1740		}
1741
1742		trace_amdgpu_vm_bo_cs(mapping);
1743	}
1744}
1745
1746/**
1747 * amdgpu_vm_bo_del - remove a bo from a specific vm
1748 *
1749 * @adev: amdgpu_device pointer
1750 * @bo_va: requested bo_va
1751 *
1752 * Remove @bo_va->bo from the requested vm.
1753 *
1754 * Object have to be reserved!
1755 */
1756void amdgpu_vm_bo_del(struct amdgpu_device *adev,
1757		      struct amdgpu_bo_va *bo_va)
1758{
1759	struct amdgpu_bo_va_mapping *mapping, *next;
1760	struct amdgpu_bo *bo = bo_va->base.bo;
1761	struct amdgpu_vm *vm = bo_va->base.vm;
1762	struct amdgpu_vm_bo_base **base;
1763
1764	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
1765
1766	if (bo) {
1767		dma_resv_assert_held(bo->tbo.base.resv);
1768		if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
1769			ttm_bo_set_bulk_move(&bo->tbo, NULL);
1770
1771		for (base = &bo_va->base.bo->vm_bo; *base;
1772		     base = &(*base)->next) {
1773			if (*base != &bo_va->base)
1774				continue;
1775
1776			*base = bo_va->base.next;
1777			break;
1778		}
1779	}
1780
1781	spin_lock(&vm->status_lock);
1782	list_del(&bo_va->base.vm_status);
1783	spin_unlock(&vm->status_lock);
1784
1785	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1786		list_del(&mapping->list);
1787		amdgpu_vm_it_remove(mapping, &vm->va);
1788		mapping->bo_va = NULL;
1789		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1790		list_add(&mapping->list, &vm->freed);
1791	}
1792	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1793		list_del(&mapping->list);
1794		amdgpu_vm_it_remove(mapping, &vm->va);
1795		amdgpu_vm_free_mapping(adev, vm, mapping,
1796				       bo_va->last_pt_update);
1797	}
1798
1799	dma_fence_put(bo_va->last_pt_update);
1800
1801	if (bo && bo_va->is_xgmi)
1802		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);
1803
1804	kfree(bo_va);
1805}
1806
1807/**
1808 * amdgpu_vm_evictable - check if we can evict a VM
1809 *
1810 * @bo: A page table of the VM.
1811 *
1812 * Check if it is possible to evict a VM.
1813 */
1814bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
1815{
1816	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
1817
1818	/* Page tables of a destroyed VM can go away immediately */
1819	if (!bo_base || !bo_base->vm)
1820		return true;
1821
1822	/* Don't evict VM page tables while they are busy */
1823	if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP))
1824		return false;
1825
1826	/* Try to block ongoing updates */
1827	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
1828		return false;
1829
1830	/* Don't evict VM page tables while they are updated */
1831	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
1832		amdgpu_vm_eviction_unlock(bo_base->vm);
1833		return false;
1834	}
1835
1836	bo_base->vm->evicting = true;
1837	amdgpu_vm_eviction_unlock(bo_base->vm);
1838	return true;
1839}
1840
1841/**
1842 * amdgpu_vm_bo_invalidate - mark the bo as invalid
1843 *
1844 * @adev: amdgpu_device pointer
 
1845 * @bo: amdgpu buffer object
1846 * @evicted: is the BO evicted
1847 *
1848 * Mark @bo as invalid.
1849 */
1850void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1851			     struct amdgpu_bo *bo, bool evicted)
1852{
1853	struct amdgpu_vm_bo_base *bo_base;
1854
1855	/* shadow bo doesn't have bo base, its validation needs its parent */
1856	if (bo->parent && (amdgpu_bo_shadowed(bo->parent) == bo))
1857		bo = bo->parent;
1858
1859	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
1860		struct amdgpu_vm *vm = bo_base->vm;
1861
1862		if (evicted && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) {
1863			amdgpu_vm_bo_evicted(bo_base);
 
 
 
 
 
 
 
1864			continue;
1865		}
1866
1867		if (bo_base->moved)
 
 
 
 
1868			continue;
1869		bo_base->moved = true;
1870
1871		if (bo->tbo.type == ttm_bo_type_kernel)
1872			amdgpu_vm_bo_relocated(bo_base);
1873		else if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)
1874			amdgpu_vm_bo_moved(bo_base);
1875		else
1876			amdgpu_vm_bo_invalidated(bo_base);
1877	}
1878}
1879
1880/**
1881 * amdgpu_vm_get_block_size - calculate VM page table size as power of two
1882 *
1883 * @vm_size: VM size
1884 *
1885 * Returns:
1886 * VM page table as power of two
1887 */
1888static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
1889{
1890	/* Total bits covered by PD + PTs */
1891	unsigned bits = ilog2(vm_size) + 18;
1892
1893	/* Make sure the PD is 4K in size up to 8GB address space.
1894	   Above that split equal between PD and PTs */
1895	if (vm_size <= 8)
1896		return (bits - 9);
1897	else
1898		return ((bits + 3) / 2);
1899}
1900
1901/**
1902 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
1903 *
1904 * @adev: amdgpu_device pointer
1905 * @min_vm_size: the minimum vm size in GB if it's set auto
1906 * @fragment_size_default: Default PTE fragment size
1907 * @max_level: max VMPT level
1908 * @max_bits: max address space size in bits
1909 *
1910 */
1911void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
1912			   uint32_t fragment_size_default, unsigned max_level,
1913			   unsigned max_bits)
1914{
1915	unsigned int max_size = 1 << (max_bits - 30);
1916	unsigned int vm_size;
1917	uint64_t tmp;
1918
1919	/* adjust vm size first */
1920	if (amdgpu_vm_size != -1) {
 
 
1921		vm_size = amdgpu_vm_size;
1922		if (vm_size > max_size) {
1923			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
1924				 amdgpu_vm_size, max_size);
1925			vm_size = max_size;
1926		}
1927	} else {
1928		struct sysinfo si;
1929		unsigned int phys_ram_gb;
1930
1931		/* Optimal VM size depends on the amount of physical
1932		 * RAM available. Underlying requirements and
1933		 * assumptions:
1934		 *
1935		 *  - Need to map system memory and VRAM from all GPUs
1936		 *     - VRAM from other GPUs not known here
1937		 *     - Assume VRAM <= system memory
1938		 *  - On GFX8 and older, VM space can be segmented for
1939		 *    different MTYPEs
1940		 *  - Need to allow room for fragmentation, guard pages etc.
1941		 *
1942		 * This adds up to a rough guess of system memory x3.
1943		 * Round up to power of two to maximize the available
1944		 * VM size with the given page table size.
1945		 */
1946		si_meminfo(&si);
1947		phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
1948			       (1 << 30) - 1) >> 30;
1949		vm_size = roundup_pow_of_two(
1950			min(max(phys_ram_gb * 3, min_vm_size), max_size));
1951	}
1952
1953	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
1954
1955	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
1956	if (amdgpu_vm_block_size != -1)
1957		tmp >>= amdgpu_vm_block_size - 9;
1958	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
1959	adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
1960	switch (adev->vm_manager.num_level) {
1961	case 3:
1962		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
1963		break;
1964	case 2:
1965		adev->vm_manager.root_level = AMDGPU_VM_PDB1;
1966		break;
1967	case 1:
1968		adev->vm_manager.root_level = AMDGPU_VM_PDB0;
1969		break;
1970	default:
1971		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
1972	}
1973	/* block size depends on vm size and hw setup*/
1974	if (amdgpu_vm_block_size != -1)
1975		adev->vm_manager.block_size =
1976			min((unsigned)amdgpu_vm_block_size, max_bits
1977			    - AMDGPU_GPU_PAGE_SHIFT
1978			    - 9 * adev->vm_manager.num_level);
1979	else if (adev->vm_manager.num_level > 1)
1980		adev->vm_manager.block_size = 9;
1981	else
1982		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
1983
1984	if (amdgpu_vm_fragment_size == -1)
1985		adev->vm_manager.fragment_size = fragment_size_default;
1986	else
1987		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
1988
1989	DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
1990		 vm_size, adev->vm_manager.num_level + 1,
1991		 adev->vm_manager.block_size,
1992		 adev->vm_manager.fragment_size);
1993}
1994
1995/**
1996 * amdgpu_vm_wait_idle - wait for the VM to become idle
1997 *
1998 * @vm: VM object to wait for
1999 * @timeout: timeout to wait for VM to become idle
2000 */
2001long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2002{
2003	timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv,
2004					DMA_RESV_USAGE_BOOKKEEP,
2005					true, timeout);
2006	if (timeout <= 0)
2007		return timeout;
2008
2009	return dma_fence_wait_timeout(vm->last_unlocked, true, timeout);
2010}
2011
2012/**
2013 * amdgpu_vm_init - initialize a vm instance
2014 *
2015 * @adev: amdgpu_device pointer
2016 * @vm: requested vm
 
2017 *
2018 * Init @vm fields.
2019 *
2020 * Returns:
2021 * 0 for success, error for failure.
2022 */
2023int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 
2024{
2025	struct amdgpu_bo *root_bo;
2026	struct amdgpu_bo_vm *root;
 
 
 
 
 
2027	int r, i;
2028
2029	vm->va = RB_ROOT_CACHED;
2030	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2031		vm->reserved_vmid[i] = NULL;
 
2032	INIT_LIST_HEAD(&vm->evicted);
2033	INIT_LIST_HEAD(&vm->relocated);
2034	INIT_LIST_HEAD(&vm->moved);
2035	INIT_LIST_HEAD(&vm->idle);
2036	INIT_LIST_HEAD(&vm->invalidated);
2037	spin_lock_init(&vm->status_lock);
2038	INIT_LIST_HEAD(&vm->freed);
2039	INIT_LIST_HEAD(&vm->done);
2040	INIT_LIST_HEAD(&vm->pt_freed);
2041	INIT_WORK(&vm->pt_free_work, amdgpu_vm_pt_free_work);
2042
2043	/* create scheduler entities for page table updates */
2044	r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
2045				  adev->vm_manager.vm_pte_scheds,
2046				  adev->vm_manager.vm_pte_num_scheds, NULL);
 
2047	if (r)
2048		return r;
2049
2050	r = drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL,
2051				  adev->vm_manager.vm_pte_scheds,
2052				  adev->vm_manager.vm_pte_num_scheds, NULL);
2053	if (r)
2054		goto error_free_immediate;
2055
2056	vm->pte_support_ats = false;
2057	vm->is_compute_context = false;
2058
2059	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2060				    AMDGPU_VM_USE_CPU_FOR_GFX);
 
2061
 
 
 
 
 
 
2062	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2063			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2064	WARN_ONCE((vm->use_cpu_for_update &&
2065		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2066		  "CPU update of VM recommended only for large BAR system\n");
 
2067
 
2068	if (vm->use_cpu_for_update)
2069		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2070	else
2071		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2072	vm->last_update = NULL;
2073	vm->last_unlocked = dma_fence_get_stub();
2074	vm->last_tlb_flush = dma_fence_get_stub();
2075
2076	mutex_init(&vm->eviction_lock);
2077	vm->evicting = false;
 
 
 
2078
2079	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
2080				false, &root);
2081	if (r)
2082		goto error_free_delayed;
2083	root_bo = &root->bo;
2084	r = amdgpu_bo_reserve(root_bo, true);
2085	if (r)
2086		goto error_free_root;
2087
2088	r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1);
 
 
2089	if (r)
2090		goto error_unreserve;
2091
2092	amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);
 
 
 
2093
2094	r = amdgpu_vm_pt_clear(adev, vm, root, false);
2095	if (r)
2096		goto error_unreserve;
2097
2098	amdgpu_bo_unreserve(vm->root.bo);
 
 
 
 
 
 
 
 
2099
2100	INIT_KFIFO(vm->faults);
 
2101
2102	return 0;
2103
2104error_unreserve:
2105	amdgpu_bo_unreserve(vm->root.bo);
2106
2107error_free_root:
2108	amdgpu_bo_unref(&root->shadow);
2109	amdgpu_bo_unref(&root_bo);
2110	vm->root.bo = NULL;
2111
2112error_free_delayed:
2113	dma_fence_put(vm->last_tlb_flush);
2114	dma_fence_put(vm->last_unlocked);
2115	drm_sched_entity_destroy(&vm->delayed);
2116
2117error_free_immediate:
2118	drm_sched_entity_destroy(&vm->immediate);
2119
2120	return r;
2121}
2122
2123/**
2124 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2125 *
2126 * @adev: amdgpu_device pointer
2127 * @vm: requested vm
2128 *
2129 * This only works on GFX VMs that don't have any BOs added and no
2130 * page tables allocated yet.
2131 *
2132 * Changes the following VM parameters:
2133 * - use_cpu_for_update
2134 * - pte_supports_ats
 
2135 *
2136 * Reinitializes the page directory to reflect the changed ATS
2137 * setting.
 
2138 *
2139 * Returns:
2140 * 0 for success, -errno for errors.
2141 */
2142int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2143{
2144	bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2145	int r;
2146
2147	r = amdgpu_bo_reserve(vm->root.bo, true);
2148	if (r)
2149		return r;
2150
2151	/* Sanity checks */
2152	if (!amdgpu_vm_pt_is_root_clean(adev, vm)) {
2153		r = -EINVAL;
2154		goto unreserve_bo;
2155	}
2156
2157	/* Check if PD needs to be reinitialized and do it before
2158	 * changing any other state, in case it fails.
2159	 */
2160	if (pte_support_ats != vm->pte_support_ats) {
2161		vm->pte_support_ats = pte_support_ats;
2162		r = amdgpu_vm_pt_clear(adev, vm, to_amdgpu_bo_vm(vm->root.bo),
2163				       false);
2164		if (r)
2165			goto unreserve_bo;
2166	}
2167
2168	/* Update VM state */
2169	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2170				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);
 
2171	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2172			 vm->use_cpu_for_update ? "CPU" : "SDMA");
2173	WARN_ONCE((vm->use_cpu_for_update &&
2174		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2175		  "CPU update of VM recommended only for large BAR system\n");
2176
2177	if (vm->use_cpu_for_update) {
2178		/* Sync with last SDMA update/clear before switching to CPU */
2179		r = amdgpu_bo_sync_wait(vm->root.bo,
2180					AMDGPU_FENCE_OWNER_UNDEFINED, true);
2181		if (r)
2182			goto unreserve_bo;
2183
2184		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2185	} else {
2186		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2187	}
2188	/*
2189	 * Make sure root PD gets mapped. As vm_update_mode could be changed
2190	 * when turning a GFX VM into a compute VM.
2191	 */
2192	r = vm->update_funcs->map_table(to_amdgpu_bo_vm(vm->root.bo));
2193	if (r)
2194		goto unreserve_bo;
2195
2196	dma_fence_put(vm->last_update);
2197	vm->last_update = NULL;
2198	vm->is_compute_context = true;
2199
2200	/* Free the shadow bo for compute VM */
2201	amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.bo)->shadow);
2202
2203	goto unreserve_bo;
2204
2205unreserve_bo:
2206	amdgpu_bo_unreserve(vm->root.bo);
2207	return r;
2208}
2209
2210/**
2211 * amdgpu_vm_release_compute - release a compute vm
2212 * @adev: amdgpu_device pointer
2213 * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
 
 
2214 *
2215 * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
2216 * pasid from vm. Compute should stop use of vm after this call.
2217 */
2218void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 
 
2219{
2220	amdgpu_vm_set_pasid(adev, vm, 0);
2221	vm->is_compute_context = false;
 
 
 
 
 
 
 
 
 
 
 
 
 
2222}
2223
2224/**
2225 * amdgpu_vm_fini - tear down a vm instance
2226 *
2227 * @adev: amdgpu_device pointer
2228 * @vm: requested vm
2229 *
2230 * Tear down @vm.
2231 * Unbind the VM and remove all bos from the vm bo list
2232 */
2233void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2234{
2235	struct amdgpu_bo_va_mapping *mapping, *tmp;
2236	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2237	struct amdgpu_bo *root;
2238	unsigned long flags;
2239	int i;
2240
2241	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2242
2243	flush_work(&vm->pt_free_work);
 
 
 
 
 
 
 
 
 
 
2244
2245	root = amdgpu_bo_ref(vm->root.bo);
2246	amdgpu_bo_reserve(root, true);
2247	amdgpu_vm_set_pasid(adev, vm, 0);
2248	dma_fence_wait(vm->last_unlocked, false);
2249	dma_fence_put(vm->last_unlocked);
2250	dma_fence_wait(vm->last_tlb_flush, false);
2251	/* Make sure that all fence callbacks have completed */
2252	spin_lock_irqsave(vm->last_tlb_flush->lock, flags);
2253	spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags);
2254	dma_fence_put(vm->last_tlb_flush);
2255
 
 
 
 
 
 
 
 
 
2256	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2257		if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2258			amdgpu_vm_prt_fini(adev, vm);
2259			prt_fini_needed = false;
2260		}
2261
2262		list_del(&mapping->list);
2263		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2264	}
2265
2266	amdgpu_vm_pt_free_root(adev, vm);
2267	amdgpu_bo_unreserve(root);
 
 
 
 
 
 
 
2268	amdgpu_bo_unref(&root);
2269	WARN_ON(vm->root.bo);
 
 
 
2270
2271	drm_sched_entity_destroy(&vm->immediate);
2272	drm_sched_entity_destroy(&vm->delayed);
 
 
 
 
 
 
 
 
 
 
 
2273
2274	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2275		dev_err(adev->dev, "still active bo inside vm\n");
 
 
 
 
2276	}
2277	rbtree_postorder_for_each_entry_safe(mapping, tmp,
2278					     &vm->va.rb_root, rb) {
2279		/* Don't remove the mapping here, we don't want to trigger a
2280		 * rebalance and the tree is about to be destroyed anyway.
2281		 */
2282		list_del(&mapping->list);
2283		kfree(mapping);
2284	}
2285
2286	dma_fence_put(vm->last_update);
2287	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2288		amdgpu_vmid_free_reserved(adev, vm, i);
2289}
2290
2291/**
2292 * amdgpu_vm_manager_init - init the VM manager
2293 *
2294 * @adev: amdgpu_device pointer
2295 *
2296 * Initialize the VM manager structures
2297 */
2298void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2299{
2300	unsigned i;
2301
2302	/* Concurrent flushes are only possible starting with Vega10 and
2303	 * are broken on Navi10 and Navi14.
2304	 */
2305	adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||
2306					      adev->asic_type == CHIP_NAVI10 ||
2307					      adev->asic_type == CHIP_NAVI14);
2308	amdgpu_vmid_mgr_init(adev);
2309
2310	adev->vm_manager.fence_context =
2311		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2312	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2313		adev->vm_manager.seqno[i] = 0;
2314
 
2315	spin_lock_init(&adev->vm_manager.prt_lock);
2316	atomic_set(&adev->vm_manager.num_prt_users, 0);
2317
2318	/* If not overridden by the user, by default, only in large BAR systems
2319	 * Compute VM tables will be updated by CPU
2320	 */
2321#ifdef CONFIG_X86_64
2322	if (amdgpu_vm_update_mode == -1) {
2323		/* For asic with VF MMIO access protection
2324		 * avoid using CPU for VM table updates
2325		 */
2326		if (amdgpu_gmc_vram_full_visible(&adev->gmc) &&
2327		    !amdgpu_sriov_vf_mmio_access_protection(adev))
2328			adev->vm_manager.vm_update_mode =
2329				AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2330		else
2331			adev->vm_manager.vm_update_mode = 0;
2332	} else
2333		adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2334#else
2335	adev->vm_manager.vm_update_mode = 0;
2336#endif
2337
2338	xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ);
 
2339}
2340
2341/**
2342 * amdgpu_vm_manager_fini - cleanup VM manager
2343 *
2344 * @adev: amdgpu_device pointer
2345 *
2346 * Cleanup the VM manager and free resources.
2347 */
2348void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2349{
2350	WARN_ON(!xa_empty(&adev->vm_manager.pasids));
2351	xa_destroy(&adev->vm_manager.pasids);
2352
2353	amdgpu_vmid_mgr_fini(adev);
2354}
2355
2356/**
2357 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
2358 *
2359 * @dev: drm device pointer
2360 * @data: drm_amdgpu_vm
2361 * @filp: drm file pointer
2362 *
2363 * Returns:
2364 * 0 for success, -errno for errors.
2365 */
2366int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2367{
2368	union drm_amdgpu_vm *args = data;
2369	struct amdgpu_device *adev = drm_to_adev(dev);
2370	struct amdgpu_fpriv *fpriv = filp->driver_priv;
2371	int r;
2372
2373	switch (args->in.op) {
2374	case AMDGPU_VM_OP_RESERVE_VMID:
2375		/* We only have requirement to reserve vmid from gfxhub */
2376		r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm,
2377					       AMDGPU_GFXHUB_0);
2378		if (r)
2379			return r;
2380		break;
2381	case AMDGPU_VM_OP_UNRESERVE_VMID:
2382		amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0);
2383		break;
2384	default:
2385		return -EINVAL;
2386	}
2387
2388	return 0;
2389}
2390
2391/**
2392 * amdgpu_vm_get_task_info - Extracts task info for a PASID.
2393 *
2394 * @adev: drm device pointer
2395 * @pasid: PASID identifier for VM
2396 * @task_info: task_info to fill.
2397 */
2398void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
2399			 struct amdgpu_task_info *task_info)
2400{
2401	struct amdgpu_vm *vm;
2402	unsigned long flags;
2403
2404	xa_lock_irqsave(&adev->vm_manager.pasids, flags);
2405
2406	vm = xa_load(&adev->vm_manager.pasids, pasid);
2407	if (vm)
2408		*task_info = vm->task_info;
2409
2410	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
2411}
2412
2413/**
2414 * amdgpu_vm_set_task_info - Sets VMs task info.
2415 *
2416 * @vm: vm for which to set the info
2417 */
2418void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
2419{
2420	if (vm->task_info.pid)
2421		return;
2422
2423	vm->task_info.pid = current->pid;
2424	get_task_comm(vm->task_info.task_name, current);
2425
2426	if (current->group_leader->mm != current->mm)
2427		return;
2428
2429	vm->task_info.tgid = current->group_leader->pid;
2430	get_task_comm(vm->task_info.process_name, current->group_leader);
2431}
2432
2433/**
2434 * amdgpu_vm_handle_fault - graceful handling of VM faults.
2435 * @adev: amdgpu device pointer
2436 * @pasid: PASID of the VM
2437 * @addr: Address of the fault
2438 * @write_fault: true is write fault, false is read fault
2439 *
2440 * Try to gracefully handle a VM fault. Return true if the fault was handled and
2441 * shouldn't be reported any more.
2442 */
2443bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
2444			    uint64_t addr, bool write_fault)
2445{
2446	bool is_compute_context = false;
2447	struct amdgpu_bo *root;
2448	unsigned long irqflags;
2449	uint64_t value, flags;
2450	struct amdgpu_vm *vm;
2451	int r;
2452
2453	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2454	vm = xa_load(&adev->vm_manager.pasids, pasid);
2455	if (vm) {
2456		root = amdgpu_bo_ref(vm->root.bo);
2457		is_compute_context = vm->is_compute_context;
2458	} else {
2459		root = NULL;
2460	}
2461	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2462
2463	if (!root)
2464		return false;
2465
2466	addr /= AMDGPU_GPU_PAGE_SIZE;
2467
2468	if (is_compute_context &&
2469	    !svm_range_restore_pages(adev, pasid, addr, write_fault)) {
2470		amdgpu_bo_unref(&root);
2471		return true;
2472	}
2473
2474	r = amdgpu_bo_reserve(root, true);
2475	if (r)
2476		goto error_unref;
2477
2478	/* Double check that the VM still exists */
2479	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2480	vm = xa_load(&adev->vm_manager.pasids, pasid);
2481	if (vm && vm->root.bo != root)
2482		vm = NULL;
2483	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2484	if (!vm)
2485		goto error_unlock;
2486
2487	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
2488		AMDGPU_PTE_SYSTEM;
2489
2490	if (is_compute_context) {
2491		/* Intentionally setting invalid PTE flag
2492		 * combination to force a no-retry-fault
2493		 */
2494		flags = AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT;
2495		value = 0;
2496	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
2497		/* Redirect the access to the dummy page */
2498		value = adev->dummy_page_addr;
2499		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
2500			AMDGPU_PTE_WRITEABLE;
2501
2502	} else {
2503		/* Let the hw retry silently on the PTE */
2504		value = 0;
2505	}
2506
2507	r = dma_resv_reserve_fences(root->tbo.base.resv, 1);
2508	if (r) {
2509		pr_debug("failed %d to reserve fence slot\n", r);
2510		goto error_unlock;
2511	}
2512
2513	r = amdgpu_vm_update_range(adev, vm, true, false, false, NULL, addr,
2514				   addr, flags, value, 0, NULL, NULL, NULL);
2515	if (r)
2516		goto error_unlock;
2517
2518	r = amdgpu_vm_update_pdes(adev, vm, true);
2519
2520error_unlock:
2521	amdgpu_bo_unreserve(root);
2522	if (r < 0)
2523		DRM_ERROR("Can't handle page fault (%d)\n", r);
2524
2525error_unref:
2526	amdgpu_bo_unref(&root);
2527
2528	return false;
2529}
2530
2531#if defined(CONFIG_DEBUG_FS)
2532/**
2533 * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
2534 *
2535 * @vm: Requested VM for printing BO info
2536 * @m: debugfs file
2537 *
2538 * Print BO information in debugfs file for the VM
2539 */
2540void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
2541{
2542	struct amdgpu_bo_va *bo_va, *tmp;
2543	u64 total_idle = 0;
2544	u64 total_evicted = 0;
2545	u64 total_relocated = 0;
2546	u64 total_moved = 0;
2547	u64 total_invalidated = 0;
2548	u64 total_done = 0;
2549	unsigned int total_idle_objs = 0;
2550	unsigned int total_evicted_objs = 0;
2551	unsigned int total_relocated_objs = 0;
2552	unsigned int total_moved_objs = 0;
2553	unsigned int total_invalidated_objs = 0;
2554	unsigned int total_done_objs = 0;
2555	unsigned int id = 0;
2556
2557	spin_lock(&vm->status_lock);
2558	seq_puts(m, "\tIdle BOs:\n");
2559	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
2560		if (!bo_va->base.bo)
2561			continue;
2562		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2563	}
2564	total_idle_objs = id;
2565	id = 0;
2566
2567	seq_puts(m, "\tEvicted BOs:\n");
2568	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
2569		if (!bo_va->base.bo)
2570			continue;
2571		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2572	}
2573	total_evicted_objs = id;
2574	id = 0;
2575
2576	seq_puts(m, "\tRelocated BOs:\n");
2577	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
2578		if (!bo_va->base.bo)
2579			continue;
2580		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2581	}
2582	total_relocated_objs = id;
2583	id = 0;
2584
2585	seq_puts(m, "\tMoved BOs:\n");
2586	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
2587		if (!bo_va->base.bo)
2588			continue;
2589		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2590	}
2591	total_moved_objs = id;
2592	id = 0;
2593
2594	seq_puts(m, "\tInvalidated BOs:\n");
2595	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
2596		if (!bo_va->base.bo)
2597			continue;
2598		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
2599	}
2600	total_invalidated_objs = id;
2601	id = 0;
2602
2603	seq_puts(m, "\tDone BOs:\n");
2604	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
2605		if (!bo_va->base.bo)
2606			continue;
2607		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
2608	}
2609	spin_unlock(&vm->status_lock);
2610	total_done_objs = id;
2611
2612	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
2613		   total_idle_objs);
2614	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,
2615		   total_evicted_objs);
2616	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,
2617		   total_relocated_objs);
2618	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,
2619		   total_moved_objs);
2620	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
2621		   total_invalidated_objs);
2622	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,
2623		   total_done_objs);
2624}
2625#endif