1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice shall be included in
  13 * all copies or substantial portions of the Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 * OTHER DEALINGS IN THE SOFTWARE.
  22 */
  23
  24#include <linux/device.h>
  25#include <linux/export.h>
  26#include <linux/err.h>
  27#include <linux/fs.h>
  28#include <linux/file.h>
  29#include <linux/sched.h>
  30#include <linux/slab.h>
  31#include <linux/uaccess.h>
  32#include <linux/compat.h>
  33#include <uapi/linux/kfd_ioctl.h>
  34#include <linux/time.h>
  35#include <linux/mm.h>
  36#include <linux/mman.h>
  37#include <linux/ptrace.h>
  38#include <linux/dma-buf.h>
  39#include <linux/fdtable.h>
  40#include <linux/processor.h>
  41#include "kfd_priv.h"
  42#include "kfd_device_queue_manager.h"
  43#include "kfd_svm.h"
  44#include "amdgpu_amdkfd.h"
  45#include "kfd_smi_events.h"
  46#include "amdgpu_dma_buf.h"
  47
  48static long kfd_ioctl(struct file *, unsigned int, unsigned long);
  49static int kfd_open(struct inode *, struct file *);
  50static int kfd_release(struct inode *, struct file *);
  51static int kfd_mmap(struct file *, struct vm_area_struct *);
  52
  53static const char kfd_dev_name[] = "kfd";
  54
  55static const struct file_operations kfd_fops = {
  56	.owner = THIS_MODULE,
  57	.unlocked_ioctl = kfd_ioctl,
  58	.compat_ioctl = compat_ptr_ioctl,
  59	.open = kfd_open,
  60	.release = kfd_release,
  61	.mmap = kfd_mmap,
  62};
  63
  64static int kfd_char_dev_major = -1;
  65static struct class *kfd_class;
  66struct device *kfd_device;
  67
  68static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
  69{
  70	struct kfd_process_device *pdd;
  71
  72	mutex_lock(&p->mutex);
  73	pdd = kfd_process_device_data_by_id(p, gpu_id);
  74
  75	if (pdd)
  76		return pdd;
  77
  78	mutex_unlock(&p->mutex);
  79	return NULL;
  80}
  81
  82static inline void kfd_unlock_pdd(struct kfd_process_device *pdd)
  83{
  84	mutex_unlock(&pdd->process->mutex);
  85}
  86
  87int kfd_chardev_init(void)
  88{
  89	int err = 0;
  90
  91	kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
  92	err = kfd_char_dev_major;
  93	if (err < 0)
  94		goto err_register_chrdev;
  95
  96	kfd_class = class_create(THIS_MODULE, kfd_dev_name);
  97	err = PTR_ERR(kfd_class);
  98	if (IS_ERR(kfd_class))
  99		goto err_class_create;
 100
 101	kfd_device = device_create(kfd_class, NULL,
 102					MKDEV(kfd_char_dev_major, 0),
 103					NULL, kfd_dev_name);
 104	err = PTR_ERR(kfd_device);
 105	if (IS_ERR(kfd_device))
 106		goto err_device_create;
 107
 108	return 0;
 109
 110err_device_create:
 111	class_destroy(kfd_class);
 112err_class_create:
 113	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 114err_register_chrdev:
 115	return err;
 116}
 117
 118void kfd_chardev_exit(void)
 119{
 120	device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
 121	class_destroy(kfd_class);
 122	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 123	kfd_device = NULL;
 124}
 125
 126
 127static int kfd_open(struct inode *inode, struct file *filep)
 128{
 129	struct kfd_process *process;
 130	bool is_32bit_user_mode;
 131
 132	if (iminor(inode) != 0)
 133		return -ENODEV;
 134
 135	is_32bit_user_mode = in_compat_syscall();
 136
 137	if (is_32bit_user_mode) {
 138		dev_warn(kfd_device,
 139			"Process %d (32-bit) failed to open /dev/kfd\n"
 140			"32-bit processes are not supported by amdkfd\n",
 141			current->pid);
 142		return -EPERM;
 143	}
 144
 145	process = kfd_create_process(filep);
 146	if (IS_ERR(process))
 147		return PTR_ERR(process);
 148
 149	if (kfd_is_locked()) {
 150		dev_dbg(kfd_device, "kfd is locked!\n"
 151				"process %d unreferenced", process->pasid);
 152		kfd_unref_process(process);
 153		return -EAGAIN;
 154	}
 155
 156	/* filep now owns the reference returned by kfd_create_process */
 157	filep->private_data = process;
 158
 159	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
 160		process->pasid, process->is_32bit_user_mode);
 161
 162	return 0;
 163}
 164
 165static int kfd_release(struct inode *inode, struct file *filep)
 166{
 167	struct kfd_process *process = filep->private_data;
 168
 169	if (process)
 170		kfd_unref_process(process);
 171
 172	return 0;
 173}
 174
 175static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
 176					void *data)
 177{
 178	struct kfd_ioctl_get_version_args *args = data;
 179
 180	args->major_version = KFD_IOCTL_MAJOR_VERSION;
 181	args->minor_version = KFD_IOCTL_MINOR_VERSION;
 182
 183	return 0;
 184}
 185
 186static int set_queue_properties_from_user(struct queue_properties *q_properties,
 187				struct kfd_ioctl_create_queue_args *args)
 188{
 189	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
 190		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 191		return -EINVAL;
 192	}
 193
 194	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
 195		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 196		return -EINVAL;
 197	}
 198
 199	if ((args->ring_base_address) &&
 200		(!access_ok((const void __user *) args->ring_base_address,
 201			sizeof(uint64_t)))) {
 202		pr_err("Can't access ring base address\n");
 203		return -EFAULT;
 204	}
 205
 206	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
 207		pr_err("Ring size must be a power of 2 or 0\n");
 208		return -EINVAL;
 209	}
 210
 211	if (!access_ok((const void __user *) args->read_pointer_address,
 212			sizeof(uint32_t))) {
 213		pr_err("Can't access read pointer\n");
 214		return -EFAULT;
 215	}
 216
 217	if (!access_ok((const void __user *) args->write_pointer_address,
 218			sizeof(uint32_t))) {
 219		pr_err("Can't access write pointer\n");
 220		return -EFAULT;
 221	}
 222
 223	if (args->eop_buffer_address &&
 224		!access_ok((const void __user *) args->eop_buffer_address,
 225			sizeof(uint32_t))) {
 226		pr_debug("Can't access eop buffer");
 227		return -EFAULT;
 228	}
 229
 230	if (args->ctx_save_restore_address &&
 231		!access_ok((const void __user *) args->ctx_save_restore_address,
 232			sizeof(uint32_t))) {
 233		pr_debug("Can't access ctx save restore buffer");
 234		return -EFAULT;
 235	}
 236
 237	q_properties->is_interop = false;
 238	q_properties->is_gws = false;
 239	q_properties->queue_percent = args->queue_percentage;
 240	q_properties->priority = args->queue_priority;
 241	q_properties->queue_address = args->ring_base_address;
 242	q_properties->queue_size = args->ring_size;
 243	q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
 244	q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
 245	q_properties->eop_ring_buffer_address = args->eop_buffer_address;
 246	q_properties->eop_ring_buffer_size = args->eop_buffer_size;
 247	q_properties->ctx_save_restore_area_address =
 248			args->ctx_save_restore_address;
 249	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
 250	q_properties->ctl_stack_size = args->ctl_stack_size;
 251	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
 252		args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 253		q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
 254	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
 255		q_properties->type = KFD_QUEUE_TYPE_SDMA;
 256	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
 257		q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
 258	else
 259		return -ENOTSUPP;
 260
 261	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 262		q_properties->format = KFD_QUEUE_FORMAT_AQL;
 263	else
 264		q_properties->format = KFD_QUEUE_FORMAT_PM4;
 265
 266	pr_debug("Queue Percentage: %d, %d\n",
 267			q_properties->queue_percent, args->queue_percentage);
 268
 269	pr_debug("Queue Priority: %d, %d\n",
 270			q_properties->priority, args->queue_priority);
 271
 272	pr_debug("Queue Address: 0x%llX, 0x%llX\n",
 273			q_properties->queue_address, args->ring_base_address);
 274
 275	pr_debug("Queue Size: 0x%llX, %u\n",
 276			q_properties->queue_size, args->ring_size);
 277
 278	pr_debug("Queue r/w Pointers: %px, %px\n",
 279			q_properties->read_ptr,
 280			q_properties->write_ptr);
 281
 282	pr_debug("Queue Format: %d\n", q_properties->format);
 283
 284	pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
 285
 286	pr_debug("Queue CTX save area: 0x%llX\n",
 287			q_properties->ctx_save_restore_area_address);
 288
 289	return 0;
 290}
 291
 292static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 293					void *data)
 294{
 295	struct kfd_ioctl_create_queue_args *args = data;
 296	struct kfd_dev *dev;
 297	int err = 0;
 298	unsigned int queue_id;
 299	struct kfd_process_device *pdd;
 300	struct queue_properties q_properties;
 301	uint32_t doorbell_offset_in_process = 0;
 302	struct amdgpu_bo *wptr_bo = NULL;
 303
 304	memset(&q_properties, 0, sizeof(struct queue_properties));
 305
 306	pr_debug("Creating queue ioctl\n");
 307
 308	err = set_queue_properties_from_user(&q_properties, args);
 309	if (err)
 310		return err;
 311
 312	pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
 313
 314	mutex_lock(&p->mutex);
 315
 316	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 317	if (!pdd) {
 318		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 319		err = -EINVAL;
 320		goto err_pdd;
 321	}
 322	dev = pdd->dev;
 323
 324	pdd = kfd_bind_process_to_device(dev, p);
 325	if (IS_ERR(pdd)) {
 326		err = -ESRCH;
 327		goto err_bind_process;
 328	}
 329
 330	if (!pdd->doorbell_index &&
 331	    kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
 332		err = -ENOMEM;
 333		goto err_alloc_doorbells;
 334	}
 335
 336	/* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work
 337	 * on unmapped queues for usermode queue oversubscription (no aggregated doorbell)
 338	 */
 339	if (dev->shared_resources.enable_mes &&
 340			((dev->adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK)
 341			>> AMDGPU_MES_API_VERSION_SHIFT) >= 2) {
 342		struct amdgpu_bo_va_mapping *wptr_mapping;
 343		struct amdgpu_vm *wptr_vm;
 344
 345		wptr_vm = drm_priv_to_vm(pdd->drm_priv);
 346		err = amdgpu_bo_reserve(wptr_vm->root.bo, false);
 347		if (err)
 348			goto err_wptr_map_gart;
 349
 350		wptr_mapping = amdgpu_vm_bo_lookup_mapping(
 351				wptr_vm, args->write_pointer_address >> PAGE_SHIFT);
 352		amdgpu_bo_unreserve(wptr_vm->root.bo);
 353		if (!wptr_mapping) {
 354			pr_err("Failed to lookup wptr bo\n");
 355			err = -EINVAL;
 356			goto err_wptr_map_gart;
 357		}
 358
 359		wptr_bo = wptr_mapping->bo_va->base.bo;
 360		if (wptr_bo->tbo.base.size > PAGE_SIZE) {
 361			pr_err("Requested GART mapping for wptr bo larger than one page\n");
 362			err = -EINVAL;
 363			goto err_wptr_map_gart;
 364		}
 365
 366		err = amdgpu_amdkfd_map_gtt_bo_to_gart(dev->adev, wptr_bo);
 367		if (err) {
 368			pr_err("Failed to map wptr bo to GART\n");
 369			goto err_wptr_map_gart;
 370		}
 371	}
 372
 373	pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
 374			p->pasid,
 375			dev->id);
 376
 377	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, wptr_bo,
 378			NULL, NULL, NULL, &doorbell_offset_in_process);
 379	if (err != 0)
 380		goto err_create_queue;
 381
 382	args->queue_id = queue_id;
 383
 384
 385	/* Return gpu_id as doorbell offset for mmap usage */
 386	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
 387	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
 388	if (KFD_IS_SOC15(dev))
 389		/* On SOC15 ASICs, include the doorbell offset within the
 390		 * process doorbell frame, which is 2 pages.
 391		 */
 392		args->doorbell_offset |= doorbell_offset_in_process;
 393
 394	mutex_unlock(&p->mutex);
 395
 396	pr_debug("Queue id %d was created successfully\n", args->queue_id);
 397
 398	pr_debug("Ring buffer address == 0x%016llX\n",
 399			args->ring_base_address);
 400
 401	pr_debug("Read ptr address    == 0x%016llX\n",
 402			args->read_pointer_address);
 403
 404	pr_debug("Write ptr address   == 0x%016llX\n",
 405			args->write_pointer_address);
 406
 407	return 0;
 408
 409err_create_queue:
 410	if (wptr_bo)
 411		amdgpu_amdkfd_free_gtt_mem(dev->adev, wptr_bo);
 412err_wptr_map_gart:
 413err_alloc_doorbells:
 414err_bind_process:
 415err_pdd:
 416	mutex_unlock(&p->mutex);
 417	return err;
 418}
 419
 420static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
 421					void *data)
 422{
 423	int retval;
 424	struct kfd_ioctl_destroy_queue_args *args = data;
 425
 426	pr_debug("Destroying queue id %d for pasid 0x%x\n",
 427				args->queue_id,
 428				p->pasid);
 429
 430	mutex_lock(&p->mutex);
 431
 432	retval = pqm_destroy_queue(&p->pqm, args->queue_id);
 433
 434	mutex_unlock(&p->mutex);
 435	return retval;
 436}
 437
 438static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 439					void *data)
 440{
 441	int retval;
 442	struct kfd_ioctl_update_queue_args *args = data;
 443	struct queue_properties properties;
 444
 445	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
 446		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 447		return -EINVAL;
 448	}
 449
 450	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
 451		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 452		return -EINVAL;
 453	}
 454
 455	if ((args->ring_base_address) &&
 456		(!access_ok((const void __user *) args->ring_base_address,
 457			sizeof(uint64_t)))) {
 458		pr_err("Can't access ring base address\n");
 459		return -EFAULT;
 460	}
 461
 462	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
 463		pr_err("Ring size must be a power of 2 or 0\n");
 464		return -EINVAL;
 465	}
 466
 467	properties.queue_address = args->ring_base_address;
 468	properties.queue_size = args->ring_size;
 469	properties.queue_percent = args->queue_percentage;
 470	properties.priority = args->queue_priority;
 471
 472	pr_debug("Updating queue id %d for pasid 0x%x\n",
 473			args->queue_id, p->pasid);
 474
 475	mutex_lock(&p->mutex);
 476
 477	retval = pqm_update_queue_properties(&p->pqm, args->queue_id, &properties);
 478
 479	mutex_unlock(&p->mutex);
 480
 481	return retval;
 482}
 483
 484static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
 485					void *data)
 486{
 487	int retval;
 488	const int max_num_cus = 1024;
 489	struct kfd_ioctl_set_cu_mask_args *args = data;
 490	struct mqd_update_info minfo = {0};
 491	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
 492	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
 493
 494	if ((args->num_cu_mask % 32) != 0) {
 495		pr_debug("num_cu_mask 0x%x must be a multiple of 32",
 496				args->num_cu_mask);
 497		return -EINVAL;
 498	}
 499
 500	minfo.cu_mask.count = args->num_cu_mask;
 501	if (minfo.cu_mask.count == 0) {
 502		pr_debug("CU mask cannot be 0");
 503		return -EINVAL;
 504	}
 505
 506	/* To prevent an unreasonably large CU mask size, set an arbitrary
 507	 * limit of max_num_cus bits.  We can then just drop any CU mask bits
 508	 * past max_num_cus bits and just use the first max_num_cus bits.
 509	 */
 510	if (minfo.cu_mask.count > max_num_cus) {
 511		pr_debug("CU mask cannot be greater than 1024 bits");
 512		minfo.cu_mask.count = max_num_cus;
 513		cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
 514	}
 515
 516	minfo.cu_mask.ptr = kzalloc(cu_mask_size, GFP_KERNEL);
 517	if (!minfo.cu_mask.ptr)
 518		return -ENOMEM;
 519
 520	retval = copy_from_user(minfo.cu_mask.ptr, cu_mask_ptr, cu_mask_size);
 521	if (retval) {
 522		pr_debug("Could not copy CU mask from userspace");
 523		retval = -EFAULT;
 524		goto out;
 525	}
 526
 527	minfo.update_flag = UPDATE_FLAG_CU_MASK;
 528
 529	mutex_lock(&p->mutex);
 530
 531	retval = pqm_update_mqd(&p->pqm, args->queue_id, &minfo);
 532
 533	mutex_unlock(&p->mutex);
 534
 535out:
 536	kfree(minfo.cu_mask.ptr);
 537	return retval;
 538}
 539
 540static int kfd_ioctl_get_queue_wave_state(struct file *filep,
 541					  struct kfd_process *p, void *data)
 542{
 543	struct kfd_ioctl_get_queue_wave_state_args *args = data;
 544	int r;
 545
 546	mutex_lock(&p->mutex);
 547
 548	r = pqm_get_wave_state(&p->pqm, args->queue_id,
 549			       (void __user *)args->ctl_stack_address,
 550			       &args->ctl_stack_used_size,
 551			       &args->save_area_used_size);
 552
 553	mutex_unlock(&p->mutex);
 554
 555	return r;
 556}
 557
 558static int kfd_ioctl_set_memory_policy(struct file *filep,
 559					struct kfd_process *p, void *data)
 560{
 561	struct kfd_ioctl_set_memory_policy_args *args = data;
 562	int err = 0;
 563	struct kfd_process_device *pdd;
 564	enum cache_policy default_policy, alternate_policy;
 565
 566	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
 567	    && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
 568		return -EINVAL;
 569	}
 570
 571	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
 572	    && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
 573		return -EINVAL;
 574	}
 575
 576	mutex_lock(&p->mutex);
 577	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 578	if (!pdd) {
 579		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 580		err = -EINVAL;
 581		goto err_pdd;
 582	}
 583
 584	pdd = kfd_bind_process_to_device(pdd->dev, p);
 585	if (IS_ERR(pdd)) {
 586		err = -ESRCH;
 587		goto out;
 588	}
 589
 590	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
 591			 ? cache_policy_coherent : cache_policy_noncoherent;
 592
 593	alternate_policy =
 594		(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
 595		   ? cache_policy_coherent : cache_policy_noncoherent;
 596
 597	if (!pdd->dev->dqm->ops.set_cache_memory_policy(pdd->dev->dqm,
 598				&pdd->qpd,
 599				default_policy,
 600				alternate_policy,
 601				(void __user *)args->alternate_aperture_base,
 602				args->alternate_aperture_size))
 603		err = -EINVAL;
 604
 605out:
 606err_pdd:
 607	mutex_unlock(&p->mutex);
 608
 609	return err;
 610}
 611
 612static int kfd_ioctl_set_trap_handler(struct file *filep,
 613					struct kfd_process *p, void *data)
 614{
 615	struct kfd_ioctl_set_trap_handler_args *args = data;
 616	int err = 0;
 617	struct kfd_process_device *pdd;
 618
 619	mutex_lock(&p->mutex);
 620
 621	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 622	if (!pdd) {
 623		err = -EINVAL;
 624		goto err_pdd;
 625	}
 626
 627	pdd = kfd_bind_process_to_device(pdd->dev, p);
 628	if (IS_ERR(pdd)) {
 629		err = -ESRCH;
 630		goto out;
 631	}
 632
 633	kfd_process_set_trap_handler(&pdd->qpd, args->tba_addr, args->tma_addr);
 634
 635out:
 636err_pdd:
 637	mutex_unlock(&p->mutex);
 638
 639	return err;
 640}
 641
 642static int kfd_ioctl_dbg_register(struct file *filep,
 643				struct kfd_process *p, void *data)
 644{
 645	return -EPERM;
 646}
 647
 648static int kfd_ioctl_dbg_unregister(struct file *filep,
 649				struct kfd_process *p, void *data)
 650{
 651	return -EPERM;
 652}
 653
 654static int kfd_ioctl_dbg_address_watch(struct file *filep,
 655					struct kfd_process *p, void *data)
 656{
 657	return -EPERM;
 658}
 659
 660/* Parse and generate fixed size data structure for wave control */
 661static int kfd_ioctl_dbg_wave_control(struct file *filep,
 662					struct kfd_process *p, void *data)
 663{
 664	return -EPERM;
 665}
 666
 667static int kfd_ioctl_get_clock_counters(struct file *filep,
 668				struct kfd_process *p, void *data)
 669{
 670	struct kfd_ioctl_get_clock_counters_args *args = data;
 671	struct kfd_process_device *pdd;
 672
 673	mutex_lock(&p->mutex);
 674	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 675	mutex_unlock(&p->mutex);
 676	if (pdd)
 677		/* Reading GPU clock counter from KGD */
 678		args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(pdd->dev->adev);
 679	else
 680		/* Node without GPU resource */
 681		args->gpu_clock_counter = 0;
 682
 683	/* No access to rdtsc. Using raw monotonic time */
 684	args->cpu_clock_counter = ktime_get_raw_ns();
 685	args->system_clock_counter = ktime_get_boottime_ns();
 686
 687	/* Since the counter is in nano-seconds we use 1GHz frequency */
 688	args->system_clock_freq = 1000000000;
 689
 690	return 0;
 691}
 692
 693
 694static int kfd_ioctl_get_process_apertures(struct file *filp,
 695				struct kfd_process *p, void *data)
 696{
 697	struct kfd_ioctl_get_process_apertures_args *args = data;
 698	struct kfd_process_device_apertures *pAperture;
 699	int i;
 700
 701	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
 702
 703	args->num_of_nodes = 0;
 704
 705	mutex_lock(&p->mutex);
 706	/* Run over all pdd of the process */
 707	for (i = 0; i < p->n_pdds; i++) {
 708		struct kfd_process_device *pdd = p->pdds[i];
 709
 710		pAperture =
 711			&args->process_apertures[args->num_of_nodes];
 712		pAperture->gpu_id = pdd->dev->id;
 713		pAperture->lds_base = pdd->lds_base;
 714		pAperture->lds_limit = pdd->lds_limit;
 715		pAperture->gpuvm_base = pdd->gpuvm_base;
 716		pAperture->gpuvm_limit = pdd->gpuvm_limit;
 717		pAperture->scratch_base = pdd->scratch_base;
 718		pAperture->scratch_limit = pdd->scratch_limit;
 719
 720		dev_dbg(kfd_device,
 721			"node id %u\n", args->num_of_nodes);
 722		dev_dbg(kfd_device,
 723			"gpu id %u\n", pdd->dev->id);
 724		dev_dbg(kfd_device,
 725			"lds_base %llX\n", pdd->lds_base);
 726		dev_dbg(kfd_device,
 727			"lds_limit %llX\n", pdd->lds_limit);
 728		dev_dbg(kfd_device,
 729			"gpuvm_base %llX\n", pdd->gpuvm_base);
 730		dev_dbg(kfd_device,
 731			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
 732		dev_dbg(kfd_device,
 733			"scratch_base %llX\n", pdd->scratch_base);
 734		dev_dbg(kfd_device,
 735			"scratch_limit %llX\n", pdd->scratch_limit);
 736
 737		if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
 738			break;
 739	}
 740	mutex_unlock(&p->mutex);
 741
 742	return 0;
 743}
 744
 745static int kfd_ioctl_get_process_apertures_new(struct file *filp,
 746				struct kfd_process *p, void *data)
 747{
 748	struct kfd_ioctl_get_process_apertures_new_args *args = data;
 749	struct kfd_process_device_apertures *pa;
 750	int ret;
 751	int i;
 752
 753	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
 754
 755	if (args->num_of_nodes == 0) {
 756		/* Return number of nodes, so that user space can alloacate
 757		 * sufficient memory
 758		 */
 759		mutex_lock(&p->mutex);
 760		args->num_of_nodes = p->n_pdds;
 761		goto out_unlock;
 762	}
 763
 764	/* Fill in process-aperture information for all available
 765	 * nodes, but not more than args->num_of_nodes as that is
 766	 * the amount of memory allocated by user
 767	 */
 768	pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
 769				args->num_of_nodes), GFP_KERNEL);
 770	if (!pa)
 771		return -ENOMEM;
 772
 773	mutex_lock(&p->mutex);
 774
 775	if (!p->n_pdds) {
 776		args->num_of_nodes = 0;
 777		kfree(pa);
 778		goto out_unlock;
 779	}
 780
 781	/* Run over all pdd of the process */
 782	for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
 783		struct kfd_process_device *pdd = p->pdds[i];
 784
 785		pa[i].gpu_id = pdd->dev->id;
 786		pa[i].lds_base = pdd->lds_base;
 787		pa[i].lds_limit = pdd->lds_limit;
 788		pa[i].gpuvm_base = pdd->gpuvm_base;
 789		pa[i].gpuvm_limit = pdd->gpuvm_limit;
 790		pa[i].scratch_base = pdd->scratch_base;
 791		pa[i].scratch_limit = pdd->scratch_limit;
 792
 793		dev_dbg(kfd_device,
 794			"gpu id %u\n", pdd->dev->id);
 795		dev_dbg(kfd_device,
 796			"lds_base %llX\n", pdd->lds_base);
 797		dev_dbg(kfd_device,
 798			"lds_limit %llX\n", pdd->lds_limit);
 799		dev_dbg(kfd_device,
 800			"gpuvm_base %llX\n", pdd->gpuvm_base);
 801		dev_dbg(kfd_device,
 802			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
 803		dev_dbg(kfd_device,
 804			"scratch_base %llX\n", pdd->scratch_base);
 805		dev_dbg(kfd_device,
 806			"scratch_limit %llX\n", pdd->scratch_limit);
 807	}
 808	mutex_unlock(&p->mutex);
 809
 810	args->num_of_nodes = i;
 811	ret = copy_to_user(
 812			(void __user *)args->kfd_process_device_apertures_ptr,
 813			pa,
 814			(i * sizeof(struct kfd_process_device_apertures)));
 815	kfree(pa);
 816	return ret ? -EFAULT : 0;
 817
 818out_unlock:
 819	mutex_unlock(&p->mutex);
 820	return 0;
 821}
 822
 823static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
 824					void *data)
 825{
 826	struct kfd_ioctl_create_event_args *args = data;
 827	int err;
 828
 829	/* For dGPUs the event page is allocated in user mode. The
 830	 * handle is passed to KFD with the first call to this IOCTL
 831	 * through the event_page_offset field.
 832	 */
 833	if (args->event_page_offset) {
 834		mutex_lock(&p->mutex);
 835		err = kfd_kmap_event_page(p, args->event_page_offset);
 836		mutex_unlock(&p->mutex);
 837		if (err)
 838			return err;
 839	}
 840
 841	err = kfd_event_create(filp, p, args->event_type,
 842				args->auto_reset != 0, args->node_id,
 843				&args->event_id, &args->event_trigger_data,
 844				&args->event_page_offset,
 845				&args->event_slot_index);
 846
 847	pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
 848	return err;
 849}
 850
 851static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
 852					void *data)
 853{
 854	struct kfd_ioctl_destroy_event_args *args = data;
 855
 856	return kfd_event_destroy(p, args->event_id);
 857}
 858
 859static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
 860				void *data)
 861{
 862	struct kfd_ioctl_set_event_args *args = data;
 863
 864	return kfd_set_event(p, args->event_id);
 865}
 866
 867static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
 868				void *data)
 869{
 870	struct kfd_ioctl_reset_event_args *args = data;
 871
 872	return kfd_reset_event(p, args->event_id);
 873}
 874
 875static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
 876				void *data)
 877{
 878	struct kfd_ioctl_wait_events_args *args = data;
 879
 880	return kfd_wait_on_events(p, args->num_events,
 881			(void __user *)args->events_ptr,
 882			(args->wait_for_all != 0),
 883			&args->timeout, &args->wait_result);
 884}
 885static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
 886					struct kfd_process *p, void *data)
 887{
 888	struct kfd_ioctl_set_scratch_backing_va_args *args = data;
 889	struct kfd_process_device *pdd;
 890	struct kfd_dev *dev;
 891	long err;
 892
 893	mutex_lock(&p->mutex);
 894	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 895	if (!pdd) {
 896		err = -EINVAL;
 897		goto err_pdd;
 898	}
 899	dev = pdd->dev;
 900
 901	pdd = kfd_bind_process_to_device(dev, p);
 902	if (IS_ERR(pdd)) {
 903		err = PTR_ERR(pdd);
 904		goto bind_process_to_device_fail;
 905	}
 906
 907	pdd->qpd.sh_hidden_private_base = args->va_addr;
 908
 909	mutex_unlock(&p->mutex);
 910
 911	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
 912	    pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
 913		dev->kfd2kgd->set_scratch_backing_va(
 914			dev->adev, args->va_addr, pdd->qpd.vmid);
 915
 916	return 0;
 917
 918bind_process_to_device_fail:
 919err_pdd:
 920	mutex_unlock(&p->mutex);
 921	return err;
 922}
 923
 924static int kfd_ioctl_get_tile_config(struct file *filep,
 925		struct kfd_process *p, void *data)
 926{
 927	struct kfd_ioctl_get_tile_config_args *args = data;
 928	struct kfd_process_device *pdd;
 929	struct tile_config config;
 930	int err = 0;
 931
 932	mutex_lock(&p->mutex);
 933	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 934	mutex_unlock(&p->mutex);
 935	if (!pdd)
 936		return -EINVAL;
 937
 938	amdgpu_amdkfd_get_tile_config(pdd->dev->adev, &config);
 939
 940	args->gb_addr_config = config.gb_addr_config;
 941	args->num_banks = config.num_banks;
 942	args->num_ranks = config.num_ranks;
 943
 944	if (args->num_tile_configs > config.num_tile_configs)
 945		args->num_tile_configs = config.num_tile_configs;
 946	err = copy_to_user((void __user *)args->tile_config_ptr,
 947			config.tile_config_ptr,
 948			args->num_tile_configs * sizeof(uint32_t));
 949	if (err) {
 950		args->num_tile_configs = 0;
 951		return -EFAULT;
 952	}
 953
 954	if (args->num_macro_tile_configs > config.num_macro_tile_configs)
 955		args->num_macro_tile_configs =
 956				config.num_macro_tile_configs;
 957	err = copy_to_user((void __user *)args->macro_tile_config_ptr,
 958			config.macro_tile_config_ptr,
 959			args->num_macro_tile_configs * sizeof(uint32_t));
 960	if (err) {
 961		args->num_macro_tile_configs = 0;
 962		return -EFAULT;
 963	}
 964
 965	return 0;
 966}
 967
 968static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
 969				void *data)
 970{
 971	struct kfd_ioctl_acquire_vm_args *args = data;
 972	struct kfd_process_device *pdd;
 973	struct file *drm_file;
 974	int ret;
 975
 976	drm_file = fget(args->drm_fd);
 977	if (!drm_file)
 978		return -EINVAL;
 979
 980	mutex_lock(&p->mutex);
 981	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 982	if (!pdd) {
 983		ret = -EINVAL;
 984		goto err_pdd;
 985	}
 986
 987	if (pdd->drm_file) {
 988		ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
 989		goto err_drm_file;
 990	}
 991
 992	ret = kfd_process_device_init_vm(pdd, drm_file);
 993	if (ret)
 994		goto err_unlock;
 995
 996	/* On success, the PDD keeps the drm_file reference */
 997	mutex_unlock(&p->mutex);
 998
 999	return 0;
1000
1001err_unlock:
1002err_pdd:
1003err_drm_file:
1004	mutex_unlock(&p->mutex);
1005	fput(drm_file);
1006	return ret;
1007}
1008
1009bool kfd_dev_is_large_bar(struct kfd_dev *dev)
1010{
1011	if (debug_largebar) {
1012		pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1013		return true;
1014	}
1015
1016	if (dev->use_iommu_v2)
1017		return false;
1018
1019	if (dev->local_mem_info.local_mem_size_private == 0 &&
1020			dev->local_mem_info.local_mem_size_public > 0)
1021		return true;
1022	return false;
1023}
1024
1025static int kfd_ioctl_get_available_memory(struct file *filep,
1026					  struct kfd_process *p, void *data)
1027{
1028	struct kfd_ioctl_get_available_memory_args *args = data;
1029	struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, args->gpu_id);
1030
1031	if (!pdd)
1032		return -EINVAL;
1033	args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev);
1034	kfd_unlock_pdd(pdd);
1035	return 0;
1036}
1037
1038static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1039					struct kfd_process *p, void *data)
1040{
1041	struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1042	struct kfd_process_device *pdd;
1043	void *mem;
1044	struct kfd_dev *dev;
1045	int idr_handle;
1046	long err;
1047	uint64_t offset = args->mmap_offset;
1048	uint32_t flags = args->flags;
1049
1050	if (args->size == 0)
1051		return -EINVAL;
1052
1053#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1054	/* Flush pending deferred work to avoid racing with deferred actions
1055	 * from previous memory map changes (e.g. munmap).
1056	 */
1057	svm_range_list_lock_and_flush_work(&p->svms, current->mm);
1058	mutex_lock(&p->svms.lock);
1059	mmap_write_unlock(current->mm);
1060	if (interval_tree_iter_first(&p->svms.objects,
1061				     args->va_addr >> PAGE_SHIFT,
1062				     (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1063		pr_err("Address: 0x%llx already allocated by SVM\n",
1064			args->va_addr);
1065		mutex_unlock(&p->svms.lock);
1066		return -EADDRINUSE;
1067	}
1068	mutex_unlock(&p->svms.lock);
1069#endif
1070	mutex_lock(&p->mutex);
1071	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1072	if (!pdd) {
1073		err = -EINVAL;
1074		goto err_pdd;
1075	}
1076
1077	dev = pdd->dev;
1078
1079	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1080		(flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1081		!kfd_dev_is_large_bar(dev)) {
1082		pr_err("Alloc host visible vram on small bar is not allowed\n");
1083		err = -EINVAL;
1084		goto err_large_bar;
1085	}
1086
1087	pdd = kfd_bind_process_to_device(dev, p);
1088	if (IS_ERR(pdd)) {
1089		err = PTR_ERR(pdd);
1090		goto err_unlock;
1091	}
1092
1093	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1094		if (args->size != kfd_doorbell_process_slice(dev)) {
1095			err = -EINVAL;
1096			goto err_unlock;
1097		}
1098		offset = kfd_get_process_doorbells(pdd);
1099		if (!offset) {
1100			err = -ENOMEM;
1101			goto err_unlock;
1102		}
1103	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1104		if (args->size != PAGE_SIZE) {
1105			err = -EINVAL;
1106			goto err_unlock;
1107		}
1108		offset = dev->adev->rmmio_remap.bus_addr;
1109		if (!offset) {
1110			err = -ENOMEM;
1111			goto err_unlock;
1112		}
1113	}
1114
1115	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1116		dev->adev, args->va_addr, args->size,
1117		pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
1118		flags, false);
1119
1120	if (err)
1121		goto err_unlock;
1122
1123	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1124	if (idr_handle < 0) {
1125		err = -EFAULT;
1126		goto err_free;
1127	}
1128
1129	/* Update the VRAM usage count */
1130	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
1131		WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + args->size);
1132
1133	mutex_unlock(&p->mutex);
1134
1135	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1136	args->mmap_offset = offset;
1137
1138	/* MMIO is mapped through kfd device
1139	 * Generate a kfd mmap offset
1140	 */
1141	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1142		args->mmap_offset = KFD_MMAP_TYPE_MMIO
1143					| KFD_MMAP_GPU_ID(args->gpu_id);
1144
1145	return 0;
1146
1147err_free:
1148	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, (struct kgd_mem *)mem,
1149					       pdd->drm_priv, NULL);
1150err_unlock:
1151err_pdd:
1152err_large_bar:
1153	mutex_unlock(&p->mutex);
1154	return err;
1155}
1156
1157static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1158					struct kfd_process *p, void *data)
1159{
1160	struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1161	struct kfd_process_device *pdd;
1162	void *mem;
1163	int ret;
1164	uint64_t size = 0;
1165
1166	mutex_lock(&p->mutex);
1167	/*
1168	 * Safeguard to prevent user space from freeing signal BO.
1169	 * It will be freed at process termination.
1170	 */
1171	if (p->signal_handle && (p->signal_handle == args->handle)) {
1172		pr_err("Free signal BO is not allowed\n");
1173		ret = -EPERM;
1174		goto err_unlock;
1175	}
1176
1177	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1178	if (!pdd) {
1179		pr_err("Process device data doesn't exist\n");
1180		ret = -EINVAL;
1181		goto err_pdd;
1182	}
1183
1184	mem = kfd_process_device_translate_handle(
1185		pdd, GET_IDR_HANDLE(args->handle));
1186	if (!mem) {
1187		ret = -EINVAL;
1188		goto err_unlock;
1189	}
1190
1191	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev,
1192				(struct kgd_mem *)mem, pdd->drm_priv, &size);
1193
1194	/* If freeing the buffer failed, leave the handle in place for
1195	 * clean-up during process tear-down.
1196	 */
1197	if (!ret)
1198		kfd_process_device_remove_obj_handle(
1199			pdd, GET_IDR_HANDLE(args->handle));
1200
1201	WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1202
1203err_unlock:
1204err_pdd:
1205	mutex_unlock(&p->mutex);
1206	return ret;
1207}
1208
1209static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1210					struct kfd_process *p, void *data)
1211{
1212	struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1213	struct kfd_process_device *pdd, *peer_pdd;
1214	void *mem;
1215	struct kfd_dev *dev;
1216	long err = 0;
1217	int i;
1218	uint32_t *devices_arr = NULL;
1219
1220	if (!args->n_devices) {
1221		pr_debug("Device IDs array empty\n");
1222		return -EINVAL;
1223	}
1224	if (args->n_success > args->n_devices) {
1225		pr_debug("n_success exceeds n_devices\n");
1226		return -EINVAL;
1227	}
1228
1229	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1230				    GFP_KERNEL);
1231	if (!devices_arr)
1232		return -ENOMEM;
1233
1234	err = copy_from_user(devices_arr,
1235			     (void __user *)args->device_ids_array_ptr,
1236			     args->n_devices * sizeof(*devices_arr));
1237	if (err != 0) {
1238		err = -EFAULT;
1239		goto copy_from_user_failed;
1240	}
1241
1242	mutex_lock(&p->mutex);
1243	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1244	if (!pdd) {
1245		err = -EINVAL;
1246		goto get_process_device_data_failed;
1247	}
1248	dev = pdd->dev;
1249
1250	pdd = kfd_bind_process_to_device(dev, p);
1251	if (IS_ERR(pdd)) {
1252		err = PTR_ERR(pdd);
1253		goto bind_process_to_device_failed;
1254	}
1255
1256	mem = kfd_process_device_translate_handle(pdd,
1257						GET_IDR_HANDLE(args->handle));
1258	if (!mem) {
1259		err = -ENOMEM;
1260		goto get_mem_obj_from_handle_failed;
1261	}
1262
1263	for (i = args->n_success; i < args->n_devices; i++) {
1264		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1265		if (!peer_pdd) {
1266			pr_debug("Getting device by id failed for 0x%x\n",
1267				 devices_arr[i]);
1268			err = -EINVAL;
1269			goto get_mem_obj_from_handle_failed;
1270		}
1271
1272		peer_pdd = kfd_bind_process_to_device(peer_pdd->dev, p);
1273		if (IS_ERR(peer_pdd)) {
1274			err = PTR_ERR(peer_pdd);
1275			goto get_mem_obj_from_handle_failed;
1276		}
1277
1278		err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1279			peer_pdd->dev->adev, (struct kgd_mem *)mem,
1280			peer_pdd->drm_priv);
1281		if (err) {
1282			struct pci_dev *pdev = peer_pdd->dev->adev->pdev;
1283
1284			dev_err(dev->adev->dev,
1285			       "Failed to map peer:%04x:%02x:%02x.%d mem_domain:%d\n",
1286			       pci_domain_nr(pdev->bus),
1287			       pdev->bus->number,
1288			       PCI_SLOT(pdev->devfn),
1289			       PCI_FUNC(pdev->devfn),
1290			       ((struct kgd_mem *)mem)->domain);
1291			goto map_memory_to_gpu_failed;
1292		}
1293		args->n_success = i+1;
1294	}
1295
1296	mutex_unlock(&p->mutex);
1297
1298	err = amdgpu_amdkfd_gpuvm_sync_memory(dev->adev, (struct kgd_mem *) mem, true);
1299	if (err) {
1300		pr_debug("Sync memory failed, wait interrupted by user signal\n");
1301		goto sync_memory_failed;
1302	}
1303
1304	/* Flush TLBs after waiting for the page table updates to complete */
1305	for (i = 0; i < args->n_devices; i++) {
1306		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1307		if (WARN_ON_ONCE(!peer_pdd))
1308			continue;
1309		kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
1310	}
1311	kfree(devices_arr);
1312
1313	return err;
1314
1315get_process_device_data_failed:
1316bind_process_to_device_failed:
1317get_mem_obj_from_handle_failed:
1318map_memory_to_gpu_failed:
1319	mutex_unlock(&p->mutex);
1320copy_from_user_failed:
1321sync_memory_failed:
1322	kfree(devices_arr);
1323
1324	return err;
1325}
1326
1327static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1328					struct kfd_process *p, void *data)
1329{
1330	struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1331	struct kfd_process_device *pdd, *peer_pdd;
1332	void *mem;
1333	long err = 0;
1334	uint32_t *devices_arr = NULL, i;
1335
1336	if (!args->n_devices) {
1337		pr_debug("Device IDs array empty\n");
1338		return -EINVAL;
1339	}
1340	if (args->n_success > args->n_devices) {
1341		pr_debug("n_success exceeds n_devices\n");
1342		return -EINVAL;
1343	}
1344
1345	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1346				    GFP_KERNEL);
1347	if (!devices_arr)
1348		return -ENOMEM;
1349
1350	err = copy_from_user(devices_arr,
1351			     (void __user *)args->device_ids_array_ptr,
1352			     args->n_devices * sizeof(*devices_arr));
1353	if (err != 0) {
1354		err = -EFAULT;
1355		goto copy_from_user_failed;
1356	}
1357
1358	mutex_lock(&p->mutex);
1359	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1360	if (!pdd) {
1361		err = -EINVAL;
1362		goto bind_process_to_device_failed;
1363	}
1364
1365	mem = kfd_process_device_translate_handle(pdd,
1366						GET_IDR_HANDLE(args->handle));
1367	if (!mem) {
1368		err = -ENOMEM;
1369		goto get_mem_obj_from_handle_failed;
1370	}
1371
1372	for (i = args->n_success; i < args->n_devices; i++) {
1373		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1374		if (!peer_pdd) {
1375			err = -EINVAL;
1376			goto get_mem_obj_from_handle_failed;
1377		}
1378		err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1379			peer_pdd->dev->adev, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1380		if (err) {
1381			pr_err("Failed to unmap from gpu %d/%d\n",
1382			       i, args->n_devices);
1383			goto unmap_memory_from_gpu_failed;
1384		}
1385		args->n_success = i+1;
1386	}
1387	mutex_unlock(&p->mutex);
1388
1389	if (kfd_flush_tlb_after_unmap(pdd->dev)) {
1390		err = amdgpu_amdkfd_gpuvm_sync_memory(pdd->dev->adev,
1391				(struct kgd_mem *) mem, true);
1392		if (err) {
1393			pr_debug("Sync memory failed, wait interrupted by user signal\n");
1394			goto sync_memory_failed;
1395		}
1396
1397		/* Flush TLBs after waiting for the page table updates to complete */
1398		for (i = 0; i < args->n_devices; i++) {
1399			peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1400			if (WARN_ON_ONCE(!peer_pdd))
1401				continue;
1402			kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
1403		}
1404	}
1405	kfree(devices_arr);
1406
1407	return 0;
1408
1409bind_process_to_device_failed:
1410get_mem_obj_from_handle_failed:
1411unmap_memory_from_gpu_failed:
1412	mutex_unlock(&p->mutex);
1413copy_from_user_failed:
1414sync_memory_failed:
1415	kfree(devices_arr);
1416	return err;
1417}
1418
1419static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1420		struct kfd_process *p, void *data)
1421{
1422	int retval;
1423	struct kfd_ioctl_alloc_queue_gws_args *args = data;
1424	struct queue *q;
1425	struct kfd_dev *dev;
1426
1427	mutex_lock(&p->mutex);
1428	q = pqm_get_user_queue(&p->pqm, args->queue_id);
1429
1430	if (q) {
1431		dev = q->device;
1432	} else {
1433		retval = -EINVAL;
1434		goto out_unlock;
1435	}
1436
1437	if (!dev->gws) {
1438		retval = -ENODEV;
1439		goto out_unlock;
1440	}
1441
1442	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1443		retval = -ENODEV;
1444		goto out_unlock;
1445	}
1446
1447	retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1448	mutex_unlock(&p->mutex);
1449
1450	args->first_gws = 0;
1451	return retval;
1452
1453out_unlock:
1454	mutex_unlock(&p->mutex);
1455	return retval;
1456}
1457
1458static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1459		struct kfd_process *p, void *data)
1460{
1461	struct kfd_ioctl_get_dmabuf_info_args *args = data;
1462	struct kfd_dev *dev = NULL;
1463	struct amdgpu_device *dmabuf_adev;
1464	void *metadata_buffer = NULL;
1465	uint32_t flags;
1466	unsigned int i;
1467	int r;
1468
1469	/* Find a KFD GPU device that supports the get_dmabuf_info query */
1470	for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1471		if (dev)
1472			break;
1473	if (!dev)
1474		return -EINVAL;
1475
1476	if (args->metadata_ptr) {
1477		metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1478		if (!metadata_buffer)
1479			return -ENOMEM;
1480	}
1481
1482	/* Get dmabuf info from KGD */
1483	r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd,
1484					  &dmabuf_adev, &args->size,
1485					  metadata_buffer, args->metadata_size,
1486					  &args->metadata_size, &flags);
1487	if (r)
1488		goto exit;
1489
1490	/* Reverse-lookup gpu_id from kgd pointer */
1491	dev = kfd_device_by_adev(dmabuf_adev);
1492	if (!dev) {
1493		r = -EINVAL;
1494		goto exit;
1495	}
1496	args->gpu_id = dev->id;
1497	args->flags = flags;
1498
1499	/* Copy metadata buffer to user mode */
1500	if (metadata_buffer) {
1501		r = copy_to_user((void __user *)args->metadata_ptr,
1502				 metadata_buffer, args->metadata_size);
1503		if (r != 0)
1504			r = -EFAULT;
1505	}
1506
1507exit:
1508	kfree(metadata_buffer);
1509
1510	return r;
1511}
1512
1513static int kfd_ioctl_import_dmabuf(struct file *filep,
1514				   struct kfd_process *p, void *data)
1515{
1516	struct kfd_ioctl_import_dmabuf_args *args = data;
1517	struct kfd_process_device *pdd;
1518	struct dma_buf *dmabuf;
1519	int idr_handle;
1520	uint64_t size;
1521	void *mem;
1522	int r;
1523
1524	dmabuf = dma_buf_get(args->dmabuf_fd);
1525	if (IS_ERR(dmabuf))
1526		return PTR_ERR(dmabuf);
1527
1528	mutex_lock(&p->mutex);
1529	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1530	if (!pdd) {
1531		r = -EINVAL;
1532		goto err_unlock;
1533	}
1534
1535	pdd = kfd_bind_process_to_device(pdd->dev, p);
1536	if (IS_ERR(pdd)) {
1537		r = PTR_ERR(pdd);
1538		goto err_unlock;
1539	}
1540
1541	r = amdgpu_amdkfd_gpuvm_import_dmabuf(pdd->dev->adev, dmabuf,
1542					      args->va_addr, pdd->drm_priv,
1543					      (struct kgd_mem **)&mem, &size,
1544					      NULL);
1545	if (r)
1546		goto err_unlock;
1547
1548	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1549	if (idr_handle < 0) {
1550		r = -EFAULT;
1551		goto err_free;
1552	}
1553
1554	mutex_unlock(&p->mutex);
1555	dma_buf_put(dmabuf);
1556
1557	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1558
1559	return 0;
1560
1561err_free:
1562	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, (struct kgd_mem *)mem,
1563					       pdd->drm_priv, NULL);
1564err_unlock:
1565	mutex_unlock(&p->mutex);
1566	dma_buf_put(dmabuf);
1567	return r;
1568}
1569
1570/* Handle requests for watching SMI events */
1571static int kfd_ioctl_smi_events(struct file *filep,
1572				struct kfd_process *p, void *data)
1573{
1574	struct kfd_ioctl_smi_events_args *args = data;
1575	struct kfd_process_device *pdd;
1576
1577	mutex_lock(&p->mutex);
1578
1579	pdd = kfd_process_device_data_by_id(p, args->gpuid);
1580	mutex_unlock(&p->mutex);
1581	if (!pdd)
1582		return -EINVAL;
1583
1584	return kfd_smi_event_open(pdd->dev, &args->anon_fd);
1585}
1586
1587#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1588
1589static int kfd_ioctl_set_xnack_mode(struct file *filep,
1590				    struct kfd_process *p, void *data)
1591{
1592	struct kfd_ioctl_set_xnack_mode_args *args = data;
1593	int r = 0;
1594
1595	mutex_lock(&p->mutex);
1596	if (args->xnack_enabled >= 0) {
1597		if (!list_empty(&p->pqm.queues)) {
1598			pr_debug("Process has user queues running\n");
1599			r = -EBUSY;
1600			goto out_unlock;
1601		}
1602
1603		if (p->xnack_enabled == args->xnack_enabled)
1604			goto out_unlock;
1605
1606		if (args->xnack_enabled && !kfd_process_xnack_mode(p, true)) {
1607			r = -EPERM;
1608			goto out_unlock;
1609		}
1610
1611		r = svm_range_switch_xnack_reserve_mem(p, args->xnack_enabled);
1612	} else {
1613		args->xnack_enabled = p->xnack_enabled;
1614	}
1615
1616out_unlock:
1617	mutex_unlock(&p->mutex);
1618
1619	return r;
1620}
1621
1622static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1623{
1624	struct kfd_ioctl_svm_args *args = data;
1625	int r = 0;
1626
1627	pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1628		 args->start_addr, args->size, args->op, args->nattr);
1629
1630	if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1631		return -EINVAL;
1632	if (!args->start_addr || !args->size)
1633		return -EINVAL;
1634
1635	r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
1636		      args->attrs);
1637
1638	return r;
1639}
1640#else
1641static int kfd_ioctl_set_xnack_mode(struct file *filep,
1642				    struct kfd_process *p, void *data)
1643{
1644	return -EPERM;
1645}
1646static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1647{
1648	return -EPERM;
1649}
1650#endif
1651
1652static int criu_checkpoint_process(struct kfd_process *p,
1653			     uint8_t __user *user_priv_data,
1654			     uint64_t *priv_offset)
1655{
1656	struct kfd_criu_process_priv_data process_priv;
1657	int ret;
1658
1659	memset(&process_priv, 0, sizeof(process_priv));
1660
1661	process_priv.version = KFD_CRIU_PRIV_VERSION;
1662	/* For CR, we don't consider negative xnack mode which is used for
1663	 * querying without changing it, here 0 simply means disabled and 1
1664	 * means enabled so retry for finding a valid PTE.
1665	 */
1666	process_priv.xnack_mode = p->xnack_enabled ? 1 : 0;
1667
1668	ret = copy_to_user(user_priv_data + *priv_offset,
1669				&process_priv, sizeof(process_priv));
1670
1671	if (ret) {
1672		pr_err("Failed to copy process information to user\n");
1673		ret = -EFAULT;
1674	}
1675
1676	*priv_offset += sizeof(process_priv);
1677	return ret;
1678}
1679
1680static int criu_checkpoint_devices(struct kfd_process *p,
1681			     uint32_t num_devices,
1682			     uint8_t __user *user_addr,
1683			     uint8_t __user *user_priv_data,
1684			     uint64_t *priv_offset)
1685{
1686	struct kfd_criu_device_priv_data *device_priv = NULL;
1687	struct kfd_criu_device_bucket *device_buckets = NULL;
1688	int ret = 0, i;
1689
1690	device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL);
1691	if (!device_buckets) {
1692		ret = -ENOMEM;
1693		goto exit;
1694	}
1695
1696	device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL);
1697	if (!device_priv) {
1698		ret = -ENOMEM;
1699		goto exit;
1700	}
1701
1702	for (i = 0; i < num_devices; i++) {
1703		struct kfd_process_device *pdd = p->pdds[i];
1704
1705		device_buckets[i].user_gpu_id = pdd->user_gpu_id;
1706		device_buckets[i].actual_gpu_id = pdd->dev->id;
1707
1708		/*
1709		 * priv_data does not contain useful information for now and is reserved for
1710		 * future use, so we do not set its contents.
1711		 */
1712	}
1713
1714	ret = copy_to_user(user_addr, device_buckets, num_devices * sizeof(*device_buckets));
1715	if (ret) {
1716		pr_err("Failed to copy device information to user\n");
1717		ret = -EFAULT;
1718		goto exit;
1719	}
1720
1721	ret = copy_to_user(user_priv_data + *priv_offset,
1722			   device_priv,
1723			   num_devices * sizeof(*device_priv));
1724	if (ret) {
1725		pr_err("Failed to copy device information to user\n");
1726		ret = -EFAULT;
1727	}
1728	*priv_offset += num_devices * sizeof(*device_priv);
1729
1730exit:
1731	kvfree(device_buckets);
1732	kvfree(device_priv);
1733	return ret;
1734}
1735
1736static uint32_t get_process_num_bos(struct kfd_process *p)
1737{
1738	uint32_t num_of_bos = 0;
1739	int i;
1740
1741	/* Run over all PDDs of the process */
1742	for (i = 0; i < p->n_pdds; i++) {
1743		struct kfd_process_device *pdd = p->pdds[i];
1744		void *mem;
1745		int id;
1746
1747		idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1748			struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
1749
1750			if ((uint64_t)kgd_mem->va > pdd->gpuvm_base)
1751				num_of_bos++;
1752		}
1753	}
1754	return num_of_bos;
1755}
1756
1757static int criu_get_prime_handle(struct drm_gem_object *gobj, int flags,
1758				      u32 *shared_fd)
1759{
1760	struct dma_buf *dmabuf;
1761	int ret;
1762
1763	dmabuf = amdgpu_gem_prime_export(gobj, flags);
1764	if (IS_ERR(dmabuf)) {
1765		ret = PTR_ERR(dmabuf);
1766		pr_err("dmabuf export failed for the BO\n");
1767		return ret;
1768	}
1769
1770	ret = dma_buf_fd(dmabuf, flags);
1771	if (ret < 0) {
1772		pr_err("dmabuf create fd failed, ret:%d\n", ret);
1773		goto out_free_dmabuf;
1774	}
1775
1776	*shared_fd = ret;
1777	return 0;
1778
1779out_free_dmabuf:
1780	dma_buf_put(dmabuf);
1781	return ret;
1782}
1783
1784static int criu_checkpoint_bos(struct kfd_process *p,
1785			       uint32_t num_bos,
1786			       uint8_t __user *user_bos,
1787			       uint8_t __user *user_priv_data,
1788			       uint64_t *priv_offset)
1789{
1790	struct kfd_criu_bo_bucket *bo_buckets;
1791	struct kfd_criu_bo_priv_data *bo_privs;
1792	int ret = 0, pdd_index, bo_index = 0, id;
1793	void *mem;
1794
1795	bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL);
1796	if (!bo_buckets)
1797		return -ENOMEM;
1798
1799	bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL);
1800	if (!bo_privs) {
1801		ret = -ENOMEM;
1802		goto exit;
1803	}
1804
1805	for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
1806		struct kfd_process_device *pdd = p->pdds[pdd_index];
1807		struct amdgpu_bo *dumper_bo;
1808		struct kgd_mem *kgd_mem;
1809
1810		idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1811			struct kfd_criu_bo_bucket *bo_bucket;
1812			struct kfd_criu_bo_priv_data *bo_priv;
1813			int i, dev_idx = 0;
1814
1815			if (!mem) {
1816				ret = -ENOMEM;
1817				goto exit;
1818			}
1819
1820			kgd_mem = (struct kgd_mem *)mem;
1821			dumper_bo = kgd_mem->bo;
1822
1823			if ((uint64_t)kgd_mem->va <= pdd->gpuvm_base)
1824				continue;
1825
1826			bo_bucket = &bo_buckets[bo_index];
1827			bo_priv = &bo_privs[bo_index];
1828
1829			bo_bucket->gpu_id = pdd->user_gpu_id;
1830			bo_bucket->addr = (uint64_t)kgd_mem->va;
1831			bo_bucket->size = amdgpu_bo_size(dumper_bo);
1832			bo_bucket->alloc_flags = (uint32_t)kgd_mem->alloc_flags;
1833			bo_priv->idr_handle = id;
1834
1835			if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1836				ret = amdgpu_ttm_tt_get_userptr(&dumper_bo->tbo,
1837								&bo_priv->user_addr);
1838				if (ret) {
1839					pr_err("Failed to obtain user address for user-pointer bo\n");
1840					goto exit;
1841				}
1842			}
1843			if (bo_bucket->alloc_flags
1844			    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
1845				ret = criu_get_prime_handle(&dumper_bo->tbo.base,
1846						bo_bucket->alloc_flags &
1847						KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
1848						&bo_bucket->dmabuf_fd);
1849				if (ret)
1850					goto exit;
1851			} else {
1852				bo_bucket->dmabuf_fd = KFD_INVALID_FD;
1853			}
1854
1855			if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
1856				bo_bucket->offset = KFD_MMAP_TYPE_DOORBELL |
1857					KFD_MMAP_GPU_ID(pdd->dev->id);
1858			else if (bo_bucket->alloc_flags &
1859				KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1860				bo_bucket->offset = KFD_MMAP_TYPE_MMIO |
1861					KFD_MMAP_GPU_ID(pdd->dev->id);
1862			else
1863				bo_bucket->offset = amdgpu_bo_mmap_offset(dumper_bo);
1864
1865			for (i = 0; i < p->n_pdds; i++) {
1866				if (amdgpu_amdkfd_bo_mapped_to_dev(p->pdds[i]->dev->adev, kgd_mem))
1867					bo_priv->mapped_gpuids[dev_idx++] = p->pdds[i]->user_gpu_id;
1868			}
1869
1870			pr_debug("bo_size = 0x%llx, bo_addr = 0x%llx bo_offset = 0x%llx\n"
1871					"gpu_id = 0x%x alloc_flags = 0x%x idr_handle = 0x%x",
1872					bo_bucket->size,
1873					bo_bucket->addr,
1874					bo_bucket->offset,
1875					bo_bucket->gpu_id,
1876					bo_bucket->alloc_flags,
1877					bo_priv->idr_handle);
1878			bo_index++;
1879		}
1880	}
1881
1882	ret = copy_to_user(user_bos, bo_buckets, num_bos * sizeof(*bo_buckets));
1883	if (ret) {
1884		pr_err("Failed to copy BO information to user\n");
1885		ret = -EFAULT;
1886		goto exit;
1887	}
1888
1889	ret = copy_to_user(user_priv_data + *priv_offset, bo_privs, num_bos * sizeof(*bo_privs));
1890	if (ret) {
1891		pr_err("Failed to copy BO priv information to user\n");
1892		ret = -EFAULT;
1893		goto exit;
1894	}
1895
1896	*priv_offset += num_bos * sizeof(*bo_privs);
1897
1898exit:
1899	while (ret && bo_index--) {
1900		if (bo_buckets[bo_index].alloc_flags
1901		    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
1902			close_fd(bo_buckets[bo_index].dmabuf_fd);
1903	}
1904
1905	kvfree(bo_buckets);
1906	kvfree(bo_privs);
1907	return ret;
1908}
1909
1910static int criu_get_process_object_info(struct kfd_process *p,
1911					uint32_t *num_devices,
1912					uint32_t *num_bos,
1913					uint32_t *num_objects,
1914					uint64_t *objs_priv_size)
1915{
1916	uint64_t queues_priv_data_size, svm_priv_data_size, priv_size;
1917	uint32_t num_queues, num_events, num_svm_ranges;
1918	int ret;
1919
1920	*num_devices = p->n_pdds;
1921	*num_bos = get_process_num_bos(p);
1922
1923	ret = kfd_process_get_queue_info(p, &num_queues, &queues_priv_data_size);
1924	if (ret)
1925		return ret;
1926
1927	num_events = kfd_get_num_events(p);
1928
1929	ret = svm_range_get_info(p, &num_svm_ranges, &svm_priv_data_size);
1930	if (ret)
1931		return ret;
1932
1933	*num_objects = num_queues + num_events + num_svm_ranges;
1934
1935	if (objs_priv_size) {
1936		priv_size = sizeof(struct kfd_criu_process_priv_data);
1937		priv_size += *num_devices * sizeof(struct kfd_criu_device_priv_data);
1938		priv_size += *num_bos * sizeof(struct kfd_criu_bo_priv_data);
1939		priv_size += queues_priv_data_size;
1940		priv_size += num_events * sizeof(struct kfd_criu_event_priv_data);
1941		priv_size += svm_priv_data_size;
1942		*objs_priv_size = priv_size;
1943	}
1944	return 0;
1945}
1946
1947static int criu_checkpoint(struct file *filep,
1948			   struct kfd_process *p,
1949			   struct kfd_ioctl_criu_args *args)
1950{
1951	int ret;
1952	uint32_t num_devices, num_bos, num_objects;
1953	uint64_t priv_size, priv_offset = 0, bo_priv_offset;
1954
1955	if (!args->devices || !args->bos || !args->priv_data)
1956		return -EINVAL;
1957
1958	mutex_lock(&p->mutex);
1959
1960	if (!p->n_pdds) {
1961		pr_err("No pdd for given process\n");
1962		ret = -ENODEV;
1963		goto exit_unlock;
1964	}
1965
1966	/* Confirm all process queues are evicted */
1967	if (!p->queues_paused) {
1968		pr_err("Cannot dump process when queues are not in evicted state\n");
1969		/* CRIU plugin did not call op PROCESS_INFO before checkpointing */
1970		ret = -EINVAL;
1971		goto exit_unlock;
1972	}
1973
1974	ret = criu_get_process_object_info(p, &num_devices, &num_bos, &num_objects, &priv_size);
1975	if (ret)
1976		goto exit_unlock;
1977
1978	if (num_devices != args->num_devices ||
1979	    num_bos != args->num_bos ||
1980	    num_objects != args->num_objects ||
1981	    priv_size != args->priv_data_size) {
1982
1983		ret = -EINVAL;
1984		goto exit_unlock;
1985	}
1986
1987	/* each function will store private data inside priv_data and adjust priv_offset */
1988	ret = criu_checkpoint_process(p, (uint8_t __user *)args->priv_data, &priv_offset);
1989	if (ret)
1990		goto exit_unlock;
1991
1992	ret = criu_checkpoint_devices(p, num_devices, (uint8_t __user *)args->devices,
1993				(uint8_t __user *)args->priv_data, &priv_offset);
1994	if (ret)
1995		goto exit_unlock;
1996
1997	/* Leave room for BOs in the private data. They need to be restored
1998	 * before events, but we checkpoint them last to simplify the error
1999	 * handling.
2000	 */
2001	bo_priv_offset = priv_offset;
2002	priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data);
2003
2004	if (num_objects) {
2005		ret = kfd_criu_checkpoint_queues(p, (uint8_t __user *)args->priv_data,
2006						 &priv_offset);
2007		if (ret)
2008			goto exit_unlock;
2009
2010		ret = kfd_criu_checkpoint_events(p, (uint8_t __user *)args->priv_data,
2011						 &priv_offset);
2012		if (ret)
2013			goto exit_unlock;
2014
2015		ret = kfd_criu_checkpoint_svm(p, (uint8_t __user *)args->priv_data, &priv_offset);
2016		if (ret)
2017			goto exit_unlock;
2018	}
2019
2020	/* This must be the last thing in this function that can fail.
2021	 * Otherwise we leak dmabuf file descriptors.
2022	 */
2023	ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos,
2024			   (uint8_t __user *)args->priv_data, &bo_priv_offset);
2025
2026exit_unlock:
2027	mutex_unlock(&p->mutex);
2028	if (ret)
2029		pr_err("Failed to dump CRIU ret:%d\n", ret);
2030	else
2031		pr_debug("CRIU dump ret:%d\n", ret);
2032
2033	return ret;
2034}
2035
2036static int criu_restore_process(struct kfd_process *p,
2037				struct kfd_ioctl_criu_args *args,
2038				uint64_t *priv_offset,
2039				uint64_t max_priv_data_size)
2040{
2041	int ret = 0;
2042	struct kfd_criu_process_priv_data process_priv;
2043
2044	if (*priv_offset + sizeof(process_priv) > max_priv_data_size)
2045		return -EINVAL;
2046
2047	ret = copy_from_user(&process_priv,
2048				(void __user *)(args->priv_data + *priv_offset),
2049				sizeof(process_priv));
2050	if (ret) {
2051		pr_err("Failed to copy process private information from user\n");
2052		ret = -EFAULT;
2053		goto exit;
2054	}
2055	*priv_offset += sizeof(process_priv);
2056
2057	if (process_priv.version != KFD_CRIU_PRIV_VERSION) {
2058		pr_err("Invalid CRIU API version (checkpointed:%d current:%d)\n",
2059			process_priv.version, KFD_CRIU_PRIV_VERSION);
2060		return -EINVAL;
2061	}
2062
2063	pr_debug("Setting XNACK mode\n");
2064	if (process_priv.xnack_mode && !kfd_process_xnack_mode(p, true)) {
2065		pr_err("xnack mode cannot be set\n");
2066		ret = -EPERM;
2067		goto exit;
2068	} else {
2069		pr_debug("set xnack mode: %d\n", process_priv.xnack_mode);
2070		p->xnack_enabled = process_priv.xnack_mode;
2071	}
2072
2073exit:
2074	return ret;
2075}
2076
2077static int criu_restore_devices(struct kfd_process *p,
2078				struct kfd_ioctl_criu_args *args,
2079				uint64_t *priv_offset,
2080				uint64_t max_priv_data_size)
2081{
2082	struct kfd_criu_device_bucket *device_buckets;
2083	struct kfd_criu_device_priv_data *device_privs;
2084	int ret = 0;
2085	uint32_t i;
2086
2087	if (args->num_devices != p->n_pdds)
2088		return -EINVAL;
2089
2090	if (*priv_offset + (args->num_devices * sizeof(*device_privs)) > max_priv_data_size)
2091		return -EINVAL;
2092
2093	device_buckets = kmalloc_array(args->num_devices, sizeof(*device_buckets), GFP_KERNEL);
2094	if (!device_buckets)
2095		return -ENOMEM;
2096
2097	ret = copy_from_user(device_buckets, (void __user *)args->devices,
2098				args->num_devices * sizeof(*device_buckets));
2099	if (ret) {
2100		pr_err("Failed to copy devices buckets from user\n");
2101		ret = -EFAULT;
2102		goto exit;
2103	}
2104
2105	for (i = 0; i < args->num_devices; i++) {
2106		struct kfd_dev *dev;
2107		struct kfd_process_device *pdd;
2108		struct file *drm_file;
2109
2110		/* device private data is not currently used */
2111
2112		if (!device_buckets[i].user_gpu_id) {
2113			pr_err("Invalid user gpu_id\n");
2114			ret = -EINVAL;
2115			goto exit;
2116		}
2117
2118		dev = kfd_device_by_id(device_buckets[i].actual_gpu_id);
2119		if (!dev) {
2120			pr_err("Failed to find device with gpu_id = %x\n",
2121				device_buckets[i].actual_gpu_id);
2122			ret = -EINVAL;
2123			goto exit;
2124		}
2125
2126		pdd = kfd_get_process_device_data(dev, p);
2127		if (!pdd) {
2128			pr_err("Failed to get pdd for gpu_id = %x\n",
2129					device_buckets[i].actual_gpu_id);
2130			ret = -EINVAL;
2131			goto exit;
2132		}
2133		pdd->user_gpu_id = device_buckets[i].user_gpu_id;
2134
2135		drm_file = fget(device_buckets[i].drm_fd);
2136		if (!drm_file) {
2137			pr_err("Invalid render node file descriptor sent from plugin (%d)\n",
2138				device_buckets[i].drm_fd);
2139			ret = -EINVAL;
2140			goto exit;
2141		}
2142
2143		if (pdd->drm_file) {
2144			ret = -EINVAL;
2145			goto exit;
2146		}
2147
2148		/* create the vm using render nodes for kfd pdd */
2149		if (kfd_process_device_init_vm(pdd, drm_file)) {
2150			pr_err("could not init vm for given pdd\n");
2151			/* On success, the PDD keeps the drm_file reference */
2152			fput(drm_file);
2153			ret = -EINVAL;
2154			goto exit;
2155		}
2156		/*
2157		 * pdd now already has the vm bound to render node so below api won't create a new
2158		 * exclusive kfd mapping but use existing one with renderDXXX but is still needed
2159		 * for iommu v2 binding  and runtime pm.
2160		 */
2161		pdd = kfd_bind_process_to_device(dev, p);
2162		if (IS_ERR(pdd)) {
2163			ret = PTR_ERR(pdd);
2164			goto exit;
2165		}
2166
2167		if (!pdd->doorbell_index &&
2168		    kfd_alloc_process_doorbells(pdd->dev, &pdd->doorbell_index) < 0) {
2169			ret = -ENOMEM;
2170			goto exit;
2171		}
2172	}
2173
2174	/*
2175	 * We are not copying device private data from user as we are not using the data for now,
2176	 * but we still adjust for its private data.
2177	 */
2178	*priv_offset += args->num_devices * sizeof(*device_privs);
2179
2180exit:
2181	kfree(device_buckets);
2182	return ret;
2183}
2184
2185static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
2186				      struct kfd_criu_bo_bucket *bo_bucket,
2187				      struct kfd_criu_bo_priv_data *bo_priv,
2188				      struct kgd_mem **kgd_mem)
2189{
2190	int idr_handle;
2191	int ret;
2192	const bool criu_resume = true;
2193	u64 offset;
2194
2195	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
2196		if (bo_bucket->size != kfd_doorbell_process_slice(pdd->dev))
2197			return -EINVAL;
2198
2199		offset = kfd_get_process_doorbells(pdd);
2200		if (!offset)
2201			return -ENOMEM;
2202	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2203		/* MMIO BOs need remapped bus address */
2204		if (bo_bucket->size != PAGE_SIZE) {
2205			pr_err("Invalid page size\n");
2206			return -EINVAL;
2207		}
2208		offset = pdd->dev->adev->rmmio_remap.bus_addr;
2209		if (!offset) {
2210			pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n");
2211			return -ENOMEM;
2212		}
2213	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
2214		offset = bo_priv->user_addr;
2215	}
2216	/* Create the BO */
2217	ret = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(pdd->dev->adev, bo_bucket->addr,
2218						      bo_bucket->size, pdd->drm_priv, kgd_mem,
2219						      &offset, bo_bucket->alloc_flags, criu_resume);
2220	if (ret) {
2221		pr_err("Could not create the BO\n");
2222		return ret;
2223	}
2224	pr_debug("New BO created: size:0x%llx addr:0x%llx offset:0x%llx\n",
2225		 bo_bucket->size, bo_bucket->addr, offset);
2226
2227	/* Restore previous IDR handle */
2228	pr_debug("Restoring old IDR handle for the BO");
2229	idr_handle = idr_alloc(&pdd->alloc_idr, *kgd_mem, bo_priv->idr_handle,
2230			       bo_priv->idr_handle + 1, GFP_KERNEL);
2231
2232	if (idr_handle < 0) {
2233		pr_err("Could not allocate idr\n");
2234		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, *kgd_mem, pdd->drm_priv,
2235						       NULL);
2236		return -ENOMEM;
2237	}
2238
2239	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
2240		bo_bucket->restored_offset = KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(pdd->dev->id);
2241	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2242		bo_bucket->restored_offset = KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(pdd->dev->id);
2243	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
2244		bo_bucket->restored_offset = offset;
2245	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
2246		bo_bucket->restored_offset = offset;
2247		/* Update the VRAM usage count */
2248		WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + bo_bucket->size);
2249	}
2250	return 0;
2251}
2252
2253static int criu_restore_bo(struct kfd_process *p,
2254			   struct kfd_criu_bo_bucket *bo_bucket,
2255			   struct kfd_criu_bo_priv_data *bo_priv)
2256{
2257	struct kfd_process_device *pdd;
2258	struct kgd_mem *kgd_mem;
2259	int ret;
2260	int j;
2261
2262	pr_debug("Restoring BO size:0x%llx addr:0x%llx gpu_id:0x%x flags:0x%x idr_handle:0x%x\n",
2263		 bo_bucket->size, bo_bucket->addr, bo_bucket->gpu_id, bo_bucket->alloc_flags,
2264		 bo_priv->idr_handle);
2265
2266	pdd = kfd_process_device_data_by_id(p, bo_bucket->gpu_id);
2267	if (!pdd) {
2268		pr_err("Failed to get pdd\n");
2269		return -ENODEV;
2270	}
2271
2272	ret = criu_restore_memory_of_gpu(pdd, bo_bucket, bo_priv, &kgd_mem);
2273	if (ret)
2274		return ret;
2275
2276	/* now map these BOs to GPU/s */
2277	for (j = 0; j < p->n_pdds; j++) {
2278		struct kfd_dev *peer;
2279		struct kfd_process_device *peer_pdd;
2280
2281		if (!bo_priv->mapped_gpuids[j])
2282			break;
2283
2284		peer_pdd = kfd_process_device_data_by_id(p, bo_priv->mapped_gpuids[j]);
2285		if (!peer_pdd)
2286			return -EINVAL;
2287
2288		peer = peer_pdd->dev;
2289
2290		peer_pdd = kfd_bind_process_to_device(peer, p);
2291		if (IS_ERR(peer_pdd))
2292			return PTR_ERR(peer_pdd);
2293
2294		ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(peer->adev, kgd_mem,
2295							    peer_pdd->drm_priv);
2296		if (ret) {
2297			pr_err("Failed to map to gpu %d/%d\n", j, p->n_pdds);
2298			return ret;
2299		}
2300	}
2301
2302	pr_debug("map memory was successful for the BO\n");
2303	/* create the dmabuf object and export the bo */
2304	if (bo_bucket->alloc_flags
2305	    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
2306		ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base, DRM_RDWR,
2307					    &bo_bucket->dmabuf_fd);
2308		if (ret)
2309			return ret;
2310	} else {
2311		bo_bucket->dmabuf_fd = KFD_INVALID_FD;
2312	}
2313
2314	return 0;
2315}
2316
2317static int criu_restore_bos(struct kfd_process *p,
2318			    struct kfd_ioctl_criu_args *args,
2319			    uint64_t *priv_offset,
2320			    uint64_t max_priv_data_size)
2321{
2322	struct kfd_criu_bo_bucket *bo_buckets = NULL;
2323	struct kfd_criu_bo_priv_data *bo_privs = NULL;
2324	int ret = 0;
2325	uint32_t i = 0;
2326
2327	if (*priv_offset + (args->num_bos * sizeof(*bo_privs)) > max_priv_data_size)
2328		return -EINVAL;
2329
2330	/* Prevent MMU notifications until stage-4 IOCTL (CRIU_RESUME) is received */
2331	amdgpu_amdkfd_block_mmu_notifications(p->kgd_process_info);
2332
2333	bo_buckets = kvmalloc_array(args->num_bos, sizeof(*bo_buckets), GFP_KERNEL);
2334	if (!bo_buckets)
2335		return -ENOMEM;
2336
2337	ret = copy_from_user(bo_buckets, (void __user *)args->bos,
2338			     args->num_bos * sizeof(*bo_buckets));
2339	if (ret) {
2340		pr_err("Failed to copy BOs information from user\n");
2341		ret = -EFAULT;
2342		goto exit;
2343	}
2344
2345	bo_privs = kvmalloc_array(args->num_bos, sizeof(*bo_privs), GFP_KERNEL);
2346	if (!bo_privs) {
2347		ret = -ENOMEM;
2348		goto exit;
2349	}
2350
2351	ret = copy_from_user(bo_privs, (void __user *)args->priv_data + *priv_offset,
2352			     args->num_bos * sizeof(*bo_privs));
2353	if (ret) {
2354		pr_err("Failed to copy BOs information from user\n");
2355		ret = -EFAULT;
2356		goto exit;
2357	}
2358	*priv_offset += args->num_bos * sizeof(*bo_privs);
2359
2360	/* Create and map new BOs */
2361	for (; i < args->num_bos; i++) {
2362		ret = criu_restore_bo(p, &bo_buckets[i], &bo_privs[i]);
2363		if (ret) {
2364			pr_debug("Failed to restore BO[%d] ret%d\n", i, ret);
2365			goto exit;
2366		}
2367	} /* done */
2368
2369	/* Copy only the buckets back so user can read bo_buckets[N].restored_offset */
2370	ret = copy_to_user((void __user *)args->bos,
2371				bo_buckets,
2372				(args->num_bos * sizeof(*bo_buckets)));
2373	if (ret)
2374		ret = -EFAULT;
2375
2376exit:
2377	while (ret && i--) {
2378		if (bo_buckets[i].alloc_flags
2379		   & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
2380			close_fd(bo_buckets[i].dmabuf_fd);
2381	}
2382	kvfree(bo_buckets);
2383	kvfree(bo_privs);
2384	return ret;
2385}
2386
2387static int criu_restore_objects(struct file *filep,
2388				struct kfd_process *p,
2389				struct kfd_ioctl_criu_args *args,
2390				uint64_t *priv_offset,
2391				uint64_t max_priv_data_size)
2392{
2393	int ret = 0;
2394	uint32_t i;
2395
2396	BUILD_BUG_ON(offsetof(struct kfd_criu_queue_priv_data, object_type));
2397	BUILD_BUG_ON(offsetof(struct kfd_criu_event_priv_data, object_type));
2398	BUILD_BUG_ON(offsetof(struct kfd_criu_svm_range_priv_data, object_type));
2399
2400	for (i = 0; i < args->num_objects; i++) {
2401		uint32_t object_type;
2402
2403		if (*priv_offset + sizeof(object_type) > max_priv_data_size) {
2404			pr_err("Invalid private data size\n");
2405			return -EINVAL;
2406		}
2407
2408		ret = get_user(object_type, (uint32_t __user *)(args->priv_data + *priv_offset));
2409		if (ret) {
2410			pr_err("Failed to copy private information from user\n");
2411			goto exit;
2412		}
2413
2414		switch (object_type) {
2415		case KFD_CRIU_OBJECT_TYPE_QUEUE:
2416			ret = kfd_criu_restore_queue(p, (uint8_t __user *)args->priv_data,
2417						     priv_offset, max_priv_data_size);
2418			if (ret)
2419				goto exit;
2420			break;
2421		case KFD_CRIU_OBJECT_TYPE_EVENT:
2422			ret = kfd_criu_restore_event(filep, p, (uint8_t __user *)args->priv_data,
2423						     priv_offset, max_priv_data_size);
2424			if (ret)
2425				goto exit;
2426			break;
2427		case KFD_CRIU_OBJECT_TYPE_SVM_RANGE:
2428			ret = kfd_criu_restore_svm(p, (uint8_t __user *)args->priv_data,
2429						     priv_offset, max_priv_data_size);
2430			if (ret)
2431				goto exit;
2432			break;
2433		default:
2434			pr_err("Invalid object type:%u at index:%d\n", object_type, i);
2435			ret = -EINVAL;
2436			goto exit;
2437		}
2438	}
2439exit:
2440	return ret;
2441}
2442
2443static int criu_restore(struct file *filep,
2444			struct kfd_process *p,
2445			struct kfd_ioctl_criu_args *args)
2446{
2447	uint64_t priv_offset = 0;
2448	int ret = 0;
2449
2450	pr_debug("CRIU restore (num_devices:%u num_bos:%u num_objects:%u priv_data_size:%llu)\n",
2451		 args->num_devices, args->num_bos, args->num_objects, args->priv_data_size);
2452
2453	if (!args->bos || !args->devices || !args->priv_data || !args->priv_data_size ||
2454	    !args->num_devices || !args->num_bos)
2455		return -EINVAL;
2456
2457	mutex_lock(&p->mutex);
2458
2459	/*
2460	 * Set the process to evicted state to avoid running any new queues before all the memory
2461	 * mappings are ready.
2462	 */
2463	ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_RESTORE);
2464	if (ret)
2465		goto exit_unlock;
2466
2467	/* Each function will adjust priv_offset based on how many bytes they consumed */
2468	ret = criu_restore_process(p, args, &priv_offset, args->priv_data_size);
2469	if (ret)
2470		goto exit_unlock;
2471
2472	ret = criu_restore_devices(p, args, &priv_offset, args->priv_data_size);
2473	if (ret)
2474		goto exit_unlock;
2475
2476	ret = criu_restore_bos(p, args, &priv_offset, args->priv_data_size);
2477	if (ret)
2478		goto exit_unlock;
2479
2480	ret = criu_restore_objects(filep, p, args, &priv_offset, args->priv_data_size);
2481	if (ret)
2482		goto exit_unlock;
2483
2484	if (priv_offset != args->priv_data_size) {
2485		pr_err("Invalid private data size\n");
2486		ret = -EINVAL;
2487	}
2488
2489exit_unlock:
2490	mutex_unlock(&p->mutex);
2491	if (ret)
2492		pr_err("Failed to restore CRIU ret:%d\n", ret);
2493	else
2494		pr_debug("CRIU restore successful\n");
2495
2496	return ret;
2497}
2498
2499static int criu_unpause(struct file *filep,
2500			struct kfd_process *p,
2501			struct kfd_ioctl_criu_args *args)
2502{
2503	int ret;
2504
2505	mutex_lock(&p->mutex);
2506
2507	if (!p->queues_paused) {
2508		mutex_unlock(&p->mutex);
2509		return -EINVAL;
2510	}
2511
2512	ret = kfd_process_restore_queues(p);
2513	if (ret)
2514		pr_err("Failed to unpause queues ret:%d\n", ret);
2515	else
2516		p->queues_paused = false;
2517
2518	mutex_unlock(&p->mutex);
2519
2520	return ret;
2521}
2522
2523static int criu_resume(struct file *filep,
2524			struct kfd_process *p,
2525			struct kfd_ioctl_criu_args *args)
2526{
2527	struct kfd_process *target = NULL;
2528	struct pid *pid = NULL;
2529	int ret = 0;
2530
2531	pr_debug("Inside %s, target pid for criu restore: %d\n", __func__,
2532		 args->pid);
2533
2534	pid = find_get_pid(args->pid);
2535	if (!pid) {
2536		pr_err("Cannot find pid info for %i\n", args->pid);
2537		return -ESRCH;
2538	}
2539
2540	pr_debug("calling kfd_lookup_process_by_pid\n");
2541	target = kfd_lookup_process_by_pid(pid);
2542
2543	put_pid(pid);
2544
2545	if (!target) {
2546		pr_debug("Cannot find process info for %i\n", args->pid);
2547		return -ESRCH;
2548	}
2549
2550	mutex_lock(&target->mutex);
2551	ret = kfd_criu_resume_svm(target);
2552	if (ret) {
2553		pr_err("kfd_criu_resume_svm failed for %i\n", args->pid);
2554		goto exit;
2555	}
2556
2557	ret =  amdgpu_amdkfd_criu_resume(target->kgd_process_info);
2558	if (ret)
2559		pr_err("amdgpu_amdkfd_criu_resume failed for %i\n", args->pid);
2560
2561exit:
2562	mutex_unlock(&target->mutex);
2563
2564	kfd_unref_process(target);
2565	return ret;
2566}
2567
2568static int criu_process_info(struct file *filep,
2569				struct kfd_process *p,
2570				struct kfd_ioctl_criu_args *args)
2571{
2572	int ret = 0;
2573
2574	mutex_lock(&p->mutex);
2575
2576	if (!p->n_pdds) {
2577		pr_err("No pdd for given process\n");
2578		ret = -ENODEV;
2579		goto err_unlock;
2580	}
2581
2582	ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_CHECKPOINT);
2583	if (ret)
2584		goto err_unlock;
2585
2586	p->queues_paused = true;
2587
2588	args->pid = task_pid_nr_ns(p->lead_thread,
2589					task_active_pid_ns(p->lead_thread));
2590
2591	ret = criu_get_process_object_info(p, &args->num_devices, &args->num_bos,
2592					   &args->num_objects, &args->priv_data_size);
2593	if (ret)
2594		goto err_unlock;
2595
2596	dev_dbg(kfd_device, "Num of devices:%u bos:%u objects:%u priv_data_size:%lld\n",
2597				args->num_devices, args->num_bos, args->num_objects,
2598				args->priv_data_size);
2599
2600err_unlock:
2601	if (ret) {
2602		kfd_process_restore_queues(p);
2603		p->queues_paused = false;
2604	}
2605	mutex_unlock(&p->mutex);
2606	return ret;
2607}
2608
2609static int kfd_ioctl_criu(struct file *filep, struct kfd_process *p, void *data)
2610{
2611	struct kfd_ioctl_criu_args *args = data;
2612	int ret;
2613
2614	dev_dbg(kfd_device, "CRIU operation: %d\n", args->op);
2615	switch (args->op) {
2616	case KFD_CRIU_OP_PROCESS_INFO:
2617		ret = criu_process_info(filep, p, args);
2618		break;
2619	case KFD_CRIU_OP_CHECKPOINT:
2620		ret = criu_checkpoint(filep, p, args);
2621		break;
2622	case KFD_CRIU_OP_UNPAUSE:
2623		ret = criu_unpause(filep, p, args);
2624		break;
2625	case KFD_CRIU_OP_RESTORE:
2626		ret = criu_restore(filep, p, args);
2627		break;
2628	case KFD_CRIU_OP_RESUME:
2629		ret = criu_resume(filep, p, args);
2630		break;
2631	default:
2632		dev_dbg(kfd_device, "Unsupported CRIU operation:%d\n", args->op);
2633		ret = -EINVAL;
2634		break;
2635	}
2636
2637	if (ret)
2638		dev_dbg(kfd_device, "CRIU operation:%d err:%d\n", args->op, ret);
2639
2640	return ret;
2641}
2642
2643#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
2644	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
2645			    .cmd_drv = 0, .name = #ioctl}
2646
2647/** Ioctl table */
2648static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
2649	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
2650			kfd_ioctl_get_version, 0),
2651
2652	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
2653			kfd_ioctl_create_queue, 0),
2654
2655	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
2656			kfd_ioctl_destroy_queue, 0),
2657
2658	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
2659			kfd_ioctl_set_memory_policy, 0),
2660
2661	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
2662			kfd_ioctl_get_clock_counters, 0),
2663
2664	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
2665			kfd_ioctl_get_process_apertures, 0),
2666
2667	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
2668			kfd_ioctl_update_queue, 0),
2669
2670	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
2671			kfd_ioctl_create_event, 0),
2672
2673	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
2674			kfd_ioctl_destroy_event, 0),
2675
2676	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
2677			kfd_ioctl_set_event, 0),
2678
2679	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
2680			kfd_ioctl_reset_event, 0),
2681
2682	AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
2683			kfd_ioctl_wait_events, 0),
2684
2685	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER_DEPRECATED,
2686			kfd_ioctl_dbg_register, 0),
2687
2688	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED,
2689			kfd_ioctl_dbg_unregister, 0),
2690
2691	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED,
2692			kfd_ioctl_dbg_address_watch, 0),
2693
2694	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED,
2695			kfd_ioctl_dbg_wave_control, 0),
2696
2697	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
2698			kfd_ioctl_set_scratch_backing_va, 0),
2699
2700	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
2701			kfd_ioctl_get_tile_config, 0),
2702
2703	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
2704			kfd_ioctl_set_trap_handler, 0),
2705
2706	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
2707			kfd_ioctl_get_process_apertures_new, 0),
2708
2709	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
2710			kfd_ioctl_acquire_vm, 0),
2711
2712	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
2713			kfd_ioctl_alloc_memory_of_gpu, 0),
2714
2715	AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
2716			kfd_ioctl_free_memory_of_gpu, 0),
2717
2718	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
2719			kfd_ioctl_map_memory_to_gpu, 0),
2720
2721	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
2722			kfd_ioctl_unmap_memory_from_gpu, 0),
2723
2724	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
2725			kfd_ioctl_set_cu_mask, 0),
2726
2727	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
2728			kfd_ioctl_get_queue_wave_state, 0),
2729
2730	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
2731				kfd_ioctl_get_dmabuf_info, 0),
2732
2733	AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
2734				kfd_ioctl_import_dmabuf, 0),
2735
2736	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
2737			kfd_ioctl_alloc_queue_gws, 0),
2738
2739	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
2740			kfd_ioctl_smi_events, 0),
2741
2742	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
2743
2744	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
2745			kfd_ioctl_set_xnack_mode, 0),
2746
2747	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CRIU_OP,
2748			kfd_ioctl_criu, KFD_IOC_FLAG_CHECKPOINT_RESTORE),
2749
2750	AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
2751			kfd_ioctl_get_available_memory, 0),
2752};
2753
2754#define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
2755
2756static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
2757{
2758	struct kfd_process *process;
2759	amdkfd_ioctl_t *func;
2760	const struct amdkfd_ioctl_desc *ioctl = NULL;
2761	unsigned int nr = _IOC_NR(cmd);
2762	char stack_kdata[128];
2763	char *kdata = NULL;
2764	unsigned int usize, asize;
2765	int retcode = -EINVAL;
2766	bool ptrace_attached = false;
2767
2768	if (nr >= AMDKFD_CORE_IOCTL_COUNT)
2769		goto err_i1;
2770
2771	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
2772		u32 amdkfd_size;
2773
2774		ioctl = &amdkfd_ioctls[nr];
2775
2776		amdkfd_size = _IOC_SIZE(ioctl->cmd);
2777		usize = asize = _IOC_SIZE(cmd);
2778		if (amdkfd_size > asize)
2779			asize = amdkfd_size;
2780
2781		cmd = ioctl->cmd;
2782	} else
2783		goto err_i1;
2784
2785	dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
2786
2787	/* Get the process struct from the filep. Only the process
2788	 * that opened /dev/kfd can use the file descriptor. Child
2789	 * processes need to create their own KFD device context.
2790	 */
2791	process = filep->private_data;
2792
2793	rcu_read_lock();
2794	if ((ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE) &&
2795	    ptrace_parent(process->lead_thread) == current)
2796		ptrace_attached = true;
2797	rcu_read_unlock();
2798
2799	if (process->lead_thread != current->group_leader
2800	    && !ptrace_attached) {
2801		dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
2802		retcode = -EBADF;
2803		goto err_i1;
2804	}
2805
2806	/* Do not trust userspace, use our own definition */
2807	func = ioctl->func;
2808
2809	if (unlikely(!func)) {
2810		dev_dbg(kfd_device, "no function\n");
2811		retcode = -EINVAL;
2812		goto err_i1;
2813	}
2814
2815	/*
2816	 * Versions of docker shipped in Ubuntu 18.xx and 20.xx do not support
2817	 * CAP_CHECKPOINT_RESTORE, so we also allow access if CAP_SYS_ADMIN as CAP_SYS_ADMIN is a
2818	 * more priviledged access.
2819	 */
2820	if (unlikely(ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE)) {
2821		if (!capable(CAP_CHECKPOINT_RESTORE) &&
2822						!capable(CAP_SYS_ADMIN)) {
2823			retcode = -EACCES;
2824			goto err_i1;
2825		}
2826	}
2827
2828	if (cmd & (IOC_IN | IOC_OUT)) {
2829		if (asize <= sizeof(stack_kdata)) {
2830			kdata = stack_kdata;
2831		} else {
2832			kdata = kmalloc(asize, GFP_KERNEL);
2833			if (!kdata) {
2834				retcode = -ENOMEM;
2835				goto err_i1;
2836			}
2837		}
2838		if (asize > usize)
2839			memset(kdata + usize, 0, asize - usize);
2840	}
2841
2842	if (cmd & IOC_IN) {
2843		if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
2844			retcode = -EFAULT;
2845			goto err_i1;
2846		}
2847	} else if (cmd & IOC_OUT) {
2848		memset(kdata, 0, usize);
2849	}
2850
2851	retcode = func(filep, process, kdata);
2852
2853	if (cmd & IOC_OUT)
2854		if (copy_to_user((void __user *)arg, kdata, usize) != 0)
2855			retcode = -EFAULT;
2856
2857err_i1:
2858	if (!ioctl)
2859		dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
2860			  task_pid_nr(current), cmd, nr);
2861
2862	if (kdata != stack_kdata)
2863		kfree(kdata);
2864
2865	if (retcode)
2866		dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
2867				nr, arg, retcode);
2868
2869	return retcode;
2870}
2871
2872static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
2873		      struct vm_area_struct *vma)
2874{
2875	phys_addr_t address;
2876
2877	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2878		return -EINVAL;
2879
2880	address = dev->adev->rmmio_remap.bus_addr;
2881
2882	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
2883				VM_DONTDUMP | VM_PFNMAP;
2884
2885	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2886
2887	pr_debug("pasid 0x%x mapping mmio page\n"
2888		 "     target user address == 0x%08llX\n"
2889		 "     physical address    == 0x%08llX\n"
2890		 "     vm_flags            == 0x%04lX\n"
2891		 "     size                == 0x%04lX\n",
2892		 process->pasid, (unsigned long long) vma->vm_start,
2893		 address, vma->vm_flags, PAGE_SIZE);
2894
2895	return io_remap_pfn_range(vma,
2896				vma->vm_start,
2897				address >> PAGE_SHIFT,
2898				PAGE_SIZE,
2899				vma->vm_page_prot);
2900}
2901
2902
2903static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
2904{
2905	struct kfd_process *process;
2906	struct kfd_dev *dev = NULL;
2907	unsigned long mmap_offset;
2908	unsigned int gpu_id;
2909
2910	process = kfd_get_process(current);
2911	if (IS_ERR(process))
2912		return PTR_ERR(process);
2913
2914	mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
2915	gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
2916	if (gpu_id)
2917		dev = kfd_device_by_id(gpu_id);
2918
2919	switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
2920	case KFD_MMAP_TYPE_DOORBELL:
2921		if (!dev)
2922			return -ENODEV;
2923		return kfd_doorbell_mmap(dev, process, vma);
2924
2925	case KFD_MMAP_TYPE_EVENTS:
2926		return kfd_event_mmap(process, vma);
2927
2928	case KFD_MMAP_TYPE_RESERVED_MEM:
2929		if (!dev)
2930			return -ENODEV;
2931		return kfd_reserved_mem_mmap(dev, process, vma);
2932	case KFD_MMAP_TYPE_MMIO:
2933		if (!dev)
2934			return -ENODEV;
2935		return kfd_mmio_mmap(dev, process, vma);
2936	}
2937
2938	return -EFAULT;
2939}