1/*
   2 * Copyright 2014 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 */
  22
  23#include <linux/mm_types.h>
  24#include <linux/slab.h>
  25#include <linux/types.h>
  26#include <linux/sched/signal.h>
  27#include <linux/sched/mm.h>
  28#include <linux/uaccess.h>
  29#include <linux/mman.h>
  30#include <linux/memory.h>
  31#include "kfd_priv.h"
  32#include "kfd_events.h"
  33#include "kfd_iommu.h"
  34#include <linux/device.h>
  35
  36/*
  37 * Wrapper around wait_queue_entry_t
  38 */
  39struct kfd_event_waiter {
  40	wait_queue_entry_t wait;
  41	struct kfd_event *event; /* Event to wait for */
  42	bool activated;		 /* Becomes true when event is signaled */
  43};
  44
  45/*
  46 * Each signal event needs a 64-bit signal slot where the signaler will write
  47 * a 1 before sending an interrupt. (This is needed because some interrupts
  48 * do not contain enough spare data bits to identify an event.)
  49 * We get whole pages and map them to the process VA.
  50 * Individual signal events use their event_id as slot index.
  51 */
  52struct kfd_signal_page {
  53	uint64_t *kernel_address;
  54	uint64_t __user *user_address;
  55	bool need_to_free_pages;
  56};
  57
  58
  59static uint64_t *page_slots(struct kfd_signal_page *page)
  60{
  61	return page->kernel_address;
  62}
  63
  64static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p)
  65{
  66	void *backing_store;
  67	struct kfd_signal_page *page;
  68
  69	page = kzalloc(sizeof(*page), GFP_KERNEL);
  70	if (!page)
  71		return NULL;
  72
  73	backing_store = (void *) __get_free_pages(GFP_KERNEL,
  74					get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
  75	if (!backing_store)
  76		goto fail_alloc_signal_store;
  77
  78	/* Initialize all events to unsignaled */
  79	memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT,
  80	       KFD_SIGNAL_EVENT_LIMIT * 8);
  81
  82	page->kernel_address = backing_store;
  83	page->need_to_free_pages = true;
  84	pr_debug("Allocated new event signal page at %p, for process %p\n",
  85			page, p);
  86
  87	return page;
  88
  89fail_alloc_signal_store:
  90	kfree(page);
  91	return NULL;
  92}
  93
  94static int allocate_event_notification_slot(struct kfd_process *p,
  95					    struct kfd_event *ev)
  96{
  97	int id;
  98
  99	if (!p->signal_page) {
 100		p->signal_page = allocate_signal_page(p);
 101		if (!p->signal_page)
 102			return -ENOMEM;
 103		/* Oldest user mode expects 256 event slots */
 104		p->signal_mapped_size = 256*8;
 105	}
 106
 107	/*
 108	 * Compatibility with old user mode: Only use signal slots
 109	 * user mode has mapped, may be less than
 110	 * KFD_SIGNAL_EVENT_LIMIT. This also allows future increase
 111	 * of the event limit without breaking user mode.
 112	 */
 113	id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,
 114		       GFP_KERNEL);
 115	if (id < 0)
 116		return id;
 117
 118	ev->event_id = id;
 119	page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT;
 120
 121	return 0;
 122}
 123
 124/*
 125 * Assumes that p->event_mutex is held and of course that p is not going
 126 * away (current or locked).
 127 */
 128static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
 129{
 130	return idr_find(&p->event_idr, id);
 131}
 132
 133/**
 134 * lookup_signaled_event_by_partial_id - Lookup signaled event from partial ID
 135 * @p:     Pointer to struct kfd_process
 136 * @id:    ID to look up
 137 * @bits:  Number of valid bits in @id
 138 *
 139 * Finds the first signaled event with a matching partial ID. If no
 140 * matching signaled event is found, returns NULL. In that case the
 141 * caller should assume that the partial ID is invalid and do an
 142 * exhaustive search of all siglaned events.
 143 *
 144 * If multiple events with the same partial ID signal at the same
 145 * time, they will be found one interrupt at a time, not necessarily
 146 * in the same order the interrupts occurred. As long as the number of
 147 * interrupts is correct, all signaled events will be seen by the
 148 * driver.
 149 */
 150static struct kfd_event *lookup_signaled_event_by_partial_id(
 151	struct kfd_process *p, uint32_t id, uint32_t bits)
 152{
 153	struct kfd_event *ev;
 154
 155	if (!p->signal_page || id >= KFD_SIGNAL_EVENT_LIMIT)
 156		return NULL;
 157
 158	/* Fast path for the common case that @id is not a partial ID
 159	 * and we only need a single lookup.
 160	 */
 161	if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) {
 162		if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
 163			return NULL;
 164
 165		return idr_find(&p->event_idr, id);
 166	}
 167
 168	/* General case for partial IDs: Iterate over all matching IDs
 169	 * and find the first one that has signaled.
 170	 */
 171	for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) {
 172		if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
 173			continue;
 174
 175		ev = idr_find(&p->event_idr, id);
 176	}
 177
 178	return ev;
 179}
 180
 181static int create_signal_event(struct file *devkfd,
 182				struct kfd_process *p,
 183				struct kfd_event *ev)
 184{
 185	int ret;
 186
 187	if (p->signal_mapped_size &&
 188	    p->signal_event_count == p->signal_mapped_size / 8) {
 189		if (!p->signal_event_limit_reached) {
 190			pr_debug("Signal event wasn't created because limit was reached\n");
 191			p->signal_event_limit_reached = true;
 192		}
 193		return -ENOSPC;
 194	}
 195
 196	ret = allocate_event_notification_slot(p, ev);
 197	if (ret) {
 198		pr_warn("Signal event wasn't created because out of kernel memory\n");
 199		return ret;
 200	}
 201
 202	p->signal_event_count++;
 203
 204	ev->user_signal_address = &p->signal_page->user_address[ev->event_id];
 205	pr_debug("Signal event number %zu created with id %d, address %p\n",
 206			p->signal_event_count, ev->event_id,
 207			ev->user_signal_address);
 208
 209	return 0;
 210}
 211
 212static int create_other_event(struct kfd_process *p, struct kfd_event *ev)
 213{
 214	/* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an
 215	 * intentional integer overflow to -1 without a compiler
 216	 * warning. idr_alloc treats a negative value as "maximum
 217	 * signed integer".
 218	 */
 219	int id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,
 220			   (uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1,
 221			   GFP_KERNEL);
 222
 223	if (id < 0)
 224		return id;
 225	ev->event_id = id;
 226
 227	return 0;
 228}
 229
 230void kfd_event_init_process(struct kfd_process *p)
 231{
 232	mutex_init(&p->event_mutex);
 233	idr_init(&p->event_idr);
 234	p->signal_page = NULL;
 235	p->signal_event_count = 0;
 236}
 237
 238static void destroy_event(struct kfd_process *p, struct kfd_event *ev)
 239{
 240	struct kfd_event_waiter *waiter;
 241
 242	/* Wake up pending waiters. They will return failure */
 243	list_for_each_entry(waiter, &ev->wq.head, wait.entry)
 244		waiter->event = NULL;
 245	wake_up_all(&ev->wq);
 246
 247	if (ev->type == KFD_EVENT_TYPE_SIGNAL ||
 248	    ev->type == KFD_EVENT_TYPE_DEBUG)
 249		p->signal_event_count--;
 250
 251	idr_remove(&p->event_idr, ev->event_id);
 252	kfree(ev);
 253}
 254
 255static void destroy_events(struct kfd_process *p)
 256{
 257	struct kfd_event *ev;
 258	uint32_t id;
 259
 260	idr_for_each_entry(&p->event_idr, ev, id)
 261		destroy_event(p, ev);
 262	idr_destroy(&p->event_idr);
 263}
 264
 265/*
 266 * We assume that the process is being destroyed and there is no need to
 267 * unmap the pages or keep bookkeeping data in order.
 268 */
 269static void shutdown_signal_page(struct kfd_process *p)
 270{
 271	struct kfd_signal_page *page = p->signal_page;
 272
 273	if (page) {
 274		if (page->need_to_free_pages)
 275			free_pages((unsigned long)page->kernel_address,
 276				   get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
 277		kfree(page);
 278	}
 279}
 280
 281void kfd_event_free_process(struct kfd_process *p)
 282{
 283	destroy_events(p);
 284	shutdown_signal_page(p);
 285}
 286
 287static bool event_can_be_gpu_signaled(const struct kfd_event *ev)
 288{
 289	return ev->type == KFD_EVENT_TYPE_SIGNAL ||
 290					ev->type == KFD_EVENT_TYPE_DEBUG;
 291}
 292
 293static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
 294{
 295	return ev->type == KFD_EVENT_TYPE_SIGNAL;
 296}
 297
 298int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
 299		       uint64_t size)
 300{
 301	struct kfd_signal_page *page;
 302
 303	if (p->signal_page)
 304		return -EBUSY;
 305
 306	page = kzalloc(sizeof(*page), GFP_KERNEL);
 307	if (!page)
 308		return -ENOMEM;
 309
 310	/* Initialize all events to unsignaled */
 311	memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT,
 312	       KFD_SIGNAL_EVENT_LIMIT * 8);
 313
 314	page->kernel_address = kernel_address;
 315
 316	p->signal_page = page;
 317	p->signal_mapped_size = size;
 318
 319	return 0;
 320}
 321
 322int kfd_event_create(struct file *devkfd, struct kfd_process *p,
 323		     uint32_t event_type, bool auto_reset, uint32_t node_id,
 324		     uint32_t *event_id, uint32_t *event_trigger_data,
 325		     uint64_t *event_page_offset, uint32_t *event_slot_index)
 326{
 327	int ret = 0;
 328	struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL);
 329
 330	if (!ev)
 331		return -ENOMEM;
 332
 333	ev->type = event_type;
 334	ev->auto_reset = auto_reset;
 335	ev->signaled = false;
 336
 337	init_waitqueue_head(&ev->wq);
 338
 339	*event_page_offset = 0;
 340
 341	mutex_lock(&p->event_mutex);
 342
 343	switch (event_type) {
 344	case KFD_EVENT_TYPE_SIGNAL:
 345	case KFD_EVENT_TYPE_DEBUG:
 346		ret = create_signal_event(devkfd, p, ev);
 347		if (!ret) {
 348			*event_page_offset = KFD_MMAP_TYPE_EVENTS;
 
 349			*event_slot_index = ev->event_id;
 350		}
 351		break;
 352	default:
 353		ret = create_other_event(p, ev);
 354		break;
 355	}
 356
 357	if (!ret) {
 358		*event_id = ev->event_id;
 359		*event_trigger_data = ev->event_id;
 360	} else {
 361		kfree(ev);
 362	}
 363
 364	mutex_unlock(&p->event_mutex);
 365
 366	return ret;
 367}
 368
 369/* Assumes that p is current. */
 370int kfd_event_destroy(struct kfd_process *p, uint32_t event_id)
 371{
 372	struct kfd_event *ev;
 373	int ret = 0;
 374
 375	mutex_lock(&p->event_mutex);
 376
 377	ev = lookup_event_by_id(p, event_id);
 378
 379	if (ev)
 380		destroy_event(p, ev);
 381	else
 382		ret = -EINVAL;
 383
 384	mutex_unlock(&p->event_mutex);
 385	return ret;
 386}
 387
 388static void set_event(struct kfd_event *ev)
 389{
 390	struct kfd_event_waiter *waiter;
 391
 392	/* Auto reset if the list is non-empty and we're waking
 393	 * someone. waitqueue_active is safe here because we're
 394	 * protected by the p->event_mutex, which is also held when
 395	 * updating the wait queues in kfd_wait_on_events.
 396	 */
 397	ev->signaled = !ev->auto_reset || !waitqueue_active(&ev->wq);
 398
 399	list_for_each_entry(waiter, &ev->wq.head, wait.entry)
 400		waiter->activated = true;
 401
 402	wake_up_all(&ev->wq);
 403}
 404
 405/* Assumes that p is current. */
 406int kfd_set_event(struct kfd_process *p, uint32_t event_id)
 407{
 408	int ret = 0;
 409	struct kfd_event *ev;
 410
 411	mutex_lock(&p->event_mutex);
 412
 413	ev = lookup_event_by_id(p, event_id);
 414
 415	if (ev && event_can_be_cpu_signaled(ev))
 416		set_event(ev);
 417	else
 418		ret = -EINVAL;
 419
 420	mutex_unlock(&p->event_mutex);
 421	return ret;
 422}
 423
 424static void reset_event(struct kfd_event *ev)
 425{
 426	ev->signaled = false;
 427}
 428
 429/* Assumes that p is current. */
 430int kfd_reset_event(struct kfd_process *p, uint32_t event_id)
 431{
 432	int ret = 0;
 433	struct kfd_event *ev;
 434
 435	mutex_lock(&p->event_mutex);
 436
 437	ev = lookup_event_by_id(p, event_id);
 438
 439	if (ev && event_can_be_cpu_signaled(ev))
 440		reset_event(ev);
 441	else
 442		ret = -EINVAL;
 443
 444	mutex_unlock(&p->event_mutex);
 445	return ret;
 446
 447}
 448
 449static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)
 450{
 451	page_slots(p->signal_page)[ev->event_id] = UNSIGNALED_EVENT_SLOT;
 452}
 453
 454static void set_event_from_interrupt(struct kfd_process *p,
 455					struct kfd_event *ev)
 456{
 457	if (ev && event_can_be_gpu_signaled(ev)) {
 458		acknowledge_signal(p, ev);
 459		set_event(ev);
 460	}
 461}
 462
 463void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
 464				uint32_t valid_id_bits)
 465{
 466	struct kfd_event *ev = NULL;
 467
 468	/*
 469	 * Because we are called from arbitrary context (workqueue) as opposed
 470	 * to process context, kfd_process could attempt to exit while we are
 471	 * running so the lookup function increments the process ref count.
 472	 */
 473	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 474
 475	if (!p)
 476		return; /* Presumably process exited. */
 477
 478	mutex_lock(&p->event_mutex);
 479
 480	if (valid_id_bits)
 481		ev = lookup_signaled_event_by_partial_id(p, partial_id,
 482							 valid_id_bits);
 483	if (ev) {
 484		set_event_from_interrupt(p, ev);
 485	} else if (p->signal_page) {
 486		/*
 487		 * Partial ID lookup failed. Assume that the event ID
 488		 * in the interrupt payload was invalid and do an
 489		 * exhaustive search of signaled events.
 490		 */
 491		uint64_t *slots = page_slots(p->signal_page);
 492		uint32_t id;
 493
 494		if (valid_id_bits)
 495			pr_debug_ratelimited("Partial ID invalid: %u (%u valid bits)\n",
 496					     partial_id, valid_id_bits);
 497
 498		if (p->signal_event_count < KFD_SIGNAL_EVENT_LIMIT / 64) {
 499			/* With relatively few events, it's faster to
 500			 * iterate over the event IDR
 501			 */
 502			idr_for_each_entry(&p->event_idr, ev, id) {
 503				if (id >= KFD_SIGNAL_EVENT_LIMIT)
 504					break;
 505
 506				if (slots[id] != UNSIGNALED_EVENT_SLOT)
 507					set_event_from_interrupt(p, ev);
 508			}
 509		} else {
 510			/* With relatively many events, it's faster to
 511			 * iterate over the signal slots and lookup
 512			 * only signaled events from the IDR.
 513			 */
 514			for (id = 0; id < KFD_SIGNAL_EVENT_LIMIT; id++)
 515				if (slots[id] != UNSIGNALED_EVENT_SLOT) {
 516					ev = lookup_event_by_id(p, id);
 517					set_event_from_interrupt(p, ev);
 518				}
 519		}
 520	}
 521
 522	mutex_unlock(&p->event_mutex);
 523	kfd_unref_process(p);
 524}
 525
 526static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
 527{
 528	struct kfd_event_waiter *event_waiters;
 529	uint32_t i;
 530
 531	event_waiters = kmalloc_array(num_events,
 532					sizeof(struct kfd_event_waiter),
 533					GFP_KERNEL);
 534
 535	for (i = 0; (event_waiters) && (i < num_events) ; i++) {
 536		init_wait(&event_waiters[i].wait);
 537		event_waiters[i].activated = false;
 538	}
 539
 540	return event_waiters;
 541}
 542
 543static int init_event_waiter_get_status(struct kfd_process *p,
 544		struct kfd_event_waiter *waiter,
 545		uint32_t event_id)
 546{
 547	struct kfd_event *ev = lookup_event_by_id(p, event_id);
 548
 549	if (!ev)
 550		return -EINVAL;
 551
 552	waiter->event = ev;
 553	waiter->activated = ev->signaled;
 554	ev->signaled = ev->signaled && !ev->auto_reset;
 555
 556	return 0;
 557}
 558
 559static void init_event_waiter_add_to_waitlist(struct kfd_event_waiter *waiter)
 560{
 561	struct kfd_event *ev = waiter->event;
 562
 563	/* Only add to the wait list if we actually need to
 564	 * wait on this event.
 565	 */
 566	if (!waiter->activated)
 567		add_wait_queue(&ev->wq, &waiter->wait);
 568}
 569
 570/* test_event_condition - Test condition of events being waited for
 571 * @all:           Return completion only if all events have signaled
 572 * @num_events:    Number of events to wait for
 573 * @event_waiters: Array of event waiters, one per event
 574 *
 575 * Returns KFD_IOC_WAIT_RESULT_COMPLETE if all (or one) event(s) have
 576 * signaled. Returns KFD_IOC_WAIT_RESULT_TIMEOUT if no (or not all)
 577 * events have signaled. Returns KFD_IOC_WAIT_RESULT_FAIL if any of
 578 * the events have been destroyed.
 579 */
 580static uint32_t test_event_condition(bool all, uint32_t num_events,
 581				struct kfd_event_waiter *event_waiters)
 582{
 583	uint32_t i;
 584	uint32_t activated_count = 0;
 585
 586	for (i = 0; i < num_events; i++) {
 587		if (!event_waiters[i].event)
 588			return KFD_IOC_WAIT_RESULT_FAIL;
 589
 590		if (event_waiters[i].activated) {
 591			if (!all)
 592				return KFD_IOC_WAIT_RESULT_COMPLETE;
 593
 594			activated_count++;
 595		}
 596	}
 597
 598	return activated_count == num_events ?
 599		KFD_IOC_WAIT_RESULT_COMPLETE : KFD_IOC_WAIT_RESULT_TIMEOUT;
 600}
 601
 602/*
 603 * Copy event specific data, if defined.
 604 * Currently only memory exception events have additional data to copy to user
 605 */
 606static int copy_signaled_event_data(uint32_t num_events,
 607		struct kfd_event_waiter *event_waiters,
 608		struct kfd_event_data __user *data)
 609{
 610	struct kfd_hsa_memory_exception_data *src;
 611	struct kfd_hsa_memory_exception_data __user *dst;
 612	struct kfd_event_waiter *waiter;
 613	struct kfd_event *event;
 614	uint32_t i;
 615
 616	for (i = 0; i < num_events; i++) {
 617		waiter = &event_waiters[i];
 618		event = waiter->event;
 619		if (waiter->activated && event->type == KFD_EVENT_TYPE_MEMORY) {
 620			dst = &data[i].memory_exception_data;
 621			src = &event->memory_exception_data;
 622			if (copy_to_user(dst, src,
 623				sizeof(struct kfd_hsa_memory_exception_data)))
 624				return -EFAULT;
 625		}
 626	}
 627
 628	return 0;
 629
 630}
 631
 632
 633
 634static long user_timeout_to_jiffies(uint32_t user_timeout_ms)
 635{
 636	if (user_timeout_ms == KFD_EVENT_TIMEOUT_IMMEDIATE)
 637		return 0;
 638
 639	if (user_timeout_ms == KFD_EVENT_TIMEOUT_INFINITE)
 640		return MAX_SCHEDULE_TIMEOUT;
 641
 642	/*
 643	 * msecs_to_jiffies interprets all values above 2^31-1 as infinite,
 644	 * but we consider them finite.
 645	 * This hack is wrong, but nobody is likely to notice.
 646	 */
 647	user_timeout_ms = min_t(uint32_t, user_timeout_ms, 0x7FFFFFFF);
 648
 649	return msecs_to_jiffies(user_timeout_ms) + 1;
 650}
 651
 652static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters)
 653{
 654	uint32_t i;
 655
 656	for (i = 0; i < num_events; i++)
 657		if (waiters[i].event)
 658			remove_wait_queue(&waiters[i].event->wq,
 659					  &waiters[i].wait);
 660
 661	kfree(waiters);
 662}
 663
 664int kfd_wait_on_events(struct kfd_process *p,
 665		       uint32_t num_events, void __user *data,
 666		       bool all, uint32_t user_timeout_ms,
 667		       uint32_t *wait_result)
 668{
 669	struct kfd_event_data __user *events =
 670			(struct kfd_event_data __user *) data;
 671	uint32_t i;
 672	int ret = 0;
 673
 674	struct kfd_event_waiter *event_waiters = NULL;
 675	long timeout = user_timeout_to_jiffies(user_timeout_ms);
 676
 677	event_waiters = alloc_event_waiters(num_events);
 678	if (!event_waiters) {
 679		ret = -ENOMEM;
 680		goto out;
 681	}
 682
 683	mutex_lock(&p->event_mutex);
 684
 685	for (i = 0; i < num_events; i++) {
 686		struct kfd_event_data event_data;
 687
 688		if (copy_from_user(&event_data, &events[i],
 689				sizeof(struct kfd_event_data))) {
 690			ret = -EFAULT;
 691			goto out_unlock;
 692		}
 693
 694		ret = init_event_waiter_get_status(p, &event_waiters[i],
 695				event_data.event_id);
 696		if (ret)
 697			goto out_unlock;
 698	}
 699
 700	/* Check condition once. */
 701	*wait_result = test_event_condition(all, num_events, event_waiters);
 702	if (*wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) {
 703		ret = copy_signaled_event_data(num_events,
 704					       event_waiters, events);
 705		goto out_unlock;
 706	} else if (WARN_ON(*wait_result == KFD_IOC_WAIT_RESULT_FAIL)) {
 707		/* This should not happen. Events shouldn't be
 708		 * destroyed while we're holding the event_mutex
 709		 */
 710		goto out_unlock;
 711	}
 712
 713	/* Add to wait lists if we need to wait. */
 714	for (i = 0; i < num_events; i++)
 715		init_event_waiter_add_to_waitlist(&event_waiters[i]);
 716
 717	mutex_unlock(&p->event_mutex);
 718
 719	while (true) {
 720		if (fatal_signal_pending(current)) {
 721			ret = -EINTR;
 722			break;
 723		}
 724
 725		if (signal_pending(current)) {
 726			/*
 727			 * This is wrong when a nonzero, non-infinite timeout
 728			 * is specified. We need to use
 729			 * ERESTARTSYS_RESTARTBLOCK, but struct restart_block
 730			 * contains a union with data for each user and it's
 731			 * in generic kernel code that I don't want to
 732			 * touch yet.
 733			 */
 734			ret = -ERESTARTSYS;
 735			break;
 736		}
 737
 738		/* Set task state to interruptible sleep before
 739		 * checking wake-up conditions. A concurrent wake-up
 740		 * will put the task back into runnable state. In that
 741		 * case schedule_timeout will not put the task to
 742		 * sleep and we'll get a chance to re-check the
 743		 * updated conditions almost immediately. Otherwise,
 744		 * this race condition would lead to a soft hang or a
 745		 * very long sleep.
 746		 */
 747		set_current_state(TASK_INTERRUPTIBLE);
 748
 749		*wait_result = test_event_condition(all, num_events,
 750						    event_waiters);
 751		if (*wait_result != KFD_IOC_WAIT_RESULT_TIMEOUT)
 752			break;
 753
 754		if (timeout <= 0)
 755			break;
 756
 757		timeout = schedule_timeout(timeout);
 758	}
 759	__set_current_state(TASK_RUNNING);
 760
 761	/* copy_signaled_event_data may sleep. So this has to happen
 762	 * after the task state is set back to RUNNING.
 763	 */
 764	if (!ret && *wait_result == KFD_IOC_WAIT_RESULT_COMPLETE)
 765		ret = copy_signaled_event_data(num_events,
 766					       event_waiters, events);
 767
 768	mutex_lock(&p->event_mutex);
 769out_unlock:
 770	free_waiters(num_events, event_waiters);
 771	mutex_unlock(&p->event_mutex);
 772out:
 773	if (ret)
 774		*wait_result = KFD_IOC_WAIT_RESULT_FAIL;
 775	else if (*wait_result == KFD_IOC_WAIT_RESULT_FAIL)
 776		ret = -EIO;
 777
 778	return ret;
 779}
 780
 781int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
 782{
 783	unsigned long pfn;
 784	struct kfd_signal_page *page;
 785	int ret;
 786
 787	/* check required size doesn't exceed the allocated size */
 788	if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) <
 789			get_order(vma->vm_end - vma->vm_start)) {
 790		pr_err("Event page mmap requested illegal size\n");
 791		return -EINVAL;
 792	}
 793
 794	page = p->signal_page;
 795	if (!page) {
 796		/* Probably KFD bug, but mmap is user-accessible. */
 797		pr_debug("Signal page could not be found\n");
 798		return -EINVAL;
 799	}
 800
 801	pfn = __pa(page->kernel_address);
 802	pfn >>= PAGE_SHIFT;
 803
 804	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE
 805		       | VM_DONTDUMP | VM_PFNMAP;
 806
 807	pr_debug("Mapping signal page\n");
 808	pr_debug("     start user address  == 0x%08lx\n", vma->vm_start);
 809	pr_debug("     end user address    == 0x%08lx\n", vma->vm_end);
 810	pr_debug("     pfn                 == 0x%016lX\n", pfn);
 811	pr_debug("     vm_flags            == 0x%08lX\n", vma->vm_flags);
 812	pr_debug("     size                == 0x%08lX\n",
 813			vma->vm_end - vma->vm_start);
 814
 815	page->user_address = (uint64_t __user *)vma->vm_start;
 816
 817	/* mapping the page to user process */
 818	ret = remap_pfn_range(vma, vma->vm_start, pfn,
 819			vma->vm_end - vma->vm_start, vma->vm_page_prot);
 820	if (!ret)
 821		p->signal_mapped_size = vma->vm_end - vma->vm_start;
 822
 823	return ret;
 824}
 825
 826/*
 827 * Assumes that p->event_mutex is held and of course
 828 * that p is not going away (current or locked).
 829 */
 830static void lookup_events_by_type_and_signal(struct kfd_process *p,
 831		int type, void *event_data)
 832{
 833	struct kfd_hsa_memory_exception_data *ev_data;
 834	struct kfd_event *ev;
 835	uint32_t id;
 836	bool send_signal = true;
 837
 838	ev_data = (struct kfd_hsa_memory_exception_data *) event_data;
 839
 840	id = KFD_FIRST_NONSIGNAL_EVENT_ID;
 841	idr_for_each_entry_continue(&p->event_idr, ev, id)
 842		if (ev->type == type) {
 843			send_signal = false;
 844			dev_dbg(kfd_device,
 845					"Event found: id %X type %d",
 846					ev->event_id, ev->type);
 847			set_event(ev);
 848			if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data)
 849				ev->memory_exception_data = *ev_data;
 850		}
 851
 852	if (type == KFD_EVENT_TYPE_MEMORY) {
 853		dev_warn(kfd_device,
 854			"Sending SIGSEGV to process %d (pasid 0x%x)",
 855				p->lead_thread->pid, p->pasid);
 856		send_sig(SIGSEGV, p->lead_thread, 0);
 857	}
 858
 859	/* Send SIGTERM no event of type "type" has been found*/
 860	if (send_signal) {
 861		if (send_sigterm) {
 862			dev_warn(kfd_device,
 863				"Sending SIGTERM to process %d (pasid 0x%x)",
 864					p->lead_thread->pid, p->pasid);
 865			send_sig(SIGTERM, p->lead_thread, 0);
 866		} else {
 867			dev_err(kfd_device,
 868				"Process %d (pasid 0x%x) got unhandled exception",
 869				p->lead_thread->pid, p->pasid);
 870		}
 871	}
 872}
 873
 874#ifdef KFD_SUPPORT_IOMMU_V2
 875void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 876		unsigned long address, bool is_write_requested,
 877		bool is_execute_requested)
 878{
 879	struct kfd_hsa_memory_exception_data memory_exception_data;
 880	struct vm_area_struct *vma;
 881
 882	/*
 883	 * Because we are called from arbitrary context (workqueue) as opposed
 884	 * to process context, kfd_process could attempt to exit while we are
 885	 * running so the lookup function increments the process ref count.
 886	 */
 887	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 888	struct mm_struct *mm;
 889
 890	if (!p)
 891		return; /* Presumably process exited. */
 892
 893	/* Take a safe reference to the mm_struct, which may otherwise
 894	 * disappear even while the kfd_process is still referenced.
 895	 */
 896	mm = get_task_mm(p->lead_thread);
 897	if (!mm) {
 898		kfd_unref_process(p);
 899		return; /* Process is exiting */
 900	}
 901
 902	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
 903
 904	mmap_read_lock(mm);
 905	vma = find_vma(mm, address);
 906
 907	memory_exception_data.gpu_id = dev->id;
 908	memory_exception_data.va = address;
 909	/* Set failure reason */
 910	memory_exception_data.failure.NotPresent = 1;
 911	memory_exception_data.failure.NoExecute = 0;
 912	memory_exception_data.failure.ReadOnly = 0;
 913	if (vma && address >= vma->vm_start) {
 914		memory_exception_data.failure.NotPresent = 0;
 915
 916		if (is_write_requested && !(vma->vm_flags & VM_WRITE))
 917			memory_exception_data.failure.ReadOnly = 1;
 918		else
 919			memory_exception_data.failure.ReadOnly = 0;
 920
 921		if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
 922			memory_exception_data.failure.NoExecute = 1;
 923		else
 924			memory_exception_data.failure.NoExecute = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 925	}
 926
 927	mmap_read_unlock(mm);
 928	mmput(mm);
 929
 930	pr_debug("notpresent %d, noexecute %d, readonly %d\n",
 931			memory_exception_data.failure.NotPresent,
 932			memory_exception_data.failure.NoExecute,
 933			memory_exception_data.failure.ReadOnly);
 934
 935	/* Workaround on Raven to not kill the process when memory is freed
 936	 * before IOMMU is able to finish processing all the excessive PPRs
 937	 */
 938	if (dev->device_info->asic_family != CHIP_RAVEN &&
 939	    dev->device_info->asic_family != CHIP_RENOIR) {
 940		mutex_lock(&p->event_mutex);
 941
 942		/* Lookup events by type and signal them */
 943		lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,
 944				&memory_exception_data);
 945
 946		mutex_unlock(&p->event_mutex);
 947	}
 
 948
 
 949	kfd_unref_process(p);
 950}
 951#endif /* KFD_SUPPORT_IOMMU_V2 */
 952
 953void kfd_signal_hw_exception_event(unsigned int pasid)
 954{
 955	/*
 956	 * Because we are called from arbitrary context (workqueue) as opposed
 957	 * to process context, kfd_process could attempt to exit while we are
 958	 * running so the lookup function increments the process ref count.
 959	 */
 960	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 961
 962	if (!p)
 963		return; /* Presumably process exited. */
 964
 965	mutex_lock(&p->event_mutex);
 966
 967	/* Lookup events by type and signal them */
 968	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL);
 969
 970	mutex_unlock(&p->event_mutex);
 971	kfd_unref_process(p);
 972}
 973
 974void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
 975				struct kfd_vm_fault_info *info)
 976{
 977	struct kfd_event *ev;
 978	uint32_t id;
 979	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 980	struct kfd_hsa_memory_exception_data memory_exception_data;
 981
 982	if (!p)
 983		return; /* Presumably process exited. */
 984	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
 985	memory_exception_data.gpu_id = dev->id;
 986	memory_exception_data.failure.imprecise = true;
 987	/* Set failure reason */
 988	if (info) {
 989		memory_exception_data.va = (info->page_addr) << PAGE_SHIFT;
 990		memory_exception_data.failure.NotPresent =
 991			info->prot_valid ? 1 : 0;
 992		memory_exception_data.failure.NoExecute =
 993			info->prot_exec ? 1 : 0;
 994		memory_exception_data.failure.ReadOnly =
 995			info->prot_write ? 1 : 0;
 996		memory_exception_data.failure.imprecise = 0;
 997	}
 998	mutex_lock(&p->event_mutex);
 999
1000	id = KFD_FIRST_NONSIGNAL_EVENT_ID;
1001	idr_for_each_entry_continue(&p->event_idr, ev, id)
1002		if (ev->type == KFD_EVENT_TYPE_MEMORY) {
1003			ev->memory_exception_data = memory_exception_data;
1004			set_event(ev);
1005		}
1006
1007	mutex_unlock(&p->event_mutex);
1008	kfd_unref_process(p);
1009}
1010
1011void kfd_signal_reset_event(struct kfd_dev *dev)
1012{
1013	struct kfd_hsa_hw_exception_data hw_exception_data;
1014	struct kfd_hsa_memory_exception_data memory_exception_data;
1015	struct kfd_process *p;
1016	struct kfd_event *ev;
1017	unsigned int temp;
1018	uint32_t id, idx;
1019	int reset_cause = atomic_read(&dev->sram_ecc_flag) ?
1020			KFD_HW_EXCEPTION_ECC :
1021			KFD_HW_EXCEPTION_GPU_HANG;
1022
1023	/* Whole gpu reset caused by GPU hang and memory is lost */
1024	memset(&hw_exception_data, 0, sizeof(hw_exception_data));
1025	hw_exception_data.gpu_id = dev->id;
1026	hw_exception_data.memory_lost = 1;
1027	hw_exception_data.reset_cause = reset_cause;
1028
1029	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
1030	memory_exception_data.ErrorType = KFD_MEM_ERR_SRAM_ECC;
1031	memory_exception_data.gpu_id = dev->id;
1032	memory_exception_data.failure.imprecise = true;
1033
1034	idx = srcu_read_lock(&kfd_processes_srcu);
1035	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1036		mutex_lock(&p->event_mutex);
1037		id = KFD_FIRST_NONSIGNAL_EVENT_ID;
1038		idr_for_each_entry_continue(&p->event_idr, ev, id) {
1039			if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
1040				ev->hw_exception_data = hw_exception_data;
1041				set_event(ev);
1042			}
1043			if (ev->type == KFD_EVENT_TYPE_MEMORY &&
1044			    reset_cause == KFD_HW_EXCEPTION_ECC) {
1045				ev->memory_exception_data = memory_exception_data;
1046				set_event(ev);
1047			}
1048		}
1049		mutex_unlock(&p->event_mutex);
1050	}
1051	srcu_read_unlock(&kfd_processes_srcu, idx);
1052}