1/*
   2 * Copyright © 2008-2015 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <linux/dma-fence-array.h>
  26#include <linux/dma-fence-chain.h>
  27#include <linux/irq_work.h>
  28#include <linux/prefetch.h>
  29#include <linux/sched.h>
  30#include <linux/sched/clock.h>
  31#include <linux/sched/signal.h>
  32#include <linux/sched/mm.h>
  33
  34#include "gem/i915_gem_context.h"
  35#include "gt/intel_breadcrumbs.h"
  36#include "gt/intel_context.h"
  37#include "gt/intel_engine.h"
  38#include "gt/intel_engine_heartbeat.h"
  39#include "gt/intel_engine_regs.h"
  40#include "gt/intel_gpu_commands.h"
  41#include "gt/intel_reset.h"
  42#include "gt/intel_ring.h"
  43#include "gt/intel_rps.h"
  44
  45#include "i915_active.h"
 
  46#include "i915_deps.h"
  47#include "i915_driver.h"
  48#include "i915_drv.h"
  49#include "i915_trace.h"
  50#include "intel_pm.h"
  51
  52struct execute_cb {
  53	struct irq_work work;
  54	struct i915_sw_fence *fence;
  55	struct i915_request *signal;
  56};
  57
  58static struct kmem_cache *slab_requests;
  59static struct kmem_cache *slab_execute_cbs;
  60
  61static const char *i915_fence_get_driver_name(struct dma_fence *fence)
  62{
  63	return dev_name(to_request(fence)->i915->drm.dev);
  64}
  65
  66static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
  67{
  68	const struct i915_gem_context *ctx;
  69
  70	/*
  71	 * The timeline struct (as part of the ppgtt underneath a context)
  72	 * may be freed when the request is no longer in use by the GPU.
  73	 * We could extend the life of a context to beyond that of all
  74	 * fences, possibly keeping the hw resource around indefinitely,
  75	 * or we just give them a false name. Since
  76	 * dma_fence_ops.get_timeline_name is a debug feature, the occasional
  77	 * lie seems justifiable.
  78	 */
  79	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  80		return "signaled";
  81
  82	ctx = i915_request_gem_context(to_request(fence));
  83	if (!ctx)
  84		return "[" DRIVER_NAME "]";
  85
  86	return ctx->name;
  87}
  88
  89static bool i915_fence_signaled(struct dma_fence *fence)
  90{
  91	return i915_request_completed(to_request(fence));
  92}
  93
  94static bool i915_fence_enable_signaling(struct dma_fence *fence)
  95{
  96	return i915_request_enable_breadcrumb(to_request(fence));
  97}
  98
  99static signed long i915_fence_wait(struct dma_fence *fence,
 100				   bool interruptible,
 101				   signed long timeout)
 102{
 103	return i915_request_wait_timeout(to_request(fence),
 104					 interruptible | I915_WAIT_PRIORITY,
 105					 timeout);
 106}
 107
 108struct kmem_cache *i915_request_slab_cache(void)
 109{
 110	return slab_requests;
 111}
 112
 113static void i915_fence_release(struct dma_fence *fence)
 114{
 115	struct i915_request *rq = to_request(fence);
 116
 117	GEM_BUG_ON(rq->guc_prio != GUC_PRIO_INIT &&
 118		   rq->guc_prio != GUC_PRIO_FINI);
 119
 120	i915_request_free_capture_list(fetch_and_zero(&rq->capture_list));
 121	if (rq->batch_res) {
 122		i915_vma_resource_put(rq->batch_res);
 123		rq->batch_res = NULL;
 124	}
 125
 126	/*
 127	 * The request is put onto a RCU freelist (i.e. the address
 128	 * is immediately reused), mark the fences as being freed now.
 129	 * Otherwise the debugobjects for the fences are only marked as
 130	 * freed when the slab cache itself is freed, and so we would get
 131	 * caught trying to reuse dead objects.
 132	 */
 133	i915_sw_fence_fini(&rq->submit);
 134	i915_sw_fence_fini(&rq->semaphore);
 135
 136	/*
 137	 * Keep one request on each engine for reserved use under mempressure
 138	 * do not use with virtual engines as this really is only needed for
 139	 * kernel contexts.
 140	 *
 141	 * We do not hold a reference to the engine here and so have to be
 142	 * very careful in what rq->engine we poke. The virtual engine is
 143	 * referenced via the rq->context and we released that ref during
 144	 * i915_request_retire(), ergo we must not dereference a virtual
 145	 * engine here. Not that we would want to, as the only consumer of
 146	 * the reserved engine->request_pool is the power management parking,
 147	 * which must-not-fail, and that is only run on the physical engines.
 148	 *
 149	 * Since the request must have been executed to be have completed,
 150	 * we know that it will have been processed by the HW and will
 151	 * not be unsubmitted again, so rq->engine and rq->execution_mask
 152	 * at this point is stable. rq->execution_mask will be a single
 153	 * bit if the last and _only_ engine it could execution on was a
 154	 * physical engine, if it's multiple bits then it started on and
 155	 * could still be on a virtual engine. Thus if the mask is not a
 156	 * power-of-two we assume that rq->engine may still be a virtual
 157	 * engine and so a dangling invalid pointer that we cannot dereference
 158	 *
 159	 * For example, consider the flow of a bonded request through a virtual
 160	 * engine. The request is created with a wide engine mask (all engines
 161	 * that we might execute on). On processing the bond, the request mask
 162	 * is reduced to one or more engines. If the request is subsequently
 163	 * bound to a single engine, it will then be constrained to only
 164	 * execute on that engine and never returned to the virtual engine
 165	 * after timeslicing away, see __unwind_incomplete_requests(). Thus we
 166	 * know that if the rq->execution_mask is a single bit, rq->engine
 167	 * can be a physical engine with the exact corresponding mask.
 168	 */
 169	if (!intel_engine_is_virtual(rq->engine) &&
 170	    is_power_of_2(rq->execution_mask) &&
 171	    !cmpxchg(&rq->engine->request_pool, NULL, rq))
 172		return;
 173
 174	kmem_cache_free(slab_requests, rq);
 175}
 176
 177const struct dma_fence_ops i915_fence_ops = {
 178	.get_driver_name = i915_fence_get_driver_name,
 179	.get_timeline_name = i915_fence_get_timeline_name,
 180	.enable_signaling = i915_fence_enable_signaling,
 181	.signaled = i915_fence_signaled,
 182	.wait = i915_fence_wait,
 183	.release = i915_fence_release,
 184};
 185
 186static void irq_execute_cb(struct irq_work *wrk)
 187{
 188	struct execute_cb *cb = container_of(wrk, typeof(*cb), work);
 189
 190	i915_sw_fence_complete(cb->fence);
 191	kmem_cache_free(slab_execute_cbs, cb);
 192}
 193
 194static __always_inline void
 195__notify_execute_cb(struct i915_request *rq, bool (*fn)(struct irq_work *wrk))
 196{
 197	struct execute_cb *cb, *cn;
 198
 199	if (llist_empty(&rq->execute_cb))
 200		return;
 201
 202	llist_for_each_entry_safe(cb, cn,
 203				  llist_del_all(&rq->execute_cb),
 204				  work.node.llist)
 205		fn(&cb->work);
 206}
 207
 208static void __notify_execute_cb_irq(struct i915_request *rq)
 209{
 210	__notify_execute_cb(rq, irq_work_queue);
 211}
 212
 213static bool irq_work_imm(struct irq_work *wrk)
 214{
 215	wrk->func(wrk);
 216	return false;
 217}
 218
 219void i915_request_notify_execute_cb_imm(struct i915_request *rq)
 220{
 221	__notify_execute_cb(rq, irq_work_imm);
 222}
 223
 224static void __i915_request_fill(struct i915_request *rq, u8 val)
 225{
 226	void *vaddr = rq->ring->vaddr;
 227	u32 head;
 228
 229	head = rq->infix;
 230	if (rq->postfix < head) {
 231		memset(vaddr + head, val, rq->ring->size - head);
 232		head = 0;
 233	}
 234	memset(vaddr + head, val, rq->postfix - head);
 235}
 236
 237/**
 238 * i915_request_active_engine
 239 * @rq: request to inspect
 240 * @active: pointer in which to return the active engine
 241 *
 242 * Fills the currently active engine to the @active pointer if the request
 243 * is active and still not completed.
 244 *
 245 * Returns true if request was active or false otherwise.
 246 */
 247bool
 248i915_request_active_engine(struct i915_request *rq,
 249			   struct intel_engine_cs **active)
 250{
 251	struct intel_engine_cs *engine, *locked;
 252	bool ret = false;
 253
 254	/*
 255	 * Serialise with __i915_request_submit() so that it sees
 256	 * is-banned?, or we know the request is already inflight.
 257	 *
 258	 * Note that rq->engine is unstable, and so we double
 259	 * check that we have acquired the lock on the final engine.
 260	 */
 261	locked = READ_ONCE(rq->engine);
 262	spin_lock_irq(&locked->sched_engine->lock);
 263	while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
 264		spin_unlock(&locked->sched_engine->lock);
 265		locked = engine;
 266		spin_lock(&locked->sched_engine->lock);
 267	}
 268
 269	if (i915_request_is_active(rq)) {
 270		if (!__i915_request_is_complete(rq))
 271			*active = locked;
 272		ret = true;
 273	}
 274
 275	spin_unlock_irq(&locked->sched_engine->lock);
 276
 277	return ret;
 278}
 279
 280static void __rq_init_watchdog(struct i915_request *rq)
 281{
 282	rq->watchdog.timer.function = NULL;
 283}
 284
 285static enum hrtimer_restart __rq_watchdog_expired(struct hrtimer *hrtimer)
 286{
 287	struct i915_request *rq =
 288		container_of(hrtimer, struct i915_request, watchdog.timer);
 289	struct intel_gt *gt = rq->engine->gt;
 290
 291	if (!i915_request_completed(rq)) {
 292		if (llist_add(&rq->watchdog.link, &gt->watchdog.list))
 293			schedule_work(&gt->watchdog.work);
 294	} else {
 295		i915_request_put(rq);
 296	}
 297
 298	return HRTIMER_NORESTART;
 299}
 300
 301static void __rq_arm_watchdog(struct i915_request *rq)
 302{
 303	struct i915_request_watchdog *wdg = &rq->watchdog;
 304	struct intel_context *ce = rq->context;
 305
 306	if (!ce->watchdog.timeout_us)
 307		return;
 308
 309	i915_request_get(rq);
 310
 311	hrtimer_init(&wdg->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 312	wdg->timer.function = __rq_watchdog_expired;
 313	hrtimer_start_range_ns(&wdg->timer,
 314			       ns_to_ktime(ce->watchdog.timeout_us *
 315					   NSEC_PER_USEC),
 316			       NSEC_PER_MSEC,
 317			       HRTIMER_MODE_REL);
 318}
 319
 320static void __rq_cancel_watchdog(struct i915_request *rq)
 321{
 322	struct i915_request_watchdog *wdg = &rq->watchdog;
 323
 324	if (wdg->timer.function && hrtimer_try_to_cancel(&wdg->timer) > 0)
 325		i915_request_put(rq);
 326}
 327
 328#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
 329
 330/**
 331 * i915_request_free_capture_list - Free a capture list
 332 * @capture: Pointer to the first list item or NULL
 333 *
 334 */
 335void i915_request_free_capture_list(struct i915_capture_list *capture)
 336{
 337	while (capture) {
 338		struct i915_capture_list *next = capture->next;
 339
 340		i915_vma_resource_put(capture->vma_res);
 341		kfree(capture);
 342		capture = next;
 343	}
 344}
 345
 346#define assert_capture_list_is_null(_rq) GEM_BUG_ON((_rq)->capture_list)
 347
 348#define clear_capture_list(_rq) ((_rq)->capture_list = NULL)
 349
 350#else
 351
 352#define i915_request_free_capture_list(_a) do {} while (0)
 353
 354#define assert_capture_list_is_null(_a) do {} while (0)
 355
 356#define clear_capture_list(_rq) do {} while (0)
 357
 358#endif
 359
 360bool i915_request_retire(struct i915_request *rq)
 361{
 362	if (!__i915_request_is_complete(rq))
 363		return false;
 364
 365	RQ_TRACE(rq, "\n");
 366
 367	GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
 368	trace_i915_request_retire(rq);
 369	i915_request_mark_complete(rq);
 370
 371	__rq_cancel_watchdog(rq);
 372
 373	/*
 374	 * We know the GPU must have read the request to have
 375	 * sent us the seqno + interrupt, so use the position
 376	 * of tail of the request to update the last known position
 377	 * of the GPU head.
 378	 *
 379	 * Note this requires that we are always called in request
 380	 * completion order.
 381	 */
 382	GEM_BUG_ON(!list_is_first(&rq->link,
 383				  &i915_request_timeline(rq)->requests));
 384	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 385		/* Poison before we release our space in the ring */
 386		__i915_request_fill(rq, POISON_FREE);
 387	rq->ring->head = rq->postfix;
 388
 389	if (!i915_request_signaled(rq)) {
 390		spin_lock_irq(&rq->lock);
 391		dma_fence_signal_locked(&rq->fence);
 392		spin_unlock_irq(&rq->lock);
 393	}
 394
 395	if (test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags))
 396		intel_rps_dec_waiters(&rq->engine->gt->rps);
 397
 398	/*
 399	 * We only loosely track inflight requests across preemption,
 400	 * and so we may find ourselves attempting to retire a _completed_
 401	 * request that we have removed from the HW and put back on a run
 402	 * queue.
 403	 *
 404	 * As we set I915_FENCE_FLAG_ACTIVE on the request, this should be
 405	 * after removing the breadcrumb and signaling it, so that we do not
 406	 * inadvertently attach the breadcrumb to a completed request.
 407	 */
 408	rq->engine->remove_active_request(rq);
 409	GEM_BUG_ON(!llist_empty(&rq->execute_cb));
 410
 411	__list_del_entry(&rq->link); /* poison neither prev/next (RCU walks) */
 412
 413	intel_context_exit(rq->context);
 414	intel_context_unpin(rq->context);
 415
 416	i915_sched_node_fini(&rq->sched);
 417	i915_request_put(rq);
 418
 419	return true;
 420}
 421
 422void i915_request_retire_upto(struct i915_request *rq)
 423{
 424	struct intel_timeline * const tl = i915_request_timeline(rq);
 425	struct i915_request *tmp;
 426
 427	RQ_TRACE(rq, "\n");
 428	GEM_BUG_ON(!__i915_request_is_complete(rq));
 429
 430	do {
 431		tmp = list_first_entry(&tl->requests, typeof(*tmp), link);
 432		GEM_BUG_ON(!i915_request_completed(tmp));
 433	} while (i915_request_retire(tmp) && tmp != rq);
 434}
 435
 436static struct i915_request * const *
 437__engine_active(struct intel_engine_cs *engine)
 438{
 439	return READ_ONCE(engine->execlists.active);
 440}
 441
 442static bool __request_in_flight(const struct i915_request *signal)
 443{
 444	struct i915_request * const *port, *rq;
 445	bool inflight = false;
 446
 447	if (!i915_request_is_ready(signal))
 448		return false;
 449
 450	/*
 451	 * Even if we have unwound the request, it may still be on
 452	 * the GPU (preempt-to-busy). If that request is inside an
 453	 * unpreemptible critical section, it will not be removed. Some
 454	 * GPU functions may even be stuck waiting for the paired request
 455	 * (__await_execution) to be submitted and cannot be preempted
 456	 * until the bond is executing.
 457	 *
 458	 * As we know that there are always preemption points between
 459	 * requests, we know that only the currently executing request
 460	 * may be still active even though we have cleared the flag.
 461	 * However, we can't rely on our tracking of ELSP[0] to know
 462	 * which request is currently active and so maybe stuck, as
 463	 * the tracking maybe an event behind. Instead assume that
 464	 * if the context is still inflight, then it is still active
 465	 * even if the active flag has been cleared.
 466	 *
 467	 * To further complicate matters, if there a pending promotion, the HW
 468	 * may either perform a context switch to the second inflight execlists,
 469	 * or it may switch to the pending set of execlists. In the case of the
 470	 * latter, it may send the ACK and we process the event copying the
 471	 * pending[] over top of inflight[], _overwriting_ our *active. Since
 472	 * this implies the HW is arbitrating and not struck in *active, we do
 473	 * not worry about complete accuracy, but we do require no read/write
 474	 * tearing of the pointer [the read of the pointer must be valid, even
 475	 * as the array is being overwritten, for which we require the writes
 476	 * to avoid tearing.]
 477	 *
 478	 * Note that the read of *execlists->active may race with the promotion
 479	 * of execlists->pending[] to execlists->inflight[], overwritting
 480	 * the value at *execlists->active. This is fine. The promotion implies
 481	 * that we received an ACK from the HW, and so the context is not
 482	 * stuck -- if we do not see ourselves in *active, the inflight status
 483	 * is valid. If instead we see ourselves being copied into *active,
 484	 * we are inflight and may signal the callback.
 485	 */
 486	if (!intel_context_inflight(signal->context))
 487		return false;
 488
 489	rcu_read_lock();
 490	for (port = __engine_active(signal->engine);
 491	     (rq = READ_ONCE(*port)); /* may race with promotion of pending[] */
 492	     port++) {
 493		if (rq->context == signal->context) {
 494			inflight = i915_seqno_passed(rq->fence.seqno,
 495						     signal->fence.seqno);
 496			break;
 497		}
 498	}
 499	rcu_read_unlock();
 500
 501	return inflight;
 502}
 503
 504static int
 505__await_execution(struct i915_request *rq,
 506		  struct i915_request *signal,
 507		  gfp_t gfp)
 508{
 509	struct execute_cb *cb;
 510
 511	if (i915_request_is_active(signal))
 512		return 0;
 513
 514	cb = kmem_cache_alloc(slab_execute_cbs, gfp);
 515	if (!cb)
 516		return -ENOMEM;
 517
 518	cb->fence = &rq->submit;
 519	i915_sw_fence_await(cb->fence);
 520	init_irq_work(&cb->work, irq_execute_cb);
 521
 522	/*
 523	 * Register the callback first, then see if the signaler is already
 524	 * active. This ensures that if we race with the
 525	 * __notify_execute_cb from i915_request_submit() and we are not
 526	 * included in that list, we get a second bite of the cherry and
 527	 * execute it ourselves. After this point, a future
 528	 * i915_request_submit() will notify us.
 529	 *
 530	 * In i915_request_retire() we set the ACTIVE bit on a completed
 531	 * request (then flush the execute_cb). So by registering the
 532	 * callback first, then checking the ACTIVE bit, we serialise with
 533	 * the completed/retired request.
 534	 */
 535	if (llist_add(&cb->work.node.llist, &signal->execute_cb)) {
 536		if (i915_request_is_active(signal) ||
 537		    __request_in_flight(signal))
 538			i915_request_notify_execute_cb_imm(signal);
 539	}
 540
 541	return 0;
 542}
 543
 544static bool fatal_error(int error)
 545{
 546	switch (error) {
 547	case 0: /* not an error! */
 548	case -EAGAIN: /* innocent victim of a GT reset (__i915_request_reset) */
 549	case -ETIMEDOUT: /* waiting for Godot (timer_i915_sw_fence_wake) */
 550		return false;
 551	default:
 552		return true;
 553	}
 554}
 555
 556void __i915_request_skip(struct i915_request *rq)
 557{
 558	GEM_BUG_ON(!fatal_error(rq->fence.error));
 559
 560	if (rq->infix == rq->postfix)
 561		return;
 562
 563	RQ_TRACE(rq, "error: %d\n", rq->fence.error);
 564
 565	/*
 566	 * As this request likely depends on state from the lost
 567	 * context, clear out all the user operations leaving the
 568	 * breadcrumb at the end (so we get the fence notifications).
 569	 */
 570	__i915_request_fill(rq, 0);
 571	rq->infix = rq->postfix;
 572}
 573
 574bool i915_request_set_error_once(struct i915_request *rq, int error)
 575{
 576	int old;
 577
 578	GEM_BUG_ON(!IS_ERR_VALUE((long)error));
 579
 580	if (i915_request_signaled(rq))
 581		return false;
 582
 583	old = READ_ONCE(rq->fence.error);
 584	do {
 585		if (fatal_error(old))
 586			return false;
 587	} while (!try_cmpxchg(&rq->fence.error, &old, error));
 588
 589	return true;
 590}
 591
 592struct i915_request *i915_request_mark_eio(struct i915_request *rq)
 593{
 594	if (__i915_request_is_complete(rq))
 595		return NULL;
 596
 597	GEM_BUG_ON(i915_request_signaled(rq));
 598
 599	/* As soon as the request is completed, it may be retired */
 600	rq = i915_request_get(rq);
 601
 602	i915_request_set_error_once(rq, -EIO);
 603	i915_request_mark_complete(rq);
 604
 605	return rq;
 606}
 607
 608bool __i915_request_submit(struct i915_request *request)
 609{
 610	struct intel_engine_cs *engine = request->engine;
 611	bool result = false;
 612
 613	RQ_TRACE(request, "\n");
 614
 615	GEM_BUG_ON(!irqs_disabled());
 616	lockdep_assert_held(&engine->sched_engine->lock);
 617
 618	/*
 619	 * With the advent of preempt-to-busy, we frequently encounter
 620	 * requests that we have unsubmitted from HW, but left running
 621	 * until the next ack and so have completed in the meantime. On
 622	 * resubmission of that completed request, we can skip
 623	 * updating the payload, and execlists can even skip submitting
 624	 * the request.
 625	 *
 626	 * We must remove the request from the caller's priority queue,
 627	 * and the caller must only call us when the request is in their
 628	 * priority queue, under the sched_engine->lock. This ensures that the
 629	 * request has *not* yet been retired and we can safely move
 630	 * the request into the engine->active.list where it will be
 631	 * dropped upon retiring. (Otherwise if resubmit a *retired*
 632	 * request, this would be a horrible use-after-free.)
 633	 */
 634	if (__i915_request_is_complete(request)) {
 635		list_del_init(&request->sched.link);
 636		goto active;
 637	}
 638
 639	if (unlikely(!intel_context_is_schedulable(request->context)))
 640		i915_request_set_error_once(request, -EIO);
 641
 642	if (unlikely(fatal_error(request->fence.error)))
 643		__i915_request_skip(request);
 644
 645	/*
 646	 * Are we using semaphores when the gpu is already saturated?
 647	 *
 648	 * Using semaphores incurs a cost in having the GPU poll a
 649	 * memory location, busywaiting for it to change. The continual
 650	 * memory reads can have a noticeable impact on the rest of the
 651	 * system with the extra bus traffic, stalling the cpu as it too
 652	 * tries to access memory across the bus (perf stat -e bus-cycles).
 653	 *
 654	 * If we installed a semaphore on this request and we only submit
 655	 * the request after the signaler completed, that indicates the
 656	 * system is overloaded and using semaphores at this time only
 657	 * increases the amount of work we are doing. If so, we disable
 658	 * further use of semaphores until we are idle again, whence we
 659	 * optimistically try again.
 660	 */
 661	if (request->sched.semaphores &&
 662	    i915_sw_fence_signaled(&request->semaphore))
 663		engine->saturated |= request->sched.semaphores;
 664
 665	engine->emit_fini_breadcrumb(request,
 666				     request->ring->vaddr + request->postfix);
 667
 668	trace_i915_request_execute(request);
 669	if (engine->bump_serial)
 670		engine->bump_serial(engine);
 671	else
 672		engine->serial++;
 673
 674	result = true;
 675
 676	GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));
 677	engine->add_active_request(request);
 678active:
 679	clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags);
 680	set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);
 681
 682	/*
 683	 * XXX Rollback bonded-execution on __i915_request_unsubmit()?
 684	 *
 685	 * In the future, perhaps when we have an active time-slicing scheduler,
 686	 * it will be interesting to unsubmit parallel execution and remove
 687	 * busywaits from the GPU until their master is restarted. This is
 688	 * quite hairy, we have to carefully rollback the fence and do a
 689	 * preempt-to-idle cycle on the target engine, all the while the
 690	 * master execute_cb may refire.
 691	 */
 692	__notify_execute_cb_irq(request);
 693
 694	/* We may be recursing from the signal callback of another i915 fence */
 695	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
 696		i915_request_enable_breadcrumb(request);
 697
 698	return result;
 699}
 700
 701void i915_request_submit(struct i915_request *request)
 702{
 703	struct intel_engine_cs *engine = request->engine;
 704	unsigned long flags;
 705
 706	/* Will be called from irq-context when using foreign fences. */
 707	spin_lock_irqsave(&engine->sched_engine->lock, flags);
 708
 709	__i915_request_submit(request);
 710
 711	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
 712}
 713
 714void __i915_request_unsubmit(struct i915_request *request)
 715{
 716	struct intel_engine_cs *engine = request->engine;
 717
 718	/*
 719	 * Only unwind in reverse order, required so that the per-context list
 720	 * is kept in seqno/ring order.
 721	 */
 722	RQ_TRACE(request, "\n");
 723
 724	GEM_BUG_ON(!irqs_disabled());
 725	lockdep_assert_held(&engine->sched_engine->lock);
 726
 727	/*
 728	 * Before we remove this breadcrumb from the signal list, we have
 729	 * to ensure that a concurrent dma_fence_enable_signaling() does not
 730	 * attach itself. We first mark the request as no longer active and
 731	 * make sure that is visible to other cores, and then remove the
 732	 * breadcrumb if attached.
 733	 */
 734	GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));
 735	clear_bit_unlock(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);
 736	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
 737		i915_request_cancel_breadcrumb(request);
 738
 739	/* We've already spun, don't charge on resubmitting. */
 740	if (request->sched.semaphores && __i915_request_has_started(request))
 741		request->sched.semaphores = 0;
 742
 743	/*
 744	 * We don't need to wake_up any waiters on request->execute, they
 745	 * will get woken by any other event or us re-adding this request
 746	 * to the engine timeline (__i915_request_submit()). The waiters
 747	 * should be quite adapt at finding that the request now has a new
 748	 * global_seqno to the one they went to sleep on.
 749	 */
 750}
 751
 752void i915_request_unsubmit(struct i915_request *request)
 753{
 754	struct intel_engine_cs *engine = request->engine;
 755	unsigned long flags;
 756
 757	/* Will be called from irq-context when using foreign fences. */
 758	spin_lock_irqsave(&engine->sched_engine->lock, flags);
 759
 760	__i915_request_unsubmit(request);
 761
 762	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
 763}
 764
 765void i915_request_cancel(struct i915_request *rq, int error)
 766{
 767	if (!i915_request_set_error_once(rq, error))
 768		return;
 769
 770	set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
 771
 772	intel_context_cancel_request(rq->context, rq);
 773}
 774
 775static int
 776submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 777{
 778	struct i915_request *request =
 779		container_of(fence, typeof(*request), submit);
 780
 781	switch (state) {
 782	case FENCE_COMPLETE:
 783		trace_i915_request_submit(request);
 784
 785		if (unlikely(fence->error))
 786			i915_request_set_error_once(request, fence->error);
 787		else
 788			__rq_arm_watchdog(request);
 789
 790		/*
 791		 * We need to serialize use of the submit_request() callback
 792		 * with its hotplugging performed during an emergency
 793		 * i915_gem_set_wedged().  We use the RCU mechanism to mark the
 794		 * critical section in order to force i915_gem_set_wedged() to
 795		 * wait until the submit_request() is completed before
 796		 * proceeding.
 797		 */
 798		rcu_read_lock();
 799		request->engine->submit_request(request);
 800		rcu_read_unlock();
 801		break;
 802
 803	case FENCE_FREE:
 804		i915_request_put(request);
 805		break;
 806	}
 807
 808	return NOTIFY_DONE;
 809}
 810
 811static int
 812semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 813{
 814	struct i915_request *rq = container_of(fence, typeof(*rq), semaphore);
 815
 816	switch (state) {
 817	case FENCE_COMPLETE:
 818		break;
 819
 820	case FENCE_FREE:
 821		i915_request_put(rq);
 822		break;
 823	}
 824
 825	return NOTIFY_DONE;
 826}
 827
 828static void retire_requests(struct intel_timeline *tl)
 829{
 830	struct i915_request *rq, *rn;
 831
 832	list_for_each_entry_safe(rq, rn, &tl->requests, link)
 833		if (!i915_request_retire(rq))
 834			break;
 835}
 836
 837static noinline struct i915_request *
 838request_alloc_slow(struct intel_timeline *tl,
 839		   struct i915_request **rsvd,
 840		   gfp_t gfp)
 841{
 842	struct i915_request *rq;
 843
 844	/* If we cannot wait, dip into our reserves */
 845	if (!gfpflags_allow_blocking(gfp)) {
 846		rq = xchg(rsvd, NULL);
 847		if (!rq) /* Use the normal failure path for one final WARN */
 848			goto out;
 849
 850		return rq;
 851	}
 852
 853	if (list_empty(&tl->requests))
 854		goto out;
 855
 856	/* Move our oldest request to the slab-cache (if not in use!) */
 857	rq = list_first_entry(&tl->requests, typeof(*rq), link);
 858	i915_request_retire(rq);
 859
 860	rq = kmem_cache_alloc(slab_requests,
 861			      gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 862	if (rq)
 863		return rq;
 864
 865	/* Ratelimit ourselves to prevent oom from malicious clients */
 866	rq = list_last_entry(&tl->requests, typeof(*rq), link);
 867	cond_synchronize_rcu(rq->rcustate);
 868
 869	/* Retire our old requests in the hope that we free some */
 870	retire_requests(tl);
 871
 872out:
 873	return kmem_cache_alloc(slab_requests, gfp);
 874}
 875
 876static void __i915_request_ctor(void *arg)
 877{
 878	struct i915_request *rq = arg;
 879
 880	spin_lock_init(&rq->lock);
 881	i915_sched_node_init(&rq->sched);
 882	i915_sw_fence_init(&rq->submit, submit_notify);
 883	i915_sw_fence_init(&rq->semaphore, semaphore_notify);
 884
 885	clear_capture_list(rq);
 886	rq->batch_res = NULL;
 887
 888	init_llist_head(&rq->execute_cb);
 889}
 890
 891#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 892#define clear_batch_ptr(_rq) ((_rq)->batch = NULL)
 893#else
 894#define clear_batch_ptr(_a) do {} while (0)
 895#endif
 896
 897struct i915_request *
 898__i915_request_create(struct intel_context *ce, gfp_t gfp)
 899{
 900	struct intel_timeline *tl = ce->timeline;
 901	struct i915_request *rq;
 902	u32 seqno;
 903	int ret;
 904
 905	might_alloc(gfp);
 906
 907	/* Check that the caller provided an already pinned context */
 908	__intel_context_pin(ce);
 909
 910	/*
 911	 * Beware: Dragons be flying overhead.
 912	 *
 913	 * We use RCU to look up requests in flight. The lookups may
 914	 * race with the request being allocated from the slab freelist.
 915	 * That is the request we are writing to here, may be in the process
 916	 * of being read by __i915_active_request_get_rcu(). As such,
 917	 * we have to be very careful when overwriting the contents. During
 918	 * the RCU lookup, we change chase the request->engine pointer,
 919	 * read the request->global_seqno and increment the reference count.
 920	 *
 921	 * The reference count is incremented atomically. If it is zero,
 922	 * the lookup knows the request is unallocated and complete. Otherwise,
 923	 * it is either still in use, or has been reallocated and reset
 924	 * with dma_fence_init(). This increment is safe for release as we
 925	 * check that the request we have a reference to and matches the active
 926	 * request.
 927	 *
 928	 * Before we increment the refcount, we chase the request->engine
 929	 * pointer. We must not call kmem_cache_zalloc() or else we set
 930	 * that pointer to NULL and cause a crash during the lookup. If
 931	 * we see the request is completed (based on the value of the
 932	 * old engine and seqno), the lookup is complete and reports NULL.
 933	 * If we decide the request is not completed (new engine or seqno),
 934	 * then we grab a reference and double check that it is still the
 935	 * active request - which it won't be and restart the lookup.
 936	 *
 937	 * Do not use kmem_cache_zalloc() here!
 938	 */
 939	rq = kmem_cache_alloc(slab_requests,
 940			      gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 941	if (unlikely(!rq)) {
 942		rq = request_alloc_slow(tl, &ce->engine->request_pool, gfp);
 943		if (!rq) {
 944			ret = -ENOMEM;
 945			goto err_unreserve;
 946		}
 947	}
 948
 949	rq->context = ce;
 950	rq->engine = ce->engine;
 951	rq->ring = ce->ring;
 952	rq->execution_mask = ce->engine->mask;
 953	rq->i915 = ce->engine->i915;
 954
 955	ret = intel_timeline_get_seqno(tl, rq, &seqno);
 956	if (ret)
 957		goto err_free;
 958
 959	dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock,
 960		       tl->fence_context, seqno);
 961
 962	RCU_INIT_POINTER(rq->timeline, tl);
 963	rq->hwsp_seqno = tl->hwsp_seqno;
 964	GEM_BUG_ON(__i915_request_is_complete(rq));
 965
 966	rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */
 967
 968	rq->guc_prio = GUC_PRIO_INIT;
 969
 970	/* We bump the ref for the fence chain */
 971	i915_sw_fence_reinit(&i915_request_get(rq)->submit);
 972	i915_sw_fence_reinit(&i915_request_get(rq)->semaphore);
 973
 974	i915_sched_node_reinit(&rq->sched);
 975
 976	/* No zalloc, everything must be cleared after use */
 977	clear_batch_ptr(rq);
 978	__rq_init_watchdog(rq);
 979	assert_capture_list_is_null(rq);
 980	GEM_BUG_ON(!llist_empty(&rq->execute_cb));
 981	GEM_BUG_ON(rq->batch_res);
 982
 983	/*
 984	 * Reserve space in the ring buffer for all the commands required to
 985	 * eventually emit this request. This is to guarantee that the
 986	 * i915_request_add() call can't fail. Note that the reserve may need
 987	 * to be redone if the request is not actually submitted straight
 988	 * away, e.g. because a GPU scheduler has deferred it.
 989	 *
 990	 * Note that due to how we add reserved_space to intel_ring_begin()
 991	 * we need to double our request to ensure that if we need to wrap
 992	 * around inside i915_request_add() there is sufficient space at
 993	 * the beginning of the ring as well.
 994	 */
 995	rq->reserved_space =
 996		2 * rq->engine->emit_fini_breadcrumb_dw * sizeof(u32);
 997
 998	/*
 999	 * Record the position of the start of the request so that
1000	 * should we detect the updated seqno part-way through the
1001	 * GPU processing the request, we never over-estimate the
1002	 * position of the head.
1003	 */
1004	rq->head = rq->ring->emit;
1005
1006	ret = rq->engine->request_alloc(rq);
1007	if (ret)
1008		goto err_unwind;
1009
1010	rq->infix = rq->ring->emit; /* end of header; start of user payload */
1011
1012	intel_context_mark_active(ce);
1013	list_add_tail_rcu(&rq->link, &tl->requests);
1014
1015	return rq;
1016
1017err_unwind:
1018	ce->ring->emit = rq->head;
1019
1020	/* Make sure we didn't add ourselves to external state before freeing */
1021	GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
1022	GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
1023
1024err_free:
1025	kmem_cache_free(slab_requests, rq);
1026err_unreserve:
1027	intel_context_unpin(ce);
1028	return ERR_PTR(ret);
1029}
1030
1031struct i915_request *
1032i915_request_create(struct intel_context *ce)
1033{
1034	struct i915_request *rq;
1035	struct intel_timeline *tl;
1036
1037	tl = intel_context_timeline_lock(ce);
1038	if (IS_ERR(tl))
1039		return ERR_CAST(tl);
1040
1041	/* Move our oldest request to the slab-cache (if not in use!) */
1042	rq = list_first_entry(&tl->requests, typeof(*rq), link);
1043	if (!list_is_last(&rq->link, &tl->requests))
1044		i915_request_retire(rq);
1045
1046	intel_context_enter(ce);
1047	rq = __i915_request_create(ce, GFP_KERNEL);
1048	intel_context_exit(ce); /* active reference transferred to request */
1049	if (IS_ERR(rq))
1050		goto err_unlock;
1051
1052	/* Check that we do not interrupt ourselves with a new request */
1053	rq->cookie = lockdep_pin_lock(&tl->mutex);
1054
1055	return rq;
1056
1057err_unlock:
1058	intel_context_timeline_unlock(tl);
1059	return rq;
1060}
1061
1062static int
1063i915_request_await_start(struct i915_request *rq, struct i915_request *signal)
1064{
1065	struct dma_fence *fence;
1066	int err;
1067
1068	if (i915_request_timeline(rq) == rcu_access_pointer(signal->timeline))
1069		return 0;
1070
1071	if (i915_request_started(signal))
1072		return 0;
1073
1074	/*
1075	 * The caller holds a reference on @signal, but we do not serialise
1076	 * against it being retired and removed from the lists.
1077	 *
1078	 * We do not hold a reference to the request before @signal, and
1079	 * so must be very careful to ensure that it is not _recycled_ as
1080	 * we follow the link backwards.
1081	 */
1082	fence = NULL;
1083	rcu_read_lock();
1084	do {
1085		struct list_head *pos = READ_ONCE(signal->link.prev);
1086		struct i915_request *prev;
1087
1088		/* Confirm signal has not been retired, the link is valid */
1089		if (unlikely(__i915_request_has_started(signal)))
1090			break;
1091
1092		/* Is signal the earliest request on its timeline? */
1093		if (pos == &rcu_dereference(signal->timeline)->requests)
1094			break;
1095
1096		/*
1097		 * Peek at the request before us in the timeline. That
1098		 * request will only be valid before it is retired, so
1099		 * after acquiring a reference to it, confirm that it is
1100		 * still part of the signaler's timeline.
1101		 */
1102		prev = list_entry(pos, typeof(*prev), link);
1103		if (!i915_request_get_rcu(prev))
1104			break;
1105
1106		/* After the strong barrier, confirm prev is still attached */
1107		if (unlikely(READ_ONCE(prev->link.next) != &signal->link)) {
1108			i915_request_put(prev);
1109			break;
1110		}
1111
1112		fence = &prev->fence;
1113	} while (0);
1114	rcu_read_unlock();
1115	if (!fence)
1116		return 0;
1117
1118	err = 0;
1119	if (!intel_timeline_sync_is_later(i915_request_timeline(rq), fence))
1120		err = i915_sw_fence_await_dma_fence(&rq->submit,
1121						    fence, 0,
1122						    I915_FENCE_GFP);
1123	dma_fence_put(fence);
1124
1125	return err;
1126}
1127
1128static intel_engine_mask_t
1129already_busywaiting(struct i915_request *rq)
1130{
1131	/*
1132	 * Polling a semaphore causes bus traffic, delaying other users of
1133	 * both the GPU and CPU. We want to limit the impact on others,
1134	 * while taking advantage of early submission to reduce GPU
1135	 * latency. Therefore we restrict ourselves to not using more
1136	 * than one semaphore from each source, and not using a semaphore
1137	 * if we have detected the engine is saturated (i.e. would not be
1138	 * submitted early and cause bus traffic reading an already passed
1139	 * semaphore).
1140	 *
1141	 * See the are-we-too-late? check in __i915_request_submit().
1142	 */
1143	return rq->sched.semaphores | READ_ONCE(rq->engine->saturated);
1144}
1145
1146static int
1147__emit_semaphore_wait(struct i915_request *to,
1148		      struct i915_request *from,
1149		      u32 seqno)
1150{
1151	const int has_token = GRAPHICS_VER(to->engine->i915) >= 12;
1152	u32 hwsp_offset;
1153	int len, err;
1154	u32 *cs;
1155
1156	GEM_BUG_ON(GRAPHICS_VER(to->engine->i915) < 8);
1157	GEM_BUG_ON(i915_request_has_initial_breadcrumb(to));
1158
1159	/* We need to pin the signaler's HWSP until we are finished reading. */
1160	err = intel_timeline_read_hwsp(from, to, &hwsp_offset);
1161	if (err)
1162		return err;
1163
1164	len = 4;
1165	if (has_token)
1166		len += 2;
1167
1168	cs = intel_ring_begin(to, len);
1169	if (IS_ERR(cs))
1170		return PTR_ERR(cs);
1171
1172	/*
1173	 * Using greater-than-or-equal here means we have to worry
1174	 * about seqno wraparound. To side step that issue, we swap
1175	 * the timeline HWSP upon wrapping, so that everyone listening
1176	 * for the old (pre-wrap) values do not see the much smaller
1177	 * (post-wrap) values than they were expecting (and so wait
1178	 * forever).
1179	 */
1180	*cs++ = (MI_SEMAPHORE_WAIT |
1181		 MI_SEMAPHORE_GLOBAL_GTT |
1182		 MI_SEMAPHORE_POLL |
1183		 MI_SEMAPHORE_SAD_GTE_SDD) +
1184		has_token;
1185	*cs++ = seqno;
1186	*cs++ = hwsp_offset;
1187	*cs++ = 0;
1188	if (has_token) {
1189		*cs++ = 0;
1190		*cs++ = MI_NOOP;
1191	}
1192
1193	intel_ring_advance(to, cs);
1194	return 0;
1195}
1196
1197static bool
1198can_use_semaphore_wait(struct i915_request *to, struct i915_request *from)
1199{
1200	return to->engine->gt->ggtt == from->engine->gt->ggtt;
1201}
1202
1203static int
1204emit_semaphore_wait(struct i915_request *to,
1205		    struct i915_request *from,
1206		    gfp_t gfp)
1207{
1208	const intel_engine_mask_t mask = READ_ONCE(from->engine)->mask;
1209	struct i915_sw_fence *wait = &to->submit;
1210
1211	if (!can_use_semaphore_wait(to, from))
1212		goto await_fence;
1213
1214	if (!intel_context_use_semaphores(to->context))
1215		goto await_fence;
1216
1217	if (i915_request_has_initial_breadcrumb(to))
1218		goto await_fence;
1219
1220	/*
1221	 * If this or its dependents are waiting on an external fence
1222	 * that may fail catastrophically, then we want to avoid using
1223	 * sempahores as they bypass the fence signaling metadata, and we
1224	 * lose the fence->error propagation.
1225	 */
1226	if (from->sched.flags & I915_SCHED_HAS_EXTERNAL_CHAIN)
1227		goto await_fence;
1228
1229	/* Just emit the first semaphore we see as request space is limited. */
1230	if (already_busywaiting(to) & mask)
1231		goto await_fence;
1232
1233	if (i915_request_await_start(to, from) < 0)
1234		goto await_fence;
1235
1236	/* Only submit our spinner after the signaler is running! */
1237	if (__await_execution(to, from, gfp))
1238		goto await_fence;
1239
1240	if (__emit_semaphore_wait(to, from, from->fence.seqno))
1241		goto await_fence;
1242
1243	to->sched.semaphores |= mask;
1244	wait = &to->semaphore;
1245
1246await_fence:
1247	return i915_sw_fence_await_dma_fence(wait,
1248					     &from->fence, 0,
1249					     I915_FENCE_GFP);
1250}
1251
1252static bool intel_timeline_sync_has_start(struct intel_timeline *tl,
1253					  struct dma_fence *fence)
1254{
1255	return __intel_timeline_sync_is_later(tl,
1256					      fence->context,
1257					      fence->seqno - 1);
1258}
1259
1260static int intel_timeline_sync_set_start(struct intel_timeline *tl,
1261					 const struct dma_fence *fence)
1262{
1263	return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1);
1264}
1265
1266static int
1267__i915_request_await_execution(struct i915_request *to,
1268			       struct i915_request *from)
1269{
1270	int err;
1271
1272	GEM_BUG_ON(intel_context_is_barrier(from->context));
1273
1274	/* Submit both requests at the same time */
1275	err = __await_execution(to, from, I915_FENCE_GFP);
1276	if (err)
1277		return err;
1278
1279	/* Squash repeated depenendices to the same timelines */
1280	if (intel_timeline_sync_has_start(i915_request_timeline(to),
1281					  &from->fence))
1282		return 0;
1283
1284	/*
1285	 * Wait until the start of this request.
1286	 *
1287	 * The execution cb fires when we submit the request to HW. But in
1288	 * many cases this may be long before the request itself is ready to
1289	 * run (consider that we submit 2 requests for the same context, where
1290	 * the request of interest is behind an indefinite spinner). So we hook
1291	 * up to both to reduce our queues and keep the execution lag minimised
1292	 * in the worst case, though we hope that the await_start is elided.
1293	 */
1294	err = i915_request_await_start(to, from);
1295	if (err < 0)
1296		return err;
1297
1298	/*
1299	 * Ensure both start together [after all semaphores in signal]
1300	 *
1301	 * Now that we are queued to the HW at roughly the same time (thanks
1302	 * to the execute cb) and are ready to run at roughly the same time
1303	 * (thanks to the await start), our signaler may still be indefinitely
1304	 * delayed by waiting on a semaphore from a remote engine. If our
1305	 * signaler depends on a semaphore, so indirectly do we, and we do not
1306	 * want to start our payload until our signaler also starts theirs.
1307	 * So we wait.
1308	 *
1309	 * However, there is also a second condition for which we need to wait
1310	 * for the precise start of the signaler. Consider that the signaler
1311	 * was submitted in a chain of requests following another context
1312	 * (with just an ordinary intra-engine fence dependency between the
1313	 * two). In this case the signaler is queued to HW, but not for
1314	 * immediate execution, and so we must wait until it reaches the
1315	 * active slot.
1316	 */
1317	if (can_use_semaphore_wait(to, from) &&
1318	    intel_engine_has_semaphores(to->engine) &&
1319	    !i915_request_has_initial_breadcrumb(to)) {
1320		err = __emit_semaphore_wait(to, from, from->fence.seqno - 1);
1321		if (err < 0)
1322			return err;
1323	}
1324
1325	/* Couple the dependency tree for PI on this exposed to->fence */
1326	if (to->engine->sched_engine->schedule) {
1327		err = i915_sched_node_add_dependency(&to->sched,
1328						     &from->sched,
1329						     I915_DEPENDENCY_WEAK);
1330		if (err < 0)
1331			return err;
1332	}
1333
1334	return intel_timeline_sync_set_start(i915_request_timeline(to),
1335					     &from->fence);
1336}
1337
1338static void mark_external(struct i915_request *rq)
1339{
1340	/*
1341	 * The downside of using semaphores is that we lose metadata passing
1342	 * along the signaling chain. This is particularly nasty when we
1343	 * need to pass along a fatal error such as EFAULT or EDEADLK. For
1344	 * fatal errors we want to scrub the request before it is executed,
1345	 * which means that we cannot preload the request onto HW and have
1346	 * it wait upon a semaphore.
1347	 */
1348	rq->sched.flags |= I915_SCHED_HAS_EXTERNAL_CHAIN;
1349}
1350
1351static int
1352__i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
1353{
1354	mark_external(rq);
1355	return i915_sw_fence_await_dma_fence(&rq->submit, fence,
1356					     i915_fence_context_timeout(rq->engine->i915,
1357									fence->context),
1358					     I915_FENCE_GFP);
1359}
1360
1361static int
1362i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
1363{
1364	struct dma_fence *iter;
1365	int err = 0;
1366
1367	if (!to_dma_fence_chain(fence))
1368		return __i915_request_await_external(rq, fence);
1369
1370	dma_fence_chain_for_each(iter, fence) {
1371		struct dma_fence_chain *chain = to_dma_fence_chain(iter);
1372
1373		if (!dma_fence_is_i915(chain->fence)) {
1374			err = __i915_request_await_external(rq, iter);
1375			break;
1376		}
1377
1378		err = i915_request_await_dma_fence(rq, chain->fence);
1379		if (err < 0)
1380			break;
1381	}
1382
1383	dma_fence_put(iter);
1384	return err;
1385}
1386
1387static inline bool is_parallel_rq(struct i915_request *rq)
1388{
1389	return intel_context_is_parallel(rq->context);
1390}
1391
1392static inline struct intel_context *request_to_parent(struct i915_request *rq)
1393{
1394	return intel_context_to_parent(rq->context);
1395}
1396
1397static bool is_same_parallel_context(struct i915_request *to,
1398				     struct i915_request *from)
1399{
1400	if (is_parallel_rq(to))
1401		return request_to_parent(to) == request_to_parent(from);
1402
1403	return false;
1404}
1405
1406int
1407i915_request_await_execution(struct i915_request *rq,
1408			     struct dma_fence *fence)
1409{
1410	struct dma_fence **child = &fence;
1411	unsigned int nchild = 1;
1412	int ret;
1413
1414	if (dma_fence_is_array(fence)) {
1415		struct dma_fence_array *array = to_dma_fence_array(fence);
1416
1417		/* XXX Error for signal-on-any fence arrays */
1418
1419		child = array->fences;
1420		nchild = array->num_fences;
1421		GEM_BUG_ON(!nchild);
1422	}
1423
1424	do {
1425		fence = *child++;
1426		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1427			continue;
1428
1429		if (fence->context == rq->fence.context)
1430			continue;
1431
1432		/*
1433		 * We don't squash repeated fence dependencies here as we
1434		 * want to run our callback in all cases.
1435		 */
1436
1437		if (dma_fence_is_i915(fence)) {
1438			if (is_same_parallel_context(rq, to_request(fence)))
1439				continue;
1440			ret = __i915_request_await_execution(rq,
1441							     to_request(fence));
1442		} else {
1443			ret = i915_request_await_external(rq, fence);
1444		}
1445		if (ret < 0)
1446			return ret;
1447	} while (--nchild);
1448
1449	return 0;
1450}
1451
1452static int
1453await_request_submit(struct i915_request *to, struct i915_request *from)
1454{
1455	/*
1456	 * If we are waiting on a virtual engine, then it may be
1457	 * constrained to execute on a single engine *prior* to submission.
1458	 * When it is submitted, it will be first submitted to the virtual
1459	 * engine and then passed to the physical engine. We cannot allow
1460	 * the waiter to be submitted immediately to the physical engine
1461	 * as it may then bypass the virtual request.
1462	 */
1463	if (to->engine == READ_ONCE(from->engine))
1464		return i915_sw_fence_await_sw_fence_gfp(&to->submit,
1465							&from->submit,
1466							I915_FENCE_GFP);
1467	else
1468		return __i915_request_await_execution(to, from);
1469}
1470
1471static int
1472i915_request_await_request(struct i915_request *to, struct i915_request *from)
1473{
1474	int ret;
1475
1476	GEM_BUG_ON(to == from);
1477	GEM_BUG_ON(to->timeline == from->timeline);
1478
1479	if (i915_request_completed(from)) {
1480		i915_sw_fence_set_error_once(&to->submit, from->fence.error);
1481		return 0;
1482	}
1483
1484	if (to->engine->sched_engine->schedule) {
1485		ret = i915_sched_node_add_dependency(&to->sched,
1486						     &from->sched,
1487						     I915_DEPENDENCY_EXTERNAL);
1488		if (ret < 0)
1489			return ret;
1490	}
1491
1492	if (!intel_engine_uses_guc(to->engine) &&
1493	    is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask)))
1494		ret = await_request_submit(to, from);
1495	else
1496		ret = emit_semaphore_wait(to, from, I915_FENCE_GFP);
1497	if (ret < 0)
1498		return ret;
1499
1500	return 0;
1501}
1502
1503int
1504i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
1505{
1506	struct dma_fence **child = &fence;
1507	unsigned int nchild = 1;
1508	int ret;
1509
1510	/*
1511	 * Note that if the fence-array was created in signal-on-any mode,
1512	 * we should *not* decompose it into its individual fences. However,
1513	 * we don't currently store which mode the fence-array is operating
1514	 * in. Fortunately, the only user of signal-on-any is private to
1515	 * amdgpu and we should not see any incoming fence-array from
1516	 * sync-file being in signal-on-any mode.
1517	 */
1518	if (dma_fence_is_array(fence)) {
1519		struct dma_fence_array *array = to_dma_fence_array(fence);
1520
1521		child = array->fences;
1522		nchild = array->num_fences;
1523		GEM_BUG_ON(!nchild);
1524	}
1525
1526	do {
1527		fence = *child++;
1528		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1529			continue;
1530
1531		/*
1532		 * Requests on the same timeline are explicitly ordered, along
1533		 * with their dependencies, by i915_request_add() which ensures
1534		 * that requests are submitted in-order through each ring.
1535		 */
1536		if (fence->context == rq->fence.context)
1537			continue;
1538
1539		/* Squash repeated waits to the same timelines */
1540		if (fence->context &&
1541		    intel_timeline_sync_is_later(i915_request_timeline(rq),
1542						 fence))
1543			continue;
1544
1545		if (dma_fence_is_i915(fence)) {
1546			if (is_same_parallel_context(rq, to_request(fence)))
1547				continue;
1548			ret = i915_request_await_request(rq, to_request(fence));
1549		} else {
1550			ret = i915_request_await_external(rq, fence);
1551		}
1552		if (ret < 0)
1553			return ret;
1554
1555		/* Record the latest fence used against each timeline */
1556		if (fence->context)
1557			intel_timeline_sync_set(i915_request_timeline(rq),
1558						fence);
1559	} while (--nchild);
1560
1561	return 0;
1562}
1563
1564/**
1565 * i915_request_await_deps - set this request to (async) wait upon a struct
1566 * i915_deps dma_fence collection
1567 * @rq: request we are wishing to use
1568 * @deps: The struct i915_deps containing the dependencies.
1569 *
1570 * Returns 0 if successful, negative error code on error.
1571 */
1572int i915_request_await_deps(struct i915_request *rq, const struct i915_deps *deps)
1573{
1574	int i, err;
1575
1576	for (i = 0; i < deps->num_deps; ++i) {
1577		err = i915_request_await_dma_fence(rq, deps->fences[i]);
1578		if (err)
1579			return err;
1580	}
1581
1582	return 0;
1583}
1584
1585/**
1586 * i915_request_await_object - set this request to (async) wait upon a bo
1587 * @to: request we are wishing to use
1588 * @obj: object which may be in use on another ring.
1589 * @write: whether the wait is on behalf of a writer
1590 *
1591 * This code is meant to abstract object synchronization with the GPU.
1592 * Conceptually we serialise writes between engines inside the GPU.
1593 * We only allow one engine to write into a buffer at any time, but
1594 * multiple readers. To ensure each has a coherent view of memory, we must:
1595 *
1596 * - If there is an outstanding write request to the object, the new
1597 *   request must wait for it to complete (either CPU or in hw, requests
1598 *   on the same ring will be naturally ordered).
1599 *
1600 * - If we are a write request (pending_write_domain is set), the new
1601 *   request must wait for outstanding read requests to complete.
1602 *
1603 * Returns 0 if successful, else propagates up the lower layer error.
1604 */
1605int
1606i915_request_await_object(struct i915_request *to,
1607			  struct drm_i915_gem_object *obj,
1608			  bool write)
1609{
1610	struct dma_resv_iter cursor;
1611	struct dma_fence *fence;
1612	int ret = 0;
1613
1614	dma_resv_for_each_fence(&cursor, obj->base.resv,
1615				dma_resv_usage_rw(write), fence) {
1616		ret = i915_request_await_dma_fence(to, fence);
1617		if (ret)
1618			break;
1619	}
1620
1621	return ret;
1622}
1623
1624static void i915_request_await_huc(struct i915_request *rq)
1625{
1626	struct intel_huc *huc = &rq->context->engine->gt->uc.huc;
1627
1628	/* don't stall kernel submissions! */
1629	if (!rcu_access_pointer(rq->context->gem_context))
1630		return;
1631
1632	if (intel_huc_wait_required(huc))
1633		i915_sw_fence_await_sw_fence(&rq->submit,
1634					     &huc->delayed_load.fence,
1635					     &rq->hucq);
1636}
1637
1638static struct i915_request *
1639__i915_request_ensure_parallel_ordering(struct i915_request *rq,
1640					struct intel_timeline *timeline)
1641{
1642	struct i915_request *prev;
1643
1644	GEM_BUG_ON(!is_parallel_rq(rq));
1645
1646	prev = request_to_parent(rq)->parallel.last_rq;
1647	if (prev) {
1648		if (!__i915_request_is_complete(prev)) {
1649			i915_sw_fence_await_sw_fence(&rq->submit,
1650						     &prev->submit,
1651						     &rq->submitq);
1652
1653			if (rq->engine->sched_engine->schedule)
1654				__i915_sched_node_add_dependency(&rq->sched,
1655								 &prev->sched,
1656								 &rq->dep,
1657								 0);
1658		}
1659		i915_request_put(prev);
1660	}
1661
1662	request_to_parent(rq)->parallel.last_rq = i915_request_get(rq);
1663
 
 
 
 
 
1664	return to_request(__i915_active_fence_set(&timeline->last_request,
1665						  &rq->fence));
1666}
1667
1668static struct i915_request *
1669__i915_request_ensure_ordering(struct i915_request *rq,
1670			       struct intel_timeline *timeline)
1671{
1672	struct i915_request *prev;
1673
1674	GEM_BUG_ON(is_parallel_rq(rq));
1675
1676	prev = to_request(__i915_active_fence_set(&timeline->last_request,
1677						  &rq->fence));
1678
1679	if (prev && !__i915_request_is_complete(prev)) {
1680		bool uses_guc = intel_engine_uses_guc(rq->engine);
1681		bool pow2 = is_power_of_2(READ_ONCE(prev->engine)->mask |
1682					  rq->engine->mask);
1683		bool same_context = prev->context == rq->context;
1684
1685		/*
1686		 * The requests are supposed to be kept in order. However,
1687		 * we need to be wary in case the timeline->last_request
1688		 * is used as a barrier for external modification to this
1689		 * context.
1690		 */
1691		GEM_BUG_ON(same_context &&
1692			   i915_seqno_passed(prev->fence.seqno,
1693					     rq->fence.seqno));
1694
1695		if ((same_context && uses_guc) || (!uses_guc && pow2))
1696			i915_sw_fence_await_sw_fence(&rq->submit,
1697						     &prev->submit,
1698						     &rq->submitq);
1699		else
1700			__i915_sw_fence_await_dma_fence(&rq->submit,
1701							&prev->fence,
1702							&rq->dmaq);
1703		if (rq->engine->sched_engine->schedule)
1704			__i915_sched_node_add_dependency(&rq->sched,
1705							 &prev->sched,
1706							 &rq->dep,
1707							 0);
1708	}
1709
 
 
 
 
1710	return prev;
1711}
1712
1713static struct i915_request *
1714__i915_request_add_to_timeline(struct i915_request *rq)
1715{
1716	struct intel_timeline *timeline = i915_request_timeline(rq);
1717	struct i915_request *prev;
1718
1719	/*
1720	 * Media workloads may require HuC, so stall them until HuC loading is
1721	 * complete. Note that HuC not being loaded when a user submission
1722	 * arrives can only happen when HuC is loaded via GSC and in that case
1723	 * we still expect the window between us starting to accept submissions
1724	 * and HuC loading completion to be small (a few hundred ms).
1725	 */
1726	if (rq->engine->class == VIDEO_DECODE_CLASS)
1727		i915_request_await_huc(rq);
1728
1729	/*
1730	 * Dependency tracking and request ordering along the timeline
1731	 * is special cased so that we can eliminate redundant ordering
1732	 * operations while building the request (we know that the timeline
1733	 * itself is ordered, and here we guarantee it).
1734	 *
1735	 * As we know we will need to emit tracking along the timeline,
1736	 * we embed the hooks into our request struct -- at the cost of
1737	 * having to have specialised no-allocation interfaces (which will
1738	 * be beneficial elsewhere).
1739	 *
1740	 * A second benefit to open-coding i915_request_await_request is
1741	 * that we can apply a slight variant of the rules specialised
1742	 * for timelines that jump between engines (such as virtual engines).
1743	 * If we consider the case of virtual engine, we must emit a dma-fence
1744	 * to prevent scheduling of the second request until the first is
1745	 * complete (to maximise our greedy late load balancing) and this
1746	 * precludes optimising to use semaphores serialisation of a single
1747	 * timeline across engines.
1748	 *
1749	 * We do not order parallel submission requests on the timeline as each
1750	 * parallel submission context has its own timeline and the ordering
1751	 * rules for parallel requests are that they must be submitted in the
1752	 * order received from the execbuf IOCTL. So rather than using the
1753	 * timeline we store a pointer to last request submitted in the
1754	 * relationship in the gem context and insert a submission fence
1755	 * between that request and request passed into this function or
1756	 * alternatively we use completion fence if gem context has a single
1757	 * timeline and this is the first submission of an execbuf IOCTL.
1758	 */
1759	if (likely(!is_parallel_rq(rq)))
1760		prev = __i915_request_ensure_ordering(rq, timeline);
1761	else
1762		prev = __i915_request_ensure_parallel_ordering(rq, timeline);
 
 
1763
1764	/*
1765	 * Make sure that no request gazumped us - if it was allocated after
1766	 * our i915_request_alloc() and called __i915_request_add() before
1767	 * us, the timeline will hold its seqno which is later than ours.
1768	 */
1769	GEM_BUG_ON(timeline->seqno != rq->fence.seqno);
1770
1771	return prev;
1772}
1773
1774/*
1775 * NB: This function is not allowed to fail. Doing so would mean the the
1776 * request is not being tracked for completion but the work itself is
1777 * going to happen on the hardware. This would be a Bad Thing(tm).
1778 */
1779struct i915_request *__i915_request_commit(struct i915_request *rq)
1780{
1781	struct intel_engine_cs *engine = rq->engine;
1782	struct intel_ring *ring = rq->ring;
1783	u32 *cs;
1784
1785	RQ_TRACE(rq, "\n");
1786
1787	/*
1788	 * To ensure that this call will not fail, space for its emissions
1789	 * should already have been reserved in the ring buffer. Let the ring
1790	 * know that it is time to use that space up.
1791	 */
1792	GEM_BUG_ON(rq->reserved_space > ring->space);
1793	rq->reserved_space = 0;
1794	rq->emitted_jiffies = jiffies;
1795
1796	/*
1797	 * Record the position of the start of the breadcrumb so that
1798	 * should we detect the updated seqno part-way through the
1799	 * GPU processing the request, we never over-estimate the
1800	 * position of the ring's HEAD.
1801	 */
1802	cs = intel_ring_begin(rq, engine->emit_fini_breadcrumb_dw);
1803	GEM_BUG_ON(IS_ERR(cs));
1804	rq->postfix = intel_ring_offset(rq, cs);
1805
1806	return __i915_request_add_to_timeline(rq);
1807}
1808
1809void __i915_request_queue_bh(struct i915_request *rq)
1810{
1811	i915_sw_fence_commit(&rq->semaphore);
1812	i915_sw_fence_commit(&rq->submit);
1813}
1814
1815void __i915_request_queue(struct i915_request *rq,
1816			  const struct i915_sched_attr *attr)
1817{
1818	/*
1819	 * Let the backend know a new request has arrived that may need
1820	 * to adjust the existing execution schedule due to a high priority
1821	 * request - i.e. we may want to preempt the current request in order
1822	 * to run a high priority dependency chain *before* we can execute this
1823	 * request.
1824	 *
1825	 * This is called before the request is ready to run so that we can
1826	 * decide whether to preempt the entire chain so that it is ready to
1827	 * run at the earliest possible convenience.
1828	 */
1829	if (attr && rq->engine->sched_engine->schedule)
1830		rq->engine->sched_engine->schedule(rq, attr);
1831
1832	local_bh_disable();
1833	__i915_request_queue_bh(rq);
1834	local_bh_enable(); /* kick tasklets */
1835}
1836
1837void i915_request_add(struct i915_request *rq)
1838{
1839	struct intel_timeline * const tl = i915_request_timeline(rq);
1840	struct i915_sched_attr attr = {};
1841	struct i915_gem_context *ctx;
1842
1843	lockdep_assert_held(&tl->mutex);
1844	lockdep_unpin_lock(&tl->mutex, rq->cookie);
1845
1846	trace_i915_request_add(rq);
1847	__i915_request_commit(rq);
1848
1849	/* XXX placeholder for selftests */
1850	rcu_read_lock();
1851	ctx = rcu_dereference(rq->context->gem_context);
1852	if (ctx)
1853		attr = ctx->sched;
1854	rcu_read_unlock();
1855
1856	__i915_request_queue(rq, &attr);
1857
1858	mutex_unlock(&tl->mutex);
1859}
1860
1861static unsigned long local_clock_ns(unsigned int *cpu)
1862{
1863	unsigned long t;
1864
1865	/*
1866	 * Cheaply and approximately convert from nanoseconds to microseconds.
1867	 * The result and subsequent calculations are also defined in the same
1868	 * approximate microseconds units. The principal source of timing
1869	 * error here is from the simple truncation.
1870	 *
1871	 * Note that local_clock() is only defined wrt to the current CPU;
1872	 * the comparisons are no longer valid if we switch CPUs. Instead of
1873	 * blocking preemption for the entire busywait, we can detect the CPU
1874	 * switch and use that as indicator of system load and a reason to
1875	 * stop busywaiting, see busywait_stop().
1876	 */
1877	*cpu = get_cpu();
1878	t = local_clock();
1879	put_cpu();
1880
1881	return t;
1882}
1883
1884static bool busywait_stop(unsigned long timeout, unsigned int cpu)
1885{
1886	unsigned int this_cpu;
1887
1888	if (time_after(local_clock_ns(&this_cpu), timeout))
1889		return true;
1890
1891	return this_cpu != cpu;
1892}
1893
1894static bool __i915_spin_request(struct i915_request * const rq, int state)
1895{
1896	unsigned long timeout_ns;
1897	unsigned int cpu;
1898
1899	/*
1900	 * Only wait for the request if we know it is likely to complete.
1901	 *
1902	 * We don't track the timestamps around requests, nor the average
1903	 * request length, so we do not have a good indicator that this
1904	 * request will complete within the timeout. What we do know is the
1905	 * order in which requests are executed by the context and so we can
1906	 * tell if the request has been started. If the request is not even
1907	 * running yet, it is a fair assumption that it will not complete
1908	 * within our relatively short timeout.
1909	 */
1910	if (!i915_request_is_running(rq))
1911		return false;
1912
1913	/*
1914	 * When waiting for high frequency requests, e.g. during synchronous
1915	 * rendering split between the CPU and GPU, the finite amount of time
1916	 * required to set up the irq and wait upon it limits the response
1917	 * rate. By busywaiting on the request completion for a short while we
1918	 * can service the high frequency waits as quick as possible. However,
1919	 * if it is a slow request, we want to sleep as quickly as possible.
1920	 * The tradeoff between waiting and sleeping is roughly the time it
1921	 * takes to sleep on a request, on the order of a microsecond.
1922	 */
1923
1924	timeout_ns = READ_ONCE(rq->engine->props.max_busywait_duration_ns);
1925	timeout_ns += local_clock_ns(&cpu);
1926	do {
1927		if (dma_fence_is_signaled(&rq->fence))
1928			return true;
1929
1930		if (signal_pending_state(state, current))
1931			break;
1932
1933		if (busywait_stop(timeout_ns, cpu))
1934			break;
1935
1936		cpu_relax();
1937	} while (!need_resched());
1938
1939	return false;
1940}
1941
1942struct request_wait {
1943	struct dma_fence_cb cb;
1944	struct task_struct *tsk;
1945};
1946
1947static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb)
1948{
1949	struct request_wait *wait = container_of(cb, typeof(*wait), cb);
1950
1951	wake_up_process(fetch_and_zero(&wait->tsk));
1952}
1953
1954/**
1955 * i915_request_wait_timeout - wait until execution of request has finished
1956 * @rq: the request to wait upon
1957 * @flags: how to wait
1958 * @timeout: how long to wait in jiffies
1959 *
1960 * i915_request_wait_timeout() waits for the request to be completed, for a
1961 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
1962 * unbounded wait).
1963 *
1964 * Returns the remaining time (in jiffies) if the request completed, which may
1965 * be zero if the request is unfinished after the timeout expires.
1966 * If the timeout is 0, it will return 1 if the fence is signaled.
1967 *
1968 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
1969 * pending before the request completes.
1970 *
1971 * NOTE: This function has the same wait semantics as dma-fence.
1972 */
1973long i915_request_wait_timeout(struct i915_request *rq,
1974			       unsigned int flags,
1975			       long timeout)
1976{
1977	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
1978		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1979	struct request_wait wait;
1980
1981	might_sleep();
1982	GEM_BUG_ON(timeout < 0);
1983
1984	if (dma_fence_is_signaled(&rq->fence))
1985		return timeout ?: 1;
1986
1987	if (!timeout)
1988		return -ETIME;
1989
1990	trace_i915_request_wait_begin(rq, flags);
1991
1992	/*
1993	 * We must never wait on the GPU while holding a lock as we
1994	 * may need to perform a GPU reset. So while we don't need to
1995	 * serialise wait/reset with an explicit lock, we do want
1996	 * lockdep to detect potential dependency cycles.
1997	 */
1998	mutex_acquire(&rq->engine->gt->reset.mutex.dep_map, 0, 0, _THIS_IP_);
1999
2000	/*
2001	 * Optimistic spin before touching IRQs.
2002	 *
2003	 * We may use a rather large value here to offset the penalty of
2004	 * switching away from the active task. Frequently, the client will
2005	 * wait upon an old swapbuffer to throttle itself to remain within a
2006	 * frame of the gpu. If the client is running in lockstep with the gpu,
2007	 * then it should not be waiting long at all, and a sleep now will incur
2008	 * extra scheduler latency in producing the next frame. To try to
2009	 * avoid adding the cost of enabling/disabling the interrupt to the
2010	 * short wait, we first spin to see if the request would have completed
2011	 * in the time taken to setup the interrupt.
2012	 *
2013	 * We need upto 5us to enable the irq, and upto 20us to hide the
2014	 * scheduler latency of a context switch, ignoring the secondary
2015	 * impacts from a context switch such as cache eviction.
2016	 *
2017	 * The scheme used for low-latency IO is called "hybrid interrupt
2018	 * polling". The suggestion there is to sleep until just before you
2019	 * expect to be woken by the device interrupt and then poll for its
2020	 * completion. That requires having a good predictor for the request
2021	 * duration, which we currently lack.
2022	 */
2023	if (CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT &&
2024	    __i915_spin_request(rq, state))
2025		goto out;
2026
2027	/*
2028	 * This client is about to stall waiting for the GPU. In many cases
2029	 * this is undesirable and limits the throughput of the system, as
2030	 * many clients cannot continue processing user input/output whilst
2031	 * blocked. RPS autotuning may take tens of milliseconds to respond
2032	 * to the GPU load and thus incurs additional latency for the client.
2033	 * We can circumvent that by promoting the GPU frequency to maximum
2034	 * before we sleep. This makes the GPU throttle up much more quickly
2035	 * (good for benchmarks and user experience, e.g. window animations),
2036	 * but at a cost of spending more power processing the workload
2037	 * (bad for battery).
2038	 */
2039	if (flags & I915_WAIT_PRIORITY && !i915_request_started(rq))
2040		intel_rps_boost(rq);
2041
2042	wait.tsk = current;
2043	if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake))
2044		goto out;
2045
2046	/*
2047	 * Flush the submission tasklet, but only if it may help this request.
2048	 *
2049	 * We sometimes experience some latency between the HW interrupts and
2050	 * tasklet execution (mostly due to ksoftirqd latency, but it can also
2051	 * be due to lazy CS events), so lets run the tasklet manually if there
2052	 * is a chance it may submit this request. If the request is not ready
2053	 * to run, as it is waiting for other fences to be signaled, flushing
2054	 * the tasklet is busy work without any advantage for this client.
2055	 *
2056	 * If the HW is being lazy, this is the last chance before we go to
2057	 * sleep to catch any pending events. We will check periodically in
2058	 * the heartbeat to flush the submission tasklets as a last resort
2059	 * for unhappy HW.
2060	 */
2061	if (i915_request_is_ready(rq))
2062		__intel_engine_flush_submission(rq->engine, false);
2063
2064	for (;;) {
2065		set_current_state(state);
2066
2067		if (dma_fence_is_signaled(&rq->fence))
2068			break;
2069
2070		if (signal_pending_state(state, current)) {
2071			timeout = -ERESTARTSYS;
2072			break;
2073		}
2074
2075		if (!timeout) {
2076			timeout = -ETIME;
2077			break;
2078		}
2079
2080		timeout = io_schedule_timeout(timeout);
2081	}
2082	__set_current_state(TASK_RUNNING);
2083
2084	if (READ_ONCE(wait.tsk))
2085		dma_fence_remove_callback(&rq->fence, &wait.cb);
2086	GEM_BUG_ON(!list_empty(&wait.cb.node));
2087
2088out:
2089	mutex_release(&rq->engine->gt->reset.mutex.dep_map, _THIS_IP_);
2090	trace_i915_request_wait_end(rq);
2091	return timeout;
2092}
2093
2094/**
2095 * i915_request_wait - wait until execution of request has finished
2096 * @rq: the request to wait upon
2097 * @flags: how to wait
2098 * @timeout: how long to wait in jiffies
2099 *
2100 * i915_request_wait() waits for the request to be completed, for a
2101 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
2102 * unbounded wait).
2103 *
2104 * Returns the remaining time (in jiffies) if the request completed, which may
2105 * be zero or -ETIME if the request is unfinished after the timeout expires.
2106 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
2107 * pending before the request completes.
2108 *
2109 * NOTE: This function behaves differently from dma-fence wait semantics for
2110 * timeout = 0. It returns 0 on success, and -ETIME if not signaled.
2111 */
2112long i915_request_wait(struct i915_request *rq,
2113		       unsigned int flags,
2114		       long timeout)
2115{
2116	long ret = i915_request_wait_timeout(rq, flags, timeout);
2117
2118	if (!ret)
2119		return -ETIME;
2120
2121	if (ret > 0 && !timeout)
2122		return 0;
2123
2124	return ret;
2125}
2126
2127static int print_sched_attr(const struct i915_sched_attr *attr,
2128			    char *buf, int x, int len)
2129{
2130	if (attr->priority == I915_PRIORITY_INVALID)
2131		return x;
2132
2133	x += snprintf(buf + x, len - x,
2134		      " prio=%d", attr->priority);
2135
2136	return x;
2137}
2138
2139static char queue_status(const struct i915_request *rq)
2140{
2141	if (i915_request_is_active(rq))
2142		return 'E';
2143
2144	if (i915_request_is_ready(rq))
2145		return intel_engine_is_virtual(rq->engine) ? 'V' : 'R';
2146
2147	return 'U';
2148}
2149
2150static const char *run_status(const struct i915_request *rq)
2151{
2152	if (__i915_request_is_complete(rq))
2153		return "!";
2154
2155	if (__i915_request_has_started(rq))
2156		return "*";
2157
2158	if (!i915_sw_fence_signaled(&rq->semaphore))
2159		return "&";
2160
2161	return "";
2162}
2163
2164static const char *fence_status(const struct i915_request *rq)
2165{
2166	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
2167		return "+";
2168
2169	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))
2170		return "-";
2171
2172	return "";
2173}
2174
2175void i915_request_show(struct drm_printer *m,
2176		       const struct i915_request *rq,
2177		       const char *prefix,
2178		       int indent)
2179{
2180	const char *name = rq->fence.ops->get_timeline_name((struct dma_fence *)&rq->fence);
2181	char buf[80] = "";
2182	int x = 0;
2183
2184	/*
2185	 * The prefix is used to show the queue status, for which we use
2186	 * the following flags:
2187	 *
2188	 *  U [Unready]
2189	 *    - initial status upon being submitted by the user
2190	 *
2191	 *    - the request is not ready for execution as it is waiting
2192	 *      for external fences
2193	 *
2194	 *  R [Ready]
2195	 *    - all fences the request was waiting on have been signaled,
2196	 *      and the request is now ready for execution and will be
2197	 *      in a backend queue
2198	 *
2199	 *    - a ready request may still need to wait on semaphores
2200	 *      [internal fences]
2201	 *
2202	 *  V [Ready/virtual]
2203	 *    - same as ready, but queued over multiple backends
2204	 *
2205	 *  E [Executing]
2206	 *    - the request has been transferred from the backend queue and
2207	 *      submitted for execution on HW
2208	 *
2209	 *    - a completed request may still be regarded as executing, its
2210	 *      status may not be updated until it is retired and removed
2211	 *      from the lists
2212	 */
2213
2214	x = print_sched_attr(&rq->sched.attr, buf, x, sizeof(buf));
2215
2216	drm_printf(m, "%s%.*s%c %llx:%lld%s%s %s @ %dms: %s\n",
2217		   prefix, indent, "                ",
2218		   queue_status(rq),
2219		   rq->fence.context, rq->fence.seqno,
2220		   run_status(rq),
2221		   fence_status(rq),
2222		   buf,
2223		   jiffies_to_msecs(jiffies - rq->emitted_jiffies),
2224		   name);
2225}
2226
2227static bool engine_match_ring(struct intel_engine_cs *engine, struct i915_request *rq)
2228{
2229	u32 ring = ENGINE_READ(engine, RING_START);
2230
2231	return ring == i915_ggtt_offset(rq->ring->vma);
2232}
2233
2234static bool match_ring(struct i915_request *rq)
2235{
2236	struct intel_engine_cs *engine;
2237	bool found;
2238	int i;
2239
2240	if (!intel_engine_is_virtual(rq->engine))
2241		return engine_match_ring(rq->engine, rq);
2242
2243	found = false;
2244	i = 0;
2245	while ((engine = intel_engine_get_sibling(rq->engine, i++))) {
2246		found = engine_match_ring(engine, rq);
2247		if (found)
2248			break;
2249	}
2250
2251	return found;
2252}
2253
2254enum i915_request_state i915_test_request_state(struct i915_request *rq)
2255{
2256	if (i915_request_completed(rq))
2257		return I915_REQUEST_COMPLETE;
2258
2259	if (!i915_request_started(rq))
2260		return I915_REQUEST_PENDING;
2261
2262	if (match_ring(rq))
2263		return I915_REQUEST_ACTIVE;
2264
2265	return I915_REQUEST_QUEUED;
2266}
2267
2268#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2269#include "selftests/mock_request.c"
2270#include "selftests/i915_request.c"
2271#endif
2272
2273void i915_request_module_exit(void)
2274{
2275	kmem_cache_destroy(slab_execute_cbs);
2276	kmem_cache_destroy(slab_requests);
2277}
2278
2279int __init i915_request_module_init(void)
2280{
2281	slab_requests =
2282		kmem_cache_create("i915_request",
2283				  sizeof(struct i915_request),
2284				  __alignof__(struct i915_request),
2285				  SLAB_HWCACHE_ALIGN |
2286				  SLAB_RECLAIM_ACCOUNT |
2287				  SLAB_TYPESAFE_BY_RCU,
2288				  __i915_request_ctor);
2289	if (!slab_requests)
2290		return -ENOMEM;
2291
2292	slab_execute_cbs = KMEM_CACHE(execute_cb,
2293					     SLAB_HWCACHE_ALIGN |
2294					     SLAB_RECLAIM_ACCOUNT |
2295					     SLAB_TYPESAFE_BY_RCU);
2296	if (!slab_execute_cbs)
2297		goto err_requests;
2298
2299	return 0;
2300
2301err_requests:
2302	kmem_cache_destroy(slab_requests);
2303	return -ENOMEM;
2304}