1/*
   2 * Copyright © 2008-2015 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <linux/dma-fence-array.h>
  26#include <linux/dma-fence-chain.h>
  27#include <linux/irq_work.h>
  28#include <linux/prefetch.h>
 
  29#include <linux/sched.h>
  30#include <linux/sched/clock.h>
  31#include <linux/sched/signal.h>
  32
  33#include "gem/i915_gem_context.h"
  34#include "gt/intel_context.h"
  35#include "gt/intel_ring.h"
  36#include "gt/intel_rps.h"
  37
  38#include "i915_active.h"
  39#include "i915_drv.h"
  40#include "i915_globals.h"
  41#include "i915_trace.h"
  42#include "intel_pm.h"
  43
  44struct execute_cb {
  45	struct irq_work work;
  46	struct i915_sw_fence *fence;
  47	void (*hook)(struct i915_request *rq, struct dma_fence *signal);
  48	struct i915_request *signal;
  49};
  50
  51static struct i915_global_request {
  52	struct i915_global base;
  53	struct kmem_cache *slab_requests;
  54	struct kmem_cache *slab_execute_cbs;
  55} global;
  56
  57static const char *i915_fence_get_driver_name(struct dma_fence *fence)
  58{
  59	return dev_name(to_request(fence)->engine->i915->drm.dev);
  60}
  61
  62static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
  63{
  64	const struct i915_gem_context *ctx;
  65
  66	/*
  67	 * The timeline struct (as part of the ppgtt underneath a context)
  68	 * may be freed when the request is no longer in use by the GPU.
  69	 * We could extend the life of a context to beyond that of all
  70	 * fences, possibly keeping the hw resource around indefinitely,
  71	 * or we just give them a false name. Since
  72	 * dma_fence_ops.get_timeline_name is a debug feature, the occasional
  73	 * lie seems justifiable.
  74	 */
  75	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  76		return "signaled";
  77
  78	ctx = i915_request_gem_context(to_request(fence));
  79	if (!ctx)
  80		return "[" DRIVER_NAME "]";
  81
  82	return ctx->name;
  83}
  84
  85static bool i915_fence_signaled(struct dma_fence *fence)
  86{
  87	return i915_request_completed(to_request(fence));
  88}
  89
  90static bool i915_fence_enable_signaling(struct dma_fence *fence)
  91{
  92	return i915_request_enable_breadcrumb(to_request(fence));
 
 
 
 
  93}
  94
  95static signed long i915_fence_wait(struct dma_fence *fence,
  96				   bool interruptible,
  97				   signed long timeout)
  98{
  99	return i915_request_wait(to_request(fence),
 100				 interruptible | I915_WAIT_PRIORITY,
 101				 timeout);
 102}
 103
 104struct kmem_cache *i915_request_slab_cache(void)
 105{
 106	return global.slab_requests;
 107}
 108
 109static void i915_fence_release(struct dma_fence *fence)
 110{
 111	struct i915_request *rq = to_request(fence);
 112
 113	/*
 114	 * The request is put onto a RCU freelist (i.e. the address
 115	 * is immediately reused), mark the fences as being freed now.
 116	 * Otherwise the debugobjects for the fences are only marked as
 117	 * freed when the slab cache itself is freed, and so we would get
 118	 * caught trying to reuse dead objects.
 119	 */
 120	i915_sw_fence_fini(&rq->submit);
 121	i915_sw_fence_fini(&rq->semaphore);
 122
 123	/*
 124	 * Keep one request on each engine for reserved use under mempressure
 125	 *
 126	 * We do not hold a reference to the engine here and so have to be
 127	 * very careful in what rq->engine we poke. The virtual engine is
 128	 * referenced via the rq->context and we released that ref during
 129	 * i915_request_retire(), ergo we must not dereference a virtual
 130	 * engine here. Not that we would want to, as the only consumer of
 131	 * the reserved engine->request_pool is the power management parking,
 132	 * which must-not-fail, and that is only run on the physical engines.
 133	 *
 134	 * Since the request must have been executed to be have completed,
 135	 * we know that it will have been processed by the HW and will
 136	 * not be unsubmitted again, so rq->engine and rq->execution_mask
 137	 * at this point is stable. rq->execution_mask will be a single
 138	 * bit if the last and _only_ engine it could execution on was a
 139	 * physical engine, if it's multiple bits then it started on and
 140	 * could still be on a virtual engine. Thus if the mask is not a
 141	 * power-of-two we assume that rq->engine may still be a virtual
 142	 * engine and so a dangling invalid pointer that we cannot dereference
 143	 *
 144	 * For example, consider the flow of a bonded request through a virtual
 145	 * engine. The request is created with a wide engine mask (all engines
 146	 * that we might execute on). On processing the bond, the request mask
 147	 * is reduced to one or more engines. If the request is subsequently
 148	 * bound to a single engine, it will then be constrained to only
 149	 * execute on that engine and never returned to the virtual engine
 150	 * after timeslicing away, see __unwind_incomplete_requests(). Thus we
 151	 * know that if the rq->execution_mask is a single bit, rq->engine
 152	 * can be a physical engine with the exact corresponding mask.
 153	 */
 154	if (is_power_of_2(rq->execution_mask) &&
 155	    !cmpxchg(&rq->engine->request_pool, NULL, rq))
 156		return;
 157
 158	kmem_cache_free(global.slab_requests, rq);
 159}
 160
 161const struct dma_fence_ops i915_fence_ops = {
 162	.get_driver_name = i915_fence_get_driver_name,
 163	.get_timeline_name = i915_fence_get_timeline_name,
 164	.enable_signaling = i915_fence_enable_signaling,
 165	.signaled = i915_fence_signaled,
 166	.wait = i915_fence_wait,
 167	.release = i915_fence_release,
 168};
 169
 170static void irq_execute_cb(struct irq_work *wrk)
 
 171{
 172	struct execute_cb *cb = container_of(wrk, typeof(*cb), work);
 
 
 
 
 173
 174	i915_sw_fence_complete(cb->fence);
 175	kmem_cache_free(global.slab_execute_cbs, cb);
 
 
 
 
 176}
 177
 178static void irq_execute_cb_hook(struct irq_work *wrk)
 
 179{
 180	struct execute_cb *cb = container_of(wrk, typeof(*cb), work);
 
 181
 182	cb->hook(container_of(cb->fence, struct i915_request, submit),
 183		 &cb->signal->fence);
 184	i915_request_put(cb->signal);
 
 
 
 185
 186	irq_execute_cb(wrk);
 
 
 
 
 
 
 
 
 
 
 187}
 188
 189static void __notify_execute_cb(struct i915_request *rq)
 
 
 
 190{
 191	struct execute_cb *cb, *cn;
 192
 193	lockdep_assert_held(&rq->lock);
 
 
 194
 195	GEM_BUG_ON(!i915_request_is_active(rq));
 196	if (llist_empty(&rq->execute_cb))
 197		return;
 198
 199	llist_for_each_entry_safe(cb, cn, rq->execute_cb.first, work.llnode)
 200		irq_work_queue(&cb->work);
 
 
 
 
 201
 202	/*
 203	 * XXX Rollback on __i915_request_unsubmit()
 204	 *
 205	 * In the future, perhaps when we have an active time-slicing scheduler,
 206	 * it will be interesting to unsubmit parallel execution and remove
 207	 * busywaits from the GPU until their master is restarted. This is
 208	 * quite hairy, we have to carefully rollback the fence and do a
 209	 * preempt-to-idle cycle on the target engine, all the while the
 210	 * master execute_cb may refire.
 211	 */
 212	init_llist_head(&rq->execute_cb);
 213}
 
 214
 215static inline void
 216remove_from_client(struct i915_request *request)
 217{
 218	struct drm_i915_file_private *file_priv;
 219
 220	if (!READ_ONCE(request->file_priv))
 221		return;
 
 
 222
 223	rcu_read_lock();
 224	file_priv = xchg(&request->file_priv, NULL);
 225	if (file_priv) {
 226		spin_lock(&file_priv->mm.lock);
 227		list_del(&request->client_link);
 228		spin_unlock(&file_priv->mm.lock);
 229	}
 230	rcu_read_unlock();
 231}
 232
 233static void free_capture_list(struct i915_request *request)
 
 234{
 235	struct i915_capture_list *capture;
 
 
 
 
 236
 237	capture = fetch_and_zero(&request->capture_list);
 238	while (capture) {
 239		struct i915_capture_list *next = capture->next;
 
 
 240
 241		kfree(capture);
 242		capture = next;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 243	}
 
 
 244}
 245
 246static void __i915_request_fill(struct i915_request *rq, u8 val)
 247{
 248	void *vaddr = rq->ring->vaddr;
 249	u32 head;
 250
 251	head = rq->infix;
 252	if (rq->postfix < head) {
 253		memset(vaddr + head, val, rq->ring->size - head);
 254		head = 0;
 255	}
 256	memset(vaddr + head, val, rq->postfix - head);
 
 257}
 258
 259static void remove_from_engine(struct i915_request *rq)
 260{
 261	struct intel_engine_cs *engine, *locked;
 
 
 
 
 
 262
 263	/*
 264	 * Virtual engines complicate acquiring the engine timeline lock,
 265	 * as their rq->engine pointer is not stable until under that
 266	 * engine lock. The simple ploy we use is to take the lock then
 267	 * check that the rq still belongs to the newly locked engine.
 
 
 
 
 
 268	 */
 269	locked = READ_ONCE(rq->engine);
 270	spin_lock_irq(&locked->active.lock);
 271	while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
 272		spin_unlock(&locked->active.lock);
 273		spin_lock(&engine->active.lock);
 274		locked = engine;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 275	}
 276	list_del_init(&rq->sched.link);
 277	clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
 278	clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);
 279	spin_unlock_irq(&locked->active.lock);
 
 280}
 281
 282bool i915_request_retire(struct i915_request *rq)
 283{
 284	if (!i915_request_completed(rq))
 285		return false;
 286
 287	RQ_TRACE(rq, "\n");
 
 
 
 
 
 
 288
 289	GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
 290	trace_i915_request_retire(rq);
 
 
 
 
 
 
 
 
 
 
 
 291
 292	/*
 293	 * We know the GPU must have read the request to have
 294	 * sent us the seqno + interrupt, so use the position
 295	 * of tail of the request to update the last known position
 296	 * of the GPU head.
 297	 *
 298	 * Note this requires that we are always called in request
 299	 * completion order.
 300	 */
 301	GEM_BUG_ON(!list_is_first(&rq->link,
 302				  &i915_request_timeline(rq)->requests));
 303	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 304		/* Poison before we release our space in the ring */
 305		__i915_request_fill(rq, POISON_FREE);
 306	rq->ring->head = rq->postfix;
 307
 308	/*
 309	 * We only loosely track inflight requests across preemption,
 310	 * and so we may find ourselves attempting to retire a _completed_
 311	 * request that we have removed from the HW and put back on a run
 312	 * queue.
 313	 */
 314	remove_from_engine(rq);
 315
 316	spin_lock_irq(&rq->lock);
 317	i915_request_mark_complete(rq);
 318	if (!i915_request_signaled(rq))
 319		dma_fence_signal_locked(&rq->fence);
 320	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))
 321		i915_request_cancel_breadcrumb(rq);
 322	if (i915_request_has_waitboost(rq)) {
 323		GEM_BUG_ON(!atomic_read(&rq->engine->gt->rps.num_waiters));
 324		atomic_dec(&rq->engine->gt->rps.num_waiters);
 325	}
 326	if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
 327		set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
 328		__notify_execute_cb(rq);
 329	}
 330	GEM_BUG_ON(!llist_empty(&rq->execute_cb));
 331	spin_unlock_irq(&rq->lock);
 332
 333	remove_from_client(rq);
 334	__list_del_entry(&rq->link); /* poison neither prev/next (RCU walks) */
 335
 336	intel_context_exit(rq->context);
 337	intel_context_unpin(rq->context);
 338
 339	free_capture_list(rq);
 340	i915_sched_node_fini(&rq->sched);
 341	i915_request_put(rq);
 342
 343	return true;
 344}
 345
 346void i915_request_retire_upto(struct i915_request *rq)
 347{
 348	struct intel_timeline * const tl = i915_request_timeline(rq);
 349	struct i915_request *tmp;
 350
 351	RQ_TRACE(rq, "\n");
 352
 353	GEM_BUG_ON(!i915_request_completed(rq));
 
 
 354
 355	do {
 356		tmp = list_first_entry(&tl->requests, typeof(*tmp), link);
 357	} while (i915_request_retire(tmp) && tmp != rq);
 358}
 359
 360static void __llist_add(struct llist_node *node, struct llist_head *head)
 361{
 362	node->next = head->first;
 363	head->first = node;
 364}
 365
 366static struct i915_request * const *
 367__engine_active(struct intel_engine_cs *engine)
 368{
 369	return READ_ONCE(engine->execlists.active);
 370}
 371
 372static bool __request_in_flight(const struct i915_request *signal)
 373{
 374	struct i915_request * const *port, *rq;
 375	bool inflight = false;
 376
 377	if (!i915_request_is_ready(signal))
 378		return false;
 
 
 
 
 
 
 379
 380	/*
 381	 * Even if we have unwound the request, it may still be on
 382	 * the GPU (preempt-to-busy). If that request is inside an
 383	 * unpreemptible critical section, it will not be removed. Some
 384	 * GPU functions may even be stuck waiting for the paired request
 385	 * (__await_execution) to be submitted and cannot be preempted
 386	 * until the bond is executing.
 387	 *
 388	 * As we know that there are always preemption points between
 389	 * requests, we know that only the currently executing request
 390	 * may be still active even though we have cleared the flag.
 391	 * However, we can't rely on our tracking of ELSP[0] to know
 392	 * which request is currently active and so maybe stuck, as
 393	 * the tracking maybe an event behind. Instead assume that
 394	 * if the context is still inflight, then it is still active
 395	 * even if the active flag has been cleared.
 396	 *
 397	 * To further complicate matters, if there a pending promotion, the HW
 398	 * may either perform a context switch to the second inflight execlists,
 399	 * or it may switch to the pending set of execlists. In the case of the
 400	 * latter, it may send the ACK and we process the event copying the
 401	 * pending[] over top of inflight[], _overwriting_ our *active. Since
 402	 * this implies the HW is arbitrating and not struck in *active, we do
 403	 * not worry about complete accuracy, but we do require no read/write
 404	 * tearing of the pointer [the read of the pointer must be valid, even
 405	 * as the array is being overwritten, for which we require the writes
 406	 * to avoid tearing.]
 407	 *
 408	 * Note that the read of *execlists->active may race with the promotion
 409	 * of execlists->pending[] to execlists->inflight[], overwritting
 410	 * the value at *execlists->active. This is fine. The promotion implies
 411	 * that we received an ACK from the HW, and so the context is not
 412	 * stuck -- if we do not see ourselves in *active, the inflight status
 413	 * is valid. If instead we see ourselves being copied into *active,
 414	 * we are inflight and may signal the callback.
 415	 */
 416	if (!intel_context_inflight(signal->context))
 417		return false;
 
 
 
 
 
 
 
 
 
 
 418
 419	rcu_read_lock();
 420	for (port = __engine_active(signal->engine);
 421	     (rq = READ_ONCE(*port)); /* may race with promotion of pending[] */
 422	     port++) {
 423		if (rq->context == signal->context) {
 424			inflight = i915_seqno_passed(rq->fence.seqno,
 425						     signal->fence.seqno);
 426			break;
 427		}
 428	}
 429	rcu_read_unlock();
 430
 431	return inflight;
 432}
 433
 434static int
 435__await_execution(struct i915_request *rq,
 436		  struct i915_request *signal,
 437		  void (*hook)(struct i915_request *rq,
 438			       struct dma_fence *signal),
 439		  gfp_t gfp)
 440{
 441	struct execute_cb *cb;
 442
 443	if (i915_request_is_active(signal)) {
 444		if (hook)
 445			hook(rq, &signal->fence);
 446		return 0;
 447	}
 448
 449	cb = kmem_cache_alloc(global.slab_execute_cbs, gfp);
 450	if (!cb)
 451		return -ENOMEM;
 452
 453	cb->fence = &rq->submit;
 454	i915_sw_fence_await(cb->fence);
 455	init_irq_work(&cb->work, irq_execute_cb);
 456
 457	if (hook) {
 458		cb->hook = hook;
 459		cb->signal = i915_request_get(signal);
 460		cb->work.func = irq_execute_cb_hook;
 461	}
 462
 463	spin_lock_irq(&signal->lock);
 464	if (i915_request_is_active(signal) || __request_in_flight(signal)) {
 465		if (hook) {
 466			hook(rq, &signal->fence);
 467			i915_request_put(signal);
 468		}
 469		i915_sw_fence_complete(cb->fence);
 470		kmem_cache_free(global.slab_execute_cbs, cb);
 471	} else {
 472		__llist_add(&cb->work.llnode, &signal->execute_cb);
 
 
 
 
 
 
 
 
 
 
 473	}
 474	spin_unlock_irq(&signal->lock);
 475
 476	return 0;
 
 477}
 478
 479static bool fatal_error(int error)
 480{
 481	switch (error) {
 482	case 0: /* not an error! */
 483	case -EAGAIN: /* innocent victim of a GT reset (__i915_request_reset) */
 484	case -ETIMEDOUT: /* waiting for Godot (timer_i915_sw_fence_wake) */
 485		return false;
 486	default:
 487		return true;
 488	}
 489}
 490
 491void __i915_request_skip(struct i915_request *rq)
 492{
 493	GEM_BUG_ON(!fatal_error(rq->fence.error));
 494
 495	if (rq->infix == rq->postfix)
 496		return;
 497
 498	/*
 499	 * As this request likely depends on state from the lost
 500	 * context, clear out all the user operations leaving the
 501	 * breadcrumb at the end (so we get the fence notifications).
 502	 */
 503	__i915_request_fill(rq, 0);
 504	rq->infix = rq->postfix;
 505}
 506
 507void i915_request_set_error_once(struct i915_request *rq, int error)
 508{
 509	int old;
 510
 511	GEM_BUG_ON(!IS_ERR_VALUE((long)error));
 512
 513	if (i915_request_signaled(rq))
 514		return;
 515
 516	old = READ_ONCE(rq->fence.error);
 517	do {
 518		if (fatal_error(old))
 519			return;
 520	} while (!try_cmpxchg(&rq->fence.error, &old, error));
 521}
 522
 523bool __i915_request_submit(struct i915_request *request)
 524{
 525	struct intel_engine_cs *engine = request->engine;
 526	bool result = false;
 527
 528	RQ_TRACE(request, "\n");
 529
 530	GEM_BUG_ON(!irqs_disabled());
 531	lockdep_assert_held(&engine->active.lock);
 532
 533	/*
 534	 * With the advent of preempt-to-busy, we frequently encounter
 535	 * requests that we have unsubmitted from HW, but left running
 536	 * until the next ack and so have completed in the meantime. On
 537	 * resubmission of that completed request, we can skip
 538	 * updating the payload, and execlists can even skip submitting
 539	 * the request.
 540	 *
 541	 * We must remove the request from the caller's priority queue,
 542	 * and the caller must only call us when the request is in their
 543	 * priority queue, under the active.lock. This ensures that the
 544	 * request has *not* yet been retired and we can safely move
 545	 * the request into the engine->active.list where it will be
 546	 * dropped upon retiring. (Otherwise if resubmit a *retired*
 547	 * request, this would be a horrible use-after-free.)
 548	 */
 549	if (i915_request_completed(request))
 550		goto xfer;
 551
 552	if (unlikely(intel_context_is_banned(request->context)))
 553		i915_request_set_error_once(request, -EIO);
 554	if (unlikely(fatal_error(request->fence.error)))
 555		__i915_request_skip(request);
 556
 557	/*
 558	 * Are we using semaphores when the gpu is already saturated?
 559	 *
 560	 * Using semaphores incurs a cost in having the GPU poll a
 561	 * memory location, busywaiting for it to change. The continual
 562	 * memory reads can have a noticeable impact on the rest of the
 563	 * system with the extra bus traffic, stalling the cpu as it too
 564	 * tries to access memory across the bus (perf stat -e bus-cycles).
 565	 *
 566	 * If we installed a semaphore on this request and we only submit
 567	 * the request after the signaler completed, that indicates the
 568	 * system is overloaded and using semaphores at this time only
 569	 * increases the amount of work we are doing. If so, we disable
 570	 * further use of semaphores until we are idle again, whence we
 571	 * optimistically try again.
 572	 */
 573	if (request->sched.semaphores &&
 574	    i915_sw_fence_signaled(&request->semaphore))
 575		engine->saturated |= request->sched.semaphores;
 576
 577	engine->emit_fini_breadcrumb(request,
 578				     request->ring->vaddr + request->postfix);
 579
 580	trace_i915_request_execute(request);
 581	engine->serial++;
 582	result = true;
 583
 584xfer:
 585	if (!test_and_set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)) {
 586		list_move_tail(&request->sched.link, &engine->active.requests);
 587		clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags);
 588	}
 589
 590	/* We may be recursing from the signal callback of another i915 fence */
 591	if (!i915_request_signaled(request)) {
 592		spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
 
 
 
 593
 594		__notify_execute_cb(request);
 595		if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 596			     &request->fence.flags) &&
 597		    !i915_request_enable_breadcrumb(request))
 598			intel_engine_signal_breadcrumbs(engine);
 599
 600		spin_unlock(&request->lock);
 601		GEM_BUG_ON(!llist_empty(&request->execute_cb));
 602	}
 603
 604	return result;
 
 
 605}
 606
 607void i915_request_submit(struct i915_request *request)
 608{
 609	struct intel_engine_cs *engine = request->engine;
 610	unsigned long flags;
 611
 612	/* Will be called from irq-context when using foreign fences. */
 613	spin_lock_irqsave(&engine->active.lock, flags);
 614
 615	__i915_request_submit(request);
 616
 617	spin_unlock_irqrestore(&engine->active.lock, flags);
 618}
 619
 620void __i915_request_unsubmit(struct i915_request *request)
 621{
 622	struct intel_engine_cs *engine = request->engine;
 623
 624	RQ_TRACE(request, "\n");
 625
 626	GEM_BUG_ON(!irqs_disabled());
 627	lockdep_assert_held(&engine->active.lock);
 628
 629	/*
 630	 * Only unwind in reverse order, required so that the per-context list
 631	 * is kept in seqno/ring order.
 632	 */
 
 
 
 
 
 633
 634	/* We may be recursing from the signal callback of another i915 fence */
 635	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
 636
 637	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
 638		i915_request_cancel_breadcrumb(request);
 639
 640	GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));
 641	clear_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);
 642
 643	spin_unlock(&request->lock);
 644
 645	/* We've already spun, don't charge on resubmitting. */
 646	if (request->sched.semaphores && i915_request_started(request))
 647		request->sched.semaphores = 0;
 
 
 
 
 648
 649	/*
 650	 * We don't need to wake_up any waiters on request->execute, they
 651	 * will get woken by any other event or us re-adding this request
 652	 * to the engine timeline (__i915_request_submit()). The waiters
 653	 * should be quite adapt at finding that the request now has a new
 654	 * global_seqno to the one they went to sleep on.
 655	 */
 656}
 657
 658void i915_request_unsubmit(struct i915_request *request)
 659{
 660	struct intel_engine_cs *engine = request->engine;
 661	unsigned long flags;
 662
 663	/* Will be called from irq-context when using foreign fences. */
 664	spin_lock_irqsave(&engine->active.lock, flags);
 665
 666	__i915_request_unsubmit(request);
 667
 668	spin_unlock_irqrestore(&engine->active.lock, flags);
 669}
 670
 671static int __i915_sw_fence_call
 672submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 673{
 674	struct i915_request *request =
 675		container_of(fence, typeof(*request), submit);
 676
 677	switch (state) {
 678	case FENCE_COMPLETE:
 679		trace_i915_request_submit(request);
 680
 681		if (unlikely(fence->error))
 682			i915_request_set_error_once(request, fence->error);
 683
 684		/*
 685		 * We need to serialize use of the submit_request() callback
 686		 * with its hotplugging performed during an emergency
 687		 * i915_gem_set_wedged().  We use the RCU mechanism to mark the
 688		 * critical section in order to force i915_gem_set_wedged() to
 689		 * wait until the submit_request() is completed before
 690		 * proceeding.
 691		 */
 692		rcu_read_lock();
 693		request->engine->submit_request(request);
 694		rcu_read_unlock();
 695		break;
 696
 697	case FENCE_FREE:
 698		i915_request_put(request);
 699		break;
 700	}
 701
 702	return NOTIFY_DONE;
 703}
 704
 705static int __i915_sw_fence_call
 706semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 707{
 708	struct i915_request *rq = container_of(fence, typeof(*rq), semaphore);
 709
 710	switch (state) {
 711	case FENCE_COMPLETE:
 712		break;
 713
 714	case FENCE_FREE:
 715		i915_request_put(rq);
 716		break;
 717	}
 718
 719	return NOTIFY_DONE;
 720}
 721
 722static void retire_requests(struct intel_timeline *tl)
 723{
 724	struct i915_request *rq, *rn;
 725
 726	list_for_each_entry_safe(rq, rn, &tl->requests, link)
 727		if (!i915_request_retire(rq))
 728			break;
 729}
 730
 731static noinline struct i915_request *
 732request_alloc_slow(struct intel_timeline *tl,
 733		   struct i915_request **rsvd,
 734		   gfp_t gfp)
 735{
 
 736	struct i915_request *rq;
 
 
 737
 738	/* If we cannot wait, dip into our reserves */
 739	if (!gfpflags_allow_blocking(gfp)) {
 740		rq = xchg(rsvd, NULL);
 741		if (!rq) /* Use the normal failure path for one final WARN */
 742			goto out;
 743
 744		return rq;
 745	}
 746
 747	if (list_empty(&tl->requests))
 748		goto out;
 749
 750	/* Move our oldest request to the slab-cache (if not in use!) */
 751	rq = list_first_entry(&tl->requests, typeof(*rq), link);
 752	i915_request_retire(rq);
 753
 754	rq = kmem_cache_alloc(global.slab_requests,
 755			      gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 756	if (rq)
 757		return rq;
 758
 759	/* Ratelimit ourselves to prevent oom from malicious clients */
 760	rq = list_last_entry(&tl->requests, typeof(*rq), link);
 761	cond_synchronize_rcu(rq->rcustate);
 762
 763	/* Retire our old requests in the hope that we free some */
 764	retire_requests(tl);
 765
 766out:
 767	return kmem_cache_alloc(global.slab_requests, gfp);
 768}
 769
 770static void __i915_request_ctor(void *arg)
 771{
 772	struct i915_request *rq = arg;
 773
 774	spin_lock_init(&rq->lock);
 775	i915_sched_node_init(&rq->sched);
 776	i915_sw_fence_init(&rq->submit, submit_notify);
 777	i915_sw_fence_init(&rq->semaphore, semaphore_notify);
 778
 779	dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock, 0, 0);
 
 
 
 
 
 780
 781	rq->file_priv = NULL;
 782	rq->capture_list = NULL;
 
 
 
 
 783
 784	init_llist_head(&rq->execute_cb);
 785}
 
 
 
 
 
 
 
 786
 787struct i915_request *
 788__i915_request_create(struct intel_context *ce, gfp_t gfp)
 789{
 790	struct intel_timeline *tl = ce->timeline;
 791	struct i915_request *rq;
 792	u32 seqno;
 793	int ret;
 794
 795	might_sleep_if(gfpflags_allow_blocking(gfp));
 
 
 796
 797	/* Check that the caller provided an already pinned context */
 798	__intel_context_pin(ce);
 
 
 
 799
 800	/*
 801	 * Beware: Dragons be flying overhead.
 802	 *
 803	 * We use RCU to look up requests in flight. The lookups may
 804	 * race with the request being allocated from the slab freelist.
 805	 * That is the request we are writing to here, may be in the process
 806	 * of being read by __i915_active_request_get_rcu(). As such,
 807	 * we have to be very careful when overwriting the contents. During
 808	 * the RCU lookup, we change chase the request->engine pointer,
 809	 * read the request->global_seqno and increment the reference count.
 810	 *
 811	 * The reference count is incremented atomically. If it is zero,
 812	 * the lookup knows the request is unallocated and complete. Otherwise,
 813	 * it is either still in use, or has been reallocated and reset
 814	 * with dma_fence_init(). This increment is safe for release as we
 815	 * check that the request we have a reference to and matches the active
 816	 * request.
 817	 *
 818	 * Before we increment the refcount, we chase the request->engine
 819	 * pointer. We must not call kmem_cache_zalloc() or else we set
 820	 * that pointer to NULL and cause a crash during the lookup. If
 821	 * we see the request is completed (based on the value of the
 822	 * old engine and seqno), the lookup is complete and reports NULL.
 823	 * If we decide the request is not completed (new engine or seqno),
 824	 * then we grab a reference and double check that it is still the
 825	 * active request - which it won't be and restart the lookup.
 826	 *
 827	 * Do not use kmem_cache_zalloc() here!
 828	 */
 829	rq = kmem_cache_alloc(global.slab_requests,
 830			      gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 831	if (unlikely(!rq)) {
 832		rq = request_alloc_slow(tl, &ce->engine->request_pool, gfp);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 833		if (!rq) {
 834			ret = -ENOMEM;
 835			goto err_unreserve;
 836		}
 837	}
 838
 839	rq->context = ce;
 840	rq->engine = ce->engine;
 841	rq->ring = ce->ring;
 842	rq->execution_mask = ce->engine->mask;
 843
 844	kref_init(&rq->fence.refcount);
 845	rq->fence.flags = 0;
 846	rq->fence.error = 0;
 847	INIT_LIST_HEAD(&rq->fence.cb_list);
 848
 849	ret = intel_timeline_get_seqno(tl, rq, &seqno);
 850	if (ret)
 851		goto err_free;
 852
 853	rq->fence.context = tl->fence_context;
 854	rq->fence.seqno = seqno;
 855
 856	RCU_INIT_POINTER(rq->timeline, tl);
 857	RCU_INIT_POINTER(rq->hwsp_cacheline, tl->hwsp_cacheline);
 858	rq->hwsp_seqno = tl->hwsp_seqno;
 859	GEM_BUG_ON(i915_request_completed(rq));
 860
 861	rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */
 862
 863	/* We bump the ref for the fence chain */
 864	i915_sw_fence_reinit(&i915_request_get(rq)->submit);
 865	i915_sw_fence_reinit(&i915_request_get(rq)->semaphore);
 866
 867	i915_sched_node_reinit(&rq->sched);
 868
 869	/* No zalloc, everything must be cleared after use */
 
 
 
 
 
 
 
 
 
 870	rq->batch = NULL;
 871	GEM_BUG_ON(rq->file_priv);
 872	GEM_BUG_ON(rq->capture_list);
 873	GEM_BUG_ON(!llist_empty(&rq->execute_cb));
 874
 875	/*
 876	 * Reserve space in the ring buffer for all the commands required to
 877	 * eventually emit this request. This is to guarantee that the
 878	 * i915_request_add() call can't fail. Note that the reserve may need
 879	 * to be redone if the request is not actually submitted straight
 880	 * away, e.g. because a GPU scheduler has deferred it.
 881	 *
 882	 * Note that due to how we add reserved_space to intel_ring_begin()
 883	 * we need to double our request to ensure that if we need to wrap
 884	 * around inside i915_request_add() there is sufficient space at
 885	 * the beginning of the ring as well.
 886	 */
 887	rq->reserved_space =
 888		2 * rq->engine->emit_fini_breadcrumb_dw * sizeof(u32);
 889
 890	/*
 891	 * Record the position of the start of the request so that
 892	 * should we detect the updated seqno part-way through the
 893	 * GPU processing the request, we never over-estimate the
 894	 * position of the head.
 895	 */
 896	rq->head = rq->ring->emit;
 897
 898	ret = rq->engine->request_alloc(rq);
 
 899	if (ret)
 900		goto err_unwind;
 901
 902	rq->infix = rq->ring->emit; /* end of header; start of user payload */
 903
 904	intel_context_mark_active(ce);
 905	list_add_tail_rcu(&rq->link, &tl->requests);
 906
 
 
 907	return rq;
 908
 909err_unwind:
 910	ce->ring->emit = rq->head;
 911
 912	/* Make sure we didn't add ourselves to external state before freeing */
 913	GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
 914	GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
 
 915
 916err_free:
 917	kmem_cache_free(global.slab_requests, rq);
 918err_unreserve:
 919	intel_context_unpin(ce);
 
 
 920	return ERR_PTR(ret);
 921}
 922
 923struct i915_request *
 924i915_request_create(struct intel_context *ce)
 925{
 926	struct i915_request *rq;
 927	struct intel_timeline *tl;
 928
 929	tl = intel_context_timeline_lock(ce);
 930	if (IS_ERR(tl))
 931		return ERR_CAST(tl);
 932
 933	/* Move our oldest request to the slab-cache (if not in use!) */
 934	rq = list_first_entry(&tl->requests, typeof(*rq), link);
 935	if (!list_is_last(&rq->link, &tl->requests))
 936		i915_request_retire(rq);
 937
 938	intel_context_enter(ce);
 939	rq = __i915_request_create(ce, GFP_KERNEL);
 940	intel_context_exit(ce); /* active reference transferred to request */
 941	if (IS_ERR(rq))
 942		goto err_unlock;
 943
 944	/* Check that we do not interrupt ourselves with a new request */
 945	rq->cookie = lockdep_pin_lock(&tl->mutex);
 946
 947	return rq;
 948
 949err_unlock:
 950	intel_context_timeline_unlock(tl);
 951	return rq;
 952}
 953
 954static int
 955i915_request_await_start(struct i915_request *rq, struct i915_request *signal)
 956{
 957	struct dma_fence *fence;
 958	int err;
 959
 960	if (i915_request_timeline(rq) == rcu_access_pointer(signal->timeline))
 961		return 0;
 962
 963	if (i915_request_started(signal))
 964		return 0;
 965
 966	fence = NULL;
 967	rcu_read_lock();
 968	spin_lock_irq(&signal->lock);
 969	do {
 970		struct list_head *pos = READ_ONCE(signal->link.prev);
 971		struct i915_request *prev;
 972
 973		/* Confirm signal has not been retired, the link is valid */
 974		if (unlikely(i915_request_started(signal)))
 975			break;
 976
 977		/* Is signal the earliest request on its timeline? */
 978		if (pos == &rcu_dereference(signal->timeline)->requests)
 979			break;
 980
 981		/*
 982		 * Peek at the request before us in the timeline. That
 983		 * request will only be valid before it is retired, so
 984		 * after acquiring a reference to it, confirm that it is
 985		 * still part of the signaler's timeline.
 986		 */
 987		prev = list_entry(pos, typeof(*prev), link);
 988		if (!i915_request_get_rcu(prev))
 989			break;
 990
 991		/* After the strong barrier, confirm prev is still attached */
 992		if (unlikely(READ_ONCE(prev->link.next) != &signal->link)) {
 993			i915_request_put(prev);
 994			break;
 995		}
 996
 997		fence = &prev->fence;
 998	} while (0);
 999	spin_unlock_irq(&signal->lock);
1000	rcu_read_unlock();
1001	if (!fence)
1002		return 0;
1003
1004	err = 0;
1005	if (!intel_timeline_sync_is_later(i915_request_timeline(rq), fence))
1006		err = i915_sw_fence_await_dma_fence(&rq->submit,
1007						    fence, 0,
1008						    I915_FENCE_GFP);
1009	dma_fence_put(fence);
1010
1011	return err;
1012}
1013
1014static intel_engine_mask_t
1015already_busywaiting(struct i915_request *rq)
1016{
1017	/*
1018	 * Polling a semaphore causes bus traffic, delaying other users of
1019	 * both the GPU and CPU. We want to limit the impact on others,
1020	 * while taking advantage of early submission to reduce GPU
1021	 * latency. Therefore we restrict ourselves to not using more
1022	 * than one semaphore from each source, and not using a semaphore
1023	 * if we have detected the engine is saturated (i.e. would not be
1024	 * submitted early and cause bus traffic reading an already passed
1025	 * semaphore).
1026	 *
1027	 * See the are-we-too-late? check in __i915_request_submit().
1028	 */
1029	return rq->sched.semaphores | READ_ONCE(rq->engine->saturated);
1030}
1031
1032static int
1033__emit_semaphore_wait(struct i915_request *to,
1034		      struct i915_request *from,
1035		      u32 seqno)
1036{
1037	const int has_token = INTEL_GEN(to->engine->i915) >= 12;
1038	u32 hwsp_offset;
1039	int len, err;
1040	u32 *cs;
1041
1042	GEM_BUG_ON(INTEL_GEN(to->engine->i915) < 8);
1043	GEM_BUG_ON(i915_request_has_initial_breadcrumb(to));
1044
1045	/* We need to pin the signaler's HWSP until we are finished reading. */
1046	err = intel_timeline_read_hwsp(from, to, &hwsp_offset);
1047	if (err)
1048		return err;
1049
1050	len = 4;
1051	if (has_token)
1052		len += 2;
1053
1054	cs = intel_ring_begin(to, len);
1055	if (IS_ERR(cs))
1056		return PTR_ERR(cs);
1057
1058	/*
1059	 * Using greater-than-or-equal here means we have to worry
1060	 * about seqno wraparound. To side step that issue, we swap
1061	 * the timeline HWSP upon wrapping, so that everyone listening
1062	 * for the old (pre-wrap) values do not see the much smaller
1063	 * (post-wrap) values than they were expecting (and so wait
1064	 * forever).
1065	 */
1066	*cs++ = (MI_SEMAPHORE_WAIT |
1067		 MI_SEMAPHORE_GLOBAL_GTT |
1068		 MI_SEMAPHORE_POLL |
1069		 MI_SEMAPHORE_SAD_GTE_SDD) +
1070		has_token;
1071	*cs++ = seqno;
1072	*cs++ = hwsp_offset;
1073	*cs++ = 0;
1074	if (has_token) {
1075		*cs++ = 0;
1076		*cs++ = MI_NOOP;
1077	}
1078
1079	intel_ring_advance(to, cs);
1080	return 0;
1081}
1082
1083static int
1084emit_semaphore_wait(struct i915_request *to,
1085		    struct i915_request *from,
1086		    gfp_t gfp)
1087{
1088	const intel_engine_mask_t mask = READ_ONCE(from->engine)->mask;
1089	struct i915_sw_fence *wait = &to->submit;
1090
1091	if (!intel_context_use_semaphores(to->context))
1092		goto await_fence;
1093
1094	if (i915_request_has_initial_breadcrumb(to))
1095		goto await_fence;
1096
1097	if (!rcu_access_pointer(from->hwsp_cacheline))
1098		goto await_fence;
1099
1100	/*
1101	 * If this or its dependents are waiting on an external fence
1102	 * that may fail catastrophically, then we want to avoid using
1103	 * sempahores as they bypass the fence signaling metadata, and we
1104	 * lose the fence->error propagation.
1105	 */
1106	if (from->sched.flags & I915_SCHED_HAS_EXTERNAL_CHAIN)
1107		goto await_fence;
1108
1109	/* Just emit the first semaphore we see as request space is limited. */
1110	if (already_busywaiting(to) & mask)
1111		goto await_fence;
1112
1113	if (i915_request_await_start(to, from) < 0)
1114		goto await_fence;
1115
1116	/* Only submit our spinner after the signaler is running! */
1117	if (__await_execution(to, from, NULL, gfp))
1118		goto await_fence;
1119
1120	if (__emit_semaphore_wait(to, from, from->fence.seqno))
1121		goto await_fence;
1122
1123	to->sched.semaphores |= mask;
1124	wait = &to->semaphore;
1125
1126await_fence:
1127	return i915_sw_fence_await_dma_fence(wait,
1128					     &from->fence, 0,
1129					     I915_FENCE_GFP);
1130}
1131
1132static bool intel_timeline_sync_has_start(struct intel_timeline *tl,
1133					  struct dma_fence *fence)
1134{
1135	return __intel_timeline_sync_is_later(tl,
1136					      fence->context,
1137					      fence->seqno - 1);
1138}
1139
1140static int intel_timeline_sync_set_start(struct intel_timeline *tl,
1141					 const struct dma_fence *fence)
1142{
1143	return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1);
1144}
1145
1146static int
1147__i915_request_await_execution(struct i915_request *to,
1148			       struct i915_request *from,
1149			       void (*hook)(struct i915_request *rq,
1150					    struct dma_fence *signal))
1151{
1152	int err;
1153
1154	GEM_BUG_ON(intel_context_is_barrier(from->context));
 
1155
1156	/* Submit both requests at the same time */
1157	err = __await_execution(to, from, hook, I915_FENCE_GFP);
1158	if (err)
1159		return err;
1160
1161	/* Squash repeated depenendices to the same timelines */
1162	if (intel_timeline_sync_has_start(i915_request_timeline(to),
1163					  &from->fence))
1164		return 0;
1165
1166	/*
1167	 * Wait until the start of this request.
1168	 *
1169	 * The execution cb fires when we submit the request to HW. But in
1170	 * many cases this may be long before the request itself is ready to
1171	 * run (consider that we submit 2 requests for the same context, where
1172	 * the request of interest is behind an indefinite spinner). So we hook
1173	 * up to both to reduce our queues and keep the execution lag minimised
1174	 * in the worst case, though we hope that the await_start is elided.
1175	 */
1176	err = i915_request_await_start(to, from);
1177	if (err < 0)
1178		return err;
1179
1180	/*
1181	 * Ensure both start together [after all semaphores in signal]
1182	 *
1183	 * Now that we are queued to the HW at roughly the same time (thanks
1184	 * to the execute cb) and are ready to run at roughly the same time
1185	 * (thanks to the await start), our signaler may still be indefinitely
1186	 * delayed by waiting on a semaphore from a remote engine. If our
1187	 * signaler depends on a semaphore, so indirectly do we, and we do not
1188	 * want to start our payload until our signaler also starts theirs.
1189	 * So we wait.
1190	 *
1191	 * However, there is also a second condition for which we need to wait
1192	 * for the precise start of the signaler. Consider that the signaler
1193	 * was submitted in a chain of requests following another context
1194	 * (with just an ordinary intra-engine fence dependency between the
1195	 * two). In this case the signaler is queued to HW, but not for
1196	 * immediate execution, and so we must wait until it reaches the
1197	 * active slot.
1198	 */
1199	if (intel_engine_has_semaphores(to->engine) &&
1200	    !i915_request_has_initial_breadcrumb(to)) {
1201		err = __emit_semaphore_wait(to, from, from->fence.seqno - 1);
1202		if (err < 0)
1203			return err;
1204	}
1205
1206	/* Couple the dependency tree for PI on this exposed to->fence */
1207	if (to->engine->schedule) {
1208		err = i915_sched_node_add_dependency(&to->sched,
1209						     &from->sched,
1210						     I915_DEPENDENCY_WEAK);
1211		if (err < 0)
1212			return err;
1213	}
1214
1215	return intel_timeline_sync_set_start(i915_request_timeline(to),
1216					     &from->fence);
1217}
1218
1219static void mark_external(struct i915_request *rq)
1220{
1221	/*
1222	 * The downside of using semaphores is that we lose metadata passing
1223	 * along the signaling chain. This is particularly nasty when we
1224	 * need to pass along a fatal error such as EFAULT or EDEADLK. For
1225	 * fatal errors we want to scrub the request before it is executed,
1226	 * which means that we cannot preload the request onto HW and have
1227	 * it wait upon a semaphore.
1228	 */
1229	rq->sched.flags |= I915_SCHED_HAS_EXTERNAL_CHAIN;
1230}
1231
1232static int
1233__i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
1234{
1235	mark_external(rq);
1236	return i915_sw_fence_await_dma_fence(&rq->submit, fence,
1237					     i915_fence_context_timeout(rq->engine->i915,
1238									fence->context),
1239					     I915_FENCE_GFP);
1240}
1241
1242static int
1243i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
1244{
1245	struct dma_fence *iter;
1246	int err = 0;
1247
1248	if (!to_dma_fence_chain(fence))
1249		return __i915_request_await_external(rq, fence);
1250
1251	dma_fence_chain_for_each(iter, fence) {
1252		struct dma_fence_chain *chain = to_dma_fence_chain(iter);
1253
1254		if (!dma_fence_is_i915(chain->fence)) {
1255			err = __i915_request_await_external(rq, iter);
1256			break;
1257		}
1258
1259		err = i915_request_await_dma_fence(rq, chain->fence);
1260		if (err < 0)
1261			break;
1262	}
1263
1264	dma_fence_put(iter);
1265	return err;
1266}
1267
1268int
1269i915_request_await_execution(struct i915_request *rq,
1270			     struct dma_fence *fence,
1271			     void (*hook)(struct i915_request *rq,
1272					  struct dma_fence *signal))
1273{
1274	struct dma_fence **child = &fence;
1275	unsigned int nchild = 1;
1276	int ret;
1277
1278	if (dma_fence_is_array(fence)) {
1279		struct dma_fence_array *array = to_dma_fence_array(fence);
1280
1281		/* XXX Error for signal-on-any fence arrays */
1282
1283		child = array->fences;
1284		nchild = array->num_fences;
1285		GEM_BUG_ON(!nchild);
1286	}
1287
1288	do {
1289		fence = *child++;
1290		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
1291			i915_sw_fence_set_error_once(&rq->submit, fence->error);
1292			continue;
1293		}
1294
1295		if (fence->context == rq->fence.context)
1296			continue;
1297
1298		/*
1299		 * We don't squash repeated fence dependencies here as we
1300		 * want to run our callback in all cases.
1301		 */
1302
1303		if (dma_fence_is_i915(fence))
1304			ret = __i915_request_await_execution(rq,
1305							     to_request(fence),
1306							     hook);
1307		else
1308			ret = i915_request_await_external(rq, fence);
1309		if (ret < 0)
1310			return ret;
1311	} while (--nchild);
1312
1313	return 0;
1314}
1315
1316static int
1317await_request_submit(struct i915_request *to, struct i915_request *from)
1318{
1319	/*
1320	 * If we are waiting on a virtual engine, then it may be
1321	 * constrained to execute on a single engine *prior* to submission.
1322	 * When it is submitted, it will be first submitted to the virtual
1323	 * engine and then passed to the physical engine. We cannot allow
1324	 * the waiter to be submitted immediately to the physical engine
1325	 * as it may then bypass the virtual request.
1326	 */
1327	if (to->engine == READ_ONCE(from->engine))
1328		return i915_sw_fence_await_sw_fence_gfp(&to->submit,
1329							&from->submit,
1330							I915_FENCE_GFP);
1331	else
1332		return __i915_request_await_execution(to, from, NULL);
1333}
1334
1335static int
1336i915_request_await_request(struct i915_request *to, struct i915_request *from)
1337{
1338	int ret;
1339
1340	GEM_BUG_ON(to == from);
1341	GEM_BUG_ON(to->timeline == from->timeline);
1342
1343	if (i915_request_completed(from)) {
1344		i915_sw_fence_set_error_once(&to->submit, from->fence.error);
1345		return 0;
1346	}
1347
1348	if (to->engine->schedule) {
1349		ret = i915_sched_node_add_dependency(&to->sched,
1350						     &from->sched,
1351						     I915_DEPENDENCY_EXTERNAL);
1352		if (ret < 0)
1353			return ret;
1354	}
1355
1356	if (is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask)))
1357		ret = await_request_submit(to, from);
1358	else
1359		ret = emit_semaphore_wait(to, from, I915_FENCE_GFP);
1360	if (ret < 0)
1361		return ret;
1362
1363	return 0;
1364}
1365
1366int
1367i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
1368{
1369	struct dma_fence **child = &fence;
1370	unsigned int nchild = 1;
1371	int ret;
1372
1373	/*
1374	 * Note that if the fence-array was created in signal-on-any mode,
1375	 * we should *not* decompose it into its individual fences. However,
1376	 * we don't currently store which mode the fence-array is operating
1377	 * in. Fortunately, the only user of signal-on-any is private to
1378	 * amdgpu and we should not see any incoming fence-array from
1379	 * sync-file being in signal-on-any mode.
1380	 */
1381	if (dma_fence_is_array(fence)) {
1382		struct dma_fence_array *array = to_dma_fence_array(fence);
1383
1384		child = array->fences;
1385		nchild = array->num_fences;
1386		GEM_BUG_ON(!nchild);
1387	}
1388
1389	do {
1390		fence = *child++;
1391		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
1392			i915_sw_fence_set_error_once(&rq->submit, fence->error);
1393			continue;
1394		}
1395
1396		/*
1397		 * Requests on the same timeline are explicitly ordered, along
1398		 * with their dependencies, by i915_request_add() which ensures
1399		 * that requests are submitted in-order through each ring.
1400		 */
1401		if (fence->context == rq->fence.context)
1402			continue;
1403
1404		/* Squash repeated waits to the same timelines */
1405		if (fence->context &&
1406		    intel_timeline_sync_is_later(i915_request_timeline(rq),
1407						 fence))
1408			continue;
1409
1410		if (dma_fence_is_i915(fence))
1411			ret = i915_request_await_request(rq, to_request(fence));
1412		else
1413			ret = i915_request_await_external(rq, fence);
 
 
1414		if (ret < 0)
1415			return ret;
1416
1417		/* Record the latest fence used against each timeline */
1418		if (fence->context)
1419			intel_timeline_sync_set(i915_request_timeline(rq),
1420						fence);
1421	} while (--nchild);
1422
1423	return 0;
1424}
1425
1426/**
1427 * i915_request_await_object - set this request to (async) wait upon a bo
1428 * @to: request we are wishing to use
1429 * @obj: object which may be in use on another ring.
1430 * @write: whether the wait is on behalf of a writer
1431 *
1432 * This code is meant to abstract object synchronization with the GPU.
1433 * Conceptually we serialise writes between engines inside the GPU.
1434 * We only allow one engine to write into a buffer at any time, but
1435 * multiple readers. To ensure each has a coherent view of memory, we must:
1436 *
1437 * - If there is an outstanding write request to the object, the new
1438 *   request must wait for it to complete (either CPU or in hw, requests
1439 *   on the same ring will be naturally ordered).
1440 *
1441 * - If we are a write request (pending_write_domain is set), the new
1442 *   request must wait for outstanding read requests to complete.
1443 *
1444 * Returns 0 if successful, else propagates up the lower layer error.
1445 */
1446int
1447i915_request_await_object(struct i915_request *to,
1448			  struct drm_i915_gem_object *obj,
1449			  bool write)
1450{
1451	struct dma_fence *excl;
1452	int ret = 0;
1453
1454	if (write) {
1455		struct dma_fence **shared;
1456		unsigned int count, i;
1457
1458		ret = dma_resv_get_fences_rcu(obj->base.resv,
1459							&excl, &count, &shared);
1460		if (ret)
1461			return ret;
1462
1463		for (i = 0; i < count; i++) {
1464			ret = i915_request_await_dma_fence(to, shared[i]);
1465			if (ret)
1466				break;
1467
1468			dma_fence_put(shared[i]);
1469		}
1470
1471		for (; i < count; i++)
1472			dma_fence_put(shared[i]);
1473		kfree(shared);
1474	} else {
1475		excl = dma_resv_get_excl_rcu(obj->base.resv);
1476	}
1477
1478	if (excl) {
1479		if (ret == 0)
1480			ret = i915_request_await_dma_fence(to, excl);
1481
1482		dma_fence_put(excl);
1483	}
1484
1485	return ret;
1486}
1487
1488static struct i915_request *
1489__i915_request_add_to_timeline(struct i915_request *rq)
 
 
 
 
1490{
1491	struct intel_timeline *timeline = i915_request_timeline(rq);
 
 
1492	struct i915_request *prev;
 
 
1493
1494	/*
1495	 * Dependency tracking and request ordering along the timeline
1496	 * is special cased so that we can eliminate redundant ordering
1497	 * operations while building the request (we know that the timeline
1498	 * itself is ordered, and here we guarantee it).
1499	 *
1500	 * As we know we will need to emit tracking along the timeline,
1501	 * we embed the hooks into our request struct -- at the cost of
1502	 * having to have specialised no-allocation interfaces (which will
1503	 * be beneficial elsewhere).
1504	 *
1505	 * A second benefit to open-coding i915_request_await_request is
1506	 * that we can apply a slight variant of the rules specialised
1507	 * for timelines that jump between engines (such as virtual engines).
1508	 * If we consider the case of virtual engine, we must emit a dma-fence
1509	 * to prevent scheduling of the second request until the first is
1510	 * complete (to maximise our greedy late load balancing) and this
1511	 * precludes optimising to use semaphores serialisation of a single
1512	 * timeline across engines.
1513	 */
1514	prev = to_request(__i915_active_fence_set(&timeline->last_request,
1515						  &rq->fence));
1516	if (prev && !i915_request_completed(prev)) {
1517		/*
1518		 * The requests are supposed to be kept in order. However,
1519		 * we need to be wary in case the timeline->last_request
1520		 * is used as a barrier for external modification to this
1521		 * context.
1522		 */
1523		GEM_BUG_ON(prev->context == rq->context &&
1524			   i915_seqno_passed(prev->fence.seqno,
1525					     rq->fence.seqno));
1526
1527		if (is_power_of_2(READ_ONCE(prev->engine)->mask | rq->engine->mask))
1528			i915_sw_fence_await_sw_fence(&rq->submit,
1529						     &prev->submit,
1530						     &rq->submitq);
1531		else
1532			__i915_sw_fence_await_dma_fence(&rq->submit,
1533							&prev->fence,
1534							&rq->dmaq);
1535		if (rq->engine->schedule)
1536			__i915_sched_node_add_dependency(&rq->sched,
1537							 &prev->sched,
1538							 &rq->dep,
1539							 0);
1540	}
1541
1542	/*
1543	 * Make sure that no request gazumped us - if it was allocated after
1544	 * our i915_request_alloc() and called __i915_request_add() before
1545	 * us, the timeline will hold its seqno which is later than ours.
1546	 */
1547	GEM_BUG_ON(timeline->seqno != rq->fence.seqno);
1548
1549	return prev;
1550}
1551
1552/*
1553 * NB: This function is not allowed to fail. Doing so would mean the the
1554 * request is not being tracked for completion but the work itself is
1555 * going to happen on the hardware. This would be a Bad Thing(tm).
1556 */
1557struct i915_request *__i915_request_commit(struct i915_request *rq)
1558{
1559	struct intel_engine_cs *engine = rq->engine;
1560	struct intel_ring *ring = rq->ring;
1561	u32 *cs;
1562
1563	RQ_TRACE(rq, "\n");
1564
1565	/*
1566	 * To ensure that this call will not fail, space for its emissions
1567	 * should already have been reserved in the ring buffer. Let the ring
1568	 * know that it is time to use that space up.
1569	 */
1570	GEM_BUG_ON(rq->reserved_space > ring->space);
1571	rq->reserved_space = 0;
1572	rq->emitted_jiffies = jiffies;
 
 
 
 
 
 
 
 
 
 
 
 
1573
1574	/*
1575	 * Record the position of the start of the breadcrumb so that
1576	 * should we detect the updated seqno part-way through the
1577	 * GPU processing the request, we never over-estimate the
1578	 * position of the ring's HEAD.
1579	 */
1580	cs = intel_ring_begin(rq, engine->emit_fini_breadcrumb_dw);
1581	GEM_BUG_ON(IS_ERR(cs));
1582	rq->postfix = intel_ring_offset(rq, cs);
1583
1584	return __i915_request_add_to_timeline(rq);
1585}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1586
1587void __i915_request_queue(struct i915_request *rq,
1588			  const struct i915_sched_attr *attr)
1589{
1590	/*
1591	 * Let the backend know a new request has arrived that may need
1592	 * to adjust the existing execution schedule due to a high priority
1593	 * request - i.e. we may want to preempt the current request in order
1594	 * to run a high priority dependency chain *before* we can execute this
1595	 * request.
1596	 *
1597	 * This is called before the request is ready to run so that we can
1598	 * decide whether to preempt the entire chain so that it is ready to
1599	 * run at the earliest possible convenience.
1600	 */
1601	if (attr && rq->engine->schedule)
1602		rq->engine->schedule(rq, attr);
1603	i915_sw_fence_commit(&rq->semaphore);
1604	i915_sw_fence_commit(&rq->submit);
1605}
1606
1607void i915_request_add(struct i915_request *rq)
1608{
1609	struct intel_timeline * const tl = i915_request_timeline(rq);
1610	struct i915_sched_attr attr = {};
1611	struct i915_gem_context *ctx;
1612
1613	lockdep_assert_held(&tl->mutex);
1614	lockdep_unpin_lock(&tl->mutex, rq->cookie);
1615
1616	trace_i915_request_add(rq);
1617	__i915_request_commit(rq);
1618
1619	/* XXX placeholder for selftests */
1620	rcu_read_lock();
1621	ctx = rcu_dereference(rq->context->gem_context);
1622	if (ctx)
1623		attr = ctx->sched;
1624	rcu_read_unlock();
1625
1626	__i915_request_queue(rq, &attr);
1627
1628	mutex_unlock(&tl->mutex);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1629}
1630
1631static unsigned long local_clock_ns(unsigned int *cpu)
1632{
1633	unsigned long t;
1634
1635	/*
1636	 * Cheaply and approximately convert from nanoseconds to microseconds.
1637	 * The result and subsequent calculations are also defined in the same
1638	 * approximate microseconds units. The principal source of timing
1639	 * error here is from the simple truncation.
1640	 *
1641	 * Note that local_clock() is only defined wrt to the current CPU;
1642	 * the comparisons are no longer valid if we switch CPUs. Instead of
1643	 * blocking preemption for the entire busywait, we can detect the CPU
1644	 * switch and use that as indicator of system load and a reason to
1645	 * stop busywaiting, see busywait_stop().
1646	 */
1647	*cpu = get_cpu();
1648	t = local_clock();
1649	put_cpu();
1650
1651	return t;
1652}
1653
1654static bool busywait_stop(unsigned long timeout, unsigned int cpu)
1655{
1656	unsigned int this_cpu;
1657
1658	if (time_after(local_clock_ns(&this_cpu), timeout))
1659		return true;
1660
1661	return this_cpu != cpu;
1662}
1663
1664static bool __i915_spin_request(const struct i915_request * const rq, int state)
 
1665{
1666	unsigned long timeout_ns;
1667	unsigned int cpu;
 
 
1668
1669	/*
1670	 * Only wait for the request if we know it is likely to complete.
1671	 *
1672	 * We don't track the timestamps around requests, nor the average
1673	 * request length, so we do not have a good indicator that this
1674	 * request will complete within the timeout. What we do know is the
1675	 * order in which requests are executed by the context and so we can
1676	 * tell if the request has been started. If the request is not even
1677	 * running yet, it is a fair assumption that it will not complete
1678	 * within our relatively short timeout.
1679	 */
1680	if (!i915_request_is_running(rq))
1681		return false;
1682
1683	/*
1684	 * When waiting for high frequency requests, e.g. during synchronous
1685	 * rendering split between the CPU and GPU, the finite amount of time
1686	 * required to set up the irq and wait upon it limits the response
1687	 * rate. By busywaiting on the request completion for a short while we
1688	 * can service the high frequency waits as quick as possible. However,
1689	 * if it is a slow request, we want to sleep as quickly as possible.
1690	 * The tradeoff between waiting and sleeping is roughly the time it
1691	 * takes to sleep on a request, on the order of a microsecond.
1692	 */
1693
1694	timeout_ns = READ_ONCE(rq->engine->props.max_busywait_duration_ns);
1695	timeout_ns += local_clock_ns(&cpu);
1696	do {
1697		if (i915_request_completed(rq))
1698			return true;
 
 
 
 
 
 
 
 
 
1699
1700		if (signal_pending_state(state, current))
1701			break;
1702
1703		if (busywait_stop(timeout_ns, cpu))
1704			break;
1705
1706		cpu_relax();
1707	} while (!need_resched());
1708
1709	return false;
1710}
1711
1712struct request_wait {
1713	struct dma_fence_cb cb;
1714	struct task_struct *tsk;
1715};
1716
1717static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb)
1718{
1719	struct request_wait *wait = container_of(cb, typeof(*wait), cb);
 
1720
1721	wake_up_process(wait->tsk);
 
 
1722}
1723
1724/**
1725 * i915_request_wait - wait until execution of request has finished
1726 * @rq: the request to wait upon
1727 * @flags: how to wait
1728 * @timeout: how long to wait in jiffies
1729 *
1730 * i915_request_wait() waits for the request to be completed, for a
1731 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
1732 * unbounded wait).
1733 *
 
 
 
 
1734 * Returns the remaining time (in jiffies) if the request completed, which may
1735 * be zero or -ETIME if the request is unfinished after the timeout expires.
1736 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
1737 * pending before the request completes.
1738 */
1739long i915_request_wait(struct i915_request *rq,
1740		       unsigned int flags,
1741		       long timeout)
1742{
1743	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
1744		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1745	struct request_wait wait;
 
 
 
1746
1747	might_sleep();
 
 
 
 
 
1748	GEM_BUG_ON(timeout < 0);
1749
1750	if (dma_fence_is_signaled(&rq->fence))
1751		return timeout;
1752
1753	if (!timeout)
1754		return -ETIME;
1755
1756	trace_i915_request_wait_begin(rq, flags);
1757
1758	/*
1759	 * We must never wait on the GPU while holding a lock as we
1760	 * may need to perform a GPU reset. So while we don't need to
1761	 * serialise wait/reset with an explicit lock, we do want
1762	 * lockdep to detect potential dependency cycles.
1763	 */
1764	mutex_acquire(&rq->engine->gt->reset.mutex.dep_map, 0, 0, _THIS_IP_);
1765
1766	/*
1767	 * Optimistic spin before touching IRQs.
1768	 *
1769	 * We may use a rather large value here to offset the penalty of
1770	 * switching away from the active task. Frequently, the client will
1771	 * wait upon an old swapbuffer to throttle itself to remain within a
1772	 * frame of the gpu. If the client is running in lockstep with the gpu,
1773	 * then it should not be waiting long at all, and a sleep now will incur
1774	 * extra scheduler latency in producing the next frame. To try to
1775	 * avoid adding the cost of enabling/disabling the interrupt to the
1776	 * short wait, we first spin to see if the request would have completed
1777	 * in the time taken to setup the interrupt.
1778	 *
1779	 * We need upto 5us to enable the irq, and upto 20us to hide the
1780	 * scheduler latency of a context switch, ignoring the secondary
1781	 * impacts from a context switch such as cache eviction.
1782	 *
1783	 * The scheme used for low-latency IO is called "hybrid interrupt
1784	 * polling". The suggestion there is to sleep until just before you
1785	 * expect to be woken by the device interrupt and then poll for its
1786	 * completion. That requires having a good predictor for the request
1787	 * duration, which we currently lack.
1788	 */
1789	if (IS_ACTIVE(CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT) &&
1790	    __i915_spin_request(rq, state)) {
1791		dma_fence_signal(&rq->fence);
1792		goto out;
1793	}
1794
1795	/*
1796	 * This client is about to stall waiting for the GPU. In many cases
1797	 * this is undesirable and limits the throughput of the system, as
1798	 * many clients cannot continue processing user input/output whilst
1799	 * blocked. RPS autotuning may take tens of milliseconds to respond
1800	 * to the GPU load and thus incurs additional latency for the client.
1801	 * We can circumvent that by promoting the GPU frequency to maximum
1802	 * before we sleep. This makes the GPU throttle up much more quickly
1803	 * (good for benchmarks and user experience, e.g. window animations),
1804	 * but at a cost of spending more power processing the workload
1805	 * (bad for battery).
1806	 */
1807	if (flags & I915_WAIT_PRIORITY) {
1808		if (!i915_request_started(rq) &&
1809		    INTEL_GEN(rq->engine->i915) >= 6)
1810			intel_rps_boost(rq);
1811	}
1812
1813	wait.tsk = current;
1814	if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake))
1815		goto out;
1816
1817	for (;;) {
 
1818		set_current_state(state);
1819
1820		if (i915_request_completed(rq)) {
1821			dma_fence_signal(&rq->fence);
1822			break;
 
 
 
 
 
 
 
 
 
 
 
 
 
1823		}
1824
1825		intel_engine_flush_submission(rq->engine);
 
1826
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1827		if (signal_pending_state(state, current)) {
1828			timeout = -ERESTARTSYS;
1829			break;
1830		}
1831
1832		if (!timeout) {
1833			timeout = -ETIME;
1834			break;
1835		}
1836
1837		timeout = io_schedule_timeout(timeout);
1838	}
1839	__set_current_state(TASK_RUNNING);
1840
1841	dma_fence_remove_callback(&rq->fence, &wait.cb);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1842
1843out:
1844	mutex_release(&rq->engine->gt->reset.mutex.dep_map, _THIS_IP_);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1845	trace_i915_request_wait_end(rq);
 
1846	return timeout;
1847}
1848
1849#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1850#include "selftests/mock_request.c"
1851#include "selftests/i915_request.c"
1852#endif
1853
1854static void i915_global_request_shrink(void)
1855{
1856	kmem_cache_shrink(global.slab_execute_cbs);
1857	kmem_cache_shrink(global.slab_requests);
1858}
 
 
 
 
 
 
1859
1860static void i915_global_request_exit(void)
1861{
1862	kmem_cache_destroy(global.slab_execute_cbs);
1863	kmem_cache_destroy(global.slab_requests);
1864}
1865
1866static struct i915_global_request global = { {
1867	.shrink = i915_global_request_shrink,
1868	.exit = i915_global_request_exit,
1869} };
1870
1871int __init i915_global_request_init(void)
1872{
1873	global.slab_requests =
1874		kmem_cache_create("i915_request",
1875				  sizeof(struct i915_request),
1876				  __alignof__(struct i915_request),
1877				  SLAB_HWCACHE_ALIGN |
1878				  SLAB_RECLAIM_ACCOUNT |
1879				  SLAB_TYPESAFE_BY_RCU,
1880				  __i915_request_ctor);
1881	if (!global.slab_requests)
1882		return -ENOMEM;
1883
1884	global.slab_execute_cbs = KMEM_CACHE(execute_cb,
1885					     SLAB_HWCACHE_ALIGN |
1886					     SLAB_RECLAIM_ACCOUNT |
1887					     SLAB_TYPESAFE_BY_RCU);
1888	if (!global.slab_execute_cbs)
1889		goto err_requests;
1890
1891	i915_global_register(&global.base);
1892	return 0;
1893
1894err_requests:
1895	kmem_cache_destroy(global.slab_requests);
1896	return -ENOMEM;
1897}