1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2014 Intel Corporation
   4 */
   5
   6/**
   7 * DOC: Logical Rings, Logical Ring Contexts and Execlists
   8 *
   9 * Motivation:
  10 * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
  11 * These expanded contexts enable a number of new abilities, especially
  12 * "Execlists" (also implemented in this file).
  13 *
  14 * One of the main differences with the legacy HW contexts is that logical
  15 * ring contexts incorporate many more things to the context's state, like
  16 * PDPs or ringbuffer control registers:
  17 *
  18 * The reason why PDPs are included in the context is straightforward: as
  19 * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
  20 * contained there mean you don't need to do a ppgtt->switch_mm yourself,
  21 * instead, the GPU will do it for you on the context switch.
  22 *
  23 * But, what about the ringbuffer control registers (head, tail, etc..)?
  24 * shouldn't we just need a set of those per engine command streamer? This is
  25 * where the name "Logical Rings" starts to make sense: by virtualizing the
  26 * rings, the engine cs shifts to a new "ring buffer" with every context
  27 * switch. When you want to submit a workload to the GPU you: A) choose your
  28 * context, B) find its appropriate virtualized ring, C) write commands to it
  29 * and then, finally, D) tell the GPU to switch to that context.
  30 *
  31 * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
  32 * to a contexts is via a context execution list, ergo "Execlists".
  33 *
  34 * LRC implementation:
  35 * Regarding the creation of contexts, we have:
  36 *
  37 * - One global default context.
  38 * - One local default context for each opened fd.
  39 * - One local extra context for each context create ioctl call.
  40 *
  41 * Now that ringbuffers belong per-context (and not per-engine, like before)
  42 * and that contexts are uniquely tied to a given engine (and not reusable,
  43 * like before) we need:
  44 *
  45 * - One ringbuffer per-engine inside each context.
  46 * - One backing object per-engine inside each context.
  47 *
  48 * The global default context starts its life with these new objects fully
  49 * allocated and populated. The local default context for each opened fd is
  50 * more complex, because we don't know at creation time which engine is going
  51 * to use them. To handle this, we have implemented a deferred creation of LR
  52 * contexts:
  53 *
  54 * The local context starts its life as a hollow or blank holder, that only
  55 * gets populated for a given engine once we receive an execbuffer. If later
  56 * on we receive another execbuffer ioctl for the same context but a different
  57 * engine, we allocate/populate a new ringbuffer and context backing object and
  58 * so on.
  59 *
  60 * Finally, regarding local contexts created using the ioctl call: as they are
  61 * only allowed with the render ring, we can allocate & populate them right
  62 * away (no need to defer anything, at least for now).
  63 *
  64 * Execlists implementation:
  65 * Execlists are the new method by which, on gen8+ hardware, workloads are
  66 * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
  67 * This method works as follows:
  68 *
  69 * When a request is committed, its commands (the BB start and any leading or
  70 * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
  71 * for the appropriate context. The tail pointer in the hardware context is not
  72 * updated at this time, but instead, kept by the driver in the ringbuffer
  73 * structure. A structure representing this request is added to a request queue
  74 * for the appropriate engine: this structure contains a copy of the context's
  75 * tail after the request was written to the ring buffer and a pointer to the
  76 * context itself.
  77 *
  78 * If the engine's request queue was empty before the request was added, the
  79 * queue is processed immediately. Otherwise the queue will be processed during
  80 * a context switch interrupt. In any case, elements on the queue will get sent
  81 * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
  82 * globally unique 20-bits submission ID.
  83 *
  84 * When execution of a request completes, the GPU updates the context status
  85 * buffer with a context complete event and generates a context switch interrupt.
  86 * During the interrupt handling, the driver examines the events in the buffer:
  87 * for each context complete event, if the announced ID matches that on the head
  88 * of the request queue, then that request is retired and removed from the queue.
  89 *
  90 * After processing, if any requests were retired and the queue is not empty
  91 * then a new execution list can be submitted. The two requests at the front of
  92 * the queue are next to be submitted but since a context may not occur twice in
  93 * an execution list, if subsequent requests have the same ID as the first then
  94 * the two requests must be combined. This is done simply by discarding requests
  95 * at the head of the queue until either only one requests is left (in which case
  96 * we use a NULL second context) or the first two requests have unique IDs.
  97 *
  98 * By always executing the first two requests in the queue the driver ensures
  99 * that the GPU is kept as busy as possible. In the case where a single context
 100 * completes but a second context is still executing, the request for this second
 101 * context will be at the head of the queue when we remove the first one. This
 102 * request will then be resubmitted along with a new request for a different context,
 103 * which will cause the hardware to continue executing the second request and queue
 104 * the new request (the GPU detects the condition of a context getting preempted
 105 * with the same context and optimizes the context switch flow by not doing
 106 * preemption, but just sampling the new tail pointer).
 107 *
 108 */
 109#include <linux/interrupt.h>
 110#include <linux/string_helpers.h>
 111
 112#include "i915_drv.h"
 113#include "i915_reg.h"
 114#include "i915_trace.h"
 115#include "i915_vgpu.h"
 116#include "gen8_engine_cs.h"
 117#include "intel_breadcrumbs.h"
 118#include "intel_context.h"
 119#include "intel_engine_heartbeat.h"
 120#include "intel_engine_pm.h"
 121#include "intel_engine_regs.h"
 122#include "intel_engine_stats.h"
 123#include "intel_execlists_submission.h"
 124#include "intel_gt.h"
 125#include "intel_gt_irq.h"
 126#include "intel_gt_pm.h"
 127#include "intel_gt_regs.h"
 128#include "intel_gt_requests.h"
 129#include "intel_lrc.h"
 130#include "intel_lrc_reg.h"
 131#include "intel_mocs.h"
 132#include "intel_reset.h"
 133#include "intel_ring.h"
 134#include "intel_workarounds.h"
 135#include "shmem_utils.h"
 136
 137#define RING_EXECLIST_QFULL		(1 << 0x2)
 138#define RING_EXECLIST1_VALID		(1 << 0x3)
 139#define RING_EXECLIST0_VALID		(1 << 0x4)
 140#define RING_EXECLIST_ACTIVE_STATUS	(3 << 0xE)
 141#define RING_EXECLIST1_ACTIVE		(1 << 0x11)
 142#define RING_EXECLIST0_ACTIVE		(1 << 0x12)
 143
 144#define GEN8_CTX_STATUS_IDLE_ACTIVE	(1 << 0)
 145#define GEN8_CTX_STATUS_PREEMPTED	(1 << 1)
 146#define GEN8_CTX_STATUS_ELEMENT_SWITCH	(1 << 2)
 147#define GEN8_CTX_STATUS_ACTIVE_IDLE	(1 << 3)
 148#define GEN8_CTX_STATUS_COMPLETE	(1 << 4)
 149#define GEN8_CTX_STATUS_LITE_RESTORE	(1 << 15)
 150
 151#define GEN8_CTX_STATUS_COMPLETED_MASK \
 152	 (GEN8_CTX_STATUS_COMPLETE | GEN8_CTX_STATUS_PREEMPTED)
 153
 154#define GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE	(0x1) /* lower csb dword */
 155#define GEN12_CTX_SWITCH_DETAIL(csb_dw)	((csb_dw) & 0xF) /* upper csb dword */
 156#define GEN12_CSB_SW_CTX_ID_MASK		GENMASK(25, 15)
 157#define GEN12_IDLE_CTX_ID		0x7FF
 158#define GEN12_CSB_CTX_VALID(csb_dw) \
 159	(FIELD_GET(GEN12_CSB_SW_CTX_ID_MASK, csb_dw) != GEN12_IDLE_CTX_ID)
 160
 161#define XEHP_CTX_STATUS_SWITCHED_TO_NEW_QUEUE	BIT(1) /* upper csb dword */
 162#define XEHP_CSB_SW_CTX_ID_MASK			GENMASK(31, 10)
 163#define XEHP_IDLE_CTX_ID			0xFFFF
 164#define XEHP_CSB_CTX_VALID(csb_dw) \
 165	(FIELD_GET(XEHP_CSB_SW_CTX_ID_MASK, csb_dw) != XEHP_IDLE_CTX_ID)
 166
 167/* Typical size of the average request (2 pipecontrols and a MI_BB) */
 168#define EXECLISTS_REQUEST_SIZE 64 /* bytes */
 169
 170struct virtual_engine {
 171	struct intel_engine_cs base;
 172	struct intel_context context;
 173	struct rcu_work rcu;
 174
 175	/*
 176	 * We allow only a single request through the virtual engine at a time
 177	 * (each request in the timeline waits for the completion fence of
 178	 * the previous before being submitted). By restricting ourselves to
 179	 * only submitting a single request, each request is placed on to a
 180	 * physical to maximise load spreading (by virtue of the late greedy
 181	 * scheduling -- each real engine takes the next available request
 182	 * upon idling).
 183	 */
 184	struct i915_request *request;
 185
 186	/*
 187	 * We keep a rbtree of available virtual engines inside each physical
 188	 * engine, sorted by priority. Here we preallocate the nodes we need
 189	 * for the virtual engine, indexed by physical_engine->id.
 190	 */
 191	struct ve_node {
 192		struct rb_node rb;
 193		int prio;
 194	} nodes[I915_NUM_ENGINES];
 195
 196	/* And finally, which physical engines this virtual engine maps onto. */
 197	unsigned int num_siblings;
 198	struct intel_engine_cs *siblings[];
 199};
 200
 201static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine)
 202{
 203	GEM_BUG_ON(!intel_engine_is_virtual(engine));
 204	return container_of(engine, struct virtual_engine, base);
 205}
 206
 207static struct intel_context *
 208execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
 209			 unsigned long flags);
 210
 211static struct i915_request *
 212__active_request(const struct intel_timeline * const tl,
 213		 struct i915_request *rq,
 214		 int error)
 215{
 216	struct i915_request *active = rq;
 217
 218	list_for_each_entry_from_reverse(rq, &tl->requests, link) {
 219		if (__i915_request_is_complete(rq))
 220			break;
 221
 222		if (error) {
 223			i915_request_set_error_once(rq, error);
 224			__i915_request_skip(rq);
 225		}
 226		active = rq;
 227	}
 228
 229	return active;
 230}
 231
 232static struct i915_request *
 233active_request(const struct intel_timeline * const tl, struct i915_request *rq)
 234{
 235	return __active_request(tl, rq, 0);
 236}
 237
 238static void ring_set_paused(const struct intel_engine_cs *engine, int state)
 239{
 240	/*
 241	 * We inspect HWS_PREEMPT with a semaphore inside
 242	 * engine->emit_fini_breadcrumb. If the dword is true,
 243	 * the ring is paused as the semaphore will busywait
 244	 * until the dword is false.
 245	 */
 246	engine->status_page.addr[I915_GEM_HWS_PREEMPT] = state;
 247	if (state)
 248		wmb();
 249}
 250
 251static struct i915_priolist *to_priolist(struct rb_node *rb)
 252{
 253	return rb_entry(rb, struct i915_priolist, node);
 254}
 255
 256static int rq_prio(const struct i915_request *rq)
 257{
 258	return READ_ONCE(rq->sched.attr.priority);
 259}
 260
 261static int effective_prio(const struct i915_request *rq)
 262{
 263	int prio = rq_prio(rq);
 264
 265	/*
 266	 * If this request is special and must not be interrupted at any
 267	 * cost, so be it. Note we are only checking the most recent request
 268	 * in the context and so may be masking an earlier vip request. It
 269	 * is hoped that under the conditions where nopreempt is used, this
 270	 * will not matter (i.e. all requests to that context will be
 271	 * nopreempt for as long as desired).
 272	 */
 273	if (i915_request_has_nopreempt(rq))
 274		prio = I915_PRIORITY_UNPREEMPTABLE;
 275
 276	return prio;
 277}
 278
 279static int queue_prio(const struct i915_sched_engine *sched_engine)
 280{
 281	struct rb_node *rb;
 282
 283	rb = rb_first_cached(&sched_engine->queue);
 284	if (!rb)
 285		return INT_MIN;
 286
 287	return to_priolist(rb)->priority;
 288}
 289
 290static int virtual_prio(const struct intel_engine_execlists *el)
 291{
 292	struct rb_node *rb = rb_first_cached(&el->virtual);
 293
 294	return rb ? rb_entry(rb, struct ve_node, rb)->prio : INT_MIN;
 295}
 296
 297static bool need_preempt(const struct intel_engine_cs *engine,
 298			 const struct i915_request *rq)
 299{
 300	int last_prio;
 301
 302	if (!intel_engine_has_semaphores(engine))
 303		return false;
 304
 305	/*
 306	 * Check if the current priority hint merits a preemption attempt.
 307	 *
 308	 * We record the highest value priority we saw during rescheduling
 309	 * prior to this dequeue, therefore we know that if it is strictly
 310	 * less than the current tail of ESLP[0], we do not need to force
 311	 * a preempt-to-idle cycle.
 312	 *
 313	 * However, the priority hint is a mere hint that we may need to
 314	 * preempt. If that hint is stale or we may be trying to preempt
 315	 * ourselves, ignore the request.
 316	 *
 317	 * More naturally we would write
 318	 *      prio >= max(0, last);
 319	 * except that we wish to prevent triggering preemption at the same
 320	 * priority level: the task that is running should remain running
 321	 * to preserve FIFO ordering of dependencies.
 322	 */
 323	last_prio = max(effective_prio(rq), I915_PRIORITY_NORMAL - 1);
 324	if (engine->sched_engine->queue_priority_hint <= last_prio)
 325		return false;
 326
 327	/*
 328	 * Check against the first request in ELSP[1], it will, thanks to the
 329	 * power of PI, be the highest priority of that context.
 330	 */
 331	if (!list_is_last(&rq->sched.link, &engine->sched_engine->requests) &&
 332	    rq_prio(list_next_entry(rq, sched.link)) > last_prio)
 333		return true;
 334
 335	/*
 336	 * If the inflight context did not trigger the preemption, then maybe
 337	 * it was the set of queued requests? Pick the highest priority in
 338	 * the queue (the first active priolist) and see if it deserves to be
 339	 * running instead of ELSP[0].
 340	 *
 341	 * The highest priority request in the queue can not be either
 342	 * ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same
 343	 * context, it's priority would not exceed ELSP[0] aka last_prio.
 344	 */
 345	return max(virtual_prio(&engine->execlists),
 346		   queue_prio(engine->sched_engine)) > last_prio;
 347}
 348
 349__maybe_unused static bool
 350assert_priority_queue(const struct i915_request *prev,
 351		      const struct i915_request *next)
 352{
 353	/*
 354	 * Without preemption, the prev may refer to the still active element
 355	 * which we refuse to let go.
 356	 *
 357	 * Even with preemption, there are times when we think it is better not
 358	 * to preempt and leave an ostensibly lower priority request in flight.
 359	 */
 360	if (i915_request_is_active(prev))
 361		return true;
 362
 363	return rq_prio(prev) >= rq_prio(next);
 364}
 365
 366static struct i915_request *
 367__unwind_incomplete_requests(struct intel_engine_cs *engine)
 368{
 369	struct i915_request *rq, *rn, *active = NULL;
 370	struct list_head *pl;
 371	int prio = I915_PRIORITY_INVALID;
 372
 373	lockdep_assert_held(&engine->sched_engine->lock);
 374
 375	list_for_each_entry_safe_reverse(rq, rn,
 376					 &engine->sched_engine->requests,
 377					 sched.link) {
 378		if (__i915_request_is_complete(rq)) {
 379			list_del_init(&rq->sched.link);
 380			continue;
 381		}
 382
 383		__i915_request_unsubmit(rq);
 384
 385		GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
 386		if (rq_prio(rq) != prio) {
 387			prio = rq_prio(rq);
 388			pl = i915_sched_lookup_priolist(engine->sched_engine,
 389							prio);
 390		}
 391		GEM_BUG_ON(i915_sched_engine_is_empty(engine->sched_engine));
 392
 393		list_move(&rq->sched.link, pl);
 394		set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
 395
 396		/* Check in case we rollback so far we wrap [size/2] */
 397		if (intel_ring_direction(rq->ring,
 398					 rq->tail,
 399					 rq->ring->tail + 8) > 0)
 400			rq->context->lrc.desc |= CTX_DESC_FORCE_RESTORE;
 401
 402		active = rq;
 403	}
 404
 405	return active;
 406}
 407
 408struct i915_request *
 409execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
 410{
 411	struct intel_engine_cs *engine =
 412		container_of(execlists, typeof(*engine), execlists);
 413
 414	return __unwind_incomplete_requests(engine);
 415}
 416
 417static void
 418execlists_context_status_change(struct i915_request *rq, unsigned long status)
 419{
 420	/*
 421	 * Only used when GVT-g is enabled now. When GVT-g is disabled,
 422	 * The compiler should eliminate this function as dead-code.
 423	 */
 424	if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
 425		return;
 426
 427	atomic_notifier_call_chain(&rq->engine->context_status_notifier,
 428				   status, rq);
 429}
 430
 431static void reset_active(struct i915_request *rq,
 432			 struct intel_engine_cs *engine)
 433{
 434	struct intel_context * const ce = rq->context;
 435	u32 head;
 436
 437	/*
 438	 * The executing context has been cancelled. We want to prevent
 439	 * further execution along this context and propagate the error on
 440	 * to anything depending on its results.
 441	 *
 442	 * In __i915_request_submit(), we apply the -EIO and remove the
 443	 * requests' payloads for any banned requests. But first, we must
 444	 * rewind the context back to the start of the incomplete request so
 445	 * that we do not jump back into the middle of the batch.
 446	 *
 447	 * We preserve the breadcrumbs and semaphores of the incomplete
 448	 * requests so that inter-timeline dependencies (i.e other timelines)
 449	 * remain correctly ordered. And we defer to __i915_request_submit()
 450	 * so that all asynchronous waits are correctly handled.
 451	 */
 452	ENGINE_TRACE(engine, "{ reset rq=%llx:%lld }\n",
 453		     rq->fence.context, rq->fence.seqno);
 454
 455	/* On resubmission of the active request, payload will be scrubbed */
 456	if (__i915_request_is_complete(rq))
 457		head = rq->tail;
 458	else
 459		head = __active_request(ce->timeline, rq, -EIO)->head;
 460	head = intel_ring_wrap(ce->ring, head);
 461
 462	/* Scrub the context image to prevent replaying the previous batch */
 463	lrc_init_regs(ce, engine, true);
 464
 465	/* We've switched away, so this should be a no-op, but intent matters */
 466	ce->lrc.lrca = lrc_update_regs(ce, engine, head);
 467}
 468
 469static bool bad_request(const struct i915_request *rq)
 470{
 471	return rq->fence.error && i915_request_started(rq);
 472}
 473
 474static struct intel_engine_cs *
 475__execlists_schedule_in(struct i915_request *rq)
 476{
 477	struct intel_engine_cs * const engine = rq->engine;
 478	struct intel_context * const ce = rq->context;
 479
 480	intel_context_get(ce);
 481
 482	if (unlikely(intel_context_is_closed(ce) &&
 483		     !intel_engine_has_heartbeat(engine)))
 484		intel_context_set_exiting(ce);
 485
 486	if (unlikely(!intel_context_is_schedulable(ce) || bad_request(rq)))
 487		reset_active(rq, engine);
 488
 489	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 490		lrc_check_regs(ce, engine, "before");
 491
 492	if (ce->tag) {
 493		/* Use a fixed tag for OA and friends */
 494		GEM_BUG_ON(ce->tag <= BITS_PER_LONG);
 495		ce->lrc.ccid = ce->tag;
 496	} else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
 497		/* We don't need a strict matching tag, just different values */
 498		unsigned int tag = ffs(READ_ONCE(engine->context_tag));
 499
 500		GEM_BUG_ON(tag == 0 || tag >= BITS_PER_LONG);
 501		clear_bit(tag - 1, &engine->context_tag);
 502		ce->lrc.ccid = tag << (XEHP_SW_CTX_ID_SHIFT - 32);
 503
 504		BUILD_BUG_ON(BITS_PER_LONG > GEN12_MAX_CONTEXT_HW_ID);
 505
 506	} else {
 507		/* We don't need a strict matching tag, just different values */
 508		unsigned int tag = __ffs(engine->context_tag);
 509
 510		GEM_BUG_ON(tag >= BITS_PER_LONG);
 511		__clear_bit(tag, &engine->context_tag);
 512		ce->lrc.ccid = (1 + tag) << (GEN11_SW_CTX_ID_SHIFT - 32);
 513
 514		BUILD_BUG_ON(BITS_PER_LONG > GEN12_MAX_CONTEXT_HW_ID);
 515	}
 516
 517	ce->lrc.ccid |= engine->execlists.ccid;
 518
 519	__intel_gt_pm_get(engine->gt);
 520	if (engine->fw_domain && !engine->fw_active++)
 521		intel_uncore_forcewake_get(engine->uncore, engine->fw_domain);
 522	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
 523	intel_engine_context_in(engine);
 524
 525	CE_TRACE(ce, "schedule-in, ccid:%x\n", ce->lrc.ccid);
 526
 527	return engine;
 528}
 529
 530static void execlists_schedule_in(struct i915_request *rq, int idx)
 531{
 532	struct intel_context * const ce = rq->context;
 533	struct intel_engine_cs *old;
 534
 535	GEM_BUG_ON(!intel_engine_pm_is_awake(rq->engine));
 536	trace_i915_request_in(rq, idx);
 537
 538	old = ce->inflight;
 539	if (!old)
 540		old = __execlists_schedule_in(rq);
 541	WRITE_ONCE(ce->inflight, ptr_inc(old));
 542
 543	GEM_BUG_ON(intel_context_inflight(ce) != rq->engine);
 544}
 545
 546static void
 547resubmit_virtual_request(struct i915_request *rq, struct virtual_engine *ve)
 548{
 549	struct intel_engine_cs *engine = rq->engine;
 550
 551	spin_lock_irq(&engine->sched_engine->lock);
 552
 553	clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
 554	WRITE_ONCE(rq->engine, &ve->base);
 555	ve->base.submit_request(rq);
 556
 557	spin_unlock_irq(&engine->sched_engine->lock);
 558}
 559
 560static void kick_siblings(struct i915_request *rq, struct intel_context *ce)
 561{
 562	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
 563	struct intel_engine_cs *engine = rq->engine;
 564
 565	/*
 566	 * After this point, the rq may be transferred to a new sibling, so
 567	 * before we clear ce->inflight make sure that the context has been
 568	 * removed from the b->signalers and furthermore we need to make sure
 569	 * that the concurrent iterator in signal_irq_work is no longer
 570	 * following ce->signal_link.
 571	 */
 572	if (!list_empty(&ce->signals))
 573		intel_context_remove_breadcrumbs(ce, engine->breadcrumbs);
 574
 575	/*
 576	 * This engine is now too busy to run this virtual request, so
 577	 * see if we can find an alternative engine for it to execute on.
 578	 * Once a request has become bonded to this engine, we treat it the
 579	 * same as other native request.
 580	 */
 581	if (i915_request_in_priority_queue(rq) &&
 582	    rq->execution_mask != engine->mask)
 583		resubmit_virtual_request(rq, ve);
 584
 585	if (READ_ONCE(ve->request))
 586		tasklet_hi_schedule(&ve->base.sched_engine->tasklet);
 587}
 588
 589static void __execlists_schedule_out(struct i915_request * const rq,
 590				     struct intel_context * const ce)
 591{
 592	struct intel_engine_cs * const engine = rq->engine;
 593	unsigned int ccid;
 594
 595	/*
 596	 * NB process_csb() is not under the engine->sched_engine->lock and hence
 597	 * schedule_out can race with schedule_in meaning that we should
 598	 * refrain from doing non-trivial work here.
 599	 */
 600
 601	CE_TRACE(ce, "schedule-out, ccid:%x\n", ce->lrc.ccid);
 602	GEM_BUG_ON(ce->inflight != engine);
 603
 604	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 605		lrc_check_regs(ce, engine, "after");
 606
 607	/*
 608	 * If we have just completed this context, the engine may now be
 609	 * idle and we want to re-enter powersaving.
 610	 */
 611	if (intel_timeline_is_last(ce->timeline, rq) &&
 612	    __i915_request_is_complete(rq))
 613		intel_engine_add_retire(engine, ce->timeline);
 614
 615	ccid = ce->lrc.ccid;
 616	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
 617		ccid >>= XEHP_SW_CTX_ID_SHIFT - 32;
 618		ccid &= XEHP_MAX_CONTEXT_HW_ID;
 619	} else {
 620		ccid >>= GEN11_SW_CTX_ID_SHIFT - 32;
 621		ccid &= GEN12_MAX_CONTEXT_HW_ID;
 622	}
 623
 624	if (ccid < BITS_PER_LONG) {
 625		GEM_BUG_ON(ccid == 0);
 626		GEM_BUG_ON(test_bit(ccid - 1, &engine->context_tag));
 627		__set_bit(ccid - 1, &engine->context_tag);
 628	}
 629	intel_engine_context_out(engine);
 630	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
 631	if (engine->fw_domain && !--engine->fw_active)
 632		intel_uncore_forcewake_put(engine->uncore, engine->fw_domain);
 633	intel_gt_pm_put_async(engine->gt);
 634
 635	/*
 636	 * If this is part of a virtual engine, its next request may
 637	 * have been blocked waiting for access to the active context.
 638	 * We have to kick all the siblings again in case we need to
 639	 * switch (e.g. the next request is not runnable on this
 640	 * engine). Hopefully, we will already have submitted the next
 641	 * request before the tasklet runs and do not need to rebuild
 642	 * each virtual tree and kick everyone again.
 643	 */
 644	if (ce->engine != engine)
 645		kick_siblings(rq, ce);
 646
 647	WRITE_ONCE(ce->inflight, NULL);
 648	intel_context_put(ce);
 649}
 650
 651static inline void execlists_schedule_out(struct i915_request *rq)
 652{
 653	struct intel_context * const ce = rq->context;
 654
 655	trace_i915_request_out(rq);
 656
 657	GEM_BUG_ON(!ce->inflight);
 658	ce->inflight = ptr_dec(ce->inflight);
 659	if (!__intel_context_inflight_count(ce->inflight))
 660		__execlists_schedule_out(rq, ce);
 661
 662	i915_request_put(rq);
 663}
 664
 665static u32 map_i915_prio_to_lrc_desc_prio(int prio)
 666{
 667	if (prio > I915_PRIORITY_NORMAL)
 668		return GEN12_CTX_PRIORITY_HIGH;
 669	else if (prio < I915_PRIORITY_NORMAL)
 670		return GEN12_CTX_PRIORITY_LOW;
 671	else
 672		return GEN12_CTX_PRIORITY_NORMAL;
 673}
 674
 675static u64 execlists_update_context(struct i915_request *rq)
 676{
 677	struct intel_context *ce = rq->context;
 678	u64 desc;
 679	u32 tail, prev;
 680
 681	desc = ce->lrc.desc;
 682	if (rq->engine->flags & I915_ENGINE_HAS_EU_PRIORITY)
 683		desc |= map_i915_prio_to_lrc_desc_prio(rq_prio(rq));
 684
 685	/*
 686	 * WaIdleLiteRestore:bdw,skl
 687	 *
 688	 * We should never submit the context with the same RING_TAIL twice
 689	 * just in case we submit an empty ring, which confuses the HW.
 690	 *
 691	 * We append a couple of NOOPs (gen8_emit_wa_tail) after the end of
 692	 * the normal request to be able to always advance the RING_TAIL on
 693	 * subsequent resubmissions (for lite restore). Should that fail us,
 694	 * and we try and submit the same tail again, force the context
 695	 * reload.
 696	 *
 697	 * If we need to return to a preempted context, we need to skip the
 698	 * lite-restore and force it to reload the RING_TAIL. Otherwise, the
 699	 * HW has a tendency to ignore us rewinding the TAIL to the end of
 700	 * an earlier request.
 701	 */
 702	GEM_BUG_ON(ce->lrc_reg_state[CTX_RING_TAIL] != rq->ring->tail);
 703	prev = rq->ring->tail;
 704	tail = intel_ring_set_tail(rq->ring, rq->tail);
 705	if (unlikely(intel_ring_direction(rq->ring, tail, prev) <= 0))
 706		desc |= CTX_DESC_FORCE_RESTORE;
 707	ce->lrc_reg_state[CTX_RING_TAIL] = tail;
 708	rq->tail = rq->wa_tail;
 709
 710	/*
 711	 * Make sure the context image is complete before we submit it to HW.
 712	 *
 713	 * Ostensibly, writes (including the WCB) should be flushed prior to
 714	 * an uncached write such as our mmio register access, the empirical
 715	 * evidence (esp. on Braswell) suggests that the WC write into memory
 716	 * may not be visible to the HW prior to the completion of the UC
 717	 * register write and that we may begin execution from the context
 718	 * before its image is complete leading to invalid PD chasing.
 719	 */
 720	wmb();
 721
 722	ce->lrc.desc &= ~CTX_DESC_FORCE_RESTORE;
 723	return desc;
 724}
 725
 726static void write_desc(struct intel_engine_execlists *execlists, u64 desc, u32 port)
 727{
 728	if (execlists->ctrl_reg) {
 729		writel(lower_32_bits(desc), execlists->submit_reg + port * 2);
 730		writel(upper_32_bits(desc), execlists->submit_reg + port * 2 + 1);
 731	} else {
 732		writel(upper_32_bits(desc), execlists->submit_reg);
 733		writel(lower_32_bits(desc), execlists->submit_reg);
 734	}
 735}
 736
 737static __maybe_unused char *
 738dump_port(char *buf, int buflen, const char *prefix, struct i915_request *rq)
 739{
 740	if (!rq)
 741		return "";
 742
 743	snprintf(buf, buflen, "%sccid:%x %llx:%lld%s prio %d",
 744		 prefix,
 745		 rq->context->lrc.ccid,
 746		 rq->fence.context, rq->fence.seqno,
 747		 __i915_request_is_complete(rq) ? "!" :
 748		 __i915_request_has_started(rq) ? "*" :
 749		 "",
 750		 rq_prio(rq));
 751
 752	return buf;
 753}
 754
 755static __maybe_unused noinline void
 756trace_ports(const struct intel_engine_execlists *execlists,
 757	    const char *msg,
 758	    struct i915_request * const *ports)
 759{
 760	const struct intel_engine_cs *engine =
 761		container_of(execlists, typeof(*engine), execlists);
 762	char __maybe_unused p0[40], p1[40];
 763
 764	if (!ports[0])
 765		return;
 766
 767	ENGINE_TRACE(engine, "%s { %s%s }\n", msg,
 768		     dump_port(p0, sizeof(p0), "", ports[0]),
 769		     dump_port(p1, sizeof(p1), ", ", ports[1]));
 770}
 771
 772static bool
 773reset_in_progress(const struct intel_engine_cs *engine)
 774{
 775	return unlikely(!__tasklet_is_enabled(&engine->sched_engine->tasklet));
 776}
 777
 778static __maybe_unused noinline bool
 779assert_pending_valid(const struct intel_engine_execlists *execlists,
 780		     const char *msg)
 781{
 782	struct intel_engine_cs *engine =
 783		container_of(execlists, typeof(*engine), execlists);
 784	struct i915_request * const *port, *rq, *prev = NULL;
 785	struct intel_context *ce = NULL;
 786	u32 ccid = -1;
 787
 788	trace_ports(execlists, msg, execlists->pending);
 789
 790	/* We may be messing around with the lists during reset, lalala */
 791	if (reset_in_progress(engine))
 792		return true;
 793
 794	if (!execlists->pending[0]) {
 795		GEM_TRACE_ERR("%s: Nothing pending for promotion!\n",
 796			      engine->name);
 797		return false;
 798	}
 799
 800	if (execlists->pending[execlists_num_ports(execlists)]) {
 801		GEM_TRACE_ERR("%s: Excess pending[%d] for promotion!\n",
 802			      engine->name, execlists_num_ports(execlists));
 803		return false;
 804	}
 805
 806	for (port = execlists->pending; (rq = *port); port++) {
 807		unsigned long flags;
 808		bool ok = true;
 809
 810		GEM_BUG_ON(!kref_read(&rq->fence.refcount));
 811		GEM_BUG_ON(!i915_request_is_active(rq));
 812
 813		if (ce == rq->context) {
 814			GEM_TRACE_ERR("%s: Dup context:%llx in pending[%zd]\n",
 815				      engine->name,
 816				      ce->timeline->fence_context,
 817				      port - execlists->pending);
 818			return false;
 819		}
 820		ce = rq->context;
 821
 822		if (ccid == ce->lrc.ccid) {
 823			GEM_TRACE_ERR("%s: Dup ccid:%x context:%llx in pending[%zd]\n",
 824				      engine->name,
 825				      ccid, ce->timeline->fence_context,
 826				      port - execlists->pending);
 827			return false;
 828		}
 829		ccid = ce->lrc.ccid;
 830
 831		/*
 832		 * Sentinels are supposed to be the last request so they flush
 833		 * the current execution off the HW. Check that they are the only
 834		 * request in the pending submission.
 835		 *
 836		 * NB: Due to the async nature of preempt-to-busy and request
 837		 * cancellation we need to handle the case where request
 838		 * becomes a sentinel in parallel to CSB processing.
 839		 */
 840		if (prev && i915_request_has_sentinel(prev) &&
 841		    !READ_ONCE(prev->fence.error)) {
 842			GEM_TRACE_ERR("%s: context:%llx after sentinel in pending[%zd]\n",
 843				      engine->name,
 844				      ce->timeline->fence_context,
 845				      port - execlists->pending);
 846			return false;
 847		}
 848		prev = rq;
 849
 850		/*
 851		 * We want virtual requests to only be in the first slot so
 852		 * that they are never stuck behind a hog and can be immediately
 853		 * transferred onto the next idle engine.
 854		 */
 855		if (rq->execution_mask != engine->mask &&
 856		    port != execlists->pending) {
 857			GEM_TRACE_ERR("%s: virtual engine:%llx not in prime position[%zd]\n",
 858				      engine->name,
 859				      ce->timeline->fence_context,
 860				      port - execlists->pending);
 861			return false;
 862		}
 863
 864		/* Hold tightly onto the lock to prevent concurrent retires! */
 865		if (!spin_trylock_irqsave(&rq->lock, flags))
 866			continue;
 867
 868		if (__i915_request_is_complete(rq))
 869			goto unlock;
 870
 871		if (i915_active_is_idle(&ce->active) &&
 872		    !intel_context_is_barrier(ce)) {
 873			GEM_TRACE_ERR("%s: Inactive context:%llx in pending[%zd]\n",
 874				      engine->name,
 875				      ce->timeline->fence_context,
 876				      port - execlists->pending);
 877			ok = false;
 878			goto unlock;
 879		}
 880
 881		if (!i915_vma_is_pinned(ce->state)) {
 882			GEM_TRACE_ERR("%s: Unpinned context:%llx in pending[%zd]\n",
 883				      engine->name,
 884				      ce->timeline->fence_context,
 885				      port - execlists->pending);
 886			ok = false;
 887			goto unlock;
 888		}
 889
 890		if (!i915_vma_is_pinned(ce->ring->vma)) {
 891			GEM_TRACE_ERR("%s: Unpinned ring:%llx in pending[%zd]\n",
 892				      engine->name,
 893				      ce->timeline->fence_context,
 894				      port - execlists->pending);
 895			ok = false;
 896			goto unlock;
 897		}
 898
 899unlock:
 900		spin_unlock_irqrestore(&rq->lock, flags);
 901		if (!ok)
 902			return false;
 903	}
 904
 905	return ce;
 906}
 907
 908static void execlists_submit_ports(struct intel_engine_cs *engine)
 909{
 910	struct intel_engine_execlists *execlists = &engine->execlists;
 911	unsigned int n;
 912
 913	GEM_BUG_ON(!assert_pending_valid(execlists, "submit"));
 914
 915	/*
 916	 * We can skip acquiring intel_runtime_pm_get() here as it was taken
 917	 * on our behalf by the request (see i915_gem_mark_busy()) and it will
 918	 * not be relinquished until the device is idle (see
 919	 * i915_gem_idle_work_handler()). As a precaution, we make sure
 920	 * that all ELSP are drained i.e. we have processed the CSB,
 921	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
 922	 */
 923	GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
 924
 925	/*
 926	 * ELSQ note: the submit queue is not cleared after being submitted
 927	 * to the HW so we need to make sure we always clean it up. This is
 928	 * currently ensured by the fact that we always write the same number
 929	 * of elsq entries, keep this in mind before changing the loop below.
 930	 */
 931	for (n = execlists_num_ports(execlists); n--; ) {
 932		struct i915_request *rq = execlists->pending[n];
 933
 934		write_desc(execlists,
 935			   rq ? execlists_update_context(rq) : 0,
 936			   n);
 937	}
 938
 939	/* we need to manually load the submit queue */
 940	if (execlists->ctrl_reg)
 941		writel(EL_CTRL_LOAD, execlists->ctrl_reg);
 942}
 943
 944static bool ctx_single_port_submission(const struct intel_context *ce)
 945{
 946	return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
 947		intel_context_force_single_submission(ce));
 948}
 949
 950static bool can_merge_ctx(const struct intel_context *prev,
 951			  const struct intel_context *next)
 952{
 953	if (prev != next)
 954		return false;
 955
 956	if (ctx_single_port_submission(prev))
 957		return false;
 958
 959	return true;
 960}
 961
 962static unsigned long i915_request_flags(const struct i915_request *rq)
 963{
 964	return READ_ONCE(rq->fence.flags);
 965}
 966
 967static bool can_merge_rq(const struct i915_request *prev,
 968			 const struct i915_request *next)
 969{
 970	GEM_BUG_ON(prev == next);
 971	GEM_BUG_ON(!assert_priority_queue(prev, next));
 972
 973	/*
 974	 * We do not submit known completed requests. Therefore if the next
 975	 * request is already completed, we can pretend to merge it in
 976	 * with the previous context (and we will skip updating the ELSP
 977	 * and tracking). Thus hopefully keeping the ELSP full with active
 978	 * contexts, despite the best efforts of preempt-to-busy to confuse
 979	 * us.
 980	 */
 981	if (__i915_request_is_complete(next))
 982		return true;
 983
 984	if (unlikely((i915_request_flags(prev) | i915_request_flags(next)) &
 985		     (BIT(I915_FENCE_FLAG_NOPREEMPT) |
 986		      BIT(I915_FENCE_FLAG_SENTINEL))))
 987		return false;
 988
 989	if (!can_merge_ctx(prev->context, next->context))
 990		return false;
 991
 992	GEM_BUG_ON(i915_seqno_passed(prev->fence.seqno, next->fence.seqno));
 993	return true;
 994}
 995
 996static bool virtual_matches(const struct virtual_engine *ve,
 997			    const struct i915_request *rq,
 998			    const struct intel_engine_cs *engine)
 999{
1000	const struct intel_engine_cs *inflight;
1001
1002	if (!rq)
1003		return false;
1004
1005	if (!(rq->execution_mask & engine->mask)) /* We peeked too soon! */
1006		return false;
1007
1008	/*
1009	 * We track when the HW has completed saving the context image
1010	 * (i.e. when we have seen the final CS event switching out of
1011	 * the context) and must not overwrite the context image before
1012	 * then. This restricts us to only using the active engine
1013	 * while the previous virtualized request is inflight (so
1014	 * we reuse the register offsets). This is a very small
1015	 * hystersis on the greedy seelction algorithm.
1016	 */
1017	inflight = intel_context_inflight(&ve->context);
1018	if (inflight && inflight != engine)
1019		return false;
1020
1021	return true;
1022}
1023
1024static struct virtual_engine *
1025first_virtual_engine(struct intel_engine_cs *engine)
1026{
1027	struct intel_engine_execlists *el = &engine->execlists;
1028	struct rb_node *rb = rb_first_cached(&el->virtual);
1029
1030	while (rb) {
1031		struct virtual_engine *ve =
1032			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
1033		struct i915_request *rq = READ_ONCE(ve->request);
1034
1035		/* lazily cleanup after another engine handled rq */
1036		if (!rq || !virtual_matches(ve, rq, engine)) {
1037			rb_erase_cached(rb, &el->virtual);
1038			RB_CLEAR_NODE(rb);
1039			rb = rb_first_cached(&el->virtual);
1040			continue;
1041		}
1042
1043		return ve;
1044	}
1045
1046	return NULL;
1047}
1048
1049static void virtual_xfer_context(struct virtual_engine *ve,
1050				 struct intel_engine_cs *engine)
1051{
1052	unsigned int n;
1053
1054	if (likely(engine == ve->siblings[0]))
1055		return;
1056
1057	GEM_BUG_ON(READ_ONCE(ve->context.inflight));
1058	if (!intel_engine_has_relative_mmio(engine))
1059		lrc_update_offsets(&ve->context, engine);
1060
1061	/*
1062	 * Move the bound engine to the top of the list for
1063	 * future execution. We then kick this tasklet first
1064	 * before checking others, so that we preferentially
1065	 * reuse this set of bound registers.
1066	 */
1067	for (n = 1; n < ve->num_siblings; n++) {
1068		if (ve->siblings[n] == engine) {
1069			swap(ve->siblings[n], ve->siblings[0]);
1070			break;
1071		}
1072	}
1073}
1074
1075static void defer_request(struct i915_request *rq, struct list_head * const pl)
1076{
1077	LIST_HEAD(list);
1078
1079	/*
1080	 * We want to move the interrupted request to the back of
1081	 * the round-robin list (i.e. its priority level), but
1082	 * in doing so, we must then move all requests that were in
1083	 * flight and were waiting for the interrupted request to
1084	 * be run after it again.
1085	 */
1086	do {
1087		struct i915_dependency *p;
1088
1089		GEM_BUG_ON(i915_request_is_active(rq));
1090		list_move_tail(&rq->sched.link, pl);
1091
1092		for_each_waiter(p, rq) {
1093			struct i915_request *w =
1094				container_of(p->waiter, typeof(*w), sched);
1095
1096			if (p->flags & I915_DEPENDENCY_WEAK)
1097				continue;
1098
1099			/* Leave semaphores spinning on the other engines */
1100			if (w->engine != rq->engine)
1101				continue;
1102
1103			/* No waiter should start before its signaler */
1104			GEM_BUG_ON(i915_request_has_initial_breadcrumb(w) &&
1105				   __i915_request_has_started(w) &&
1106				   !__i915_request_is_complete(rq));
1107
1108			if (!i915_request_is_ready(w))
1109				continue;
1110
1111			if (rq_prio(w) < rq_prio(rq))
1112				continue;
1113
1114			GEM_BUG_ON(rq_prio(w) > rq_prio(rq));
1115			GEM_BUG_ON(i915_request_is_active(w));
1116			list_move_tail(&w->sched.link, &list);
1117		}
1118
1119		rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
1120	} while (rq);
1121}
1122
1123static void defer_active(struct intel_engine_cs *engine)
1124{
1125	struct i915_request *rq;
1126
1127	rq = __unwind_incomplete_requests(engine);
1128	if (!rq)
1129		return;
1130
1131	defer_request(rq, i915_sched_lookup_priolist(engine->sched_engine,
1132						     rq_prio(rq)));
1133}
1134
1135static bool
1136timeslice_yield(const struct intel_engine_execlists *el,
1137		const struct i915_request *rq)
1138{
1139	/*
1140	 * Once bitten, forever smitten!
1141	 *
1142	 * If the active context ever busy-waited on a semaphore,
1143	 * it will be treated as a hog until the end of its timeslice (i.e.
1144	 * until it is scheduled out and replaced by a new submission,
1145	 * possibly even its own lite-restore). The HW only sends an interrupt
1146	 * on the first miss, and we do know if that semaphore has been
1147	 * signaled, or even if it is now stuck on another semaphore. Play
1148	 * safe, yield if it might be stuck -- it will be given a fresh
1149	 * timeslice in the near future.
1150	 */
1151	return rq->context->lrc.ccid == READ_ONCE(el->yield);
1152}
1153
1154static bool needs_timeslice(const struct intel_engine_cs *engine,
1155			    const struct i915_request *rq)
1156{
1157	if (!intel_engine_has_timeslices(engine))
1158		return false;
1159
1160	/* If not currently active, or about to switch, wait for next event */
1161	if (!rq || __i915_request_is_complete(rq))
1162		return false;
1163
1164	/* We do not need to start the timeslice until after the ACK */
1165	if (READ_ONCE(engine->execlists.pending[0]))
1166		return false;
1167
1168	/* If ELSP[1] is occupied, always check to see if worth slicing */
1169	if (!list_is_last_rcu(&rq->sched.link,
1170			      &engine->sched_engine->requests)) {
1171		ENGINE_TRACE(engine, "timeslice required for second inflight context\n");
1172		return true;
1173	}
1174
1175	/* Otherwise, ELSP[0] is by itself, but may be waiting in the queue */
1176	if (!i915_sched_engine_is_empty(engine->sched_engine)) {
1177		ENGINE_TRACE(engine, "timeslice required for queue\n");
1178		return true;
1179	}
1180
1181	if (!RB_EMPTY_ROOT(&engine->execlists.virtual.rb_root)) {
1182		ENGINE_TRACE(engine, "timeslice required for virtual\n");
1183		return true;
1184	}
1185
1186	return false;
1187}
1188
1189static bool
1190timeslice_expired(struct intel_engine_cs *engine, const struct i915_request *rq)
1191{
1192	const struct intel_engine_execlists *el = &engine->execlists;
1193
1194	if (i915_request_has_nopreempt(rq) && __i915_request_has_started(rq))
1195		return false;
1196
1197	if (!needs_timeslice(engine, rq))
1198		return false;
1199
1200	return timer_expired(&el->timer) || timeslice_yield(el, rq);
1201}
1202
1203static unsigned long timeslice(const struct intel_engine_cs *engine)
1204{
1205	return READ_ONCE(engine->props.timeslice_duration_ms);
1206}
1207
1208static void start_timeslice(struct intel_engine_cs *engine)
1209{
1210	struct intel_engine_execlists *el = &engine->execlists;
1211	unsigned long duration;
1212
1213	/* Disable the timer if there is nothing to switch to */
1214	duration = 0;
1215	if (needs_timeslice(engine, *el->active)) {
1216		/* Avoid continually prolonging an active timeslice */
1217		if (timer_active(&el->timer)) {
1218			/*
1219			 * If we just submitted a new ELSP after an old
1220			 * context, that context may have already consumed
1221			 * its timeslice, so recheck.
1222			 */
1223			if (!timer_pending(&el->timer))
1224				tasklet_hi_schedule(&engine->sched_engine->tasklet);
1225			return;
1226		}
1227
1228		duration = timeslice(engine);
1229	}
1230
1231	set_timer_ms(&el->timer, duration);
1232}
1233
1234static void record_preemption(struct intel_engine_execlists *execlists)
1235{
1236	(void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++);
1237}
1238
1239static unsigned long active_preempt_timeout(struct intel_engine_cs *engine,
1240					    const struct i915_request *rq)
1241{
1242	if (!rq)
1243		return 0;
1244
1245	/* Only allow ourselves to force reset the currently active context */
1246	engine->execlists.preempt_target = rq;
1247
1248	/* Force a fast reset for terminated contexts (ignoring sysfs!) */
1249	if (unlikely(intel_context_is_banned(rq->context) || bad_request(rq)))
1250		return INTEL_CONTEXT_BANNED_PREEMPT_TIMEOUT_MS;
1251
1252	return READ_ONCE(engine->props.preempt_timeout_ms);
1253}
1254
1255static void set_preempt_timeout(struct intel_engine_cs *engine,
1256				const struct i915_request *rq)
1257{
1258	if (!intel_engine_has_preempt_reset(engine))
1259		return;
1260
1261	set_timer_ms(&engine->execlists.preempt,
1262		     active_preempt_timeout(engine, rq));
1263}
1264
1265static bool completed(const struct i915_request *rq)
1266{
1267	if (i915_request_has_sentinel(rq))
1268		return false;
1269
1270	return __i915_request_is_complete(rq);
1271}
1272
1273static void execlists_dequeue(struct intel_engine_cs *engine)
1274{
1275	struct intel_engine_execlists * const execlists = &engine->execlists;
1276	struct i915_sched_engine * const sched_engine = engine->sched_engine;
1277	struct i915_request **port = execlists->pending;
1278	struct i915_request ** const last_port = port + execlists->port_mask;
1279	struct i915_request *last, * const *active;
1280	struct virtual_engine *ve;
1281	struct rb_node *rb;
1282	bool submit = false;
1283
1284	/*
1285	 * Hardware submission is through 2 ports. Conceptually each port
1286	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
1287	 * static for a context, and unique to each, so we only execute
1288	 * requests belonging to a single context from each ring. RING_HEAD
1289	 * is maintained by the CS in the context image, it marks the place
1290	 * where it got up to last time, and through RING_TAIL we tell the CS
1291	 * where we want to execute up to this time.
1292	 *
1293	 * In this list the requests are in order of execution. Consecutive
1294	 * requests from the same context are adjacent in the ringbuffer. We
1295	 * can combine these requests into a single RING_TAIL update:
1296	 *
1297	 *              RING_HEAD...req1...req2
1298	 *                                    ^- RING_TAIL
1299	 * since to execute req2 the CS must first execute req1.
1300	 *
1301	 * Our goal then is to point each port to the end of a consecutive
1302	 * sequence of requests as being the most optimal (fewest wake ups
1303	 * and context switches) submission.
1304	 */
1305
1306	spin_lock(&sched_engine->lock);
1307
1308	/*
1309	 * If the queue is higher priority than the last
1310	 * request in the currently active context, submit afresh.
1311	 * We will resubmit again afterwards in case we need to split
1312	 * the active context to interject the preemption request,
1313	 * i.e. we will retrigger preemption following the ack in case
1314	 * of trouble.
1315	 *
1316	 */
1317	active = execlists->active;
1318	while ((last = *active) && completed(last))
1319		active++;
1320
1321	if (last) {
1322		if (need_preempt(engine, last)) {
1323			ENGINE_TRACE(engine,
1324				     "preempting last=%llx:%lld, prio=%d, hint=%d\n",
1325				     last->fence.context,
1326				     last->fence.seqno,
1327				     last->sched.attr.priority,
1328				     sched_engine->queue_priority_hint);
1329			record_preemption(execlists);
1330
1331			/*
1332			 * Don't let the RING_HEAD advance past the breadcrumb
1333			 * as we unwind (and until we resubmit) so that we do
1334			 * not accidentally tell it to go backwards.
1335			 */
1336			ring_set_paused(engine, 1);
1337
1338			/*
1339			 * Note that we have not stopped the GPU at this point,
1340			 * so we are unwinding the incomplete requests as they
1341			 * remain inflight and so by the time we do complete
1342			 * the preemption, some of the unwound requests may
1343			 * complete!
1344			 */
1345			__unwind_incomplete_requests(engine);
1346
1347			last = NULL;
1348		} else if (timeslice_expired(engine, last)) {
1349			ENGINE_TRACE(engine,
1350				     "expired:%s last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n",
1351				     str_yes_no(timer_expired(&execlists->timer)),
1352				     last->fence.context, last->fence.seqno,
1353				     rq_prio(last),
1354				     sched_engine->queue_priority_hint,
1355				     str_yes_no(timeslice_yield(execlists, last)));
1356
1357			/*
1358			 * Consume this timeslice; ensure we start a new one.
1359			 *
1360			 * The timeslice expired, and we will unwind the
1361			 * running contexts and recompute the next ELSP.
1362			 * If that submit will be the same pair of contexts
1363			 * (due to dependency ordering), we will skip the
1364			 * submission. If we don't cancel the timer now,
1365			 * we will see that the timer has expired and
1366			 * reschedule the tasklet; continually until the
1367			 * next context switch or other preemption event.
1368			 *
1369			 * Since we have decided to reschedule based on
1370			 * consumption of this timeslice, if we submit the
1371			 * same context again, grant it a full timeslice.
1372			 */
1373			cancel_timer(&execlists->timer);
1374			ring_set_paused(engine, 1);
1375			defer_active(engine);
1376
1377			/*
1378			 * Unlike for preemption, if we rewind and continue
1379			 * executing the same context as previously active,
1380			 * the order of execution will remain the same and
1381			 * the tail will only advance. We do not need to
1382			 * force a full context restore, as a lite-restore
1383			 * is sufficient to resample the monotonic TAIL.
1384			 *
1385			 * If we switch to any other context, similarly we
1386			 * will not rewind TAIL of current context, and
1387			 * normal save/restore will preserve state and allow
1388			 * us to later continue executing the same request.
1389			 */
1390			last = NULL;
1391		} else {
1392			/*
1393			 * Otherwise if we already have a request pending
1394			 * for execution after the current one, we can
1395			 * just wait until the next CS event before
1396			 * queuing more. In either case we will force a
1397			 * lite-restore preemption event, but if we wait
1398			 * we hopefully coalesce several updates into a single
1399			 * submission.
1400			 */
1401			if (active[1]) {
1402				/*
1403				 * Even if ELSP[1] is occupied and not worthy
1404				 * of timeslices, our queue might be.
1405				 */
1406				spin_unlock(&sched_engine->lock);
1407				return;
1408			}
1409		}
1410	}
1411
1412	/* XXX virtual is always taking precedence */
1413	while ((ve = first_virtual_engine(engine))) {
1414		struct i915_request *rq;
1415
1416		spin_lock(&ve->base.sched_engine->lock);
1417
1418		rq = ve->request;
1419		if (unlikely(!virtual_matches(ve, rq, engine)))
1420			goto unlock; /* lost the race to a sibling */
1421
1422		GEM_BUG_ON(rq->engine != &ve->base);
1423		GEM_BUG_ON(rq->context != &ve->context);
1424
1425		if (unlikely(rq_prio(rq) < queue_prio(sched_engine))) {
1426			spin_unlock(&ve->base.sched_engine->lock);
1427			break;
1428		}
1429
1430		if (last && !can_merge_rq(last, rq)) {
1431			spin_unlock(&ve->base.sched_engine->lock);
1432			spin_unlock(&engine->sched_engine->lock);
1433			return; /* leave this for another sibling */
1434		}
1435
1436		ENGINE_TRACE(engine,
1437			     "virtual rq=%llx:%lld%s, new engine? %s\n",
1438			     rq->fence.context,
1439			     rq->fence.seqno,
1440			     __i915_request_is_complete(rq) ? "!" :
1441			     __i915_request_has_started(rq) ? "*" :
1442			     "",
1443			     str_yes_no(engine != ve->siblings[0]));
1444
1445		WRITE_ONCE(ve->request, NULL);
1446		WRITE_ONCE(ve->base.sched_engine->queue_priority_hint, INT_MIN);
1447
1448		rb = &ve->nodes[engine->id].rb;
1449		rb_erase_cached(rb, &execlists->virtual);
1450		RB_CLEAR_NODE(rb);
1451
1452		GEM_BUG_ON(!(rq->execution_mask & engine->mask));
1453		WRITE_ONCE(rq->engine, engine);
1454
1455		if (__i915_request_submit(rq)) {
1456			/*
1457			 * Only after we confirm that we will submit
1458			 * this request (i.e. it has not already
1459			 * completed), do we want to update the context.
1460			 *
1461			 * This serves two purposes. It avoids
1462			 * unnecessary work if we are resubmitting an
1463			 * already completed request after timeslicing.
1464			 * But more importantly, it prevents us altering
1465			 * ve->siblings[] on an idle context, where
1466			 * we may be using ve->siblings[] in
1467			 * virtual_context_enter / virtual_context_exit.
1468			 */
1469			virtual_xfer_context(ve, engine);
1470			GEM_BUG_ON(ve->siblings[0] != engine);
1471
1472			submit = true;
1473			last = rq;
1474		}
1475
1476		i915_request_put(rq);
1477unlock:
1478		spin_unlock(&ve->base.sched_engine->lock);
1479
1480		/*
1481		 * Hmm, we have a bunch of virtual engine requests,
1482		 * but the first one was already completed (thanks
1483		 * preempt-to-busy!). Keep looking at the veng queue
1484		 * until we have no more relevant requests (i.e.
1485		 * the normal submit queue has higher priority).
1486		 */
1487		if (submit)
1488			break;
1489	}
1490
1491	while ((rb = rb_first_cached(&sched_engine->queue))) {
1492		struct i915_priolist *p = to_priolist(rb);
1493		struct i915_request *rq, *rn;
1494
1495		priolist_for_each_request_consume(rq, rn, p) {
1496			bool merge = true;
1497
1498			/*
1499			 * Can we combine this request with the current port?
1500			 * It has to be the same context/ringbuffer and not
1501			 * have any exceptions (e.g. GVT saying never to
1502			 * combine contexts).
1503			 *
1504			 * If we can combine the requests, we can execute both
1505			 * by updating the RING_TAIL to point to the end of the
1506			 * second request, and so we never need to tell the
1507			 * hardware about the first.
1508			 */
1509			if (last && !can_merge_rq(last, rq)) {
1510				/*
1511				 * If we are on the second port and cannot
1512				 * combine this request with the last, then we
1513				 * are done.
1514				 */
1515				if (port == last_port)
1516					goto done;
1517
1518				/*
1519				 * We must not populate both ELSP[] with the
1520				 * same LRCA, i.e. we must submit 2 different
1521				 * contexts if we submit 2 ELSP.
1522				 */
1523				if (last->context == rq->context)
1524					goto done;
1525
1526				if (i915_request_has_sentinel(last))
1527					goto done;
1528
1529				/*
1530				 * We avoid submitting virtual requests into
1531				 * the secondary ports so that we can migrate
1532				 * the request immediately to another engine
1533				 * rather than wait for the primary request.
1534				 */
1535				if (rq->execution_mask != engine->mask)
1536					goto done;
1537
1538				/*
1539				 * If GVT overrides us we only ever submit
1540				 * port[0], leaving port[1] empty. Note that we
1541				 * also have to be careful that we don't queue
1542				 * the same context (even though a different
1543				 * request) to the second port.
1544				 */
1545				if (ctx_single_port_submission(last->context) ||
1546				    ctx_single_port_submission(rq->context))
1547					goto done;
1548
1549				merge = false;
1550			}
1551
1552			if (__i915_request_submit(rq)) {
1553				if (!merge) {
1554					*port++ = i915_request_get(last);
1555					last = NULL;
1556				}
1557
1558				GEM_BUG_ON(last &&
1559					   !can_merge_ctx(last->context,
1560							  rq->context));
1561				GEM_BUG_ON(last &&
1562					   i915_seqno_passed(last->fence.seqno,
1563							     rq->fence.seqno));
1564
1565				submit = true;
1566				last = rq;
1567			}
1568		}
1569
1570		rb_erase_cached(&p->node, &sched_engine->queue);
1571		i915_priolist_free(p);
1572	}
1573done:
1574	*port++ = i915_request_get(last);
1575
1576	/*
1577	 * Here be a bit of magic! Or sleight-of-hand, whichever you prefer.
1578	 *
1579	 * We choose the priority hint such that if we add a request of greater
1580	 * priority than this, we kick the submission tasklet to decide on
1581	 * the right order of submitting the requests to hardware. We must
1582	 * also be prepared to reorder requests as they are in-flight on the
1583	 * HW. We derive the priority hint then as the first "hole" in
1584	 * the HW submission ports and if there are no available slots,
1585	 * the priority of the lowest executing request, i.e. last.
1586	 *
1587	 * When we do receive a higher priority request ready to run from the
1588	 * user, see queue_request(), the priority hint is bumped to that
1589	 * request triggering preemption on the next dequeue (or subsequent
1590	 * interrupt for secondary ports).
1591	 */
1592	sched_engine->queue_priority_hint = queue_prio(sched_engine);
1593	i915_sched_engine_reset_on_empty(sched_engine);
1594	spin_unlock(&sched_engine->lock);
1595
1596	/*
1597	 * We can skip poking the HW if we ended up with exactly the same set
1598	 * of requests as currently running, e.g. trying to timeslice a pair
1599	 * of ordered contexts.
1600	 */
1601	if (submit &&
1602	    memcmp(active,
1603		   execlists->pending,
1604		   (port - execlists->pending) * sizeof(*port))) {
1605		*port = NULL;
1606		while (port-- != execlists->pending)
1607			execlists_schedule_in(*port, port - execlists->pending);
1608
1609		WRITE_ONCE(execlists->yield, -1);
1610		set_preempt_timeout(engine, *active);
1611		execlists_submit_ports(engine);
1612	} else {
1613		ring_set_paused(engine, 0);
1614		while (port-- != execlists->pending)
1615			i915_request_put(*port);
1616		*execlists->pending = NULL;
1617	}
1618}
1619
1620static void execlists_dequeue_irq(struct intel_engine_cs *engine)
1621{
1622	local_irq_disable(); /* Suspend interrupts across request submission */
1623	execlists_dequeue(engine);
1624	local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */
1625}
1626
1627static void clear_ports(struct i915_request **ports, int count)
1628{
1629	memset_p((void **)ports, NULL, count);
1630}
1631
1632static void
1633copy_ports(struct i915_request **dst, struct i915_request **src, int count)
1634{
1635	/* A memcpy_p() would be very useful here! */
1636	while (count--)
1637		WRITE_ONCE(*dst++, *src++); /* avoid write tearing */
1638}
1639
1640static struct i915_request **
1641cancel_port_requests(struct intel_engine_execlists * const execlists,
1642		     struct i915_request **inactive)
1643{
1644	struct i915_request * const *port;
1645
1646	for (port = execlists->pending; *port; port++)
1647		*inactive++ = *port;
1648	clear_ports(execlists->pending, ARRAY_SIZE(execlists->pending));
1649
1650	/* Mark the end of active before we overwrite *active */
1651	for (port = xchg(&execlists->active, execlists->pending); *port; port++)
1652		*inactive++ = *port;
1653	clear_ports(execlists->inflight, ARRAY_SIZE(execlists->inflight));
1654
1655	smp_wmb(); /* complete the seqlock for execlists_active() */
1656	WRITE_ONCE(execlists->active, execlists->inflight);
1657
1658	/* Having cancelled all outstanding process_csb(), stop their timers */
1659	GEM_BUG_ON(execlists->pending[0]);
1660	cancel_timer(&execlists->timer);
1661	cancel_timer(&execlists->preempt);
1662
1663	return inactive;
1664}
1665
1666/*
1667 * Starting with Gen12, the status has a new format:
1668 *
1669 *     bit  0:     switched to new queue
1670 *     bit  1:     reserved
1671 *     bit  2:     semaphore wait mode (poll or signal), only valid when
1672 *                 switch detail is set to "wait on semaphore"
1673 *     bits 3-5:   engine class
1674 *     bits 6-11:  engine instance
1675 *     bits 12-14: reserved
1676 *     bits 15-25: sw context id of the lrc the GT switched to
1677 *     bits 26-31: sw counter of the lrc the GT switched to
1678 *     bits 32-35: context switch detail
1679 *                  - 0: ctx complete
1680 *                  - 1: wait on sync flip
1681 *                  - 2: wait on vblank
1682 *                  - 3: wait on scanline
1683 *                  - 4: wait on semaphore
1684 *                  - 5: context preempted (not on SEMAPHORE_WAIT or
1685 *                       WAIT_FOR_EVENT)
1686 *     bit  36:    reserved
1687 *     bits 37-43: wait detail (for switch detail 1 to 4)
1688 *     bits 44-46: reserved
1689 *     bits 47-57: sw context id of the lrc the GT switched away from
1690 *     bits 58-63: sw counter of the lrc the GT switched away from
1691 *
1692 * Xe_HP csb shuffles things around compared to TGL:
1693 *
1694 *     bits 0-3:   context switch detail (same possible values as TGL)
1695 *     bits 4-9:   engine instance
1696 *     bits 10-25: sw context id of the lrc the GT switched to
1697 *     bits 26-31: sw counter of the lrc the GT switched to
1698 *     bit  32:    semaphore wait mode (poll or signal), Only valid when
1699 *                 switch detail is set to "wait on semaphore"
1700 *     bit  33:    switched to new queue
1701 *     bits 34-41: wait detail (for switch detail 1 to 4)
1702 *     bits 42-57: sw context id of the lrc the GT switched away from
1703 *     bits 58-63: sw counter of the lrc the GT switched away from
1704 */
1705static inline bool
1706__gen12_csb_parse(bool ctx_to_valid, bool ctx_away_valid, bool new_queue,
1707		  u8 switch_detail)
1708{
1709	/*
1710	 * The context switch detail is not guaranteed to be 5 when a preemption
1711	 * occurs, so we can't just check for that. The check below works for
1712	 * all the cases we care about, including preemptions of WAIT
1713	 * instructions and lite-restore. Preempt-to-idle via the CTRL register
1714	 * would require some extra handling, but we don't support that.
1715	 */
1716	if (!ctx_away_valid || new_queue) {
1717		GEM_BUG_ON(!ctx_to_valid);
1718		return true;
1719	}
1720
1721	/*
1722	 * switch detail = 5 is covered by the case above and we do not expect a
1723	 * context switch on an unsuccessful wait instruction since we always
1724	 * use polling mode.
1725	 */
1726	GEM_BUG_ON(switch_detail);
1727	return false;
1728}
1729
1730static bool xehp_csb_parse(const u64 csb)
1731{
1732	return __gen12_csb_parse(XEHP_CSB_CTX_VALID(lower_32_bits(csb)), /* cxt to */
1733				 XEHP_CSB_CTX_VALID(upper_32_bits(csb)), /* cxt away */
1734				 upper_32_bits(csb) & XEHP_CTX_STATUS_SWITCHED_TO_NEW_QUEUE,
1735				 GEN12_CTX_SWITCH_DETAIL(lower_32_bits(csb)));
1736}
1737
1738static bool gen12_csb_parse(const u64 csb)
1739{
1740	return __gen12_csb_parse(GEN12_CSB_CTX_VALID(lower_32_bits(csb)), /* cxt to */
1741				 GEN12_CSB_CTX_VALID(upper_32_bits(csb)), /* cxt away */
1742				 lower_32_bits(csb) & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE,
1743				 GEN12_CTX_SWITCH_DETAIL(upper_32_bits(csb)));
1744}
1745
1746static bool gen8_csb_parse(const u64 csb)
1747{
1748	return csb & (GEN8_CTX_STATUS_IDLE_ACTIVE | GEN8_CTX_STATUS_PREEMPTED);
1749}
1750
1751static noinline u64
1752wa_csb_read(const struct intel_engine_cs *engine, u64 * const csb)
1753{
1754	u64 entry;
1755
1756	/*
1757	 * Reading from the HWSP has one particular advantage: we can detect
1758	 * a stale entry. Since the write into HWSP is broken, we have no reason
1759	 * to trust the HW at all, the mmio entry may equally be unordered, so
1760	 * we prefer the path that is self-checking and as a last resort,
1761	 * return the mmio value.
1762	 *
1763	 * tgl,dg1:HSDES#22011327657
1764	 */
1765	preempt_disable();
1766	if (wait_for_atomic_us((entry = READ_ONCE(*csb)) != -1, 10)) {
1767		int idx = csb - engine->execlists.csb_status;
1768		int status;
1769
1770		status = GEN8_EXECLISTS_STATUS_BUF;
1771		if (idx >= 6) {
1772			status = GEN11_EXECLISTS_STATUS_BUF2;
1773			idx -= 6;
1774		}
1775		status += sizeof(u64) * idx;
1776
1777		entry = intel_uncore_read64(engine->uncore,
1778					    _MMIO(engine->mmio_base + status));
1779	}
1780	preempt_enable();
1781
1782	return entry;
1783}
1784
1785static u64 csb_read(const struct intel_engine_cs *engine, u64 * const csb)
1786{
1787	u64 entry = READ_ONCE(*csb);
1788
1789	/*
1790	 * Unfortunately, the GPU does not always serialise its write
1791	 * of the CSB entries before its write of the CSB pointer, at least
1792	 * from the perspective of the CPU, using what is known as a Global
1793	 * Observation Point. We may read a new CSB tail pointer, but then
1794	 * read the stale CSB entries, causing us to misinterpret the
1795	 * context-switch events, and eventually declare the GPU hung.
1796	 *
1797	 * icl:HSDES#1806554093
1798	 * tgl:HSDES#22011248461
1799	 */
1800	if (unlikely(entry == -1))
1801		entry = wa_csb_read(engine, csb);
1802
1803	/* Consume this entry so that we can spot its future reuse. */
1804	WRITE_ONCE(*csb, -1);
1805
1806	/* ELSP is an implicit wmb() before the GPU wraps and overwrites csb */
1807	return entry;
1808}
1809
1810static void new_timeslice(struct intel_engine_execlists *el)
1811{
1812	/* By cancelling, we will start afresh in start_timeslice() */
1813	cancel_timer(&el->timer);
1814}
1815
1816static struct i915_request **
1817process_csb(struct intel_engine_cs *engine, struct i915_request **inactive)
1818{
1819	struct intel_engine_execlists * const execlists = &engine->execlists;
1820	u64 * const buf = execlists->csb_status;
1821	const u8 num_entries = execlists->csb_size;
1822	struct i915_request **prev;
1823	u8 head, tail;
1824
1825	/*
1826	 * As we modify our execlists state tracking we require exclusive
1827	 * access. Either we are inside the tasklet, or the tasklet is disabled
1828	 * and we assume that is only inside the reset paths and so serialised.
1829	 */
1830	GEM_BUG_ON(!tasklet_is_locked(&engine->sched_engine->tasklet) &&
1831		   !reset_in_progress(engine));
1832
1833	/*
1834	 * Note that csb_write, csb_status may be either in HWSP or mmio.
1835	 * When reading from the csb_write mmio register, we have to be
1836	 * careful to only use the GEN8_CSB_WRITE_PTR portion, which is
1837	 * the low 4bits. As it happens we know the next 4bits are always
1838	 * zero and so we can simply masked off the low u8 of the register
1839	 * and treat it identically to reading from the HWSP (without having
1840	 * to use explicit shifting and masking, and probably bifurcating
1841	 * the code to handle the legacy mmio read).
1842	 */
1843	head = execlists->csb_head;
1844	tail = READ_ONCE(*execlists->csb_write);
1845	if (unlikely(head == tail))
1846		return inactive;
1847
1848	/*
1849	 * We will consume all events from HW, or at least pretend to.
1850	 *
1851	 * The sequence of events from the HW is deterministic, and derived
1852	 * from our writes to the ELSP, with a smidgen of variability for
1853	 * the arrival of the asynchronous requests wrt to the inflight
1854	 * execution. If the HW sends an event that does not correspond with
1855	 * the one we are expecting, we have to abandon all hope as we lose
1856	 * all tracking of what the engine is actually executing. We will
1857	 * only detect we are out of sequence with the HW when we get an
1858	 * 'impossible' event because we have already drained our own
1859	 * preemption/promotion queue. If this occurs, we know that we likely
1860	 * lost track of execution earlier and must unwind and restart, the
1861	 * simplest way is by stop processing the event queue and force the
1862	 * engine to reset.
1863	 */
1864	execlists->csb_head = tail;
1865	ENGINE_TRACE(engine, "cs-irq head=%d, tail=%d\n", head, tail);
1866
1867	/*
1868	 * Hopefully paired with a wmb() in HW!
1869	 *
1870	 * We must complete the read of the write pointer before any reads
1871	 * from the CSB, so that we do not see stale values. Without an rmb
1872	 * (lfence) the HW may speculatively perform the CSB[] reads *before*
1873	 * we perform the READ_ONCE(*csb_write).
1874	 */
1875	rmb();
1876
1877	/* Remember who was last running under the timer */
1878	prev = inactive;
1879	*prev = NULL;
1880
1881	do {
1882		bool promote;
1883		u64 csb;
1884
1885		if (++head == num_entries)
1886			head = 0;
1887
1888		/*
1889		 * We are flying near dragons again.
1890		 *
1891		 * We hold a reference to the request in execlist_port[]
1892		 * but no more than that. We are operating in softirq
1893		 * context and so cannot hold any mutex or sleep. That
1894		 * prevents us stopping the requests we are processing
1895		 * in port[] from being retired simultaneously (the
1896		 * breadcrumb will be complete before we see the
1897		 * context-switch). As we only hold the reference to the
1898		 * request, any pointer chasing underneath the request
1899		 * is subject to a potential use-after-free. Thus we
1900		 * store all of the bookkeeping within port[] as
1901		 * required, and avoid using unguarded pointers beneath
1902		 * request itself. The same applies to the atomic
1903		 * status notifier.
1904		 */
1905
1906		csb = csb_read(engine, buf + head);
1907		ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n",
1908			     head, upper_32_bits(csb), lower_32_bits(csb));
1909
1910		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
1911			promote = xehp_csb_parse(csb);
1912		else if (GRAPHICS_VER(engine->i915) >= 12)
1913			promote = gen12_csb_parse(csb);
1914		else
1915			promote = gen8_csb_parse(csb);
1916		if (promote) {
1917			struct i915_request * const *old = execlists->active;
1918
1919			if (GEM_WARN_ON(!*execlists->pending)) {
1920				execlists->error_interrupt |= ERROR_CSB;
1921				break;
1922			}
1923
1924			ring_set_paused(engine, 0);
1925
1926			/* Point active to the new ELSP; prevent overwriting */
1927			WRITE_ONCE(execlists->active, execlists->pending);
1928			smp_wmb(); /* notify execlists_active() */
1929
1930			/* cancel old inflight, prepare for switch */
1931			trace_ports(execlists, "preempted", old);
1932			while (*old)
1933				*inactive++ = *old++;
1934
1935			/* switch pending to inflight */
1936			GEM_BUG_ON(!assert_pending_valid(execlists, "promote"));
1937			copy_ports(execlists->inflight,
1938				   execlists->pending,
1939				   execlists_num_ports(execlists));
1940			smp_wmb(); /* complete the seqlock */
1941			WRITE_ONCE(execlists->active, execlists->inflight);
1942
1943			/* XXX Magic delay for tgl */
1944			ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
1945
1946			WRITE_ONCE(execlists->pending[0], NULL);
1947		} else {
1948			if (GEM_WARN_ON(!*execlists->active)) {
1949				execlists->error_interrupt |= ERROR_CSB;
1950				break;
1951			}
1952
1953			/* port0 completed, advanced to port1 */
1954			trace_ports(execlists, "completed", execlists->active);
1955
1956			/*
1957			 * We rely on the hardware being strongly
1958			 * ordered, that the breadcrumb write is
1959			 * coherent (visible from the CPU) before the
1960			 * user interrupt is processed. One might assume
1961			 * that the breadcrumb write being before the
1962			 * user interrupt and the CS event for the context
1963			 * switch would therefore be before the CS event
1964			 * itself...
1965			 */
1966			if (GEM_SHOW_DEBUG() &&
1967			    !__i915_request_is_complete(*execlists->active)) {
1968				struct i915_request *rq = *execlists->active;
1969				const u32 *regs __maybe_unused =
1970					rq->context->lrc_reg_state;
1971
1972				ENGINE_TRACE(engine,
1973					     "context completed before request!\n");
1974				ENGINE_TRACE(engine,
1975					     "ring:{start:0x%08x, head:%04x, tail:%04x, ctl:%08x, mode:%08x}\n",
1976					     ENGINE_READ(engine, RING_START),
1977					     ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR,
1978					     ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR,
1979					     ENGINE_READ(engine, RING_CTL),
1980					     ENGINE_READ(engine, RING_MI_MODE));
1981				ENGINE_TRACE(engine,
1982					     "rq:{start:%08x, head:%04x, tail:%04x, seqno:%llx:%d, hwsp:%d}, ",
1983					     i915_ggtt_offset(rq->ring->vma),
1984					     rq->head, rq->tail,
1985					     rq->fence.context,
1986					     lower_32_bits(rq->fence.seqno),
1987					     hwsp_seqno(rq));
1988				ENGINE_TRACE(engine,
1989					     "ctx:{start:%08x, head:%04x, tail:%04x}, ",
1990					     regs[CTX_RING_START],
1991					     regs[CTX_RING_HEAD],
1992					     regs[CTX_RING_TAIL]);
1993			}
1994
1995			*inactive++ = *execlists->active++;
1996
1997			GEM_BUG_ON(execlists->active - execlists->inflight >
1998				   execlists_num_ports(execlists));
1999		}
2000	} while (head != tail);
2001
2002	/*
2003	 * Gen11 has proven to fail wrt global observation point between
2004	 * entry and tail update, failing on the ordering and thus
2005	 * we see an old entry in the context status buffer.
2006	 *
2007	 * Forcibly evict out entries for the next gpu csb update,
2008	 * to increase the odds that we get a fresh entries with non
2009	 * working hardware. The cost for doing so comes out mostly with
2010	 * the wash as hardware, working or not, will need to do the
2011	 * invalidation before.
2012	 */
2013	drm_clflush_virt_range(&buf[0], num_entries * sizeof(buf[0]));
2014
2015	/*
2016	 * We assume that any event reflects a change in context flow
2017	 * and merits a fresh timeslice. We reinstall the timer after
2018	 * inspecting the queue to see if we need to resumbit.
2019	 */
2020	if (*prev != *execlists->active) { /* elide lite-restores */
2021		/*
2022		 * Note the inherent discrepancy between the HW runtime,
2023		 * recorded as part of the context switch, and the CPU
2024		 * adjustment for active contexts. We have to hope that
2025		 * the delay in processing the CS event is very small
2026		 * and consistent. It works to our advantage to have
2027		 * the CPU adjustment _undershoot_ (i.e. start later than)
2028		 * the CS timestamp so we never overreport the runtime
2029		 * and correct overselves later when updating from HW.
2030		 */
2031		if (*prev)
2032			lrc_runtime_stop((*prev)->context);
2033		if (*execlists->active)
2034			lrc_runtime_start((*execlists->active)->context);
2035		new_timeslice(execlists);
2036	}
2037
2038	return inactive;
2039}
2040
2041static void post_process_csb(struct i915_request **port,
2042			     struct i915_request **last)
2043{
2044	while (port != last)
2045		execlists_schedule_out(*port++);
2046}
2047
2048static void __execlists_hold(struct i915_request *rq)
2049{
2050	LIST_HEAD(list);
2051
2052	do {
2053		struct i915_dependency *p;
2054
2055		if (i915_request_is_active(rq))
2056			__i915_request_unsubmit(rq);
2057
2058		clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2059		list_move_tail(&rq->sched.link,
2060			       &rq->engine->sched_engine->hold);
2061		i915_request_set_hold(rq);
2062		RQ_TRACE(rq, "on hold\n");
2063
2064		for_each_waiter(p, rq) {
2065			struct i915_request *w =
2066				container_of(p->waiter, typeof(*w), sched);
2067
2068			if (p->flags & I915_DEPENDENCY_WEAK)
2069				continue;
2070
2071			/* Leave semaphores spinning on the other engines */
2072			if (w->engine != rq->engine)
2073				continue;
2074
2075			if (!i915_request_is_ready(w))
2076				continue;
2077
2078			if (__i915_request_is_complete(w))
2079				continue;
2080
2081			if (i915_request_on_hold(w))
2082				continue;
2083
2084			list_move_tail(&w->sched.link, &list);
2085		}
2086
2087		rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
2088	} while (rq);
2089}
2090
2091static bool execlists_hold(struct intel_engine_cs *engine,
2092			   struct i915_request *rq)
2093{
2094	if (i915_request_on_hold(rq))
2095		return false;
2096
2097	spin_lock_irq(&engine->sched_engine->lock);
2098
2099	if (__i915_request_is_complete(rq)) { /* too late! */
2100		rq = NULL;
2101		goto unlock;
2102	}
2103
2104	/*
2105	 * Transfer this request onto the hold queue to prevent it
2106	 * being resumbitted to HW (and potentially completed) before we have
2107	 * released it. Since we may have already submitted following
2108	 * requests, we need to remove those as well.
2109	 */
2110	GEM_BUG_ON(i915_request_on_hold(rq));
2111	GEM_BUG_ON(rq->engine != engine);
2112	__execlists_hold(rq);
2113	GEM_BUG_ON(list_empty(&engine->sched_engine->hold));
2114
2115unlock:
2116	spin_unlock_irq(&engine->sched_engine->lock);
2117	return rq;
2118}
2119
2120static bool hold_request(const struct i915_request *rq)
2121{
2122	struct i915_dependency *p;
2123	bool result = false;
2124
2125	/*
2126	 * If one of our ancestors is on hold, we must also be on hold,
2127	 * otherwise we will bypass it and execute before it.
2128	 */
2129	rcu_read_lock();
2130	for_each_signaler(p, rq) {
2131		const struct i915_request *s =
2132			container_of(p->signaler, typeof(*s), sched);
2133
2134		if (s->engine != rq->engine)
2135			continue;
2136
2137		result = i915_request_on_hold(s);
2138		if (result)
2139			break;
2140	}
2141	rcu_read_unlock();
2142
2143	return result;
2144}
2145
2146static void __execlists_unhold(struct i915_request *rq)
2147{
2148	LIST_HEAD(list);
2149
2150	do {
2151		struct i915_dependency *p;
2152
2153		RQ_TRACE(rq, "hold release\n");
2154
2155		GEM_BUG_ON(!i915_request_on_hold(rq));
2156		GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
2157
2158		i915_request_clear_hold(rq);
2159		list_move_tail(&rq->sched.link,
2160			       i915_sched_lookup_priolist(rq->engine->sched_engine,
2161							  rq_prio(rq)));
2162		set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2163
2164		/* Also release any children on this engine that are ready */
2165		for_each_waiter(p, rq) {
2166			struct i915_request *w =
2167				container_of(p->waiter, typeof(*w), sched);
2168
2169			if (p->flags & I915_DEPENDENCY_WEAK)
2170				continue;
2171
2172			if (w->engine != rq->engine)
2173				continue;
2174
2175			if (!i915_request_on_hold(w))
2176				continue;
2177
2178			/* Check that no other parents are also on hold */
2179			if (hold_request(w))
2180				continue;
2181
2182			list_move_tail(&w->sched.link, &list);
2183		}
2184
2185		rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
2186	} while (rq);
2187}
2188
2189static void execlists_unhold(struct intel_engine_cs *engine,
2190			     struct i915_request *rq)
2191{
2192	spin_lock_irq(&engine->sched_engine->lock);
2193
2194	/*
2195	 * Move this request back to the priority queue, and all of its
2196	 * children and grandchildren that were suspended along with it.
2197	 */
2198	__execlists_unhold(rq);
2199
2200	if (rq_prio(rq) > engine->sched_engine->queue_priority_hint) {
2201		engine->sched_engine->queue_priority_hint = rq_prio(rq);
2202		tasklet_hi_schedule(&engine->sched_engine->tasklet);
2203	}
2204
2205	spin_unlock_irq(&engine->sched_engine->lock);
2206}
2207
2208struct execlists_capture {
2209	struct work_struct work;
2210	struct i915_request *rq;
2211	struct i915_gpu_coredump *error;
2212};
2213
2214static void execlists_capture_work(struct work_struct *work)
2215{
2216	struct execlists_capture *cap = container_of(work, typeof(*cap), work);
2217	const gfp_t gfp = __GFP_KSWAPD_RECLAIM | __GFP_RETRY_MAYFAIL |
2218		__GFP_NOWARN;
2219	struct intel_engine_cs *engine = cap->rq->engine;
2220	struct intel_gt_coredump *gt = cap->error->gt;
2221	struct intel_engine_capture_vma *vma;
2222
2223	/* Compress all the objects attached to the request, slow! */
2224	vma = intel_engine_coredump_add_request(gt->engine, cap->rq, gfp);
2225	if (vma) {
2226		struct i915_vma_compress *compress =
2227			i915_vma_capture_prepare(gt);
2228
2229		intel_engine_coredump_add_vma(gt->engine, vma, compress);
2230		i915_vma_capture_finish(gt, compress);
2231	}
2232
2233	gt->simulated = gt->engine->simulated;
2234	cap->error->simulated = gt->simulated;
2235
2236	/* Publish the error state, and announce it to the world */
2237	i915_error_state_store(cap->error);
2238	i915_gpu_coredump_put(cap->error);
2239
2240	/* Return this request and all that depend upon it for signaling */
2241	execlists_unhold(engine, cap->rq);
2242	i915_request_put(cap->rq);
2243
2244	kfree(cap);
2245}
2246
2247static struct execlists_capture *capture_regs(struct intel_engine_cs *engine)
2248{
2249	const gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;
2250	struct execlists_capture *cap;
2251
2252	cap = kmalloc(sizeof(*cap), gfp);
2253	if (!cap)
2254		return NULL;
2255
2256	cap->error = i915_gpu_coredump_alloc(engine->i915, gfp);
2257	if (!cap->error)
2258		goto err_cap;
2259
2260	cap->error->gt = intel_gt_coredump_alloc(engine->gt, gfp, CORE_DUMP_FLAG_NONE);
2261	if (!cap->error->gt)
2262		goto err_gpu;
2263
2264	cap->error->gt->engine = intel_engine_coredump_alloc(engine, gfp, CORE_DUMP_FLAG_NONE);
2265	if (!cap->error->gt->engine)
2266		goto err_gt;
2267
2268	cap->error->gt->engine->hung = true;
2269
2270	return cap;
2271
2272err_gt:
2273	kfree(cap->error->gt);
2274err_gpu:
2275	kfree(cap->error);
2276err_cap:
2277	kfree(cap);
2278	return NULL;
2279}
2280
2281static struct i915_request *
2282active_context(struct intel_engine_cs *engine, u32 ccid)
2283{
2284	const struct intel_engine_execlists * const el = &engine->execlists;
2285	struct i915_request * const *port, *rq;
2286
2287	/*
2288	 * Use the most recent result from process_csb(), but just in case
2289	 * we trigger an error (via interrupt) before the first CS event has
2290	 * been written, peek at the next submission.
2291	 */
2292
2293	for (port = el->active; (rq = *port); port++) {
2294		if (rq->context->lrc.ccid == ccid) {
2295			ENGINE_TRACE(engine,
2296				     "ccid:%x found at active:%zd\n",
2297				     ccid, port - el->active);
2298			return rq;
2299		}
2300	}
2301
2302	for (port = el->pending; (rq = *port); port++) {
2303		if (rq->context->lrc.ccid == ccid) {
2304			ENGINE_TRACE(engine,
2305				     "ccid:%x found at pending:%zd\n",
2306				     ccid, port - el->pending);
2307			return rq;
2308		}
2309	}
2310
2311	ENGINE_TRACE(engine, "ccid:%x not found\n", ccid);
2312	return NULL;
2313}
2314
2315static u32 active_ccid(struct intel_engine_cs *engine)
2316{
2317	return ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI);
2318}
2319
2320static void execlists_capture(struct intel_engine_cs *engine)
2321{
2322	struct execlists_capture *cap;
2323
2324	if (!IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR))
2325		return;
2326
2327	/*
2328	 * We need to _quickly_ capture the engine state before we reset.
2329	 * We are inside an atomic section (softirq) here and we are delaying
2330	 * the forced preemption event.
2331	 */
2332	cap = capture_regs(engine);
2333	if (!cap)
2334		return;
2335
2336	spin_lock_irq(&engine->sched_engine->lock);
2337	cap->rq = active_context(engine, active_ccid(engine));
2338	if (cap->rq) {
2339		cap->rq = active_request(cap->rq->context->timeline, cap->rq);
2340		cap->rq = i915_request_get_rcu(cap->rq);
2341	}
2342	spin_unlock_irq(&engine->sched_engine->lock);
2343	if (!cap->rq)
2344		goto err_free;
2345
2346	/*
2347	 * Remove the request from the execlists queue, and take ownership
2348	 * of the request. We pass it to our worker who will _slowly_ compress
2349	 * all the pages the _user_ requested for debugging their batch, after
2350	 * which we return it to the queue for signaling.
2351	 *
2352	 * By removing them from the execlists queue, we also remove the
2353	 * requests from being processed by __unwind_incomplete_requests()
2354	 * during the intel_engine_reset(), and so they will *not* be replayed
2355	 * afterwards.
2356	 *
2357	 * Note that because we have not yet reset the engine at this point,
2358	 * it is possible for the request that we have identified as being
2359	 * guilty, did in fact complete and we will then hit an arbitration
2360	 * point allowing the outstanding preemption to succeed. The likelihood
2361	 * of that is very low (as capturing of the engine registers should be
2362	 * fast enough to run inside an irq-off atomic section!), so we will
2363	 * simply hold that request accountable for being non-preemptible
2364	 * long enough to force the reset.
2365	 */
2366	if (!execlists_hold(engine, cap->rq))
2367		goto err_rq;
2368
2369	INIT_WORK(&cap->work, execlists_capture_work);
2370	schedule_work(&cap->work);
2371	return;
2372
2373err_rq:
2374	i915_request_put(cap->rq);
2375err_free:
2376	i915_gpu_coredump_put(cap->error);
2377	kfree(cap);
2378}
2379
2380static void execlists_reset(struct intel_engine_cs *engine, const char *msg)
2381{
2382	const unsigned int bit = I915_RESET_ENGINE + engine->id;
2383	unsigned long *lock = &engine->gt->reset.flags;
2384
2385	if (!intel_has_reset_engine(engine->gt))
2386		return;
2387
2388	if (test_and_set_bit(bit, lock))
2389		return;
2390
2391	ENGINE_TRACE(engine, "reset for %s\n", msg);
2392
2393	/* Mark this tasklet as disabled to avoid waiting for it to complete */
2394	tasklet_disable_nosync(&engine->sched_engine->tasklet);
2395
2396	ring_set_paused(engine, 1); /* Freeze the current request in place */
2397	execlists_capture(engine);
2398	intel_engine_reset(engine, msg);
2399
2400	tasklet_enable(&engine->sched_engine->tasklet);
2401	clear_and_wake_up_bit(bit, lock);
2402}
2403
2404static bool preempt_timeout(const struct intel_engine_cs *const engine)
2405{
2406	const struct timer_list *t = &engine->execlists.preempt;
2407
2408	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2409		return false;
2410
2411	if (!timer_expired(t))
2412		return false;
2413
2414	return engine->execlists.pending[0];
2415}
2416
2417/*
2418 * Check the unread Context Status Buffers and manage the submission of new
2419 * contexts to the ELSP accordingly.
2420 */
2421static void execlists_submission_tasklet(struct tasklet_struct *t)
2422{
2423	struct i915_sched_engine *sched_engine =
2424		from_tasklet(sched_engine, t, tasklet);
2425	struct intel_engine_cs * const engine = sched_engine->private_data;
2426	struct i915_request *post[2 * EXECLIST_MAX_PORTS];
2427	struct i915_request **inactive;
2428
2429	rcu_read_lock();
2430	inactive = process_csb(engine, post);
2431	GEM_BUG_ON(inactive - post > ARRAY_SIZE(post));
2432
2433	if (unlikely(preempt_timeout(engine))) {
2434		const struct i915_request *rq = *engine->execlists.active;
2435
2436		/*
2437		 * If after the preempt-timeout expired, we are still on the
2438		 * same active request/context as before we initiated the
2439		 * preemption, reset the engine.
2440		 *
2441		 * However, if we have processed a CS event to switch contexts,
2442		 * but not yet processed the CS event for the pending
2443		 * preemption, reset the timer allowing the new context to
2444		 * gracefully exit.
2445		 */
2446		cancel_timer(&engine->execlists.preempt);
2447		if (rq == engine->execlists.preempt_target)
2448			engine->execlists.error_interrupt |= ERROR_PREEMPT;
2449		else
2450			set_timer_ms(&engine->execlists.preempt,
2451				     active_preempt_timeout(engine, rq));
2452	}
2453
2454	if (unlikely(READ_ONCE(engine->execlists.error_interrupt))) {
2455		const char *msg;
2456
2457		/* Generate the error message in priority wrt to the user! */
2458		if (engine->execlists.error_interrupt & GENMASK(15, 0))
2459			msg = "CS error"; /* thrown by a user payload */
2460		else if (engine->execlists.error_interrupt & ERROR_CSB)
2461			msg = "invalid CSB event";
2462		else if (engine->execlists.error_interrupt & ERROR_PREEMPT)
2463			msg = "preemption time out";
2464		else
2465			msg = "internal error";
2466
2467		engine->execlists.error_interrupt = 0;
2468		execlists_reset(engine, msg);
2469	}
2470
2471	if (!engine->execlists.pending[0]) {
2472		execlists_dequeue_irq(engine);
2473		start_timeslice(engine);
2474	}
2475
2476	post_process_csb(post, inactive);
2477	rcu_read_unlock();
2478}
2479
2480static void execlists_irq_handler(struct intel_engine_cs *engine, u16 iir)
2481{
2482	bool tasklet = false;
2483
2484	if (unlikely(iir & GT_CS_MASTER_ERROR_INTERRUPT)) {
2485		u32 eir;
2486
2487		/* Upper 16b are the enabling mask, rsvd for internal errors */
2488		eir = ENGINE_READ(engine, RING_EIR) & GENMASK(15, 0);
2489		ENGINE_TRACE(engine, "CS error: %x\n", eir);
2490
2491		/* Disable the error interrupt until after the reset */
2492		if (likely(eir)) {
2493			ENGINE_WRITE(engine, RING_EMR, ~0u);
2494			ENGINE_WRITE(engine, RING_EIR, eir);
2495			WRITE_ONCE(engine->execlists.error_interrupt, eir);
2496			tasklet = true;
2497		}
2498	}
2499
2500	if (iir & GT_WAIT_SEMAPHORE_INTERRUPT) {
2501		WRITE_ONCE(engine->execlists.yield,
2502			   ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI));
2503		ENGINE_TRACE(engine, "semaphore yield: %08x\n",
2504			     engine->execlists.yield);
2505		if (del_timer(&engine->execlists.timer))
2506			tasklet = true;
2507	}
2508
2509	if (iir & GT_CONTEXT_SWITCH_INTERRUPT)
2510		tasklet = true;
2511
2512	if (iir & GT_RENDER_USER_INTERRUPT)
2513		intel_engine_signal_breadcrumbs(engine);
2514
2515	if (tasklet)
2516		tasklet_hi_schedule(&engine->sched_engine->tasklet);
2517}
2518
2519static void __execlists_kick(struct intel_engine_execlists *execlists)
2520{
2521	struct intel_engine_cs *engine =
2522		container_of(execlists, typeof(*engine), execlists);
2523
2524	/* Kick the tasklet for some interrupt coalescing and reset handling */
2525	tasklet_hi_schedule(&engine->sched_engine->tasklet);
2526}
2527
2528#define execlists_kick(t, member) \
2529	__execlists_kick(container_of(t, struct intel_engine_execlists, member))
2530
2531static void execlists_timeslice(struct timer_list *timer)
2532{
2533	execlists_kick(timer, timer);
2534}
2535
2536static void execlists_preempt(struct timer_list *timer)
2537{
2538	execlists_kick(timer, preempt);
2539}
2540
2541static void queue_request(struct intel_engine_cs *engine,
2542			  struct i915_request *rq)
2543{
2544	GEM_BUG_ON(!list_empty(&rq->sched.link));
2545	list_add_tail(&rq->sched.link,
2546		      i915_sched_lookup_priolist(engine->sched_engine,
2547						 rq_prio(rq)));
2548	set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2549}
2550
2551static bool submit_queue(struct intel_engine_cs *engine,
2552			 const struct i915_request *rq)
2553{
2554	struct i915_sched_engine *sched_engine = engine->sched_engine;
2555
2556	if (rq_prio(rq) <= sched_engine->queue_priority_hint)
2557		return false;
2558
2559	sched_engine->queue_priority_hint = rq_prio(rq);
2560	return true;
2561}
2562
2563static bool ancestor_on_hold(const struct intel_engine_cs *engine,
2564			     const struct i915_request *rq)
2565{
2566	GEM_BUG_ON(i915_request_on_hold(rq));
2567	return !list_empty(&engine->sched_engine->hold) && hold_request(rq);
2568}
2569
2570static void execlists_submit_request(struct i915_request *request)
2571{
2572	struct intel_engine_cs *engine = request->engine;
2573	unsigned long flags;
2574
2575	/* Will be called from irq-context when using foreign fences. */
2576	spin_lock_irqsave(&engine->sched_engine->lock, flags);
2577
2578	if (unlikely(ancestor_on_hold(engine, request))) {
2579		RQ_TRACE(request, "ancestor on hold\n");
2580		list_add_tail(&request->sched.link,
2581			      &engine->sched_engine->hold);
2582		i915_request_set_hold(request);
2583	} else {
2584		queue_request(engine, request);
2585
2586		GEM_BUG_ON(i915_sched_engine_is_empty(engine->sched_engine));
2587		GEM_BUG_ON(list_empty(&request->sched.link));
2588
2589		if (submit_queue(engine, request))
2590			__execlists_kick(&engine->execlists);
2591	}
2592
2593	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
2594}
2595
2596static int
2597__execlists_context_pre_pin(struct intel_context *ce,
2598			    struct intel_engine_cs *engine,
2599			    struct i915_gem_ww_ctx *ww, void **vaddr)
2600{
2601	int err;
2602
2603	err = lrc_pre_pin(ce, engine, ww, vaddr);
2604	if (err)
2605		return err;
2606
2607	if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) {
2608		lrc_init_state(ce, engine, *vaddr);
2609
2610		__i915_gem_object_flush_map(ce->state->obj, 0, engine->context_size);
2611	}
2612
2613	return 0;
2614}
2615
2616static int execlists_context_pre_pin(struct intel_context *ce,
2617				     struct i915_gem_ww_ctx *ww,
2618				     void **vaddr)
2619{
2620	return __execlists_context_pre_pin(ce, ce->engine, ww, vaddr);
2621}
2622
2623static int execlists_context_pin(struct intel_context *ce, void *vaddr)
2624{
2625	return lrc_pin(ce, ce->engine, vaddr);
2626}
2627
2628static int execlists_context_alloc(struct intel_context *ce)
2629{
2630	return lrc_alloc(ce, ce->engine);
2631}
2632
2633static void execlists_context_cancel_request(struct intel_context *ce,
2634					     struct i915_request *rq)
2635{
2636	struct intel_engine_cs *engine = NULL;
2637
2638	i915_request_active_engine(rq, &engine);
2639
2640	if (engine && intel_engine_pulse(engine))
2641		intel_gt_handle_error(engine->gt, engine->mask, 0,
2642				      "request cancellation by %s",
2643				      current->comm);
2644}
2645
2646static struct intel_context *
2647execlists_create_parallel(struct intel_engine_cs **engines,
2648			  unsigned int num_siblings,
2649			  unsigned int width)
2650{
2651	struct intel_context *parent = NULL, *ce, *err;
2652	int i;
2653
2654	GEM_BUG_ON(num_siblings != 1);
2655
2656	for (i = 0; i < width; ++i) {
2657		ce = intel_context_create(engines[i]);
2658		if (IS_ERR(ce)) {
2659			err = ce;
2660			goto unwind;
2661		}
2662
2663		if (i == 0)
2664			parent = ce;
2665		else
2666			intel_context_bind_parent_child(parent, ce);
2667	}
2668
2669	parent->parallel.fence_context = dma_fence_context_alloc(1);
2670
2671	intel_context_set_nopreempt(parent);
2672	for_each_child(parent, ce)
2673		intel_context_set_nopreempt(ce);
2674
2675	return parent;
2676
2677unwind:
2678	if (parent)
2679		intel_context_put(parent);
2680	return err;
2681}
2682
2683static const struct intel_context_ops execlists_context_ops = {
2684	.flags = COPS_HAS_INFLIGHT | COPS_RUNTIME_CYCLES,
2685
2686	.alloc = execlists_context_alloc,
2687
2688	.cancel_request = execlists_context_cancel_request,
2689
2690	.pre_pin = execlists_context_pre_pin,
2691	.pin = execlists_context_pin,
2692	.unpin = lrc_unpin,
2693	.post_unpin = lrc_post_unpin,
2694
2695	.enter = intel_context_enter_engine,
2696	.exit = intel_context_exit_engine,
2697
2698	.reset = lrc_reset,
2699	.destroy = lrc_destroy,
2700
2701	.create_parallel = execlists_create_parallel,
2702	.create_virtual = execlists_create_virtual,
2703};
2704
2705static int emit_pdps(struct i915_request *rq)
2706{
2707	const struct intel_engine_cs * const engine = rq->engine;
2708	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(rq->context->vm);
2709	int err, i;
2710	u32 *cs;
2711
2712	GEM_BUG_ON(intel_vgpu_active(rq->engine->i915));
2713
2714	/*
2715	 * Beware ye of the dragons, this sequence is magic!
2716	 *
2717	 * Small changes to this sequence can cause anything from
2718	 * GPU hangs to forcewake errors and machine lockups!
2719	 */
2720
2721	cs = intel_ring_begin(rq, 2);
2722	if (IS_ERR(cs))
2723		return PTR_ERR(cs);
2724
2725	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
2726	*cs++ = MI_NOOP;
2727	intel_ring_advance(rq, cs);
2728
2729	/* Flush any residual operations from the context load */
2730	err = engine->emit_flush(rq, EMIT_FLUSH);
2731	if (err)
2732		return err;
2733
2734	/* Magic required to prevent forcewake errors! */
2735	err = engine->emit_flush(rq, EMIT_INVALIDATE);
2736	if (err)
2737		return err;
2738
2739	cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
2740	if (IS_ERR(cs))
2741		return PTR_ERR(cs);
2742
2743	/* Ensure the LRI have landed before we invalidate & continue */
2744	*cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;
2745	for (i = GEN8_3LVL_PDPES; i--; ) {
2746		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
2747		u32 base = engine->mmio_base;
2748
2749		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
2750		*cs++ = upper_32_bits(pd_daddr);
2751		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
2752		*cs++ = lower_32_bits(pd_daddr);
2753	}
2754	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2755	intel_ring_advance(rq, cs);
2756
2757	intel_ring_advance(rq, cs);
2758
2759	return 0;
2760}
2761
2762static int execlists_request_alloc(struct i915_request *request)
2763{
2764	int ret;
2765
2766	GEM_BUG_ON(!intel_context_is_pinned(request->context));
2767
2768	/*
2769	 * Flush enough space to reduce the likelihood of waiting after
2770	 * we start building the request - in which case we will just
2771	 * have to repeat work.
2772	 */
2773	request->reserved_space += EXECLISTS_REQUEST_SIZE;
2774
2775	/*
2776	 * Note that after this point, we have committed to using
2777	 * this request as it is being used to both track the
2778	 * state of engine initialisation and liveness of the
2779	 * golden renderstate above. Think twice before you try
2780	 * to cancel/unwind this request now.
2781	 */
2782
2783	if (!i915_vm_is_4lvl(request->context->vm)) {
2784		ret = emit_pdps(request);
2785		if (ret)
2786			return ret;
2787	}
2788
2789	/* Unconditionally invalidate GPU caches and TLBs. */
2790	ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
2791	if (ret)
2792		return ret;
2793
2794	request->reserved_space -= EXECLISTS_REQUEST_SIZE;
2795	return 0;
2796}
2797
2798static void reset_csb_pointers(struct intel_engine_cs *engine)
2799{
2800	struct intel_engine_execlists * const execlists = &engine->execlists;
2801	const unsigned int reset_value = execlists->csb_size - 1;
2802
2803	ring_set_paused(engine, 0);
2804
2805	/*
2806	 * Sometimes Icelake forgets to reset its pointers on a GPU reset.
2807	 * Bludgeon them with a mmio update to be sure.
2808	 */
2809	ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR,
2810		     0xffff << 16 | reset_value << 8 | reset_value);
2811	ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
2812
2813	/*
2814	 * After a reset, the HW starts writing into CSB entry [0]. We
2815	 * therefore have to set our HEAD pointer back one entry so that
2816	 * the *first* entry we check is entry 0. To complicate this further,
2817	 * as we don't wait for the first interrupt after reset, we have to
2818	 * fake the HW write to point back to the last entry so that our
2819	 * inline comparison of our cached head position against the last HW
2820	 * write works even before the first interrupt.
2821	 */
2822	execlists->csb_head = reset_value;
2823	WRITE_ONCE(*execlists->csb_write, reset_value);
2824	wmb(); /* Make sure this is visible to HW (paranoia?) */
2825
2826	/* Check that the GPU does indeed update the CSB entries! */
2827	memset(execlists->csb_status, -1, (reset_value + 1) * sizeof(u64));
2828	drm_clflush_virt_range(execlists->csb_status,
2829			       execlists->csb_size *
2830			       sizeof(execlists->csb_status));
2831
2832	/* Once more for luck and our trusty paranoia */
2833	ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR,
2834		     0xffff << 16 | reset_value << 8 | reset_value);
2835	ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
2836
2837	GEM_BUG_ON(READ_ONCE(*execlists->csb_write) != reset_value);
2838}
2839
2840static void sanitize_hwsp(struct intel_engine_cs *engine)
2841{
2842	struct intel_timeline *tl;
2843
2844	list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
2845		intel_timeline_reset_seqno(tl);
2846}
2847
2848static void execlists_sanitize(struct intel_engine_cs *engine)
2849{
2850	GEM_BUG_ON(execlists_active(&engine->execlists));
2851
2852	/*
2853	 * Poison residual state on resume, in case the suspend didn't!
2854	 *
2855	 * We have to assume that across suspend/resume (or other loss
2856	 * of control) that the contents of our pinned buffers has been
2857	 * lost, replaced by garbage. Since this doesn't always happen,
2858	 * let's poison such state so that we more quickly spot when
2859	 * we falsely assume it has been preserved.
2860	 */
2861	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
2862		memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
2863
2864	reset_csb_pointers(engine);
2865
2866	/*
2867	 * The kernel_context HWSP is stored in the status_page. As above,
2868	 * that may be lost on resume/initialisation, and so we need to
2869	 * reset the value in the HWSP.
2870	 */
2871	sanitize_hwsp(engine);
2872
2873	/* And scrub the dirty cachelines for the HWSP */
2874	drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE);
2875
2876	intel_engine_reset_pinned_contexts(engine);
2877}
2878
2879static void enable_error_interrupt(struct intel_engine_cs *engine)
2880{
2881	u32 status;
2882
2883	engine->execlists.error_interrupt = 0;
2884	ENGINE_WRITE(engine, RING_EMR, ~0u);
2885	ENGINE_WRITE(engine, RING_EIR, ~0u); /* clear all existing errors */
2886
2887	status = ENGINE_READ(engine, RING_ESR);
2888	if (unlikely(status)) {
2889		drm_err(&engine->i915->drm,
2890			"engine '%s' resumed still in error: %08x\n",
2891			engine->name, status);
2892		__intel_gt_reset(engine->gt, engine->mask);
2893	}
2894
2895	/*
2896	 * On current gen8+, we have 2 signals to play with
2897	 *
2898	 * - I915_ERROR_INSTUCTION (bit 0)
2899	 *
2900	 *    Generate an error if the command parser encounters an invalid
2901	 *    instruction
2902	 *
2903	 *    This is a fatal error.
2904	 *
2905	 * - CP_PRIV (bit 2)
2906	 *
2907	 *    Generate an error on privilege violation (where the CP replaces
2908	 *    the instruction with a no-op). This also fires for writes into
2909	 *    read-only scratch pages.
2910	 *
2911	 *    This is a non-fatal error, parsing continues.
2912	 *
2913	 * * there are a few others defined for odd HW that we do not use
2914	 *
2915	 * Since CP_PRIV fires for cases where we have chosen to ignore the
2916	 * error (as the HW is validating and suppressing the mistakes), we
2917	 * only unmask the instruction error bit.
2918	 */
2919	ENGINE_WRITE(engine, RING_EMR, ~I915_ERROR_INSTRUCTION);
2920}
2921
2922static void enable_execlists(struct intel_engine_cs *engine)
2923{
2924	u32 mode;
2925
2926	assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
2927
2928	intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */
2929
2930	if (GRAPHICS_VER(engine->i915) >= 11)
2931		mode = _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE);
2932	else
2933		mode = _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE);
2934	ENGINE_WRITE_FW(engine, RING_MODE_GEN7, mode);
2935
2936	ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
2937
2938	ENGINE_WRITE_FW(engine,
2939			RING_HWS_PGA,
2940			i915_ggtt_offset(engine->status_page.vma));
2941	ENGINE_POSTING_READ(engine, RING_HWS_PGA);
2942
2943	enable_error_interrupt(engine);
2944}
2945
2946static int execlists_resume(struct intel_engine_cs *engine)
2947{
2948	intel_mocs_init_engine(engine);
2949	intel_breadcrumbs_reset(engine->breadcrumbs);
2950
2951	enable_execlists(engine);
2952
2953	if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)
2954		xehp_enable_ccs_engines(engine);
2955
2956	return 0;
2957}
2958
2959static void execlists_reset_prepare(struct intel_engine_cs *engine)
2960{
2961	ENGINE_TRACE(engine, "depth<-%d\n",
2962		     atomic_read(&engine->sched_engine->tasklet.count));
2963
2964	/*
2965	 * Prevent request submission to the hardware until we have
2966	 * completed the reset in i915_gem_reset_finish(). If a request
2967	 * is completed by one engine, it may then queue a request
2968	 * to a second via its execlists->tasklet *just* as we are
2969	 * calling engine->resume() and also writing the ELSP.
2970	 * Turning off the execlists->tasklet until the reset is over
2971	 * prevents the race.
2972	 */
2973	__tasklet_disable_sync_once(&engine->sched_engine->tasklet);
2974	GEM_BUG_ON(!reset_in_progress(engine));
2975
2976	/*
2977	 * We stop engines, otherwise we might get failed reset and a
2978	 * dead gpu (on elk). Also as modern gpu as kbl can suffer
2979	 * from system hang if batchbuffer is progressing when
2980	 * the reset is issued, regardless of READY_TO_RESET ack.
2981	 * Thus assume it is best to stop engines on all gens
2982	 * where we have a gpu reset.
2983	 *
2984	 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES)
2985	 *
2986	 * FIXME: Wa for more modern gens needs to be validated
2987	 */
2988	ring_set_paused(engine, 1);
2989	intel_engine_stop_cs(engine);
2990
2991	/*
2992	 * Wa_22011802037:gen11/gen12: In addition to stopping the cs, we need
2993	 * to wait for any pending mi force wakeups
2994	 */
2995	if (IS_GRAPHICS_VER(engine->i915, 11, 12))
2996		intel_engine_wait_for_pending_mi_fw(engine);
2997
2998	engine->execlists.reset_ccid = active_ccid(engine);
2999}
3000
3001static struct i915_request **
3002reset_csb(struct intel_engine_cs *engine, struct i915_request **inactive)
3003{
3004	struct intel_engine_execlists * const execlists = &engine->execlists;
3005
3006	drm_clflush_virt_range(execlists->csb_write,
3007			       sizeof(execlists->csb_write[0]));
3008
3009	inactive = process_csb(engine, inactive); /* drain preemption events */
3010
3011	/* Following the reset, we need to reload the CSB read/write pointers */
3012	reset_csb_pointers(engine);
3013
3014	return inactive;
3015}
3016
3017static void
3018execlists_reset_active(struct intel_engine_cs *engine, bool stalled)
3019{
3020	struct intel_context *ce;
3021	struct i915_request *rq;
3022	u32 head;
3023
3024	/*
3025	 * Save the currently executing context, even if we completed
3026	 * its request, it was still running at the time of the
3027	 * reset and will have been clobbered.
3028	 */
3029	rq = active_context(engine, engine->execlists.reset_ccid);
3030	if (!rq)
3031		return;
3032
3033	ce = rq->context;
3034	GEM_BUG_ON(!i915_vma_is_pinned(ce->state));
3035
3036	if (__i915_request_is_complete(rq)) {
3037		/* Idle context; tidy up the ring so we can restart afresh */
3038		head = intel_ring_wrap(ce->ring, rq->tail);
3039		goto out_replay;
3040	}
3041
3042	/* We still have requests in-flight; the engine should be active */
3043	GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
3044
3045	/* Context has requests still in-flight; it should not be idle! */
3046	GEM_BUG_ON(i915_active_is_idle(&ce->active));
3047
3048	rq = active_request(ce->timeline, rq);
3049	head = intel_ring_wrap(ce->ring, rq->head);
3050	GEM_BUG_ON(head == ce->ring->tail);
3051
3052	/*
3053	 * If this request hasn't started yet, e.g. it is waiting on a
3054	 * semaphore, we need to avoid skipping the request or else we
3055	 * break the signaling chain. However, if the context is corrupt
3056	 * the request will not restart and we will be stuck with a wedged
3057	 * device. It is quite often the case that if we issue a reset
3058	 * while the GPU is loading the context image, that the context
3059	 * image becomes corrupt.
3060	 *
3061	 * Otherwise, if we have not started yet, the request should replay
3062	 * perfectly and we do not need to flag the result as being erroneous.
3063	 */
3064	if (!__i915_request_has_started(rq))
3065		goto out_replay;
3066
3067	/*
3068	 * If the request was innocent, we leave the request in the ELSP
3069	 * and will try to replay it on restarting. The context image may
3070	 * have been corrupted by the reset, in which case we may have
3071	 * to service a new GPU hang, but more likely we can continue on
3072	 * without impact.
3073	 *
3074	 * If the request was guilty, we presume the context is corrupt
3075	 * and have to at least restore the RING register in the context
3076	 * image back to the expected values to skip over the guilty request.
3077	 */
3078	__i915_request_reset(rq, stalled);
3079
3080	/*
3081	 * We want a simple context + ring to execute the breadcrumb update.
3082	 * We cannot rely on the context being intact across the GPU hang,
3083	 * so clear it and rebuild just what we need for the breadcrumb.
3084	 * All pending requests for this context will be zapped, and any
3085	 * future request will be after userspace has had the opportunity
3086	 * to recreate its own state.
3087	 */
3088out_replay:
3089	ENGINE_TRACE(engine, "replay {head:%04x, tail:%04x}\n",
3090		     head, ce->ring->tail);
3091	lrc_reset_regs(ce, engine);
3092	ce->lrc.lrca = lrc_update_regs(ce, engine, head);
3093}
3094
3095static void execlists_reset_csb(struct intel_engine_cs *engine, bool stalled)
3096{
3097	struct intel_engine_execlists * const execlists = &engine->execlists;
3098	struct i915_request *post[2 * EXECLIST_MAX_PORTS];
3099	struct i915_request **inactive;
3100
3101	rcu_read_lock();
3102	inactive = reset_csb(engine, post);
3103
3104	execlists_reset_active(engine, true);
3105
3106	inactive = cancel_port_requests(execlists, inactive);
3107	post_process_csb(post, inactive);
3108	rcu_read_unlock();
3109}
3110
3111static void execlists_reset_rewind(struct intel_engine_cs *engine, bool stalled)
3112{
3113	unsigned long flags;
3114
3115	ENGINE_TRACE(engine, "\n");
3116
3117	/* Process the csb, find the guilty context and throw away */
3118	execlists_reset_csb(engine, stalled);
3119
3120	/* Push back any incomplete requests for replay after the reset. */
3121	rcu_read_lock();
3122	spin_lock_irqsave(&engine->sched_engine->lock, flags);
3123	__unwind_incomplete_requests(engine);
3124	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
3125	rcu_read_unlock();
3126}
3127
3128static void nop_submission_tasklet(struct tasklet_struct *t)
3129{
3130	struct i915_sched_engine *sched_engine =
3131		from_tasklet(sched_engine, t, tasklet);
3132	struct intel_engine_cs * const engine = sched_engine->private_data;
3133
3134	/* The driver is wedged; don't process any more events. */
3135	WRITE_ONCE(engine->sched_engine->queue_priority_hint, INT_MIN);
3136}
3137
3138static void execlists_reset_cancel(struct intel_engine_cs *engine)
3139{
3140	struct intel_engine_execlists * const execlists = &engine->execlists;
3141	struct i915_sched_engine * const sched_engine = engine->sched_engine;
3142	struct i915_request *rq, *rn;
3143	struct rb_node *rb;
3144	unsigned long flags;
3145
3146	ENGINE_TRACE(engine, "\n");
3147
3148	/*
3149	 * Before we call engine->cancel_requests(), we should have exclusive
3150	 * access to the submission state. This is arranged for us by the
3151	 * caller disabling the interrupt generation, the tasklet and other
3152	 * threads that may then access the same state, giving us a free hand
3153	 * to reset state. However, we still need to let lockdep be aware that
3154	 * we know this state may be accessed in hardirq context, so we
3155	 * disable the irq around this manipulation and we want to keep
3156	 * the spinlock focused on its duties and not accidentally conflate
3157	 * coverage to the submission's irq state. (Similarly, although we
3158	 * shouldn't need to disable irq around the manipulation of the
3159	 * submission's irq state, we also wish to remind ourselves that
3160	 * it is irq state.)
3161	 */
3162	execlists_reset_csb(engine, true);
3163
3164	rcu_read_lock();
3165	spin_lock_irqsave(&engine->sched_engine->lock, flags);
3166
3167	/* Mark all executing requests as skipped. */
3168	list_for_each_entry(rq, &engine->sched_engine->requests, sched.link)
3169		i915_request_put(i915_request_mark_eio(rq));
3170	intel_engine_signal_breadcrumbs(engine);
3171
3172	/* Flush the queued requests to the timeline list (for retiring). */
3173	while ((rb = rb_first_cached(&sched_engine->queue))) {
3174		struct i915_priolist *p = to_priolist(rb);
3175
3176		priolist_for_each_request_consume(rq, rn, p) {
3177			if (i915_request_mark_eio(rq)) {
3178				__i915_request_submit(rq);
3179				i915_request_put(rq);
3180			}
3181		}
3182
3183		rb_erase_cached(&p->node, &sched_engine->queue);
3184		i915_priolist_free(p);
3185	}
3186
3187	/* On-hold requests will be flushed to timeline upon their release */
3188	list_for_each_entry(rq, &sched_engine->hold, sched.link)
3189		i915_request_put(i915_request_mark_eio(rq));
3190
3191	/* Cancel all attached virtual engines */
3192	while ((rb = rb_first_cached(&execlists->virtual))) {
3193		struct virtual_engine *ve =
3194			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
3195
3196		rb_erase_cached(rb, &execlists->virtual);
3197		RB_CLEAR_NODE(rb);
3198
3199		spin_lock(&ve->base.sched_engine->lock);
3200		rq = fetch_and_zero(&ve->request);
3201		if (rq) {
3202			if (i915_request_mark_eio(rq)) {
3203				rq->engine = engine;
3204				__i915_request_submit(rq);
3205				i915_request_put(rq);
3206			}
3207			i915_request_put(rq);
3208
3209			ve->base.sched_engine->queue_priority_hint = INT_MIN;
3210		}
3211		spin_unlock(&ve->base.sched_engine->lock);
3212	}
3213
3214	/* Remaining _unready_ requests will be nop'ed when submitted */
3215
3216	sched_engine->queue_priority_hint = INT_MIN;
3217	sched_engine->queue = RB_ROOT_CACHED;
3218
3219	GEM_BUG_ON(__tasklet_is_enabled(&engine->sched_engine->tasklet));
3220	engine->sched_engine->tasklet.callback = nop_submission_tasklet;
3221
3222	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
3223	rcu_read_unlock();
3224}
3225
3226static void execlists_reset_finish(struct intel_engine_cs *engine)
3227{
3228	struct intel_engine_execlists * const execlists = &engine->execlists;
3229
3230	/*
3231	 * After a GPU reset, we may have requests to replay. Do so now while
3232	 * we still have the forcewake to be sure that the GPU is not allowed
3233	 * to sleep before we restart and reload a context.
3234	 *
3235	 * If the GPU reset fails, the engine may still be alive with requests
3236	 * inflight. We expect those to complete, or for the device to be
3237	 * reset as the next level of recovery, and as a final resort we
3238	 * will declare the device wedged.
3239	 */
3240	GEM_BUG_ON(!reset_in_progress(engine));
3241
3242	/* And kick in case we missed a new request submission. */
3243	if (__tasklet_enable(&engine->sched_engine->tasklet))
3244		__execlists_kick(execlists);
3245
3246	ENGINE_TRACE(engine, "depth->%d\n",
3247		     atomic_read(&engine->sched_engine->tasklet.count));
3248}
3249
3250static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
3251{
3252	ENGINE_WRITE(engine, RING_IMR,
3253		     ~(engine->irq_enable_mask | engine->irq_keep_mask));
3254	ENGINE_POSTING_READ(engine, RING_IMR);
3255}
3256
3257static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
3258{
3259	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
3260}
3261
3262static void execlists_park(struct intel_engine_cs *engine)
3263{
3264	cancel_timer(&engine->execlists.timer);
3265	cancel_timer(&engine->execlists.preempt);
3266}
3267
3268static void add_to_engine(struct i915_request *rq)
3269{
3270	lockdep_assert_held(&rq->engine->sched_engine->lock);
3271	list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests);
3272}
3273
3274static void remove_from_engine(struct i915_request *rq)
3275{
3276	struct intel_engine_cs *engine, *locked;
3277
3278	/*
3279	 * Virtual engines complicate acquiring the engine timeline lock,
3280	 * as their rq->engine pointer is not stable until under that
3281	 * engine lock. The simple ploy we use is to take the lock then
3282	 * check that the rq still belongs to the newly locked engine.
3283	 */
3284	locked = READ_ONCE(rq->engine);
3285	spin_lock_irq(&locked->sched_engine->lock);
3286	while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
3287		spin_unlock(&locked->sched_engine->lock);
3288		spin_lock(&engine->sched_engine->lock);
3289		locked = engine;
3290	}
3291	list_del_init(&rq->sched.link);
3292
3293	clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
3294	clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);
3295
3296	/* Prevent further __await_execution() registering a cb, then flush */
3297	set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
3298
3299	spin_unlock_irq(&locked->sched_engine->lock);
3300
3301	i915_request_notify_execute_cb_imm(rq);
3302}
3303
3304static bool can_preempt(struct intel_engine_cs *engine)
3305{
3306	if (GRAPHICS_VER(engine->i915) > 8)
3307		return true;
3308
3309	/* GPGPU on bdw requires extra w/a; not implemented */
3310	return engine->class != RENDER_CLASS;
3311}
3312
3313static void kick_execlists(const struct i915_request *rq, int prio)
3314{
3315	struct intel_engine_cs *engine = rq->engine;
3316	struct i915_sched_engine *sched_engine = engine->sched_engine;
3317	const struct i915_request *inflight;
3318
3319	/*
3320	 * We only need to kick the tasklet once for the high priority
3321	 * new context we add into the queue.
3322	 */
3323	if (prio <= sched_engine->queue_priority_hint)
3324		return;
3325
3326	rcu_read_lock();
3327
3328	/* Nothing currently active? We're overdue for a submission! */
3329	inflight = execlists_active(&engine->execlists);
3330	if (!inflight)
3331		goto unlock;
3332
3333	/*
3334	 * If we are already the currently executing context, don't
3335	 * bother evaluating if we should preempt ourselves.
3336	 */
3337	if (inflight->context == rq->context)
3338		goto unlock;
3339
3340	ENGINE_TRACE(engine,
3341		     "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",
3342		     prio,
3343		     rq->fence.context, rq->fence.seqno,
3344		     inflight->fence.context, inflight->fence.seqno,
3345		     inflight->sched.attr.priority);
3346
3347	sched_engine->queue_priority_hint = prio;
3348
3349	/*
3350	 * Allow preemption of low -> normal -> high, but we do
3351	 * not allow low priority tasks to preempt other low priority
3352	 * tasks under the impression that latency for low priority
3353	 * tasks does not matter (as much as background throughput),
3354	 * so kiss.
3355	 */
3356	if (prio >= max(I915_PRIORITY_NORMAL, rq_prio(inflight)))
3357		tasklet_hi_schedule(&sched_engine->tasklet);
3358
3359unlock:
3360	rcu_read_unlock();
3361}
3362
3363static void execlists_set_default_submission(struct intel_engine_cs *engine)
3364{
3365	engine->submit_request = execlists_submit_request;
3366	engine->sched_engine->schedule = i915_schedule;
3367	engine->sched_engine->kick_backend = kick_execlists;
3368	engine->sched_engine->tasklet.callback = execlists_submission_tasklet;
3369}
3370
3371static void execlists_shutdown(struct intel_engine_cs *engine)
3372{
3373	/* Synchronise with residual timers and any softirq they raise */
3374	del_timer_sync(&engine->execlists.timer);
3375	del_timer_sync(&engine->execlists.preempt);
3376	tasklet_kill(&engine->sched_engine->tasklet);
3377}
3378
3379static void execlists_release(struct intel_engine_cs *engine)
3380{
3381	engine->sanitize = NULL; /* no longer in control, nothing to sanitize */
3382
3383	execlists_shutdown(engine);
3384
3385	intel_engine_cleanup_common(engine);
3386	lrc_fini_wa_ctx(engine);
3387}
3388
3389static ktime_t __execlists_engine_busyness(struct intel_engine_cs *engine,
3390					   ktime_t *now)
3391{
3392	struct intel_engine_execlists_stats *stats = &engine->stats.execlists;
3393	ktime_t total = stats->total;
3394
3395	/*
3396	 * If the engine is executing something at the moment
3397	 * add it to the total.
3398	 */
3399	*now = ktime_get();
3400	if (READ_ONCE(stats->active))
3401		total = ktime_add(total, ktime_sub(*now, stats->start));
3402
3403	return total;
3404}
3405
3406static ktime_t execlists_engine_busyness(struct intel_engine_cs *engine,
3407					 ktime_t *now)
3408{
3409	struct intel_engine_execlists_stats *stats = &engine->stats.execlists;
3410	unsigned int seq;
3411	ktime_t total;
3412
3413	do {
3414		seq = read_seqcount_begin(&stats->lock);
3415		total = __execlists_engine_busyness(engine, now);
3416	} while (read_seqcount_retry(&stats->lock, seq));
3417
3418	return total;
3419}
3420
3421static void
3422logical_ring_default_vfuncs(struct intel_engine_cs *engine)
3423{
3424	/* Default vfuncs which can be overridden by each engine. */
3425
3426	engine->resume = execlists_resume;
3427
3428	engine->cops = &execlists_context_ops;
3429	engine->request_alloc = execlists_request_alloc;
3430	engine->add_active_request = add_to_engine;
3431	engine->remove_active_request = remove_from_engine;
3432
3433	engine->reset.prepare = execlists_reset_prepare;
3434	engine->reset.rewind = execlists_reset_rewind;
3435	engine->reset.cancel = execlists_reset_cancel;
3436	engine->reset.finish = execlists_reset_finish;
3437
3438	engine->park = execlists_park;
3439	engine->unpark = NULL;
3440
3441	engine->emit_flush = gen8_emit_flush_xcs;
3442	engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;
3443	engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs;
3444	if (GRAPHICS_VER(engine->i915) >= 12) {
3445		engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_xcs;
3446		engine->emit_flush = gen12_emit_flush_xcs;
3447	}
3448	engine->set_default_submission = execlists_set_default_submission;
3449
3450	if (GRAPHICS_VER(engine->i915) < 11) {
3451		engine->irq_enable = gen8_logical_ring_enable_irq;
3452		engine->irq_disable = gen8_logical_ring_disable_irq;
3453	} else {
3454		/*
3455		 * TODO: On Gen11 interrupt masks need to be clear
3456		 * to allow C6 entry. Keep interrupts enabled at
3457		 * and take the hit of generating extra interrupts
3458		 * until a more refined solution exists.
3459		 */
3460	}
3461	intel_engine_set_irq_handler(engine, execlists_irq_handler);
3462
3463	engine->flags |= I915_ENGINE_SUPPORTS_STATS;
3464	if (!intel_vgpu_active(engine->i915)) {
3465		engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
3466		if (can_preempt(engine)) {
3467			engine->flags |= I915_ENGINE_HAS_PREEMPTION;
3468			if (CONFIG_DRM_I915_TIMESLICE_DURATION)
3469				engine->flags |= I915_ENGINE_HAS_TIMESLICES;
3470		}
3471	}
3472
3473	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
3474		if (intel_engine_has_preemption(engine))
3475			engine->emit_bb_start = xehp_emit_bb_start;
3476		else
3477			engine->emit_bb_start = xehp_emit_bb_start_noarb;
3478	} else {
3479		if (intel_engine_has_preemption(engine))
3480			engine->emit_bb_start = gen8_emit_bb_start;
3481		else
3482			engine->emit_bb_start = gen8_emit_bb_start_noarb;
3483	}
3484
3485	engine->busyness = execlists_engine_busyness;
3486}
3487
3488static void logical_ring_default_irqs(struct intel_engine_cs *engine)
3489{
3490	unsigned int shift = 0;
3491
3492	if (GRAPHICS_VER(engine->i915) < 11) {
3493		const u8 irq_shifts[] = {
3494			[RCS0]  = GEN8_RCS_IRQ_SHIFT,
3495			[BCS0]  = GEN8_BCS_IRQ_SHIFT,
3496			[VCS0]  = GEN8_VCS0_IRQ_SHIFT,
3497			[VCS1]  = GEN8_VCS1_IRQ_SHIFT,
3498			[VECS0] = GEN8_VECS_IRQ_SHIFT,
3499		};
3500
3501		shift = irq_shifts[engine->id];
3502	}
3503
3504	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
3505	engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
3506	engine->irq_keep_mask |= GT_CS_MASTER_ERROR_INTERRUPT << shift;
3507	engine->irq_keep_mask |= GT_WAIT_SEMAPHORE_INTERRUPT << shift;
3508}
3509
3510static void rcs_submission_override(struct intel_engine_cs *engine)
3511{
3512	switch (GRAPHICS_VER(engine->i915)) {
3513	case 12:
3514		engine->emit_flush = gen12_emit_flush_rcs;
3515		engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;
3516		break;
3517	case 11:
3518		engine->emit_flush = gen11_emit_flush_rcs;
3519		engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;
3520		break;
3521	default:
3522		engine->emit_flush = gen8_emit_flush_rcs;
3523		engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;
3524		break;
3525	}
3526}
3527
3528int intel_execlists_submission_setup(struct intel_engine_cs *engine)
3529{
3530	struct intel_engine_execlists * const execlists = &engine->execlists;
3531	struct drm_i915_private *i915 = engine->i915;
3532	struct intel_uncore *uncore = engine->uncore;
3533	u32 base = engine->mmio_base;
3534
3535	tasklet_setup(&engine->sched_engine->tasklet, execlists_submission_tasklet);
3536	timer_setup(&engine->execlists.timer, execlists_timeslice, 0);
3537	timer_setup(&engine->execlists.preempt, execlists_preempt, 0);
3538
3539	logical_ring_default_vfuncs(engine);
3540	logical_ring_default_irqs(engine);
3541
3542	if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)
3543		rcs_submission_override(engine);
3544
3545	lrc_init_wa_ctx(engine);
3546
3547	if (HAS_LOGICAL_RING_ELSQ(i915)) {
3548		execlists->submit_reg = uncore->regs +
3549			i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(base));
3550		execlists->ctrl_reg = uncore->regs +
3551			i915_mmio_reg_offset(RING_EXECLIST_CONTROL(base));
3552
3553		engine->fw_domain = intel_uncore_forcewake_for_reg(engine->uncore,
3554				    RING_EXECLIST_CONTROL(engine->mmio_base),
3555				    FW_REG_WRITE);
3556	} else {
3557		execlists->submit_reg = uncore->regs +
3558			i915_mmio_reg_offset(RING_ELSP(base));
3559	}
3560
3561	execlists->csb_status =
3562		(u64 *)&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
3563
3564	execlists->csb_write =
3565		&engine->status_page.addr[INTEL_HWS_CSB_WRITE_INDEX(i915)];
3566
3567	if (GRAPHICS_VER(i915) < 11)
3568		execlists->csb_size = GEN8_CSB_ENTRIES;
3569	else
3570		execlists->csb_size = GEN11_CSB_ENTRIES;
3571
3572	engine->context_tag = GENMASK(BITS_PER_LONG - 2, 0);
3573	if (GRAPHICS_VER(engine->i915) >= 11 &&
3574	    GRAPHICS_VER_FULL(engine->i915) < IP_VER(12, 50)) {
3575		execlists->ccid |= engine->instance << (GEN11_ENGINE_INSTANCE_SHIFT - 32);
3576		execlists->ccid |= engine->class << (GEN11_ENGINE_CLASS_SHIFT - 32);
3577	}
3578
3579	/* Finally, take ownership and responsibility for cleanup! */
3580	engine->sanitize = execlists_sanitize;
3581	engine->release = execlists_release;
3582
3583	return 0;
3584}
3585
3586static struct list_head *virtual_queue(struct virtual_engine *ve)
3587{
3588	return &ve->base.sched_engine->default_priolist.requests;
3589}
3590
3591static void rcu_virtual_context_destroy(struct work_struct *wrk)
3592{
3593	struct virtual_engine *ve =
3594		container_of(wrk, typeof(*ve), rcu.work);
3595	unsigned int n;
3596
3597	GEM_BUG_ON(ve->context.inflight);
3598
3599	/* Preempt-to-busy may leave a stale request behind. */
3600	if (unlikely(ve->request)) {
3601		struct i915_request *old;
3602
3603		spin_lock_irq(&ve->base.sched_engine->lock);
3604
3605		old = fetch_and_zero(&ve->request);
3606		if (old) {
3607			GEM_BUG_ON(!__i915_request_is_complete(old));
3608			__i915_request_submit(old);
3609			i915_request_put(old);
3610		}
3611
3612		spin_unlock_irq(&ve->base.sched_engine->lock);
3613	}
3614
3615	/*
3616	 * Flush the tasklet in case it is still running on another core.
3617	 *
3618	 * This needs to be done before we remove ourselves from the siblings'
3619	 * rbtrees as in the case it is running in parallel, it may reinsert
3620	 * the rb_node into a sibling.
3621	 */
3622	tasklet_kill(&ve->base.sched_engine->tasklet);
3623
3624	/* Decouple ourselves from the siblings, no more access allowed. */
3625	for (n = 0; n < ve->num_siblings; n++) {
3626		struct intel_engine_cs *sibling = ve->siblings[n];
3627		struct rb_node *node = &ve->nodes[sibling->id].rb;
3628
3629		if (RB_EMPTY_NODE(node))
3630			continue;
3631
3632		spin_lock_irq(&sibling->sched_engine->lock);
3633
3634		/* Detachment is lazily performed in the sched_engine->tasklet */
3635		if (!RB_EMPTY_NODE(node))
3636			rb_erase_cached(node, &sibling->execlists.virtual);
3637
3638		spin_unlock_irq(&sibling->sched_engine->lock);
3639	}
3640	GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.sched_engine->tasklet));
3641	GEM_BUG_ON(!list_empty(virtual_queue(ve)));
3642
3643	lrc_fini(&ve->context);
3644	intel_context_fini(&ve->context);
3645
3646	if (ve->base.breadcrumbs)
3647		intel_breadcrumbs_put(ve->base.breadcrumbs);
3648	if (ve->base.sched_engine)
3649		i915_sched_engine_put(ve->base.sched_engine);
3650	intel_engine_free_request_pool(&ve->base);
3651
3652	kfree(ve);
3653}
3654
3655static void virtual_context_destroy(struct kref *kref)
3656{
3657	struct virtual_engine *ve =
3658		container_of(kref, typeof(*ve), context.ref);
3659
3660	GEM_BUG_ON(!list_empty(&ve->context.signals));
3661
3662	/*
3663	 * When destroying the virtual engine, we have to be aware that
3664	 * it may still be in use from an hardirq/softirq context causing
3665	 * the resubmission of a completed request (background completion
3666	 * due to preempt-to-busy). Before we can free the engine, we need
3667	 * to flush the submission code and tasklets that are still potentially
3668	 * accessing the engine. Flushing the tasklets requires process context,
3669	 * and since we can guard the resubmit onto the engine with an RCU read
3670	 * lock, we can delegate the free of the engine to an RCU worker.
3671	 */
3672	INIT_RCU_WORK(&ve->rcu, rcu_virtual_context_destroy);
3673	queue_rcu_work(system_wq, &ve->rcu);
3674}
3675
3676static void virtual_engine_initial_hint(struct virtual_engine *ve)
3677{
3678	int swp;
3679
3680	/*
3681	 * Pick a random sibling on starting to help spread the load around.
3682	 *
3683	 * New contexts are typically created with exactly the same order
3684	 * of siblings, and often started in batches. Due to the way we iterate
3685	 * the array of sibling when submitting requests, sibling[0] is
3686	 * prioritised for dequeuing. If we make sure that sibling[0] is fairly
3687	 * randomised across the system, we also help spread the load by the
3688	 * first engine we inspect being different each time.
3689	 *
3690	 * NB This does not force us to execute on this engine, it will just
3691	 * typically be the first we inspect for submission.
3692	 */
3693	swp = get_random_u32_below(ve->num_siblings);
3694	if (swp)
3695		swap(ve->siblings[swp], ve->siblings[0]);
3696}
3697
3698static int virtual_context_alloc(struct intel_context *ce)
3699{
3700	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3701
3702	return lrc_alloc(ce, ve->siblings[0]);
3703}
3704
3705static int virtual_context_pre_pin(struct intel_context *ce,
3706				   struct i915_gem_ww_ctx *ww,
3707				   void **vaddr)
3708{
3709	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3710
3711	 /* Note: we must use a real engine class for setting up reg state */
3712	return __execlists_context_pre_pin(ce, ve->siblings[0], ww, vaddr);
3713}
3714
3715static int virtual_context_pin(struct intel_context *ce, void *vaddr)
3716{
3717	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3718
3719	return lrc_pin(ce, ve->siblings[0], vaddr);
3720}
3721
3722static void virtual_context_enter(struct intel_context *ce)
3723{
3724	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3725	unsigned int n;
3726
3727	for (n = 0; n < ve->num_siblings; n++)
3728		intel_engine_pm_get(ve->siblings[n]);
3729
3730	intel_timeline_enter(ce->timeline);
3731}
3732
3733static void virtual_context_exit(struct intel_context *ce)
3734{
3735	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3736	unsigned int n;
3737
3738	intel_timeline_exit(ce->timeline);
3739
3740	for (n = 0; n < ve->num_siblings; n++)
3741		intel_engine_pm_put(ve->siblings[n]);
3742}
3743
3744static struct intel_engine_cs *
3745virtual_get_sibling(struct intel_engine_cs *engine, unsigned int sibling)
3746{
3747	struct virtual_engine *ve = to_virtual_engine(engine);
3748
3749	if (sibling >= ve->num_siblings)
3750		return NULL;
3751
3752	return ve->siblings[sibling];
3753}
3754
3755static const struct intel_context_ops virtual_context_ops = {
3756	.flags = COPS_HAS_INFLIGHT | COPS_RUNTIME_CYCLES,
3757
3758	.alloc = virtual_context_alloc,
3759
3760	.cancel_request = execlists_context_cancel_request,
3761
3762	.pre_pin = virtual_context_pre_pin,
3763	.pin = virtual_context_pin,
3764	.unpin = lrc_unpin,
3765	.post_unpin = lrc_post_unpin,
3766
3767	.enter = virtual_context_enter,
3768	.exit = virtual_context_exit,
3769
3770	.destroy = virtual_context_destroy,
3771
3772	.get_sibling = virtual_get_sibling,
3773};
3774
3775static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
3776{
3777	struct i915_request *rq;
3778	intel_engine_mask_t mask;
3779
3780	rq = READ_ONCE(ve->request);
3781	if (!rq)
3782		return 0;
3783
3784	/* The rq is ready for submission; rq->execution_mask is now stable. */
3785	mask = rq->execution_mask;
3786	if (unlikely(!mask)) {
3787		/* Invalid selection, submit to a random engine in error */
3788		i915_request_set_error_once(rq, -ENODEV);
3789		mask = ve->siblings[0]->mask;
3790	}
3791
3792	ENGINE_TRACE(&ve->base, "rq=%llx:%lld, mask=%x, prio=%d\n",
3793		     rq->fence.context, rq->fence.seqno,
3794		     mask, ve->base.sched_engine->queue_priority_hint);
3795
3796	return mask;
3797}
3798
3799static void virtual_submission_tasklet(struct tasklet_struct *t)
3800{
3801	struct i915_sched_engine *sched_engine =
3802		from_tasklet(sched_engine, t, tasklet);
3803	struct virtual_engine * const ve =
3804		(struct virtual_engine *)sched_engine->private_data;
3805	const int prio = READ_ONCE(sched_engine->queue_priority_hint);
3806	intel_engine_mask_t mask;
3807	unsigned int n;
3808
3809	rcu_read_lock();
3810	mask = virtual_submission_mask(ve);
3811	rcu_read_unlock();
3812	if (unlikely(!mask))
3813		return;
3814
3815	for (n = 0; n < ve->num_siblings; n++) {
3816		struct intel_engine_cs *sibling = READ_ONCE(ve->siblings[n]);
3817		struct ve_node * const node = &ve->nodes[sibling->id];
3818		struct rb_node **parent, *rb;
3819		bool first;
3820
3821		if (!READ_ONCE(ve->request))
3822			break; /* already handled by a sibling's tasklet */
3823
3824		spin_lock_irq(&sibling->sched_engine->lock);
3825
3826		if (unlikely(!(mask & sibling->mask))) {
3827			if (!RB_EMPTY_NODE(&node->rb)) {
3828				rb_erase_cached(&node->rb,
3829						&sibling->execlists.virtual);
3830				RB_CLEAR_NODE(&node->rb);
3831			}
3832
3833			goto unlock_engine;
3834		}
3835
3836		if (unlikely(!RB_EMPTY_NODE(&node->rb))) {
3837			/*
3838			 * Cheat and avoid rebalancing the tree if we can
3839			 * reuse this node in situ.
3840			 */
3841			first = rb_first_cached(&sibling->execlists.virtual) ==
3842				&node->rb;
3843			if (prio == node->prio || (prio > node->prio && first))
3844				goto submit_engine;
3845
3846			rb_erase_cached(&node->rb, &sibling->execlists.virtual);
3847		}
3848
3849		rb = NULL;
3850		first = true;
3851		parent = &sibling->execlists.virtual.rb_root.rb_node;
3852		while (*parent) {
3853			struct ve_node *other;
3854
3855			rb = *parent;
3856			other = rb_entry(rb, typeof(*other), rb);
3857			if (prio > other->prio) {
3858				parent = &rb->rb_left;
3859			} else {
3860				parent = &rb->rb_right;
3861				first = false;
3862			}
3863		}
3864
3865		rb_link_node(&node->rb, rb, parent);
3866		rb_insert_color_cached(&node->rb,
3867				       &sibling->execlists.virtual,
3868				       first);
3869
3870submit_engine:
3871		GEM_BUG_ON(RB_EMPTY_NODE(&node->rb));
3872		node->prio = prio;
3873		if (first && prio > sibling->sched_engine->queue_priority_hint)
3874			tasklet_hi_schedule(&sibling->sched_engine->tasklet);
3875
3876unlock_engine:
3877		spin_unlock_irq(&sibling->sched_engine->lock);
3878
3879		if (intel_context_inflight(&ve->context))
3880			break;
3881	}
3882}
3883
3884static void virtual_submit_request(struct i915_request *rq)
3885{
3886	struct virtual_engine *ve = to_virtual_engine(rq->engine);
3887	unsigned long flags;
3888
3889	ENGINE_TRACE(&ve->base, "rq=%llx:%lld\n",
3890		     rq->fence.context,
3891		     rq->fence.seqno);
3892
3893	GEM_BUG_ON(ve->base.submit_request != virtual_submit_request);
3894
3895	spin_lock_irqsave(&ve->base.sched_engine->lock, flags);
3896
3897	/* By the time we resubmit a request, it may be completed */
3898	if (__i915_request_is_complete(rq)) {
3899		__i915_request_submit(rq);
3900		goto unlock;
3901	}
3902
3903	if (ve->request) { /* background completion from preempt-to-busy */
3904		GEM_BUG_ON(!__i915_request_is_complete(ve->request));
3905		__i915_request_submit(ve->request);
3906		i915_request_put(ve->request);
3907	}
3908
3909	ve->base.sched_engine->queue_priority_hint = rq_prio(rq);
3910	ve->request = i915_request_get(rq);
3911
3912	GEM_BUG_ON(!list_empty(virtual_queue(ve)));
3913	list_move_tail(&rq->sched.link, virtual_queue(ve));
3914
3915	tasklet_hi_schedule(&ve->base.sched_engine->tasklet);
3916
3917unlock:
3918	spin_unlock_irqrestore(&ve->base.sched_engine->lock, flags);
3919}
3920
3921static struct intel_context *
3922execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
3923			 unsigned long flags)
3924{
3925	struct drm_i915_private *i915 = siblings[0]->i915;
3926	struct virtual_engine *ve;
3927	unsigned int n;
3928	int err;
3929
3930	ve = kzalloc(struct_size(ve, siblings, count), GFP_KERNEL);
3931	if (!ve)
3932		return ERR_PTR(-ENOMEM);
3933
3934	ve->base.i915 = i915;
3935	ve->base.gt = siblings[0]->gt;
3936	ve->base.uncore = siblings[0]->uncore;
3937	ve->base.id = -1;
3938
3939	ve->base.class = OTHER_CLASS;
3940	ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;
3941	ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
3942	ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
3943
3944	/*
3945	 * The decision on whether to submit a request using semaphores
3946	 * depends on the saturated state of the engine. We only compute
3947	 * this during HW submission of the request, and we need for this
3948	 * state to be globally applied to all requests being submitted
3949	 * to this engine. Virtual engines encompass more than one physical
3950	 * engine and so we cannot accurately tell in advance if one of those
3951	 * engines is already saturated and so cannot afford to use a semaphore
3952	 * and be pessimized in priority for doing so -- if we are the only
3953	 * context using semaphores after all other clients have stopped, we
3954	 * will be starved on the saturated system. Such a global switch for
3955	 * semaphores is less than ideal, but alas is the current compromise.
3956	 */
3957	ve->base.saturated = ALL_ENGINES;
3958
3959	snprintf(ve->base.name, sizeof(ve->base.name), "virtual");
3960
3961	intel_engine_init_execlists(&ve->base);
3962
3963	ve->base.sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL);
3964	if (!ve->base.sched_engine) {
3965		err = -ENOMEM;
3966		goto err_put;
3967	}
3968	ve->base.sched_engine->private_data = &ve->base;
3969
3970	ve->base.cops = &virtual_context_ops;
3971	ve->base.request_alloc = execlists_request_alloc;
3972
3973	ve->base.sched_engine->schedule = i915_schedule;
3974	ve->base.sched_engine->kick_backend = kick_execlists;
3975	ve->base.submit_request = virtual_submit_request;
3976
3977	INIT_LIST_HEAD(virtual_queue(ve));
3978	tasklet_setup(&ve->base.sched_engine->tasklet, virtual_submission_tasklet);
3979
3980	intel_context_init(&ve->context, &ve->base);
3981
3982	ve->base.breadcrumbs = intel_breadcrumbs_create(NULL);
3983	if (!ve->base.breadcrumbs) {
3984		err = -ENOMEM;
3985		goto err_put;
3986	}
3987
3988	for (n = 0; n < count; n++) {
3989		struct intel_engine_cs *sibling = siblings[n];
3990
3991		GEM_BUG_ON(!is_power_of_2(sibling->mask));
3992		if (sibling->mask & ve->base.mask) {
3993			drm_dbg(&i915->drm,
3994				"duplicate %s entry in load balancer\n",
3995				sibling->name);
3996			err = -EINVAL;
3997			goto err_put;
3998		}
3999
4000		/*
4001		 * The virtual engine implementation is tightly coupled to
4002		 * the execlists backend -- we push out request directly
4003		 * into a tree inside each physical engine. We could support
4004		 * layering if we handle cloning of the requests and
4005		 * submitting a copy into each backend.
4006		 */
4007		if (sibling->sched_engine->tasklet.callback !=
4008		    execlists_submission_tasklet) {
4009			err = -ENODEV;
4010			goto err_put;
4011		}
4012
4013		GEM_BUG_ON(RB_EMPTY_NODE(&ve->nodes[sibling->id].rb));
4014		RB_CLEAR_NODE(&ve->nodes[sibling->id].rb);
4015
4016		ve->siblings[ve->num_siblings++] = sibling;
4017		ve->base.mask |= sibling->mask;
4018		ve->base.logical_mask |= sibling->logical_mask;
4019
4020		/*
4021		 * All physical engines must be compatible for their emission
4022		 * functions (as we build the instructions during request
4023		 * construction and do not alter them before submission
4024		 * on the physical engine). We use the engine class as a guide
4025		 * here, although that could be refined.
4026		 */
4027		if (ve->base.class != OTHER_CLASS) {
4028			if (ve->base.class != sibling->class) {
4029				drm_dbg(&i915->drm,
4030					"invalid mixing of engine class, sibling %d, already %d\n",
4031					sibling->class, ve->base.class);
4032				err = -EINVAL;
4033				goto err_put;
4034			}
4035			continue;
4036		}
4037
4038		ve->base.class = sibling->class;
4039		ve->base.uabi_class = sibling->uabi_class;
4040		snprintf(ve->base.name, sizeof(ve->base.name),
4041			 "v%dx%d", ve->base.class, count);
4042		ve->base.context_size = sibling->context_size;
4043
4044		ve->base.add_active_request = sibling->add_active_request;
4045		ve->base.remove_active_request = sibling->remove_active_request;
4046		ve->base.emit_bb_start = sibling->emit_bb_start;
4047		ve->base.emit_flush = sibling->emit_flush;
4048		ve->base.emit_init_breadcrumb = sibling->emit_init_breadcrumb;
4049		ve->base.emit_fini_breadcrumb = sibling->emit_fini_breadcrumb;
4050		ve->base.emit_fini_breadcrumb_dw =
4051			sibling->emit_fini_breadcrumb_dw;
4052
4053		ve->base.flags = sibling->flags;
4054	}
4055
4056	ve->base.flags |= I915_ENGINE_IS_VIRTUAL;
4057
4058	virtual_engine_initial_hint(ve);
4059	return &ve->context;
4060
4061err_put:
4062	intel_context_put(&ve->context);
4063	return ERR_PTR(err);
4064}
4065
4066void intel_execlists_show_requests(struct intel_engine_cs *engine,
4067				   struct drm_printer *m,
4068				   void (*show_request)(struct drm_printer *m,
4069							const struct i915_request *rq,
4070							const char *prefix,
4071							int indent),
4072				   unsigned int max)
4073{
4074	const struct intel_engine_execlists *execlists = &engine->execlists;
4075	struct i915_sched_engine *sched_engine = engine->sched_engine;
4076	struct i915_request *rq, *last;
4077	unsigned long flags;
4078	unsigned int count;
4079	struct rb_node *rb;
4080
4081	spin_lock_irqsave(&sched_engine->lock, flags);
4082
4083	last = NULL;
4084	count = 0;
4085	list_for_each_entry(rq, &sched_engine->requests, sched.link) {
4086		if (count++ < max - 1)
4087			show_request(m, rq, "\t\t", 0);
4088		else
4089			last = rq;
4090	}
4091	if (last) {
4092		if (count > max) {
4093			drm_printf(m,
4094				   "\t\t...skipping %d executing requests...\n",
4095				   count - max);
4096		}
4097		show_request(m, last, "\t\t", 0);
4098	}
4099
4100	if (sched_engine->queue_priority_hint != INT_MIN)
4101		drm_printf(m, "\t\tQueue priority hint: %d\n",
4102			   READ_ONCE(sched_engine->queue_priority_hint));
4103
4104	last = NULL;
4105	count = 0;
4106	for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) {
4107		struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
4108
4109		priolist_for_each_request(rq, p) {
4110			if (count++ < max - 1)
4111				show_request(m, rq, "\t\t", 0);
4112			else
4113				last = rq;
4114		}
4115	}
4116	if (last) {
4117		if (count > max) {
4118			drm_printf(m,
4119				   "\t\t...skipping %d queued requests...\n",
4120				   count - max);
4121		}
4122		show_request(m, last, "\t\t", 0);
4123	}
4124
4125	last = NULL;
4126	count = 0;
4127	for (rb = rb_first_cached(&execlists->virtual); rb; rb = rb_next(rb)) {
4128		struct virtual_engine *ve =
4129			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
4130		struct i915_request *rq = READ_ONCE(ve->request);
4131
4132		if (rq) {
4133			if (count++ < max - 1)
4134				show_request(m, rq, "\t\t", 0);
4135			else
4136				last = rq;
4137		}
4138	}
4139	if (last) {
4140		if (count > max) {
4141			drm_printf(m,
4142				   "\t\t...skipping %d virtual requests...\n",
4143				   count - max);
4144		}
4145		show_request(m, last, "\t\t", 0);
4146	}
4147
4148	spin_unlock_irqrestore(&sched_engine->lock, flags);
4149}
4150
4151static unsigned long list_count(struct list_head *list)
4152{
4153	struct list_head *pos;
4154	unsigned long count = 0;
4155
4156	list_for_each(pos, list)
4157		count++;
4158
4159	return count;
4160}
4161
4162void intel_execlists_dump_active_requests(struct intel_engine_cs *engine,
4163					  struct i915_request *hung_rq,
4164					  struct drm_printer *m)
4165{
4166	unsigned long flags;
4167
4168	spin_lock_irqsave(&engine->sched_engine->lock, flags);
4169
4170	intel_engine_dump_active_requests(&engine->sched_engine->requests, hung_rq, m);
4171
4172	drm_printf(m, "\tOn hold?: %lu\n",
4173		   list_count(&engine->sched_engine->hold));
4174
4175	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
4176}
4177
4178#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
4179#include "selftest_execlists.c"
4180#endif