1/*
   2 * SPDX-License-Identifier: MIT
   3 *
   4 * Copyright © 2008,2010 Intel Corporation
   5 */
   6
   7#include <linux/dma-resv.h>
   8#include <linux/highmem.h>
   9#include <linux/sync_file.h>
  10#include <linux/uaccess.h>
  11
  12#include <drm/drm_syncobj.h>
  13
  14#include "display/intel_frontbuffer.h"
  15
  16#include "gem/i915_gem_ioctls.h"
  17#include "gt/intel_context.h"
  18#include "gt/intel_gpu_commands.h"
  19#include "gt/intel_gt.h"
  20#include "gt/intel_gt_buffer_pool.h"
  21#include "gt/intel_gt_pm.h"
  22#include "gt/intel_ring.h"
  23
  24#include "pxp/intel_pxp.h"
  25
  26#include "i915_cmd_parser.h"
  27#include "i915_drv.h"
  28#include "i915_file_private.h"
  29#include "i915_gem_clflush.h"
  30#include "i915_gem_context.h"
  31#include "i915_gem_evict.h"
  32#include "i915_gem_ioctls.h"
  33#include "i915_reg.h"
  34#include "i915_trace.h"
  35#include "i915_user_extensions.h"
  36
  37struct eb_vma {
  38	struct i915_vma *vma;
  39	unsigned int flags;
  40
  41	/** This vma's place in the execbuf reservation list */
  42	struct drm_i915_gem_exec_object2 *exec;
  43	struct list_head bind_link;
  44	struct list_head reloc_link;
  45
  46	struct hlist_node node;
  47	u32 handle;
  48};
  49
  50enum {
  51	FORCE_CPU_RELOC = 1,
  52	FORCE_GTT_RELOC,
  53	FORCE_GPU_RELOC,
  54#define DBG_FORCE_RELOC 0 /* choose one of the above! */
  55};
  56
  57/* __EXEC_OBJECT_ flags > BIT(29) defined in i915_vma.h */
  58#define __EXEC_OBJECT_HAS_PIN		BIT(29)
  59#define __EXEC_OBJECT_HAS_FENCE		BIT(28)
  60#define __EXEC_OBJECT_USERPTR_INIT	BIT(27)
  61#define __EXEC_OBJECT_NEEDS_MAP		BIT(26)
  62#define __EXEC_OBJECT_NEEDS_BIAS	BIT(25)
  63#define __EXEC_OBJECT_INTERNAL_FLAGS	(~0u << 25) /* all of the above + */
  64#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
  65
  66#define __EXEC_HAS_RELOC	BIT(31)
  67#define __EXEC_ENGINE_PINNED	BIT(30)
  68#define __EXEC_USERPTR_USED	BIT(29)
  69#define __EXEC_INTERNAL_FLAGS	(~0u << 29)
  70#define UPDATE			PIN_OFFSET_FIXED
  71
  72#define BATCH_OFFSET_BIAS (256*1024)
  73
  74#define __I915_EXEC_ILLEGAL_FLAGS \
  75	(__I915_EXEC_UNKNOWN_FLAGS | \
  76	 I915_EXEC_CONSTANTS_MASK  | \
  77	 I915_EXEC_RESOURCE_STREAMER)
  78
  79/* Catch emission of unexpected errors for CI! */
  80#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
  81#undef EINVAL
  82#define EINVAL ({ \
  83	DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \
  84	22; \
  85})
  86#endif
  87
  88/**
  89 * DOC: User command execution
  90 *
  91 * Userspace submits commands to be executed on the GPU as an instruction
  92 * stream within a GEM object we call a batchbuffer. This instructions may
  93 * refer to other GEM objects containing auxiliary state such as kernels,
  94 * samplers, render targets and even secondary batchbuffers. Userspace does
  95 * not know where in the GPU memory these objects reside and so before the
  96 * batchbuffer is passed to the GPU for execution, those addresses in the
  97 * batchbuffer and auxiliary objects are updated. This is known as relocation,
  98 * or patching. To try and avoid having to relocate each object on the next
  99 * execution, userspace is told the location of those objects in this pass,
 100 * but this remains just a hint as the kernel may choose a new location for
 101 * any object in the future.
 102 *
 103 * At the level of talking to the hardware, submitting a batchbuffer for the
 104 * GPU to execute is to add content to a buffer from which the HW
 105 * command streamer is reading.
 106 *
 107 * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e.
 108 *    Execlists, this command is not placed on the same buffer as the
 109 *    remaining items.
 110 *
 111 * 2. Add a command to invalidate caches to the buffer.
 112 *
 113 * 3. Add a batchbuffer start command to the buffer; the start command is
 114 *    essentially a token together with the GPU address of the batchbuffer
 115 *    to be executed.
 116 *
 117 * 4. Add a pipeline flush to the buffer.
 118 *
 119 * 5. Add a memory write command to the buffer to record when the GPU
 120 *    is done executing the batchbuffer. The memory write writes the
 121 *    global sequence number of the request, ``i915_request::global_seqno``;
 122 *    the i915 driver uses the current value in the register to determine
 123 *    if the GPU has completed the batchbuffer.
 124 *
 125 * 6. Add a user interrupt command to the buffer. This command instructs
 126 *    the GPU to issue an interrupt when the command, pipeline flush and
 127 *    memory write are completed.
 128 *
 129 * 7. Inform the hardware of the additional commands added to the buffer
 130 *    (by updating the tail pointer).
 131 *
 132 * Processing an execbuf ioctl is conceptually split up into a few phases.
 133 *
 134 * 1. Validation - Ensure all the pointers, handles and flags are valid.
 135 * 2. Reservation - Assign GPU address space for every object
 136 * 3. Relocation - Update any addresses to point to the final locations
 137 * 4. Serialisation - Order the request with respect to its dependencies
 138 * 5. Construction - Construct a request to execute the batchbuffer
 139 * 6. Submission (at some point in the future execution)
 140 *
 141 * Reserving resources for the execbuf is the most complicated phase. We
 142 * neither want to have to migrate the object in the address space, nor do
 143 * we want to have to update any relocations pointing to this object. Ideally,
 144 * we want to leave the object where it is and for all the existing relocations
 145 * to match. If the object is given a new address, or if userspace thinks the
 146 * object is elsewhere, we have to parse all the relocation entries and update
 147 * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that
 148 * all the target addresses in all of its objects match the value in the
 149 * relocation entries and that they all match the presumed offsets given by the
 150 * list of execbuffer objects. Using this knowledge, we know that if we haven't
 151 * moved any buffers, all the relocation entries are valid and we can skip
 152 * the update. (If userspace is wrong, the likely outcome is an impromptu GPU
 153 * hang.) The requirement for using I915_EXEC_NO_RELOC are:
 154 *
 155 *      The addresses written in the objects must match the corresponding
 156 *      reloc.presumed_offset which in turn must match the corresponding
 157 *      execobject.offset.
 158 *
 159 *      Any render targets written to in the batch must be flagged with
 160 *      EXEC_OBJECT_WRITE.
 161 *
 162 *      To avoid stalling, execobject.offset should match the current
 163 *      address of that object within the active context.
 164 *
 165 * The reservation is done is multiple phases. First we try and keep any
 166 * object already bound in its current location - so as long as meets the
 167 * constraints imposed by the new execbuffer. Any object left unbound after the
 168 * first pass is then fitted into any available idle space. If an object does
 169 * not fit, all objects are removed from the reservation and the process rerun
 170 * after sorting the objects into a priority order (more difficult to fit
 171 * objects are tried first). Failing that, the entire VM is cleared and we try
 172 * to fit the execbuf once last time before concluding that it simply will not
 173 * fit.
 174 *
 175 * A small complication to all of this is that we allow userspace not only to
 176 * specify an alignment and a size for the object in the address space, but
 177 * we also allow userspace to specify the exact offset. This objects are
 178 * simpler to place (the location is known a priori) all we have to do is make
 179 * sure the space is available.
 180 *
 181 * Once all the objects are in place, patching up the buried pointers to point
 182 * to the final locations is a fairly simple job of walking over the relocation
 183 * entry arrays, looking up the right address and rewriting the value into
 184 * the object. Simple! ... The relocation entries are stored in user memory
 185 * and so to access them we have to copy them into a local buffer. That copy
 186 * has to avoid taking any pagefaults as they may lead back to a GEM object
 187 * requiring the struct_mutex (i.e. recursive deadlock). So once again we split
 188 * the relocation into multiple passes. First we try to do everything within an
 189 * atomic context (avoid the pagefaults) which requires that we never wait. If
 190 * we detect that we may wait, or if we need to fault, then we have to fallback
 191 * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm
 192 * bells yet?) Dropping the mutex means that we lose all the state we have
 193 * built up so far for the execbuf and we must reset any global data. However,
 194 * we do leave the objects pinned in their final locations - which is a
 195 * potential issue for concurrent execbufs. Once we have left the mutex, we can
 196 * allocate and copy all the relocation entries into a large array at our
 197 * leisure, reacquire the mutex, reclaim all the objects and other state and
 198 * then proceed to update any incorrect addresses with the objects.
 199 *
 200 * As we process the relocation entries, we maintain a record of whether the
 201 * object is being written to. Using NORELOC, we expect userspace to provide
 202 * this information instead. We also check whether we can skip the relocation
 203 * by comparing the expected value inside the relocation entry with the target's
 204 * final address. If they differ, we have to map the current object and rewrite
 205 * the 4 or 8 byte pointer within.
 206 *
 207 * Serialising an execbuf is quite simple according to the rules of the GEM
 208 * ABI. Execution within each context is ordered by the order of submission.
 209 * Writes to any GEM object are in order of submission and are exclusive. Reads
 210 * from a GEM object are unordered with respect to other reads, but ordered by
 211 * writes. A write submitted after a read cannot occur before the read, and
 212 * similarly any read submitted after a write cannot occur before the write.
 213 * Writes are ordered between engines such that only one write occurs at any
 214 * time (completing any reads beforehand) - using semaphores where available
 215 * and CPU serialisation otherwise. Other GEM access obey the same rules, any
 216 * write (either via mmaps using set-domain, or via pwrite) must flush all GPU
 217 * reads before starting, and any read (either using set-domain or pread) must
 218 * flush all GPU writes before starting. (Note we only employ a barrier before,
 219 * we currently rely on userspace not concurrently starting a new execution
 220 * whilst reading or writing to an object. This may be an advantage or not
 221 * depending on how much you trust userspace not to shoot themselves in the
 222 * foot.) Serialisation may just result in the request being inserted into
 223 * a DAG awaiting its turn, but most simple is to wait on the CPU until
 224 * all dependencies are resolved.
 225 *
 226 * After all of that, is just a matter of closing the request and handing it to
 227 * the hardware (well, leaving it in a queue to be executed). However, we also
 228 * offer the ability for batchbuffers to be run with elevated privileges so
 229 * that they access otherwise hidden registers. (Used to adjust L3 cache etc.)
 230 * Before any batch is given extra privileges we first must check that it
 231 * contains no nefarious instructions, we check that each instruction is from
 232 * our whitelist and all registers are also from an allowed list. We first
 233 * copy the user's batchbuffer to a shadow (so that the user doesn't have
 234 * access to it, either by the CPU or GPU as we scan it) and then parse each
 235 * instruction. If everything is ok, we set a flag telling the hardware to run
 236 * the batchbuffer in trusted mode, otherwise the ioctl is rejected.
 237 */
 238
 239struct eb_fence {
 240	struct drm_syncobj *syncobj; /* Use with ptr_mask_bits() */
 241	struct dma_fence *dma_fence;
 242	u64 value;
 243	struct dma_fence_chain *chain_fence;
 244};
 245
 246struct i915_execbuffer {
 247	struct drm_i915_private *i915; /** i915 backpointer */
 248	struct drm_file *file; /** per-file lookup tables and limits */
 249	struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */
 250	struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */
 251	struct eb_vma *vma;
 252
 253	struct intel_gt *gt; /* gt for the execbuf */
 254	struct intel_context *context; /* logical state for the request */
 255	struct i915_gem_context *gem_context; /** caller's context */
 256
 257	/** our requests to build */
 258	struct i915_request *requests[MAX_ENGINE_INSTANCE + 1];
 259	/** identity of the batch obj/vma */
 260	struct eb_vma *batches[MAX_ENGINE_INSTANCE + 1];
 261	struct i915_vma *trampoline; /** trampoline used for chaining */
 262
 263	/** used for excl fence in dma_resv objects when > 1 BB submitted */
 264	struct dma_fence *composite_fence;
 265
 266	/** actual size of execobj[] as we may extend it for the cmdparser */
 267	unsigned int buffer_count;
 268
 269	/* number of batches in execbuf IOCTL */
 270	unsigned int num_batches;
 271
 272	/** list of vma not yet bound during reservation phase */
 273	struct list_head unbound;
 274
 275	/** list of vma that have execobj.relocation_count */
 276	struct list_head relocs;
 277
 278	struct i915_gem_ww_ctx ww;
 279
 280	/**
 281	 * Track the most recently used object for relocations, as we
 282	 * frequently have to perform multiple relocations within the same
 283	 * obj/page
 284	 */
 285	struct reloc_cache {
 286		struct drm_mm_node node; /** temporary GTT binding */
 287		unsigned long vaddr; /** Current kmap address */
 288		unsigned long page; /** Currently mapped page index */
 289		unsigned int graphics_ver; /** Cached value of GRAPHICS_VER */
 290		bool use_64bit_reloc : 1;
 291		bool has_llc : 1;
 292		bool has_fence : 1;
 293		bool needs_unfenced : 1;
 294	} reloc_cache;
 295
 296	u64 invalid_flags; /** Set of execobj.flags that are invalid */
 297
 298	/** Length of batch within object */
 299	u64 batch_len[MAX_ENGINE_INSTANCE + 1];
 300	u32 batch_start_offset; /** Location within object of batch */
 301	u32 batch_flags; /** Flags composed for emit_bb_start() */
 302	struct intel_gt_buffer_pool_node *batch_pool; /** pool node for batch buffer */
 303
 304	/**
 305	 * Indicate either the size of the hastable used to resolve
 306	 * relocation handles, or if negative that we are using a direct
 307	 * index into the execobj[].
 308	 */
 309	int lut_size;
 310	struct hlist_head *buckets; /** ht for relocation handles */
 311
 312	struct eb_fence *fences;
 313	unsigned long num_fences;
 314#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
 315	struct i915_capture_list *capture_lists[MAX_ENGINE_INSTANCE + 1];
 316#endif
 317};
 318
 319static int eb_parse(struct i915_execbuffer *eb);
 320static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle);
 321static void eb_unpin_engine(struct i915_execbuffer *eb);
 322static void eb_capture_release(struct i915_execbuffer *eb);
 323
 324static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
 325{
 326	return intel_engine_requires_cmd_parser(eb->context->engine) ||
 327		(intel_engine_using_cmd_parser(eb->context->engine) &&
 328		 eb->args->batch_len);
 329}
 330
 331static int eb_create(struct i915_execbuffer *eb)
 332{
 333	if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
 334		unsigned int size = 1 + ilog2(eb->buffer_count);
 335
 336		/*
 337		 * Without a 1:1 association between relocation handles and
 338		 * the execobject[] index, we instead create a hashtable.
 339		 * We size it dynamically based on available memory, starting
 340		 * first with 1:1 assocative hash and scaling back until
 341		 * the allocation succeeds.
 342		 *
 343		 * Later on we use a positive lut_size to indicate we are
 344		 * using this hashtable, and a negative value to indicate a
 345		 * direct lookup.
 346		 */
 347		do {
 348			gfp_t flags;
 349
 350			/* While we can still reduce the allocation size, don't
 351			 * raise a warning and allow the allocation to fail.
 352			 * On the last pass though, we want to try as hard
 353			 * as possible to perform the allocation and warn
 354			 * if it fails.
 355			 */
 356			flags = GFP_KERNEL;
 357			if (size > 1)
 358				flags |= __GFP_NORETRY | __GFP_NOWARN;
 359
 360			eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
 361					      flags);
 362			if (eb->buckets)
 363				break;
 364		} while (--size);
 365
 366		if (unlikely(!size))
 367			return -ENOMEM;
 368
 369		eb->lut_size = size;
 370	} else {
 371		eb->lut_size = -eb->buffer_count;
 372	}
 373
 374	return 0;
 375}
 376
 377static bool
 378eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
 379		 const struct i915_vma *vma,
 380		 unsigned int flags)
 381{
 382	if (vma->node.size < entry->pad_to_size)
 383		return true;
 384
 385	if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
 386		return true;
 387
 388	if (flags & EXEC_OBJECT_PINNED &&
 389	    vma->node.start != entry->offset)
 390		return true;
 391
 392	if (flags & __EXEC_OBJECT_NEEDS_BIAS &&
 393	    vma->node.start < BATCH_OFFSET_BIAS)
 394		return true;
 395
 396	if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
 397	    (vma->node.start + vma->node.size + 4095) >> 32)
 398		return true;
 399
 400	if (flags & __EXEC_OBJECT_NEEDS_MAP &&
 401	    !i915_vma_is_map_and_fenceable(vma))
 402		return true;
 403
 404	return false;
 405}
 406
 407static u64 eb_pin_flags(const struct drm_i915_gem_exec_object2 *entry,
 408			unsigned int exec_flags)
 409{
 410	u64 pin_flags = 0;
 411
 412	if (exec_flags & EXEC_OBJECT_NEEDS_GTT)
 413		pin_flags |= PIN_GLOBAL;
 414
 415	/*
 416	 * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
 417	 * limit address to the first 4GBs for unflagged objects.
 418	 */
 419	if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
 420		pin_flags |= PIN_ZONE_4G;
 421
 422	if (exec_flags & __EXEC_OBJECT_NEEDS_MAP)
 423		pin_flags |= PIN_MAPPABLE;
 424
 425	if (exec_flags & EXEC_OBJECT_PINNED)
 426		pin_flags |= entry->offset | PIN_OFFSET_FIXED;
 427	else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS)
 428		pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
 429
 430	return pin_flags;
 431}
 432
 433static inline int
 434eb_pin_vma(struct i915_execbuffer *eb,
 435	   const struct drm_i915_gem_exec_object2 *entry,
 436	   struct eb_vma *ev)
 437{
 438	struct i915_vma *vma = ev->vma;
 439	u64 pin_flags;
 440	int err;
 441
 442	if (vma->node.size)
 443		pin_flags = vma->node.start;
 444	else
 445		pin_flags = entry->offset & PIN_OFFSET_MASK;
 446
 447	pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED | PIN_VALIDATE;
 448	if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT))
 449		pin_flags |= PIN_GLOBAL;
 450
 451	/* Attempt to reuse the current location if available */
 452	err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags);
 453	if (err == -EDEADLK)
 454		return err;
 455
 456	if (unlikely(err)) {
 457		if (entry->flags & EXEC_OBJECT_PINNED)
 458			return err;
 459
 460		/* Failing that pick any _free_ space if suitable */
 461		err = i915_vma_pin_ww(vma, &eb->ww,
 462					     entry->pad_to_size,
 463					     entry->alignment,
 464					     eb_pin_flags(entry, ev->flags) |
 465					     PIN_USER | PIN_NOEVICT | PIN_VALIDATE);
 466		if (unlikely(err))
 467			return err;
 468	}
 469
 470	if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) {
 471		err = i915_vma_pin_fence(vma);
 472		if (unlikely(err))
 473			return err;
 474
 475		if (vma->fence)
 476			ev->flags |= __EXEC_OBJECT_HAS_FENCE;
 477	}
 478
 479	ev->flags |= __EXEC_OBJECT_HAS_PIN;
 480	if (eb_vma_misplaced(entry, vma, ev->flags))
 481		return -EBADSLT;
 482
 483	return 0;
 484}
 485
 486static inline void
 487eb_unreserve_vma(struct eb_vma *ev)
 488{
 489	if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE))
 490		__i915_vma_unpin_fence(ev->vma);
 491
 492	ev->flags &= ~__EXEC_OBJECT_RESERVED;
 493}
 494
 495static int
 496eb_validate_vma(struct i915_execbuffer *eb,
 497		struct drm_i915_gem_exec_object2 *entry,
 498		struct i915_vma *vma)
 499{
 500	/* Relocations are disallowed for all platforms after TGL-LP.  This
 501	 * also covers all platforms with local memory.
 502	 */
 503	if (entry->relocation_count &&
 504	    GRAPHICS_VER(eb->i915) >= 12 && !IS_TIGERLAKE(eb->i915))
 505		return -EINVAL;
 506
 507	if (unlikely(entry->flags & eb->invalid_flags))
 508		return -EINVAL;
 509
 510	if (unlikely(entry->alignment &&
 511		     !is_power_of_2_u64(entry->alignment)))
 512		return -EINVAL;
 513
 514	/*
 515	 * Offset can be used as input (EXEC_OBJECT_PINNED), reject
 516	 * any non-page-aligned or non-canonical addresses.
 517	 */
 518	if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
 519		     entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK)))
 520		return -EINVAL;
 521
 522	/* pad_to_size was once a reserved field, so sanitize it */
 523	if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
 524		if (unlikely(offset_in_page(entry->pad_to_size)))
 525			return -EINVAL;
 526	} else {
 527		entry->pad_to_size = 0;
 528	}
 529	/*
 530	 * From drm_mm perspective address space is continuous,
 531	 * so from this point we're always using non-canonical
 532	 * form internally.
 533	 */
 534	entry->offset = gen8_noncanonical_addr(entry->offset);
 535
 536	if (!eb->reloc_cache.has_fence) {
 537		entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
 538	} else {
 539		if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
 540		     eb->reloc_cache.needs_unfenced) &&
 541		    i915_gem_object_is_tiled(vma->obj))
 542			entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
 543	}
 544
 545	return 0;
 546}
 547
 548static inline bool
 549is_batch_buffer(struct i915_execbuffer *eb, unsigned int buffer_idx)
 550{
 551	return eb->args->flags & I915_EXEC_BATCH_FIRST ?
 552		buffer_idx < eb->num_batches :
 553		buffer_idx >= eb->args->buffer_count - eb->num_batches;
 554}
 555
 556static int
 557eb_add_vma(struct i915_execbuffer *eb,
 558	   unsigned int *current_batch,
 559	   unsigned int i,
 560	   struct i915_vma *vma)
 561{
 562	struct drm_i915_private *i915 = eb->i915;
 563	struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
 564	struct eb_vma *ev = &eb->vma[i];
 565
 566	ev->vma = vma;
 567	ev->exec = entry;
 568	ev->flags = entry->flags;
 569
 570	if (eb->lut_size > 0) {
 571		ev->handle = entry->handle;
 572		hlist_add_head(&ev->node,
 573			       &eb->buckets[hash_32(entry->handle,
 574						    eb->lut_size)]);
 575	}
 576
 577	if (entry->relocation_count)
 578		list_add_tail(&ev->reloc_link, &eb->relocs);
 579
 580	/*
 581	 * SNA is doing fancy tricks with compressing batch buffers, which leads
 582	 * to negative relocation deltas. Usually that works out ok since the
 583	 * relocate address is still positive, except when the batch is placed
 584	 * very low in the GTT. Ensure this doesn't happen.
 585	 *
 586	 * Note that actual hangs have only been observed on gen7, but for
 587	 * paranoia do it everywhere.
 588	 */
 589	if (is_batch_buffer(eb, i)) {
 590		if (entry->relocation_count &&
 591		    !(ev->flags & EXEC_OBJECT_PINNED))
 592			ev->flags |= __EXEC_OBJECT_NEEDS_BIAS;
 593		if (eb->reloc_cache.has_fence)
 594			ev->flags |= EXEC_OBJECT_NEEDS_FENCE;
 595
 596		eb->batches[*current_batch] = ev;
 597
 598		if (unlikely(ev->flags & EXEC_OBJECT_WRITE)) {
 599			drm_dbg(&i915->drm,
 600				"Attempting to use self-modifying batch buffer\n");
 601			return -EINVAL;
 602		}
 603
 604		if (range_overflows_t(u64,
 605				      eb->batch_start_offset,
 606				      eb->args->batch_len,
 607				      ev->vma->size)) {
 608			drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n");
 609			return -EINVAL;
 610		}
 611
 612		if (eb->args->batch_len == 0)
 613			eb->batch_len[*current_batch] = ev->vma->size -
 614				eb->batch_start_offset;
 615		else
 616			eb->batch_len[*current_batch] = eb->args->batch_len;
 617		if (unlikely(eb->batch_len[*current_batch] == 0)) { /* impossible! */
 618			drm_dbg(&i915->drm, "Invalid batch length\n");
 619			return -EINVAL;
 620		}
 621
 622		++*current_batch;
 623	}
 624
 625	return 0;
 626}
 627
 628static inline int use_cpu_reloc(const struct reloc_cache *cache,
 629				const struct drm_i915_gem_object *obj)
 630{
 631	if (!i915_gem_object_has_struct_page(obj))
 632		return false;
 633
 634	if (DBG_FORCE_RELOC == FORCE_CPU_RELOC)
 635		return true;
 636
 637	if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
 638		return false;
 639
 640	return (cache->has_llc ||
 641		obj->cache_dirty ||
 642		obj->cache_level != I915_CACHE_NONE);
 643}
 644
 645static int eb_reserve_vma(struct i915_execbuffer *eb,
 646			  struct eb_vma *ev,
 647			  u64 pin_flags)
 648{
 649	struct drm_i915_gem_exec_object2 *entry = ev->exec;
 650	struct i915_vma *vma = ev->vma;
 651	int err;
 652
 653	if (drm_mm_node_allocated(&vma->node) &&
 654	    eb_vma_misplaced(entry, vma, ev->flags)) {
 655		err = i915_vma_unbind(vma);
 656		if (err)
 657			return err;
 658	}
 659
 660	err = i915_vma_pin_ww(vma, &eb->ww,
 661			   entry->pad_to_size, entry->alignment,
 662			   eb_pin_flags(entry, ev->flags) | pin_flags);
 663	if (err)
 664		return err;
 665
 666	if (entry->offset != vma->node.start) {
 667		entry->offset = vma->node.start | UPDATE;
 668		eb->args->flags |= __EXEC_HAS_RELOC;
 669	}
 670
 671	if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) {
 672		err = i915_vma_pin_fence(vma);
 673		if (unlikely(err))
 674			return err;
 675
 676		if (vma->fence)
 677			ev->flags |= __EXEC_OBJECT_HAS_FENCE;
 678	}
 679
 680	ev->flags |= __EXEC_OBJECT_HAS_PIN;
 681	GEM_BUG_ON(eb_vma_misplaced(entry, vma, ev->flags));
 682
 683	return 0;
 684}
 685
 686static bool eb_unbind(struct i915_execbuffer *eb, bool force)
 687{
 688	const unsigned int count = eb->buffer_count;
 689	unsigned int i;
 690	struct list_head last;
 691	bool unpinned = false;
 692
 693	/* Resort *all* the objects into priority order */
 694	INIT_LIST_HEAD(&eb->unbound);
 695	INIT_LIST_HEAD(&last);
 696
 697	for (i = 0; i < count; i++) {
 698		struct eb_vma *ev = &eb->vma[i];
 699		unsigned int flags = ev->flags;
 700
 701		if (!force && flags & EXEC_OBJECT_PINNED &&
 702		    flags & __EXEC_OBJECT_HAS_PIN)
 703			continue;
 704
 705		unpinned = true;
 706		eb_unreserve_vma(ev);
 707
 708		if (flags & EXEC_OBJECT_PINNED)
 709			/* Pinned must have their slot */
 710			list_add(&ev->bind_link, &eb->unbound);
 711		else if (flags & __EXEC_OBJECT_NEEDS_MAP)
 712			/* Map require the lowest 256MiB (aperture) */
 713			list_add_tail(&ev->bind_link, &eb->unbound);
 714		else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
 715			/* Prioritise 4GiB region for restricted bo */
 716			list_add(&ev->bind_link, &last);
 717		else
 718			list_add_tail(&ev->bind_link, &last);
 719	}
 720
 721	list_splice_tail(&last, &eb->unbound);
 722	return unpinned;
 723}
 724
 725static int eb_reserve(struct i915_execbuffer *eb)
 726{
 727	struct eb_vma *ev;
 728	unsigned int pass;
 729	int err = 0;
 730	bool unpinned;
 731
 732	/*
 733	 * We have one more buffers that we couldn't bind, which could be due to
 734	 * various reasons. To resolve this we have 4 passes, with every next
 735	 * level turning the screws tighter:
 736	 *
 737	 * 0. Unbind all objects that do not match the GTT constraints for the
 738	 * execbuffer (fenceable, mappable, alignment etc). Bind all new
 739	 * objects.  This avoids unnecessary unbinding of later objects in order
 740	 * to make room for the earlier objects *unless* we need to defragment.
 741	 *
 742	 * 1. Reorder the buffers, where objects with the most restrictive
 743	 * placement requirements go first (ignoring fixed location buffers for
 744	 * now).  For example, objects needing the mappable aperture (the first
 745	 * 256M of GTT), should go first vs objects that can be placed just
 746	 * about anywhere. Repeat the previous pass.
 747	 *
 748	 * 2. Consider buffers that are pinned at a fixed location. Also try to
 749	 * evict the entire VM this time, leaving only objects that we were
 750	 * unable to lock. Try again to bind the buffers. (still using the new
 751	 * buffer order).
 752	 *
 753	 * 3. We likely have object lock contention for one or more stubborn
 754	 * objects in the VM, for which we need to evict to make forward
 755	 * progress (perhaps we are fighting the shrinker?). When evicting the
 756	 * VM this time around, anything that we can't lock we now track using
 757	 * the busy_bo, using the full lock (after dropping the vm->mutex to
 758	 * prevent deadlocks), instead of trylock. We then continue to evict the
 759	 * VM, this time with the stubborn object locked, which we can now
 760	 * hopefully unbind (if still bound in the VM). Repeat until the VM is
 761	 * evicted. Finally we should be able bind everything.
 762	 */
 763	for (pass = 0; pass <= 3; pass++) {
 764		int pin_flags = PIN_USER | PIN_VALIDATE;
 765
 766		if (pass == 0)
 767			pin_flags |= PIN_NONBLOCK;
 768
 769		if (pass >= 1)
 770			unpinned = eb_unbind(eb, pass >= 2);
 771
 772		if (pass == 2) {
 773			err = mutex_lock_interruptible(&eb->context->vm->mutex);
 774			if (!err) {
 775				err = i915_gem_evict_vm(eb->context->vm, &eb->ww, NULL);
 776				mutex_unlock(&eb->context->vm->mutex);
 777			}
 778			if (err)
 779				return err;
 780		}
 781
 782		if (pass == 3) {
 783retry:
 784			err = mutex_lock_interruptible(&eb->context->vm->mutex);
 785			if (!err) {
 786				struct drm_i915_gem_object *busy_bo = NULL;
 787
 788				err = i915_gem_evict_vm(eb->context->vm, &eb->ww, &busy_bo);
 789				mutex_unlock(&eb->context->vm->mutex);
 790				if (err && busy_bo) {
 791					err = i915_gem_object_lock(busy_bo, &eb->ww);
 792					i915_gem_object_put(busy_bo);
 793					if (!err)
 794						goto retry;
 795				}
 796			}
 797			if (err)
 798				return err;
 799		}
 800
 801		list_for_each_entry(ev, &eb->unbound, bind_link) {
 802			err = eb_reserve_vma(eb, ev, pin_flags);
 803			if (err)
 804				break;
 805		}
 806
 807		if (err != -ENOSPC)
 808			break;
 809	}
 810
 811	return err;
 812}
 813
 814static int eb_select_context(struct i915_execbuffer *eb)
 815{
 816	struct i915_gem_context *ctx;
 817
 818	ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
 819	if (unlikely(IS_ERR(ctx)))
 820		return PTR_ERR(ctx);
 821
 822	eb->gem_context = ctx;
 823	if (i915_gem_context_has_full_ppgtt(ctx))
 824		eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
 825
 826	return 0;
 827}
 828
 829static int __eb_add_lut(struct i915_execbuffer *eb,
 830			u32 handle, struct i915_vma *vma)
 831{
 832	struct i915_gem_context *ctx = eb->gem_context;
 833	struct i915_lut_handle *lut;
 834	int err;
 835
 836	lut = i915_lut_handle_alloc();
 837	if (unlikely(!lut))
 838		return -ENOMEM;
 839
 840	i915_vma_get(vma);
 841	if (!atomic_fetch_inc(&vma->open_count))
 842		i915_vma_reopen(vma);
 843	lut->handle = handle;
 844	lut->ctx = ctx;
 845
 846	/* Check that the context hasn't been closed in the meantime */
 847	err = -EINTR;
 848	if (!mutex_lock_interruptible(&ctx->lut_mutex)) {
 849		if (likely(!i915_gem_context_is_closed(ctx)))
 850			err = radix_tree_insert(&ctx->handles_vma, handle, vma);
 851		else
 852			err = -ENOENT;
 853		if (err == 0) { /* And nor has this handle */
 854			struct drm_i915_gem_object *obj = vma->obj;
 855
 856			spin_lock(&obj->lut_lock);
 857			if (idr_find(&eb->file->object_idr, handle) == obj) {
 858				list_add(&lut->obj_link, &obj->lut_list);
 859			} else {
 860				radix_tree_delete(&ctx->handles_vma, handle);
 861				err = -ENOENT;
 862			}
 863			spin_unlock(&obj->lut_lock);
 864		}
 865		mutex_unlock(&ctx->lut_mutex);
 866	}
 867	if (unlikely(err))
 868		goto err;
 869
 870	return 0;
 871
 872err:
 873	i915_vma_close(vma);
 874	i915_vma_put(vma);
 875	i915_lut_handle_free(lut);
 876	return err;
 877}
 878
 879static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle)
 880{
 881	struct i915_address_space *vm = eb->context->vm;
 882
 883	do {
 884		struct drm_i915_gem_object *obj;
 885		struct i915_vma *vma;
 886		int err;
 887
 888		rcu_read_lock();
 889		vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle);
 890		if (likely(vma && vma->vm == vm))
 891			vma = i915_vma_tryget(vma);
 892		rcu_read_unlock();
 893		if (likely(vma))
 894			return vma;
 895
 896		obj = i915_gem_object_lookup(eb->file, handle);
 897		if (unlikely(!obj))
 898			return ERR_PTR(-ENOENT);
 899
 900		/*
 901		 * If the user has opted-in for protected-object tracking, make
 902		 * sure the object encryption can be used.
 903		 * We only need to do this when the object is first used with
 904		 * this context, because the context itself will be banned when
 905		 * the protected objects become invalid.
 906		 */
 907		if (i915_gem_context_uses_protected_content(eb->gem_context) &&
 908		    i915_gem_object_is_protected(obj)) {
 909			err = intel_pxp_key_check(&vm->gt->pxp, obj, true);
 910			if (err) {
 911				i915_gem_object_put(obj);
 912				return ERR_PTR(err);
 913			}
 914		}
 915
 916		vma = i915_vma_instance(obj, vm, NULL);
 917		if (IS_ERR(vma)) {
 918			i915_gem_object_put(obj);
 919			return vma;
 920		}
 921
 922		err = __eb_add_lut(eb, handle, vma);
 923		if (likely(!err))
 924			return vma;
 925
 926		i915_gem_object_put(obj);
 927		if (err != -EEXIST)
 928			return ERR_PTR(err);
 929	} while (1);
 930}
 931
 932static int eb_lookup_vmas(struct i915_execbuffer *eb)
 933{
 934	unsigned int i, current_batch = 0;
 935	int err = 0;
 936
 937	INIT_LIST_HEAD(&eb->relocs);
 938
 939	for (i = 0; i < eb->buffer_count; i++) {
 940		struct i915_vma *vma;
 941
 942		vma = eb_lookup_vma(eb, eb->exec[i].handle);
 943		if (IS_ERR(vma)) {
 944			err = PTR_ERR(vma);
 945			goto err;
 946		}
 947
 948		err = eb_validate_vma(eb, &eb->exec[i], vma);
 949		if (unlikely(err)) {
 950			i915_vma_put(vma);
 951			goto err;
 952		}
 953
 954		err = eb_add_vma(eb, &current_batch, i, vma);
 955		if (err)
 956			return err;
 957
 958		if (i915_gem_object_is_userptr(vma->obj)) {
 959			err = i915_gem_object_userptr_submit_init(vma->obj);
 960			if (err) {
 961				if (i + 1 < eb->buffer_count) {
 962					/*
 963					 * Execbuffer code expects last vma entry to be NULL,
 964					 * since we already initialized this entry,
 965					 * set the next value to NULL or we mess up
 966					 * cleanup handling.
 967					 */
 968					eb->vma[i + 1].vma = NULL;
 969				}
 970
 971				return err;
 972			}
 973
 974			eb->vma[i].flags |= __EXEC_OBJECT_USERPTR_INIT;
 975			eb->args->flags |= __EXEC_USERPTR_USED;
 976		}
 977	}
 978
 979	return 0;
 980
 981err:
 982	eb->vma[i].vma = NULL;
 983	return err;
 984}
 985
 986static int eb_lock_vmas(struct i915_execbuffer *eb)
 987{
 988	unsigned int i;
 989	int err;
 990
 991	for (i = 0; i < eb->buffer_count; i++) {
 992		struct eb_vma *ev = &eb->vma[i];
 993		struct i915_vma *vma = ev->vma;
 994
 995		err = i915_gem_object_lock(vma->obj, &eb->ww);
 996		if (err)
 997			return err;
 998	}
 999
1000	return 0;
1001}
1002
1003static int eb_validate_vmas(struct i915_execbuffer *eb)
1004{
1005	unsigned int i;
1006	int err;
1007
1008	INIT_LIST_HEAD(&eb->unbound);
1009
1010	err = eb_lock_vmas(eb);
1011	if (err)
1012		return err;
1013
1014	for (i = 0; i < eb->buffer_count; i++) {
1015		struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
1016		struct eb_vma *ev = &eb->vma[i];
1017		struct i915_vma *vma = ev->vma;
1018
1019		err = eb_pin_vma(eb, entry, ev);
1020		if (err == -EDEADLK)
1021			return err;
1022
1023		if (!err) {
1024			if (entry->offset != vma->node.start) {
1025				entry->offset = vma->node.start | UPDATE;
1026				eb->args->flags |= __EXEC_HAS_RELOC;
1027			}
1028		} else {
1029			eb_unreserve_vma(ev);
1030
1031			list_add_tail(&ev->bind_link, &eb->unbound);
1032			if (drm_mm_node_allocated(&vma->node)) {
1033				err = i915_vma_unbind(vma);
1034				if (err)
1035					return err;
1036			}
1037		}
1038
1039		/* Reserve enough slots to accommodate composite fences */
1040		err = dma_resv_reserve_fences(vma->obj->base.resv, eb->num_batches);
1041		if (err)
1042			return err;
1043
1044		GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
1045			   eb_vma_misplaced(&eb->exec[i], vma, ev->flags));
1046	}
1047
1048	if (!list_empty(&eb->unbound))
1049		return eb_reserve(eb);
1050
1051	return 0;
1052}
1053
1054static struct eb_vma *
1055eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
1056{
1057	if (eb->lut_size < 0) {
1058		if (handle >= -eb->lut_size)
1059			return NULL;
1060		return &eb->vma[handle];
1061	} else {
1062		struct hlist_head *head;
1063		struct eb_vma *ev;
1064
1065		head = &eb->buckets[hash_32(handle, eb->lut_size)];
1066		hlist_for_each_entry(ev, head, node) {
1067			if (ev->handle == handle)
1068				return ev;
1069		}
1070		return NULL;
1071	}
1072}
1073
1074static void eb_release_vmas(struct i915_execbuffer *eb, bool final)
1075{
1076	const unsigned int count = eb->buffer_count;
1077	unsigned int i;
1078
1079	for (i = 0; i < count; i++) {
1080		struct eb_vma *ev = &eb->vma[i];
1081		struct i915_vma *vma = ev->vma;
1082
1083		if (!vma)
1084			break;
1085
1086		eb_unreserve_vma(ev);
1087
1088		if (final)
1089			i915_vma_put(vma);
1090	}
1091
1092	eb_capture_release(eb);
1093	eb_unpin_engine(eb);
1094}
1095
1096static void eb_destroy(const struct i915_execbuffer *eb)
1097{
1098	if (eb->lut_size > 0)
1099		kfree(eb->buckets);
1100}
1101
1102static inline u64
1103relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
1104		  const struct i915_vma *target)
1105{
1106	return gen8_canonical_addr((int)reloc->delta + target->node.start);
1107}
1108
1109static void reloc_cache_init(struct reloc_cache *cache,
1110			     struct drm_i915_private *i915)
1111{
1112	cache->page = -1;
1113	cache->vaddr = 0;
1114	/* Must be a variable in the struct to allow GCC to unroll. */
1115	cache->graphics_ver = GRAPHICS_VER(i915);
1116	cache->has_llc = HAS_LLC(i915);
1117	cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
1118	cache->has_fence = cache->graphics_ver < 4;
1119	cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
1120	cache->node.flags = 0;
1121}
1122
1123static inline void *unmask_page(unsigned long p)
1124{
1125	return (void *)(uintptr_t)(p & PAGE_MASK);
1126}
1127
1128static inline unsigned int unmask_flags(unsigned long p)
1129{
1130	return p & ~PAGE_MASK;
1131}
1132
1133#define KMAP 0x4 /* after CLFLUSH_FLAGS */
1134
1135static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
1136{
1137	struct drm_i915_private *i915 =
1138		container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
1139	return to_gt(i915)->ggtt;
1140}
1141
1142static void reloc_cache_unmap(struct reloc_cache *cache)
1143{
1144	void *vaddr;
1145
1146	if (!cache->vaddr)
1147		return;
1148
1149	vaddr = unmask_page(cache->vaddr);
1150	if (cache->vaddr & KMAP)
1151		kunmap_atomic(vaddr);
1152	else
1153		io_mapping_unmap_atomic((void __iomem *)vaddr);
1154}
1155
1156static void reloc_cache_remap(struct reloc_cache *cache,
1157			      struct drm_i915_gem_object *obj)
1158{
1159	void *vaddr;
1160
1161	if (!cache->vaddr)
1162		return;
1163
1164	if (cache->vaddr & KMAP) {
1165		struct page *page = i915_gem_object_get_page(obj, cache->page);
1166
1167		vaddr = kmap_atomic(page);
1168		cache->vaddr = unmask_flags(cache->vaddr) |
1169			(unsigned long)vaddr;
1170	} else {
1171		struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1172		unsigned long offset;
1173
1174		offset = cache->node.start;
1175		if (!drm_mm_node_allocated(&cache->node))
1176			offset += cache->page << PAGE_SHIFT;
1177
1178		cache->vaddr = (unsigned long)
1179			io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1180	}
1181}
1182
1183static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb)
1184{
1185	void *vaddr;
1186
1187	if (!cache->vaddr)
1188		return;
1189
1190	vaddr = unmask_page(cache->vaddr);
1191	if (cache->vaddr & KMAP) {
1192		struct drm_i915_gem_object *obj =
1193			(struct drm_i915_gem_object *)cache->node.mm;
1194		if (cache->vaddr & CLFLUSH_AFTER)
1195			mb();
1196
1197		kunmap_atomic(vaddr);
1198		i915_gem_object_finish_access(obj);
1199	} else {
1200		struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1201
1202		intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1203		io_mapping_unmap_atomic((void __iomem *)vaddr);
1204
1205		if (drm_mm_node_allocated(&cache->node)) {
1206			ggtt->vm.clear_range(&ggtt->vm,
1207					     cache->node.start,
1208					     cache->node.size);
1209			mutex_lock(&ggtt->vm.mutex);
1210			drm_mm_remove_node(&cache->node);
1211			mutex_unlock(&ggtt->vm.mutex);
1212		} else {
1213			i915_vma_unpin((struct i915_vma *)cache->node.mm);
1214		}
1215	}
1216
1217	cache->vaddr = 0;
1218	cache->page = -1;
1219}
1220
1221static void *reloc_kmap(struct drm_i915_gem_object *obj,
1222			struct reloc_cache *cache,
1223			unsigned long pageno)
1224{
1225	void *vaddr;
1226	struct page *page;
1227
1228	if (cache->vaddr) {
1229		kunmap_atomic(unmask_page(cache->vaddr));
1230	} else {
1231		unsigned int flushes;
1232		int err;
1233
1234		err = i915_gem_object_prepare_write(obj, &flushes);
1235		if (err)
1236			return ERR_PTR(err);
1237
1238		BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
1239		BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
1240
1241		cache->vaddr = flushes | KMAP;
1242		cache->node.mm = (void *)obj;
1243		if (flushes)
1244			mb();
1245	}
1246
1247	page = i915_gem_object_get_page(obj, pageno);
1248	if (!obj->mm.dirty)
1249		set_page_dirty(page);
1250
1251	vaddr = kmap_atomic(page);
1252	cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
1253	cache->page = pageno;
1254
1255	return vaddr;
1256}
1257
1258static void *reloc_iomap(struct i915_vma *batch,
1259			 struct i915_execbuffer *eb,
1260			 unsigned long page)
1261{
1262	struct drm_i915_gem_object *obj = batch->obj;
1263	struct reloc_cache *cache = &eb->reloc_cache;
1264	struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1265	unsigned long offset;
1266	void *vaddr;
1267
1268	if (cache->vaddr) {
1269		intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1270		io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
1271	} else {
1272		struct i915_vma *vma = ERR_PTR(-ENODEV);
1273		int err;
1274
1275		if (i915_gem_object_is_tiled(obj))
1276			return ERR_PTR(-EINVAL);
1277
1278		if (use_cpu_reloc(cache, obj))
1279			return NULL;
1280
1281		err = i915_gem_object_set_to_gtt_domain(obj, true);
1282		if (err)
1283			return ERR_PTR(err);
1284
1285		/*
1286		 * i915_gem_object_ggtt_pin_ww may attempt to remove the batch
1287		 * VMA from the object list because we no longer pin.
1288		 *
1289		 * Only attempt to pin the batch buffer to ggtt if the current batch
1290		 * is not inside ggtt, or the batch buffer is not misplaced.
1291		 */
1292		if (!i915_is_ggtt(batch->vm) ||
1293		    !i915_vma_misplaced(batch, 0, 0, PIN_MAPPABLE)) {
1294			vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0,
1295							  PIN_MAPPABLE |
1296							  PIN_NONBLOCK /* NOWARN */ |
1297							  PIN_NOEVICT);
1298		}
1299
1300		if (vma == ERR_PTR(-EDEADLK))
1301			return vma;
1302
1303		if (IS_ERR(vma)) {
1304			memset(&cache->node, 0, sizeof(cache->node));
1305			mutex_lock(&ggtt->vm.mutex);
1306			err = drm_mm_insert_node_in_range
1307				(&ggtt->vm.mm, &cache->node,
1308				 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
1309				 0, ggtt->mappable_end,
1310				 DRM_MM_INSERT_LOW);
1311			mutex_unlock(&ggtt->vm.mutex);
1312			if (err) /* no inactive aperture space, use cpu reloc */
1313				return NULL;
1314		} else {
1315			cache->node.start = vma->node.start;
1316			cache->node.mm = (void *)vma;
1317		}
1318	}
1319
1320	offset = cache->node.start;
1321	if (drm_mm_node_allocated(&cache->node)) {
1322		ggtt->vm.insert_page(&ggtt->vm,
1323				     i915_gem_object_get_dma_address(obj, page),
1324				     offset, I915_CACHE_NONE, 0);
1325	} else {
1326		offset += page << PAGE_SHIFT;
1327	}
1328
1329	vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap,
1330							 offset);
1331	cache->page = page;
1332	cache->vaddr = (unsigned long)vaddr;
1333
1334	return vaddr;
1335}
1336
1337static void *reloc_vaddr(struct i915_vma *vma,
1338			 struct i915_execbuffer *eb,
1339			 unsigned long page)
1340{
1341	struct reloc_cache *cache = &eb->reloc_cache;
1342	void *vaddr;
1343
1344	if (cache->page == page) {
1345		vaddr = unmask_page(cache->vaddr);
1346	} else {
1347		vaddr = NULL;
1348		if ((cache->vaddr & KMAP) == 0)
1349			vaddr = reloc_iomap(vma, eb, page);
1350		if (!vaddr)
1351			vaddr = reloc_kmap(vma->obj, cache, page);
1352	}
1353
1354	return vaddr;
1355}
1356
1357static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
1358{
1359	if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) {
1360		if (flushes & CLFLUSH_BEFORE)
1361			drm_clflush_virt_range(addr, sizeof(*addr));
1362
1363		*addr = value;
1364
1365		/*
1366		 * Writes to the same cacheline are serialised by the CPU
1367		 * (including clflush). On the write path, we only require
1368		 * that it hits memory in an orderly fashion and place
1369		 * mb barriers at the start and end of the relocation phase
1370		 * to ensure ordering of clflush wrt to the system.
1371		 */
1372		if (flushes & CLFLUSH_AFTER)
1373			drm_clflush_virt_range(addr, sizeof(*addr));
1374	} else
1375		*addr = value;
1376}
1377
1378static u64
1379relocate_entry(struct i915_vma *vma,
1380	       const struct drm_i915_gem_relocation_entry *reloc,
1381	       struct i915_execbuffer *eb,
1382	       const struct i915_vma *target)
1383{
1384	u64 target_addr = relocation_target(reloc, target);
1385	u64 offset = reloc->offset;
1386	bool wide = eb->reloc_cache.use_64bit_reloc;
1387	void *vaddr;
1388
1389repeat:
1390	vaddr = reloc_vaddr(vma, eb,
1391			    offset >> PAGE_SHIFT);
1392	if (IS_ERR(vaddr))
1393		return PTR_ERR(vaddr);
1394
1395	GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32)));
1396	clflush_write32(vaddr + offset_in_page(offset),
1397			lower_32_bits(target_addr),
1398			eb->reloc_cache.vaddr);
1399
1400	if (wide) {
1401		offset += sizeof(u32);
1402		target_addr >>= 32;
1403		wide = false;
1404		goto repeat;
1405	}
1406
1407	return target->node.start | UPDATE;
1408}
1409
1410static u64
1411eb_relocate_entry(struct i915_execbuffer *eb,
1412		  struct eb_vma *ev,
1413		  const struct drm_i915_gem_relocation_entry *reloc)
1414{
1415	struct drm_i915_private *i915 = eb->i915;
1416	struct eb_vma *target;
1417	int err;
1418
1419	/* we've already hold a reference to all valid objects */
1420	target = eb_get_vma(eb, reloc->target_handle);
1421	if (unlikely(!target))
1422		return -ENOENT;
1423
1424	/* Validate that the target is in a valid r/w GPU domain */
1425	if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
1426		drm_dbg(&i915->drm, "reloc with multiple write domains: "
1427			  "target %d offset %d "
1428			  "read %08x write %08x",
1429			  reloc->target_handle,
1430			  (int) reloc->offset,
1431			  reloc->read_domains,
1432			  reloc->write_domain);
1433		return -EINVAL;
1434	}
1435	if (unlikely((reloc->write_domain | reloc->read_domains)
1436		     & ~I915_GEM_GPU_DOMAINS)) {
1437		drm_dbg(&i915->drm, "reloc with read/write non-GPU domains: "
1438			  "target %d offset %d "
1439			  "read %08x write %08x",
1440			  reloc->target_handle,
1441			  (int) reloc->offset,
1442			  reloc->read_domains,
1443			  reloc->write_domain);
1444		return -EINVAL;
1445	}
1446
1447	if (reloc->write_domain) {
1448		target->flags |= EXEC_OBJECT_WRITE;
1449
1450		/*
1451		 * Sandybridge PPGTT errata: We need a global gtt mapping
1452		 * for MI and pipe_control writes because the gpu doesn't
1453		 * properly redirect them through the ppgtt for non_secure
1454		 * batchbuffers.
1455		 */
1456		if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
1457		    GRAPHICS_VER(eb->i915) == 6 &&
1458		    !i915_vma_is_bound(target->vma, I915_VMA_GLOBAL_BIND)) {
1459			struct i915_vma *vma = target->vma;
1460
1461			reloc_cache_unmap(&eb->reloc_cache);
1462			mutex_lock(&vma->vm->mutex);
1463			err = i915_vma_bind(target->vma,
1464					    target->vma->obj->cache_level,
1465					    PIN_GLOBAL, NULL, NULL);
1466			mutex_unlock(&vma->vm->mutex);
1467			reloc_cache_remap(&eb->reloc_cache, ev->vma->obj);
1468			if (err)
1469				return err;
1470		}
1471	}
1472
1473	/*
1474	 * If the relocation already has the right value in it, no
1475	 * more work needs to be done.
1476	 */
1477	if (!DBG_FORCE_RELOC &&
1478	    gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset)
1479		return 0;
1480
1481	/* Check that the relocation address is valid... */
1482	if (unlikely(reloc->offset >
1483		     ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
1484		drm_dbg(&i915->drm, "Relocation beyond object bounds: "
1485			  "target %d offset %d size %d.\n",
1486			  reloc->target_handle,
1487			  (int)reloc->offset,
1488			  (int)ev->vma->size);
1489		return -EINVAL;
1490	}
1491	if (unlikely(reloc->offset & 3)) {
1492		drm_dbg(&i915->drm, "Relocation not 4-byte aligned: "
1493			  "target %d offset %d.\n",
1494			  reloc->target_handle,
1495			  (int)reloc->offset);
1496		return -EINVAL;
1497	}
1498
1499	/*
1500	 * If we write into the object, we need to force the synchronisation
1501	 * barrier, either with an asynchronous clflush or if we executed the
1502	 * patching using the GPU (though that should be serialised by the
1503	 * timeline). To be completely sure, and since we are required to
1504	 * do relocations we are already stalling, disable the user's opt
1505	 * out of our synchronisation.
1506	 */
1507	ev->flags &= ~EXEC_OBJECT_ASYNC;
1508
1509	/* and update the user's relocation entry */
1510	return relocate_entry(ev->vma, reloc, eb, target->vma);
1511}
1512
1513static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev)
1514{
1515#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
1516	struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
1517	const struct drm_i915_gem_exec_object2 *entry = ev->exec;
1518	struct drm_i915_gem_relocation_entry __user *urelocs =
1519		u64_to_user_ptr(entry->relocs_ptr);
1520	unsigned long remain = entry->relocation_count;
1521
1522	if (unlikely(remain > N_RELOC(ULONG_MAX)))
1523		return -EINVAL;
1524
1525	/*
1526	 * We must check that the entire relocation array is safe
1527	 * to read. However, if the array is not writable the user loses
1528	 * the updated relocation values.
1529	 */
1530	if (unlikely(!access_ok(urelocs, remain * sizeof(*urelocs))))
1531		return -EFAULT;
1532
1533	do {
1534		struct drm_i915_gem_relocation_entry *r = stack;
1535		unsigned int count =
1536			min_t(unsigned long, remain, ARRAY_SIZE(stack));
1537		unsigned int copied;
1538
1539		/*
1540		 * This is the fast path and we cannot handle a pagefault
1541		 * whilst holding the struct mutex lest the user pass in the
1542		 * relocations contained within a mmaped bo. For in such a case
1543		 * we, the page fault handler would call i915_gem_fault() and
1544		 * we would try to acquire the struct mutex again. Obviously
1545		 * this is bad and so lockdep complains vehemently.
1546		 */
1547		pagefault_disable();
1548		copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
1549		pagefault_enable();
1550		if (unlikely(copied)) {
1551			remain = -EFAULT;
1552			goto out;
1553		}
1554
1555		remain -= count;
1556		do {
1557			u64 offset = eb_relocate_entry(eb, ev, r);
1558
1559			if (likely(offset == 0)) {
1560			} else if ((s64)offset < 0) {
1561				remain = (int)offset;
1562				goto out;
1563			} else {
1564				/*
1565				 * Note that reporting an error now
1566				 * leaves everything in an inconsistent
1567				 * state as we have *already* changed
1568				 * the relocation value inside the
1569				 * object. As we have not changed the
1570				 * reloc.presumed_offset or will not
1571				 * change the execobject.offset, on the
1572				 * call we may not rewrite the value
1573				 * inside the object, leaving it
1574				 * dangling and causing a GPU hang. Unless
1575				 * userspace dynamically rebuilds the
1576				 * relocations on each execbuf rather than
1577				 * presume a static tree.
1578				 *
1579				 * We did previously check if the relocations
1580				 * were writable (access_ok), an error now
1581				 * would be a strange race with mprotect,
1582				 * having already demonstrated that we
1583				 * can read from this userspace address.
1584				 */
1585				offset = gen8_canonical_addr(offset & ~UPDATE);
1586				__put_user(offset,
1587					   &urelocs[r - stack].presumed_offset);
1588			}
1589		} while (r++, --count);
1590		urelocs += ARRAY_SIZE(stack);
1591	} while (remain);
1592out:
1593	reloc_cache_reset(&eb->reloc_cache, eb);
1594	return remain;
1595}
1596
1597static int
1598eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev)
1599{
1600	const struct drm_i915_gem_exec_object2 *entry = ev->exec;
1601	struct drm_i915_gem_relocation_entry *relocs =
1602		u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1603	unsigned int i;
1604	int err;
1605
1606	for (i = 0; i < entry->relocation_count; i++) {
1607		u64 offset = eb_relocate_entry(eb, ev, &relocs[i]);
1608
1609		if ((s64)offset < 0) {
1610			err = (int)offset;
1611			goto err;
1612		}
1613	}
1614	err = 0;
1615err:
1616	reloc_cache_reset(&eb->reloc_cache, eb);
1617	return err;
1618}
1619
1620static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
1621{
1622	const char __user *addr, *end;
1623	unsigned long size;
1624	char __maybe_unused c;
1625
1626	size = entry->relocation_count;
1627	if (size == 0)
1628		return 0;
1629
1630	if (size > N_RELOC(ULONG_MAX))
1631		return -EINVAL;
1632
1633	addr = u64_to_user_ptr(entry->relocs_ptr);
1634	size *= sizeof(struct drm_i915_gem_relocation_entry);
1635	if (!access_ok(addr, size))
1636		return -EFAULT;
1637
1638	end = addr + size;
1639	for (; addr < end; addr += PAGE_SIZE) {
1640		int err = __get_user(c, addr);
1641		if (err)
1642			return err;
1643	}
1644	return __get_user(c, end - 1);
1645}
1646
1647static int eb_copy_relocations(const struct i915_execbuffer *eb)
1648{
1649	struct drm_i915_gem_relocation_entry *relocs;
1650	const unsigned int count = eb->buffer_count;
1651	unsigned int i;
1652	int err;
1653
1654	for (i = 0; i < count; i++) {
1655		const unsigned int nreloc = eb->exec[i].relocation_count;
1656		struct drm_i915_gem_relocation_entry __user *urelocs;
1657		unsigned long size;
1658		unsigned long copied;
1659
1660		if (nreloc == 0)
1661			continue;
1662
1663		err = check_relocations(&eb->exec[i]);
1664		if (err)
1665			goto err;
1666
1667		urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
1668		size = nreloc * sizeof(*relocs);
1669
1670		relocs = kvmalloc_array(size, 1, GFP_KERNEL);
1671		if (!relocs) {
1672			err = -ENOMEM;
1673			goto err;
1674		}
1675
1676		/* copy_from_user is limited to < 4GiB */
1677		copied = 0;
1678		do {
1679			unsigned int len =
1680				min_t(u64, BIT_ULL(31), size - copied);
1681
1682			if (__copy_from_user((char *)relocs + copied,
1683					     (char __user *)urelocs + copied,
1684					     len))
1685				goto end;
1686
1687			copied += len;
1688		} while (copied < size);
1689
1690		/*
1691		 * As we do not update the known relocation offsets after
1692		 * relocating (due to the complexities in lock handling),
1693		 * we need to mark them as invalid now so that we force the
1694		 * relocation processing next time. Just in case the target
1695		 * object is evicted and then rebound into its old
1696		 * presumed_offset before the next execbuffer - if that
1697		 * happened we would make the mistake of assuming that the
1698		 * relocations were valid.
1699		 */
1700		if (!user_access_begin(urelocs, size))
1701			goto end;
1702
1703		for (copied = 0; copied < nreloc; copied++)
1704			unsafe_put_user(-1,
1705					&urelocs[copied].presumed_offset,
1706					end_user);
1707		user_access_end();
1708
1709		eb->exec[i].relocs_ptr = (uintptr_t)relocs;
1710	}
1711
1712	return 0;
1713
1714end_user:
1715	user_access_end();
1716end:
1717	kvfree(relocs);
1718	err = -EFAULT;
1719err:
1720	while (i--) {
1721		relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
1722		if (eb->exec[i].relocation_count)
1723			kvfree(relocs);
1724	}
1725	return err;
1726}
1727
1728static int eb_prefault_relocations(const struct i915_execbuffer *eb)
1729{
1730	const unsigned int count = eb->buffer_count;
1731	unsigned int i;
1732
1733	for (i = 0; i < count; i++) {
1734		int err;
1735
1736		err = check_relocations(&eb->exec[i]);
1737		if (err)
1738			return err;
1739	}
1740
1741	return 0;
1742}
1743
1744static int eb_reinit_userptr(struct i915_execbuffer *eb)
1745{
1746	const unsigned int count = eb->buffer_count;
1747	unsigned int i;
1748	int ret;
1749
1750	if (likely(!(eb->args->flags & __EXEC_USERPTR_USED)))
1751		return 0;
1752
1753	for (i = 0; i < count; i++) {
1754		struct eb_vma *ev = &eb->vma[i];
1755
1756		if (!i915_gem_object_is_userptr(ev->vma->obj))
1757			continue;
1758
1759		ret = i915_gem_object_userptr_submit_init(ev->vma->obj);
1760		if (ret)
1761			return ret;
1762
1763		ev->flags |= __EXEC_OBJECT_USERPTR_INIT;
1764	}
1765
1766	return 0;
1767}
1768
1769static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb)
1770{
1771	bool have_copy = false;
1772	struct eb_vma *ev;
1773	int err = 0;
1774
1775repeat:
1776	if (signal_pending(current)) {
1777		err = -ERESTARTSYS;
1778		goto out;
1779	}
1780
1781	/* We may process another execbuffer during the unlock... */
1782	eb_release_vmas(eb, false);
1783	i915_gem_ww_ctx_fini(&eb->ww);
1784
1785	/*
1786	 * We take 3 passes through the slowpatch.
1787	 *
1788	 * 1 - we try to just prefault all the user relocation entries and
1789	 * then attempt to reuse the atomic pagefault disabled fast path again.
1790	 *
1791	 * 2 - we copy the user entries to a local buffer here outside of the
1792	 * local and allow ourselves to wait upon any rendering before
1793	 * relocations
1794	 *
1795	 * 3 - we already have a local copy of the relocation entries, but
1796	 * were interrupted (EAGAIN) whilst waiting for the objects, try again.
1797	 */
1798	if (!err) {
1799		err = eb_prefault_relocations(eb);
1800	} else if (!have_copy) {
1801		err = eb_copy_relocations(eb);
1802		have_copy = err == 0;
1803	} else {
1804		cond_resched();
1805		err = 0;
1806	}
1807
1808	if (!err)
1809		err = eb_reinit_userptr(eb);
1810
1811	i915_gem_ww_ctx_init(&eb->ww, true);
1812	if (err)
1813		goto out;
1814
1815	/* reacquire the objects */
1816repeat_validate:
1817	err = eb_pin_engine(eb, false);
1818	if (err)
1819		goto err;
1820
1821	err = eb_validate_vmas(eb);
1822	if (err)
1823		goto err;
1824
1825	GEM_BUG_ON(!eb->batches[0]);
1826
1827	list_for_each_entry(ev, &eb->relocs, reloc_link) {
1828		if (!have_copy) {
1829			err = eb_relocate_vma(eb, ev);
1830			if (err)
1831				break;
1832		} else {
1833			err = eb_relocate_vma_slow(eb, ev);
1834			if (err)
1835				break;
1836		}
1837	}
1838
1839	if (err == -EDEADLK)
1840		goto err;
1841
1842	if (err && !have_copy)
1843		goto repeat;
1844
1845	if (err)
1846		goto err;
1847
1848	/* as last step, parse the command buffer */
1849	err = eb_parse(eb);
1850	if (err)
1851		goto err;
1852
1853	/*
1854	 * Leave the user relocations as are, this is the painfully slow path,
1855	 * and we want to avoid the complication of dropping the lock whilst
1856	 * having buffers reserved in the aperture and so causing spurious
1857	 * ENOSPC for random operations.
1858	 */
1859
1860err:
1861	if (err == -EDEADLK) {
1862		eb_release_vmas(eb, false);
1863		err = i915_gem_ww_ctx_backoff(&eb->ww);
1864		if (!err)
1865			goto repeat_validate;
1866	}
1867
1868	if (err == -EAGAIN)
1869		goto repeat;
1870
1871out:
1872	if (have_copy) {
1873		const unsigned int count = eb->buffer_count;
1874		unsigned int i;
1875
1876		for (i = 0; i < count; i++) {
1877			const struct drm_i915_gem_exec_object2 *entry =
1878				&eb->exec[i];
1879			struct drm_i915_gem_relocation_entry *relocs;
1880
1881			if (!entry->relocation_count)
1882				continue;
1883
1884			relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1885			kvfree(relocs);
1886		}
1887	}
1888
1889	return err;
1890}
1891
1892static int eb_relocate_parse(struct i915_execbuffer *eb)
1893{
1894	int err;
1895	bool throttle = true;
1896
1897retry:
1898	err = eb_pin_engine(eb, throttle);
1899	if (err) {
1900		if (err != -EDEADLK)
1901			return err;
1902
1903		goto err;
1904	}
1905
1906	/* only throttle once, even if we didn't need to throttle */
1907	throttle = false;
1908
1909	err = eb_validate_vmas(eb);
1910	if (err == -EAGAIN)
1911		goto slow;
1912	else if (err)
1913		goto err;
1914
1915	/* The objects are in their final locations, apply the relocations. */
1916	if (eb->args->flags & __EXEC_HAS_RELOC) {
1917		struct eb_vma *ev;
1918
1919		list_for_each_entry(ev, &eb->relocs, reloc_link) {
1920			err = eb_relocate_vma(eb, ev);
1921			if (err)
1922				break;
1923		}
1924
1925		if (err == -EDEADLK)
1926			goto err;
1927		else if (err)
1928			goto slow;
1929	}
1930
1931	if (!err)
1932		err = eb_parse(eb);
1933
1934err:
1935	if (err == -EDEADLK) {
1936		eb_release_vmas(eb, false);
1937		err = i915_gem_ww_ctx_backoff(&eb->ww);
1938		if (!err)
1939			goto retry;
1940	}
1941
1942	return err;
1943
1944slow:
1945	err = eb_relocate_parse_slow(eb);
1946	if (err)
1947		/*
1948		 * If the user expects the execobject.offset and
1949		 * reloc.presumed_offset to be an exact match,
1950		 * as for using NO_RELOC, then we cannot update
1951		 * the execobject.offset until we have completed
1952		 * relocation.
1953		 */
1954		eb->args->flags &= ~__EXEC_HAS_RELOC;
1955
1956	return err;
1957}
1958
1959/*
1960 * Using two helper loops for the order of which requests / batches are created
1961 * and added the to backend. Requests are created in order from the parent to
1962 * the last child. Requests are added in the reverse order, from the last child
1963 * to parent. This is done for locking reasons as the timeline lock is acquired
1964 * during request creation and released when the request is added to the
1965 * backend. To make lockdep happy (see intel_context_timeline_lock) this must be
1966 * the ordering.
1967 */
1968#define for_each_batch_create_order(_eb, _i) \
1969	for ((_i) = 0; (_i) < (_eb)->num_batches; ++(_i))
1970#define for_each_batch_add_order(_eb, _i) \
1971	BUILD_BUG_ON(!typecheck(int, _i)); \
1972	for ((_i) = (_eb)->num_batches - 1; (_i) >= 0; --(_i))
1973
1974static struct i915_request *
1975eb_find_first_request_added(struct i915_execbuffer *eb)
1976{
1977	int i;
1978
1979	for_each_batch_add_order(eb, i)
1980		if (eb->requests[i])
1981			return eb->requests[i];
1982
1983	GEM_BUG_ON("Request not found");
1984
1985	return NULL;
1986}
1987
1988#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
1989
1990/* Stage with GFP_KERNEL allocations before we enter the signaling critical path */
1991static int eb_capture_stage(struct i915_execbuffer *eb)
1992{
1993	const unsigned int count = eb->buffer_count;
1994	unsigned int i = count, j;
1995
1996	while (i--) {
1997		struct eb_vma *ev = &eb->vma[i];
1998		struct i915_vma *vma = ev->vma;
1999		unsigned int flags = ev->flags;
2000
2001		if (!(flags & EXEC_OBJECT_CAPTURE))
2002			continue;
2003
2004		if (i915_gem_context_is_recoverable(eb->gem_context) &&
2005		    (IS_DGFX(eb->i915) || GRAPHICS_VER_FULL(eb->i915) > IP_VER(12, 0)))
2006			return -EINVAL;
2007
2008		for_each_batch_create_order(eb, j) {
2009			struct i915_capture_list *capture;
2010
2011			capture = kmalloc(sizeof(*capture), GFP_KERNEL);
2012			if (!capture)
2013				continue;
2014
2015			capture->next = eb->capture_lists[j];
2016			capture->vma_res = i915_vma_resource_get(vma->resource);
2017			eb->capture_lists[j] = capture;
2018		}
2019	}
2020
2021	return 0;
2022}
2023
2024/* Commit once we're in the critical path */
2025static void eb_capture_commit(struct i915_execbuffer *eb)
2026{
2027	unsigned int j;
2028
2029	for_each_batch_create_order(eb, j) {
2030		struct i915_request *rq = eb->requests[j];
2031
2032		if (!rq)
2033			break;
2034
2035		rq->capture_list = eb->capture_lists[j];
2036		eb->capture_lists[j] = NULL;
2037	}
2038}
2039
2040/*
2041 * Release anything that didn't get committed due to errors.
2042 * The capture_list will otherwise be freed at request retire.
2043 */
2044static void eb_capture_release(struct i915_execbuffer *eb)
2045{
2046	unsigned int j;
2047
2048	for_each_batch_create_order(eb, j) {
2049		if (eb->capture_lists[j]) {
2050			i915_request_free_capture_list(eb->capture_lists[j]);
2051			eb->capture_lists[j] = NULL;
2052		}
2053	}
2054}
2055
2056static void eb_capture_list_clear(struct i915_execbuffer *eb)
2057{
2058	memset(eb->capture_lists, 0, sizeof(eb->capture_lists));
2059}
2060
2061#else
2062
2063static int eb_capture_stage(struct i915_execbuffer *eb)
2064{
2065	return 0;
2066}
2067
2068static void eb_capture_commit(struct i915_execbuffer *eb)
2069{
2070}
2071
2072static void eb_capture_release(struct i915_execbuffer *eb)
2073{
2074}
2075
2076static void eb_capture_list_clear(struct i915_execbuffer *eb)
2077{
2078}
2079
2080#endif
2081
2082static int eb_move_to_gpu(struct i915_execbuffer *eb)
2083{
2084	const unsigned int count = eb->buffer_count;
2085	unsigned int i = count;
2086	int err = 0, j;
2087
2088	while (i--) {
2089		struct eb_vma *ev = &eb->vma[i];
2090		struct i915_vma *vma = ev->vma;
2091		unsigned int flags = ev->flags;
2092		struct drm_i915_gem_object *obj = vma->obj;
2093
2094		assert_vma_held(vma);
2095
2096		/*
2097		 * If the GPU is not _reading_ through the CPU cache, we need
2098		 * to make sure that any writes (both previous GPU writes from
2099		 * before a change in snooping levels and normal CPU writes)
2100		 * caught in that cache are flushed to main memory.
2101		 *
2102		 * We want to say
2103		 *   obj->cache_dirty &&
2104		 *   !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)
2105		 * but gcc's optimiser doesn't handle that as well and emits
2106		 * two jumps instead of one. Maybe one day...
2107		 *
2108		 * FIXME: There is also sync flushing in set_pages(), which
2109		 * serves a different purpose(some of the time at least).
2110		 *
2111		 * We should consider:
2112		 *
2113		 *   1. Rip out the async flush code.
2114		 *
2115		 *   2. Or make the sync flushing use the async clflush path
2116		 *   using mandatory fences underneath. Currently the below
2117		 *   async flush happens after we bind the object.
2118		 */
2119		if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) {
2120			if (i915_gem_clflush_object(obj, 0))
2121				flags &= ~EXEC_OBJECT_ASYNC;
2122		}
2123
2124		/* We only need to await on the first request */
2125		if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) {
2126			err = i915_request_await_object
2127				(eb_find_first_request_added(eb), obj,
2128				 flags & EXEC_OBJECT_WRITE);
2129		}
2130
2131		for_each_batch_add_order(eb, j) {
2132			if (err)
2133				break;
2134			if (!eb->requests[j])
2135				continue;
2136
2137			err = _i915_vma_move_to_active(vma, eb->requests[j],
2138						       j ? NULL :
2139						       eb->composite_fence ?
2140						       eb->composite_fence :
2141						       &eb->requests[j]->fence,
2142						       flags | __EXEC_OBJECT_NO_RESERVE |
2143						       __EXEC_OBJECT_NO_REQUEST_AWAIT);
2144		}
2145	}
2146
2147#ifdef CONFIG_MMU_NOTIFIER
2148	if (!err && (eb->args->flags & __EXEC_USERPTR_USED)) {
2149		read_lock(&eb->i915->mm.notifier_lock);
2150
2151		/*
2152		 * count is always at least 1, otherwise __EXEC_USERPTR_USED
2153		 * could not have been set
2154		 */
2155		for (i = 0; i < count; i++) {
2156			struct eb_vma *ev = &eb->vma[i];
2157			struct drm_i915_gem_object *obj = ev->vma->obj;
2158
2159			if (!i915_gem_object_is_userptr(obj))
2160				continue;
2161
2162			err = i915_gem_object_userptr_submit_done(obj);
2163			if (err)
2164				break;
2165		}
2166
2167		read_unlock(&eb->i915->mm.notifier_lock);
2168	}
2169#endif
2170
2171	if (unlikely(err))
2172		goto err_skip;
2173
2174	/* Unconditionally flush any chipset caches (for streaming writes). */
2175	intel_gt_chipset_flush(eb->gt);
2176	eb_capture_commit(eb);
2177
2178	return 0;
2179
2180err_skip:
2181	for_each_batch_create_order(eb, j) {
2182		if (!eb->requests[j])
2183			break;
2184
2185		i915_request_set_error_once(eb->requests[j], err);
2186	}
2187	return err;
2188}
2189
2190static int i915_gem_check_execbuffer(struct drm_i915_private *i915,
2191				     struct drm_i915_gem_execbuffer2 *exec)
2192{
2193	if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
2194		return -EINVAL;
2195
2196	/* Kernel clipping was a DRI1 misfeature */
2197	if (!(exec->flags & (I915_EXEC_FENCE_ARRAY |
2198			     I915_EXEC_USE_EXTENSIONS))) {
2199		if (exec->num_cliprects || exec->cliprects_ptr)
2200			return -EINVAL;
2201	}
2202
2203	if (exec->DR4 == 0xffffffff) {
2204		drm_dbg(&i915->drm, "UXA submitting garbage DR4, fixing up\n");
2205		exec->DR4 = 0;
2206	}
2207	if (exec->DR1 || exec->DR4)
2208		return -EINVAL;
2209
2210	if ((exec->batch_start_offset | exec->batch_len) & 0x7)
2211		return -EINVAL;
2212
2213	return 0;
2214}
2215
2216static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
2217{
2218	u32 *cs;
2219	int i;
2220
2221	if (GRAPHICS_VER(rq->engine->i915) != 7 || rq->engine->id != RCS0) {
2222		drm_dbg(&rq->engine->i915->drm, "sol reset is gen7/rcs only\n");
2223		return -EINVAL;
2224	}
2225
2226	cs = intel_ring_begin(rq, 4 * 2 + 2);
2227	if (IS_ERR(cs))
2228		return PTR_ERR(cs);
2229
2230	*cs++ = MI_LOAD_REGISTER_IMM(4);
2231	for (i = 0; i < 4; i++) {
2232		*cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
2233		*cs++ = 0;
2234	}
2235	*cs++ = MI_NOOP;
2236	intel_ring_advance(rq, cs);
2237
2238	return 0;
2239}
2240
2241static struct i915_vma *
2242shadow_batch_pin(struct i915_execbuffer *eb,
2243		 struct drm_i915_gem_object *obj,
2244		 struct i915_address_space *vm,
2245		 unsigned int flags)
2246{
2247	struct i915_vma *vma;
2248	int err;
2249
2250	vma = i915_vma_instance(obj, vm, NULL);
2251	if (IS_ERR(vma))
2252		return vma;
2253
2254	err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags | PIN_VALIDATE);
2255	if (err)
2256		return ERR_PTR(err);
2257
2258	return vma;
2259}
2260
2261static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma)
2262{
2263	/*
2264	 * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
2265	 * batch" bit. Hence we need to pin secure batches into the global gtt.
2266	 * hsw should have this fixed, but bdw mucks it up again. */
2267	if (eb->batch_flags & I915_DISPATCH_SECURE)
2268		return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, PIN_VALIDATE);
2269
2270	return NULL;
2271}
2272
2273static int eb_parse(struct i915_execbuffer *eb)
2274{
2275	struct drm_i915_private *i915 = eb->i915;
2276	struct intel_gt_buffer_pool_node *pool = eb->batch_pool;
2277	struct i915_vma *shadow, *trampoline, *batch;
2278	unsigned long len;
2279	int err;
2280
2281	if (!eb_use_cmdparser(eb)) {
2282		batch = eb_dispatch_secure(eb, eb->batches[0]->vma);
2283		if (IS_ERR(batch))
2284			return PTR_ERR(batch);
2285
2286		goto secure_batch;
2287	}
2288
2289	if (intel_context_is_parallel(eb->context))
2290		return -EINVAL;
2291
2292	len = eb->batch_len[0];
2293	if (!CMDPARSER_USES_GGTT(eb->i915)) {
2294		/*
2295		 * ppGTT backed shadow buffers must be mapped RO, to prevent
2296		 * post-scan tampering
2297		 */
2298		if (!eb->context->vm->has_read_only) {
2299			drm_dbg(&i915->drm,
2300				"Cannot prevent post-scan tampering without RO capable vm\n");
2301			return -EINVAL;
2302		}
2303	} else {
2304		len += I915_CMD_PARSER_TRAMPOLINE_SIZE;
2305	}
2306	if (unlikely(len < eb->batch_len[0])) /* last paranoid check of overflow */
2307		return -EINVAL;
2308
2309	if (!pool) {
2310		pool = intel_gt_get_buffer_pool(eb->gt, len,
2311						I915_MAP_WB);
2312		if (IS_ERR(pool))
2313			return PTR_ERR(pool);
2314		eb->batch_pool = pool;
2315	}
2316
2317	err = i915_gem_object_lock(pool->obj, &eb->ww);
2318	if (err)
2319		return err;
2320
2321	shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER);
2322	if (IS_ERR(shadow))
2323		return PTR_ERR(shadow);
2324
2325	intel_gt_buffer_pool_mark_used(pool);
2326	i915_gem_object_set_readonly(shadow->obj);
2327	shadow->private = pool;
2328
2329	trampoline = NULL;
2330	if (CMDPARSER_USES_GGTT(eb->i915)) {
2331		trampoline = shadow;
2332
2333		shadow = shadow_batch_pin(eb, pool->obj,
2334					  &eb->gt->ggtt->vm,
2335					  PIN_GLOBAL);
2336		if (IS_ERR(shadow))
2337			return PTR_ERR(shadow);
2338
2339		shadow->private = pool;
2340
2341		eb->batch_flags |= I915_DISPATCH_SECURE;
2342	}
2343
2344	batch = eb_dispatch_secure(eb, shadow);
2345	if (IS_ERR(batch))
2346		return PTR_ERR(batch);
2347
2348	err = dma_resv_reserve_fences(shadow->obj->base.resv, 1);
2349	if (err)
2350		return err;
2351
2352	err = intel_engine_cmd_parser(eb->context->engine,
2353				      eb->batches[0]->vma,
2354				      eb->batch_start_offset,
2355				      eb->batch_len[0],
2356				      shadow, trampoline);
2357	if (err)
2358		return err;
2359
2360	eb->batches[0] = &eb->vma[eb->buffer_count++];
2361	eb->batches[0]->vma = i915_vma_get(shadow);
2362	eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN;
2363
2364	eb->trampoline = trampoline;
2365	eb->batch_start_offset = 0;
2366
2367secure_batch:
2368	if (batch) {
2369		if (intel_context_is_parallel(eb->context))
2370			return -EINVAL;
2371
2372		eb->batches[0] = &eb->vma[eb->buffer_count++];
2373		eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN;
2374		eb->batches[0]->vma = i915_vma_get(batch);
2375	}
2376	return 0;
2377}
2378
2379static int eb_request_submit(struct i915_execbuffer *eb,
2380			     struct i915_request *rq,
2381			     struct i915_vma *batch,
2382			     u64 batch_len)
2383{
2384	int err;
2385
2386	if (intel_context_nopreempt(rq->context))
2387		__set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
2388
2389	if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
2390		err = i915_reset_gen7_sol_offsets(rq);
2391		if (err)
2392			return err;
2393	}
2394
2395	/*
2396	 * After we completed waiting for other engines (using HW semaphores)
2397	 * then we can signal that this request/batch is ready to run. This
2398	 * allows us to determine if the batch is still waiting on the GPU
2399	 * or actually running by checking the breadcrumb.
2400	 */
2401	if (rq->context->engine->emit_init_breadcrumb) {
2402		err = rq->context->engine->emit_init_breadcrumb(rq);
2403		if (err)
2404			return err;
2405	}
2406
2407	err = rq->context->engine->emit_bb_start(rq,
2408						 batch->node.start +
2409						 eb->batch_start_offset,
2410						 batch_len,
2411						 eb->batch_flags);
2412	if (err)
2413		return err;
2414
2415	if (eb->trampoline) {
2416		GEM_BUG_ON(intel_context_is_parallel(rq->context));
2417		GEM_BUG_ON(eb->batch_start_offset);
2418		err = rq->context->engine->emit_bb_start(rq,
2419							 eb->trampoline->node.start +
2420							 batch_len, 0, 0);
2421		if (err)
2422			return err;
2423	}
2424
2425	return 0;
2426}
2427
2428static int eb_submit(struct i915_execbuffer *eb)
2429{
2430	unsigned int i;
2431	int err;
2432
2433	err = eb_move_to_gpu(eb);
2434
2435	for_each_batch_create_order(eb, i) {
2436		if (!eb->requests[i])
2437			break;
2438
2439		trace_i915_request_queue(eb->requests[i], eb->batch_flags);
2440		if (!err)
2441			err = eb_request_submit(eb, eb->requests[i],
2442						eb->batches[i]->vma,
2443						eb->batch_len[i]);
2444	}
2445
2446	return err;
2447}
2448
2449static int num_vcs_engines(struct drm_i915_private *i915)
2450{
2451	return hweight_long(VDBOX_MASK(to_gt(i915)));
2452}
2453
2454/*
2455 * Find one BSD ring to dispatch the corresponding BSD command.
2456 * The engine index is returned.
2457 */
2458static unsigned int
2459gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
2460			 struct drm_file *file)
2461{
2462	struct drm_i915_file_private *file_priv = file->driver_priv;
2463
2464	/* Check whether the file_priv has already selected one ring. */
2465	if ((int)file_priv->bsd_engine < 0)
2466		file_priv->bsd_engine =
2467			get_random_u32_below(num_vcs_engines(dev_priv));
2468
2469	return file_priv->bsd_engine;
2470}
2471
2472static const enum intel_engine_id user_ring_map[] = {
2473	[I915_EXEC_DEFAULT]	= RCS0,
2474	[I915_EXEC_RENDER]	= RCS0,
2475	[I915_EXEC_BLT]		= BCS0,
2476	[I915_EXEC_BSD]		= VCS0,
2477	[I915_EXEC_VEBOX]	= VECS0
2478};
2479
2480static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce)
2481{
2482	struct intel_ring *ring = ce->ring;
2483	struct intel_timeline *tl = ce->timeline;
2484	struct i915_request *rq;
2485
2486	/*
2487	 * Completely unscientific finger-in-the-air estimates for suitable
2488	 * maximum user request size (to avoid blocking) and then backoff.
2489	 */
2490	if (intel_ring_update_space(ring) >= PAGE_SIZE)
2491		return NULL;
2492
2493	/*
2494	 * Find a request that after waiting upon, there will be at least half
2495	 * the ring available. The hysteresis allows us to compete for the
2496	 * shared ring and should mean that we sleep less often prior to
2497	 * claiming our resources, but not so long that the ring completely
2498	 * drains before we can submit our next request.
2499	 */
2500	list_for_each_entry(rq, &tl->requests, link) {
2501		if (rq->ring != ring)
2502			continue;
2503
2504		if (__intel_ring_space(rq->postfix,
2505				       ring->emit, ring->size) > ring->size / 2)
2506			break;
2507	}
2508	if (&rq->link == &tl->requests)
2509		return NULL; /* weird, we will check again later for real */
2510
2511	return i915_request_get(rq);
2512}
2513
2514static int eb_pin_timeline(struct i915_execbuffer *eb, struct intel_context *ce,
2515			   bool throttle)
2516{
2517	struct intel_timeline *tl;
2518	struct i915_request *rq = NULL;
2519
2520	/*
2521	 * Take a local wakeref for preparing to dispatch the execbuf as
2522	 * we expect to access the hardware fairly frequently in the
2523	 * process, and require the engine to be kept awake between accesses.
2524	 * Upon dispatch, we acquire another prolonged wakeref that we hold
2525	 * until the timeline is idle, which in turn releases the wakeref
2526	 * taken on the engine, and the parent device.
2527	 */
2528	tl = intel_context_timeline_lock(ce);
2529	if (IS_ERR(tl))
2530		return PTR_ERR(tl);
2531
2532	intel_context_enter(ce);
2533	if (throttle)
2534		rq = eb_throttle(eb, ce);
2535	intel_context_timeline_unlock(tl);
2536
2537	if (rq) {
2538		bool nonblock = eb->file->filp->f_flags & O_NONBLOCK;
2539		long timeout = nonblock ? 0 : MAX_SCHEDULE_TIMEOUT;
2540
2541		if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE,
2542				      timeout) < 0) {
2543			i915_request_put(rq);
2544
2545			/*
2546			 * Error path, cannot use intel_context_timeline_lock as
2547			 * that is user interruptable and this clean up step
2548			 * must be done.
2549			 */
2550			mutex_lock(&ce->timeline->mutex);
2551			intel_context_exit(ce);
2552			mutex_unlock(&ce->timeline->mutex);
2553
2554			if (nonblock)
2555				return -EWOULDBLOCK;
2556			else
2557				return -EINTR;
2558		}
2559		i915_request_put(rq);
2560	}
2561
2562	return 0;
2563}
2564
2565static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle)
2566{
2567	struct intel_context *ce = eb->context, *child;
2568	int err;
2569	int i = 0, j = 0;
2570
2571	GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED);
2572
2573	if (unlikely(intel_context_is_banned(ce)))
2574		return -EIO;
2575
2576	/*
2577	 * Pinning the contexts may generate requests in order to acquire
2578	 * GGTT space, so do this first before we reserve a seqno for
2579	 * ourselves.
2580	 */
2581	err = intel_context_pin_ww(ce, &eb->ww);
2582	if (err)
2583		return err;
2584	for_each_child(ce, child) {
2585		err = intel_context_pin_ww(child, &eb->ww);
2586		GEM_BUG_ON(err);	/* perma-pinned should incr a counter */
2587	}
2588
2589	for_each_child(ce, child) {
2590		err = eb_pin_timeline(eb, child, throttle);
2591		if (err)
2592			goto unwind;
2593		++i;
2594	}
2595	err = eb_pin_timeline(eb, ce, throttle);
2596	if (err)
2597		goto unwind;
2598
2599	eb->args->flags |= __EXEC_ENGINE_PINNED;
2600	return 0;
2601
2602unwind:
2603	for_each_child(ce, child) {
2604		if (j++ < i) {
2605			mutex_lock(&child->timeline->mutex);
2606			intel_context_exit(child);
2607			mutex_unlock(&child->timeline->mutex);
2608		}
2609	}
2610	for_each_child(ce, child)
2611		intel_context_unpin(child);
2612	intel_context_unpin(ce);
2613	return err;
2614}
2615
2616static void eb_unpin_engine(struct i915_execbuffer *eb)
2617{
2618	struct intel_context *ce = eb->context, *child;
2619
2620	if (!(eb->args->flags & __EXEC_ENGINE_PINNED))
2621		return;
2622
2623	eb->args->flags &= ~__EXEC_ENGINE_PINNED;
2624
2625	for_each_child(ce, child) {
2626		mutex_lock(&child->timeline->mutex);
2627		intel_context_exit(child);
2628		mutex_unlock(&child->timeline->mutex);
2629
2630		intel_context_unpin(child);
2631	}
2632
2633	mutex_lock(&ce->timeline->mutex);
2634	intel_context_exit(ce);
2635	mutex_unlock(&ce->timeline->mutex);
2636
2637	intel_context_unpin(ce);
2638}
2639
2640static unsigned int
2641eb_select_legacy_ring(struct i915_execbuffer *eb)
2642{
2643	struct drm_i915_private *i915 = eb->i915;
2644	struct drm_i915_gem_execbuffer2 *args = eb->args;
2645	unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
2646
2647	if (user_ring_id != I915_EXEC_BSD &&
2648	    (args->flags & I915_EXEC_BSD_MASK)) {
2649		drm_dbg(&i915->drm,
2650			"execbuf with non bsd ring but with invalid "
2651			"bsd dispatch flags: %d\n", (int)(args->flags));
2652		return -1;
2653	}
2654
2655	if (user_ring_id == I915_EXEC_BSD && num_vcs_engines(i915) > 1) {
2656		unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
2657
2658		if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
2659			bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file);
2660		} else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
2661			   bsd_idx <= I915_EXEC_BSD_RING2) {
2662			bsd_idx >>= I915_EXEC_BSD_SHIFT;
2663			bsd_idx--;
2664		} else {
2665			drm_dbg(&i915->drm,
2666				"execbuf with unknown bsd ring: %u\n",
2667				bsd_idx);
2668			return -1;
2669		}
2670
2671		return _VCS(bsd_idx);
2672	}
2673
2674	if (user_ring_id >= ARRAY_SIZE(user_ring_map)) {
2675		drm_dbg(&i915->drm, "execbuf with unknown ring: %u\n",
2676			user_ring_id);
2677		return -1;
2678	}
2679
2680	return user_ring_map[user_ring_id];
2681}
2682
2683static int
2684eb_select_engine(struct i915_execbuffer *eb)
2685{
2686	struct intel_context *ce, *child;
2687	unsigned int idx;
2688	int err;
2689
2690	if (i915_gem_context_user_engines(eb->gem_context))
2691		idx = eb->args->flags & I915_EXEC_RING_MASK;
2692	else
2693		idx = eb_select_legacy_ring(eb);
2694
2695	ce = i915_gem_context_get_engine(eb->gem_context, idx);
2696	if (IS_ERR(ce))
2697		return PTR_ERR(ce);
2698
2699	if (intel_context_is_parallel(ce)) {
2700		if (eb->buffer_count < ce->parallel.number_children + 1) {
2701			intel_context_put(ce);
2702			return -EINVAL;
2703		}
2704		if (eb->batch_start_offset || eb->args->batch_len) {
2705			intel_context_put(ce);
2706			return -EINVAL;
2707		}
2708	}
2709	eb->num_batches = ce->parallel.number_children + 1;
2710
2711	for_each_child(ce, child)
2712		intel_context_get(child);
2713	intel_gt_pm_get(ce->engine->gt);
2714
2715	if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
2716		err = intel_context_alloc_state(ce);
2717		if (err)
2718			goto err;
2719	}
2720	for_each_child(ce, child) {
2721		if (!test_bit(CONTEXT_ALLOC_BIT, &child->flags)) {
2722			err = intel_context_alloc_state(child);
2723			if (err)
2724				goto err;
2725		}
2726	}
2727
2728	/*
2729	 * ABI: Before userspace accesses the GPU (e.g. execbuffer), report
2730	 * EIO if the GPU is already wedged.
2731	 */
2732	err = intel_gt_terminally_wedged(ce->engine->gt);
2733	if (err)
2734		goto err;
2735
2736	if (!i915_vm_tryget(ce->vm)) {
2737		err = -ENOENT;
2738		goto err;
2739	}
2740
2741	eb->context = ce;
2742	eb->gt = ce->engine->gt;
2743
2744	/*
2745	 * Make sure engine pool stays alive even if we call intel_context_put
2746	 * during ww handling. The pool is destroyed when last pm reference
2747	 * is dropped, which breaks our -EDEADLK handling.
2748	 */
2749	return err;
2750
2751err:
2752	intel_gt_pm_put(ce->engine->gt);
2753	for_each_child(ce, child)
2754		intel_context_put(child);
2755	intel_context_put(ce);
2756	return err;
2757}
2758
2759static void
2760eb_put_engine(struct i915_execbuffer *eb)
2761{
2762	struct intel_context *child;
2763
2764	i915_vm_put(eb->context->vm);
2765	intel_gt_pm_put(eb->gt);
2766	for_each_child(eb->context, child)
2767		intel_context_put(child);
2768	intel_context_put(eb->context);
2769}
2770
2771static void
2772__free_fence_array(struct eb_fence *fences, unsigned int n)
2773{
2774	while (n--) {
2775		drm_syncobj_put(ptr_mask_bits(fences[n].syncobj, 2));
2776		dma_fence_put(fences[n].dma_fence);
2777		dma_fence_chain_free(fences[n].chain_fence);
2778	}
2779	kvfree(fences);
2780}
2781
2782static int
2783add_timeline_fence_array(struct i915_execbuffer *eb,
2784			 const struct drm_i915_gem_execbuffer_ext_timeline_fences *timeline_fences)
2785{
2786	struct drm_i915_gem_exec_fence __user *user_fences;
2787	u64 __user *user_values;
2788	struct eb_fence *f;
2789	u64 nfences;
2790	int err = 0;
2791
2792	nfences = timeline_fences->fence_count;
2793	if (!nfences)
2794		return 0;
2795
2796	/* Check multiplication overflow for access_ok() and kvmalloc_array() */
2797	BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
2798	if (nfences > min_t(unsigned long,
2799			    ULONG_MAX / sizeof(*user_fences),
2800			    SIZE_MAX / sizeof(*f)) - eb->num_fences)
2801		return -EINVAL;
2802
2803	user_fences = u64_to_user_ptr(timeline_fences->handles_ptr);
2804	if (!access_ok(user_fences, nfences * sizeof(*user_fences)))
2805		return -EFAULT;
2806
2807	user_values = u64_to_user_ptr(timeline_fences->values_ptr);
2808	if (!access_ok(user_values, nfences * sizeof(*user_values)))
2809		return -EFAULT;
2810
2811	f = krealloc(eb->fences,
2812		     (eb->num_fences + nfences) * sizeof(*f),
2813		     __GFP_NOWARN | GFP_KERNEL);
2814	if (!f)
2815		return -ENOMEM;
2816
2817	eb->fences = f;
2818	f += eb->num_fences;
2819
2820	BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
2821		     ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
2822
2823	while (nfences--) {
2824		struct drm_i915_gem_exec_fence user_fence;
2825		struct drm_syncobj *syncobj;
2826		struct dma_fence *fence = NULL;
2827		u64 point;
2828
2829		if (__copy_from_user(&user_fence,
2830				     user_fences++,
2831				     sizeof(user_fence)))
2832			return -EFAULT;
2833
2834		if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS)
2835			return -EINVAL;
2836
2837		if (__get_user(point, user_values++))
2838			return -EFAULT;
2839
2840		syncobj = drm_syncobj_find(eb->file, user_fence.handle);
2841		if (!syncobj) {
2842			drm_dbg(&eb->i915->drm,
2843				"Invalid syncobj handle provided\n");
2844			return -ENOENT;
2845		}
2846
2847		fence = drm_syncobj_fence_get(syncobj);
2848
2849		if (!fence && user_fence.flags &&
2850		    !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2851			drm_dbg(&eb->i915->drm,
2852				"Syncobj handle has no fence\n");
2853			drm_syncobj_put(syncobj);
2854			return -EINVAL;
2855		}
2856
2857		if (fence)
2858			err = dma_fence_chain_find_seqno(&fence, point);
2859
2860		if (err && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2861			drm_dbg(&eb->i915->drm,
2862				"Syncobj handle missing requested point %llu\n",
2863				point);
2864			dma_fence_put(fence);
2865			drm_syncobj_put(syncobj);
2866			return err;
2867		}
2868
2869		/*
2870		 * A point might have been signaled already and
2871		 * garbage collected from the timeline. In this case
2872		 * just ignore the point and carry on.
2873		 */
2874		if (!fence && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2875			drm_syncobj_put(syncobj);
2876			continue;
2877		}
2878
2879		/*
2880		 * For timeline syncobjs we need to preallocate chains for
2881		 * later signaling.
2882		 */
2883		if (point != 0 && user_fence.flags & I915_EXEC_FENCE_SIGNAL) {
2884			/*
2885			 * Waiting and signaling the same point (when point !=
2886			 * 0) would break the timeline.
2887			 */
2888			if (user_fence.flags & I915_EXEC_FENCE_WAIT) {
2889				drm_dbg(&eb->i915->drm,
2890					"Trying to wait & signal the same timeline point.\n");
2891				dma_fence_put(fence);
2892				drm_syncobj_put(syncobj);
2893				return -EINVAL;
2894			}
2895
2896			f->chain_fence = dma_fence_chain_alloc();
2897			if (!f->chain_fence) {
2898				drm_syncobj_put(syncobj);
2899				dma_fence_put(fence);
2900				return -ENOMEM;
2901			}
2902		} else {
2903			f->chain_fence = NULL;
2904		}
2905
2906		f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2);
2907		f->dma_fence = fence;
2908		f->value = point;
2909		f++;
2910		eb->num_fences++;
2911	}
2912
2913	return 0;
2914}
2915
2916static int add_fence_array(struct i915_execbuffer *eb)
2917{
2918	struct drm_i915_gem_execbuffer2 *args = eb->args;
2919	struct drm_i915_gem_exec_fence __user *user;
2920	unsigned long num_fences = args->num_cliprects;
2921	struct eb_fence *f;
2922
2923	if (!(args->flags & I915_EXEC_FENCE_ARRAY))
2924		return 0;
2925
2926	if (!num_fences)
2927		return 0;
2928
2929	/* Check multiplication overflow for access_ok() and kvmalloc_array() */
2930	BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
2931	if (num_fences > min_t(unsigned long,
2932			       ULONG_MAX / sizeof(*user),
2933			       SIZE_MAX / sizeof(*f) - eb->num_fences))
2934		return -EINVAL;
2935
2936	user = u64_to_user_ptr(args->cliprects_ptr);
2937	if (!access_ok(user, num_fences * sizeof(*user)))
2938		return -EFAULT;
2939
2940	f = krealloc(eb->fences,
2941		     (eb->num_fences + num_fences) * sizeof(*f),
2942		     __GFP_NOWARN | GFP_KERNEL);
2943	if (!f)
2944		return -ENOMEM;
2945
2946	eb->fences = f;
2947	f += eb->num_fences;
2948	while (num_fences--) {
2949		struct drm_i915_gem_exec_fence user_fence;
2950		struct drm_syncobj *syncobj;
2951		struct dma_fence *fence = NULL;
2952
2953		if (__copy_from_user(&user_fence, user++, sizeof(user_fence)))
2954			return -EFAULT;
2955
2956		if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS)
2957			return -EINVAL;
2958
2959		syncobj = drm_syncobj_find(eb->file, user_fence.handle);
2960		if (!syncobj) {
2961			drm_dbg(&eb->i915->drm,
2962				"Invalid syncobj handle provided\n");
2963			return -ENOENT;
2964		}
2965
2966		if (user_fence.flags & I915_EXEC_FENCE_WAIT) {
2967			fence = drm_syncobj_fence_get(syncobj);
2968			if (!fence) {
2969				drm_dbg(&eb->i915->drm,
2970					"Syncobj handle has no fence\n");
2971				drm_syncobj_put(syncobj);
2972				return -EINVAL;
2973			}
2974		}
2975
2976		BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
2977			     ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
2978
2979		f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2);
2980		f->dma_fence = fence;
2981		f->value = 0;
2982		f->chain_fence = NULL;
2983		f++;
2984		eb->num_fences++;
2985	}
2986
2987	return 0;
2988}
2989
2990static void put_fence_array(struct eb_fence *fences, int num_fences)
2991{
2992	if (fences)
2993		__free_fence_array(fences, num_fences);
2994}
2995
2996static int
2997await_fence_array(struct i915_execbuffer *eb,
2998		  struct i915_request *rq)
2999{
3000	unsigned int n;
3001	int err;
3002
3003	for (n = 0; n < eb->num_fences; n++) {
3004		if (!eb->fences[n].dma_fence)
3005			continue;
3006
3007		err = i915_request_await_dma_fence(rq, eb->fences[n].dma_fence);
3008		if (err < 0)
3009			return err;
3010	}
3011
3012	return 0;
3013}
3014
3015static void signal_fence_array(const struct i915_execbuffer *eb,
3016			       struct dma_fence * const fence)
3017{
3018	unsigned int n;
3019
3020	for (n = 0; n < eb->num_fences; n++) {
3021		struct drm_syncobj *syncobj;
3022		unsigned int flags;
3023
3024		syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2);
3025		if (!(flags & I915_EXEC_FENCE_SIGNAL))
3026			continue;
3027
3028		if (eb->fences[n].chain_fence) {
3029			drm_syncobj_add_point(syncobj,
3030					      eb->fences[n].chain_fence,
3031					      fence,
3032					      eb->fences[n].value);
3033			/*
3034			 * The chain's ownership is transferred to the
3035			 * timeline.
3036			 */
3037			eb->fences[n].chain_fence = NULL;
3038		} else {
3039			drm_syncobj_replace_fence(syncobj, fence);
3040		}
3041	}
3042}
3043
3044static int
3045parse_timeline_fences(struct i915_user_extension __user *ext, void *data)
3046{
3047	struct i915_execbuffer *eb = data;
3048	struct drm_i915_gem_execbuffer_ext_timeline_fences timeline_fences;
3049
3050	if (copy_from_user(&timeline_fences, ext, sizeof(timeline_fences)))
3051		return -EFAULT;
3052
3053	return add_timeline_fence_array(eb, &timeline_fences);
3054}
3055
3056static void retire_requests(struct intel_timeline *tl, struct i915_request *end)
3057{
3058	struct i915_request *rq, *rn;
3059
3060	list_for_each_entry_safe(rq, rn, &tl->requests, link)
3061		if (rq == end || !i915_request_retire(rq))
3062			break;
3063}
3064
3065static int eb_request_add(struct i915_execbuffer *eb, struct i915_request *rq,
3066			  int err, bool last_parallel)
3067{
3068	struct intel_timeline * const tl = i915_request_timeline(rq);
3069	struct i915_sched_attr attr = {};
3070	struct i915_request *prev;
3071
3072	lockdep_assert_held(&tl->mutex);
3073	lockdep_unpin_lock(&tl->mutex, rq->cookie);
3074
3075	trace_i915_request_add(rq);
3076
3077	prev = __i915_request_commit(rq);
3078
3079	/* Check that the context wasn't destroyed before submission */
3080	if (likely(!intel_context_is_closed(eb->context))) {
3081		attr = eb->gem_context->sched;
3082	} else {
3083		/* Serialise with context_close via the add_to_timeline */
3084		i915_request_set_error_once(rq, -ENOENT);
3085		__i915_request_skip(rq);
3086		err = -ENOENT; /* override any transient errors */
3087	}
3088
3089	if (intel_context_is_parallel(eb->context)) {
3090		if (err) {
3091			__i915_request_skip(rq);
3092			set_bit(I915_FENCE_FLAG_SKIP_PARALLEL,
3093				&rq->fence.flags);
3094		}
3095		if (last_parallel)
3096			set_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL,
3097				&rq->fence.flags);
3098	}
3099
3100	__i915_request_queue(rq, &attr);
3101
3102	/* Try to clean up the client's timeline after submitting the request */
3103	if (prev)
3104		retire_requests(tl, prev);
3105
3106	mutex_unlock(&tl->mutex);
3107
3108	return err;
3109}
3110
3111static int eb_requests_add(struct i915_execbuffer *eb, int err)
3112{
3113	int i;
3114
3115	/*
3116	 * We iterate in reverse order of creation to release timeline mutexes in
3117	 * same order.
3118	 */
3119	for_each_batch_add_order(eb, i) {
3120		struct i915_request *rq = eb->requests[i];
3121
3122		if (!rq)
3123			continue;
3124		err |= eb_request_add(eb, rq, err, i == 0);
3125	}
3126
3127	return err;
3128}
3129
3130static const i915_user_extension_fn execbuf_extensions[] = {
3131	[DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES] = parse_timeline_fences,
3132};
3133
3134static int
3135parse_execbuf2_extensions(struct drm_i915_gem_execbuffer2 *args,
3136			  struct i915_execbuffer *eb)
3137{
3138	if (!(args->flags & I915_EXEC_USE_EXTENSIONS))
3139		return 0;
3140
3141	/* The execbuf2 extension mechanism reuses cliprects_ptr. So we cannot
3142	 * have another flag also using it at the same time.
3143	 */
3144	if (eb->args->flags & I915_EXEC_FENCE_ARRAY)
3145		return -EINVAL;
3146
3147	if (args->num_cliprects != 0)
3148		return -EINVAL;
3149
3150	return i915_user_extensions(u64_to_user_ptr(args->cliprects_ptr),
3151				    execbuf_extensions,
3152				    ARRAY_SIZE(execbuf_extensions),
3153				    eb);
3154}
3155
3156static void eb_requests_get(struct i915_execbuffer *eb)
3157{
3158	unsigned int i;
3159
3160	for_each_batch_create_order(eb, i) {
3161		if (!eb->requests[i])
3162			break;
3163
3164		i915_request_get(eb->requests[i]);
3165	}
3166}
3167
3168static void eb_requests_put(struct i915_execbuffer *eb)
3169{
3170	unsigned int i;
3171
3172	for_each_batch_create_order(eb, i) {
3173		if (!eb->requests[i])
3174			break;
3175
3176		i915_request_put(eb->requests[i]);
3177	}
3178}
3179
3180static struct sync_file *
3181eb_composite_fence_create(struct i915_execbuffer *eb, int out_fence_fd)
3182{
3183	struct sync_file *out_fence = NULL;
3184	struct dma_fence_array *fence_array;
3185	struct dma_fence **fences;
3186	unsigned int i;
3187
3188	GEM_BUG_ON(!intel_context_is_parent(eb->context));
3189
3190	fences = kmalloc_array(eb->num_batches, sizeof(*fences), GFP_KERNEL);
3191	if (!fences)
3192		return ERR_PTR(-ENOMEM);
3193
3194	for_each_batch_create_order(eb, i) {
3195		fences[i] = &eb->requests[i]->fence;
3196		__set_bit(I915_FENCE_FLAG_COMPOSITE,
3197			  &eb->requests[i]->fence.flags);
3198	}
3199
3200	fence_array = dma_fence_array_create(eb->num_batches,
3201					     fences,
3202					     eb->context->parallel.fence_context,
3203					     eb->context->parallel.seqno++,
3204					     false);
3205	if (!fence_array) {
3206		kfree(fences);
3207		return ERR_PTR(-ENOMEM);
3208	}
3209
3210	/* Move ownership to the dma_fence_array created above */
3211	for_each_batch_create_order(eb, i)
3212		dma_fence_get(fences[i]);
3213
3214	if (out_fence_fd != -1) {
3215		out_fence = sync_file_create(&fence_array->base);
3216		/* sync_file now owns fence_arry, drop creation ref */
3217		dma_fence_put(&fence_array->base);
3218		if (!out_fence)
3219			return ERR_PTR(-ENOMEM);
3220	}
3221
3222	eb->composite_fence = &fence_array->base;
3223
3224	return out_fence;
3225}
3226
3227static struct sync_file *
3228eb_fences_add(struct i915_execbuffer *eb, struct i915_request *rq,
3229	      struct dma_fence *in_fence, int out_fence_fd)
3230{
3231	struct sync_file *out_fence = NULL;
3232	int err;
3233
3234	if (unlikely(eb->gem_context->syncobj)) {
3235		struct dma_fence *fence;
3236
3237		fence = drm_syncobj_fence_get(eb->gem_context->syncobj);
3238		err = i915_request_await_dma_fence(rq, fence);
3239		dma_fence_put(fence);
3240		if (err)
3241			return ERR_PTR(err);
3242	}
3243
3244	if (in_fence) {
3245		if (eb->args->flags & I915_EXEC_FENCE_SUBMIT)
3246			err = i915_request_await_execution(rq, in_fence);
3247		else
3248			err = i915_request_await_dma_fence(rq, in_fence);
3249		if (err < 0)
3250			return ERR_PTR(err);
3251	}
3252
3253	if (eb->fences) {
3254		err = await_fence_array(eb, rq);
3255		if (err)
3256			return ERR_PTR(err);
3257	}
3258
3259	if (intel_context_is_parallel(eb->context)) {
3260		out_fence = eb_composite_fence_create(eb, out_fence_fd);
3261		if (IS_ERR(out_fence))
3262			return ERR_PTR(-ENOMEM);
3263	} else if (out_fence_fd != -1) {
3264		out_fence = sync_file_create(&rq->fence);
3265		if (!out_fence)
3266			return ERR_PTR(-ENOMEM);
3267	}
3268
3269	return out_fence;
3270}
3271
3272static struct intel_context *
3273eb_find_context(struct i915_execbuffer *eb, unsigned int context_number)
3274{
3275	struct intel_context *child;
3276
3277	if (likely(context_number == 0))
3278		return eb->context;
3279
3280	for_each_child(eb->context, child)
3281		if (!--context_number)
3282			return child;
3283
3284	GEM_BUG_ON("Context not found");
3285
3286	return NULL;
3287}
3288
3289static struct sync_file *
3290eb_requests_create(struct i915_execbuffer *eb, struct dma_fence *in_fence,
3291		   int out_fence_fd)
3292{
3293	struct sync_file *out_fence = NULL;
3294	unsigned int i;
3295
3296	for_each_batch_create_order(eb, i) {
3297		/* Allocate a request for this batch buffer nice and early. */
3298		eb->requests[i] = i915_request_create(eb_find_context(eb, i));
3299		if (IS_ERR(eb->requests[i])) {
3300			out_fence = ERR_CAST(eb->requests[i]);
3301			eb->requests[i] = NULL;
3302			return out_fence;
3303		}
3304
3305		/*
3306		 * Only the first request added (committed to backend) has to
3307		 * take the in fences into account as all subsequent requests
3308		 * will have fences inserted inbetween them.
3309		 */
3310		if (i + 1 == eb->num_batches) {
3311			out_fence = eb_fences_add(eb, eb->requests[i],
3312						  in_fence, out_fence_fd);
3313			if (IS_ERR(out_fence))
3314				return out_fence;
3315		}
3316
3317		/*
3318		 * Not really on stack, but we don't want to call
3319		 * kfree on the batch_snapshot when we put it, so use the
3320		 * _onstack interface.
3321		 */
3322		if (eb->batches[i]->vma)
3323			eb->requests[i]->batch_res =
3324				i915_vma_resource_get(eb->batches[i]->vma->resource);
3325		if (eb->batch_pool) {
3326			GEM_BUG_ON(intel_context_is_parallel(eb->context));
3327			intel_gt_buffer_pool_mark_active(eb->batch_pool,
3328							 eb->requests[i]);
3329		}
3330	}
3331
3332	return out_fence;
3333}
3334
3335static int
3336i915_gem_do_execbuffer(struct drm_device *dev,
3337		       struct drm_file *file,
3338		       struct drm_i915_gem_execbuffer2 *args,
3339		       struct drm_i915_gem_exec_object2 *exec)
3340{
3341	struct drm_i915_private *i915 = to_i915(dev);
3342	struct i915_execbuffer eb;
3343	struct dma_fence *in_fence = NULL;
3344	struct sync_file *out_fence = NULL;
3345	int out_fence_fd = -1;
3346	int err;
3347
3348	BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS);
3349	BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
3350		     ~__EXEC_OBJECT_UNKNOWN_FLAGS);
3351
3352	eb.i915 = i915;
3353	eb.file = file;
3354	eb.args = args;
3355	if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
3356		args->flags |= __EXEC_HAS_RELOC;
3357
3358	eb.exec = exec;
3359	eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1);
3360	eb.vma[0].vma = NULL;
3361	eb.batch_pool = NULL;
3362
3363	eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
3364	reloc_cache_init(&eb.reloc_cache, eb.i915);
3365
3366	eb.buffer_count = args->buffer_count;
3367	eb.batch_start_offset = args->batch_start_offset;
3368	eb.trampoline = NULL;
3369
3370	eb.fences = NULL;
3371	eb.num_fences = 0;
3372
3373	eb_capture_list_clear(&eb);
3374
3375	memset(eb.requests, 0, sizeof(struct i915_request *) *
3376	       ARRAY_SIZE(eb.requests));
3377	eb.composite_fence = NULL;
3378
3379	eb.batch_flags = 0;
3380	if (args->flags & I915_EXEC_SECURE) {
3381		if (GRAPHICS_VER(i915) >= 11)
3382			return -ENODEV;
3383
3384		/* Return -EPERM to trigger fallback code on old binaries. */
3385		if (!HAS_SECURE_BATCHES(i915))
3386			return -EPERM;
3387
3388		if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
3389			return -EPERM;
3390
3391		eb.batch_flags |= I915_DISPATCH_SECURE;
3392	}
3393	if (args->flags & I915_EXEC_IS_PINNED)
3394		eb.batch_flags |= I915_DISPATCH_PINNED;
3395
3396	err = parse_execbuf2_extensions(args, &eb);
3397	if (err)
3398		goto err_ext;
3399
3400	err = add_fence_array(&eb);
3401	if (err)
3402		goto err_ext;
3403
3404#define IN_FENCES (I915_EXEC_FENCE_IN | I915_EXEC_FENCE_SUBMIT)
3405	if (args->flags & IN_FENCES) {
3406		if ((args->flags & IN_FENCES) == IN_FENCES)
3407			return -EINVAL;
3408
3409		in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
3410		if (!in_fence) {
3411			err = -EINVAL;
3412			goto err_ext;
3413		}
3414	}
3415#undef IN_FENCES
3416
3417	if (args->flags & I915_EXEC_FENCE_OUT) {
3418		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
3419		if (out_fence_fd < 0) {
3420			err = out_fence_fd;
3421			goto err_in_fence;
3422		}
3423	}
3424
3425	err = eb_create(&eb);
3426	if (err)
3427		goto err_out_fence;
3428
3429	GEM_BUG_ON(!eb.lut_size);
3430
3431	err = eb_select_context(&eb);
3432	if (unlikely(err))
3433		goto err_destroy;
3434
3435	err = eb_select_engine(&eb);
3436	if (unlikely(err))
3437		goto err_context;
3438
3439	err = eb_lookup_vmas(&eb);
3440	if (err) {
3441		eb_release_vmas(&eb, true);
3442		goto err_engine;
3443	}
3444
3445	i915_gem_ww_ctx_init(&eb.ww, true);
3446
3447	err = eb_relocate_parse(&eb);
3448	if (err) {
3449		/*
3450		 * If the user expects the execobject.offset and
3451		 * reloc.presumed_offset to be an exact match,
3452		 * as for using NO_RELOC, then we cannot update
3453		 * the execobject.offset until we have completed
3454		 * relocation.
3455		 */
3456		args->flags &= ~__EXEC_HAS_RELOC;
3457		goto err_vma;
3458	}
3459
3460	ww_acquire_done(&eb.ww.ctx);
3461	err = eb_capture_stage(&eb);
3462	if (err)
3463		goto err_vma;
3464
3465	out_fence = eb_requests_create(&eb, in_fence, out_fence_fd);
3466	if (IS_ERR(out_fence)) {
3467		err = PTR_ERR(out_fence);
3468		out_fence = NULL;
3469		if (eb.requests[0])
3470			goto err_request;
3471		else
3472			goto err_vma;
3473	}
3474
3475	err = eb_submit(&eb);
3476
3477err_request:
3478	eb_requests_get(&eb);
3479	err = eb_requests_add(&eb, err);
3480
3481	if (eb.fences)
3482		signal_fence_array(&eb, eb.composite_fence ?
3483				   eb.composite_fence :
3484				   &eb.requests[0]->fence);
3485
3486	if (unlikely(eb.gem_context->syncobj)) {
3487		drm_syncobj_replace_fence(eb.gem_context->syncobj,
3488					  eb.composite_fence ?
3489					  eb.composite_fence :
3490					  &eb.requests[0]->fence);
3491	}
3492
3493	if (out_fence) {
3494		if (err == 0) {
3495			fd_install(out_fence_fd, out_fence->file);
3496			args->rsvd2 &= GENMASK_ULL(31, 0); /* keep in-fence */
3497			args->rsvd2 |= (u64)out_fence_fd << 32;
3498			out_fence_fd = -1;
3499		} else {
3500			fput(out_fence->file);
3501		}
3502	}
3503
3504	if (!out_fence && eb.composite_fence)
3505		dma_fence_put(eb.composite_fence);
3506
3507	eb_requests_put(&eb);
3508
3509err_vma:
3510	eb_release_vmas(&eb, true);
3511	WARN_ON(err == -EDEADLK);
3512	i915_gem_ww_ctx_fini(&eb.ww);
3513
3514	if (eb.batch_pool)
3515		intel_gt_buffer_pool_put(eb.batch_pool);
3516err_engine:
3517	eb_put_engine(&eb);
3518err_context:
3519	i915_gem_context_put(eb.gem_context);
3520err_destroy:
3521	eb_destroy(&eb);
3522err_out_fence:
3523	if (out_fence_fd != -1)
3524		put_unused_fd(out_fence_fd);
3525err_in_fence:
3526	dma_fence_put(in_fence);
3527err_ext:
3528	put_fence_array(eb.fences, eb.num_fences);
3529	return err;
3530}
3531
3532static size_t eb_element_size(void)
3533{
3534	return sizeof(struct drm_i915_gem_exec_object2) + sizeof(struct eb_vma);
3535}
3536
3537static bool check_buffer_count(size_t count)
3538{
3539	const size_t sz = eb_element_size();
3540
3541	/*
3542	 * When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup
3543	 * array size (see eb_create()). Otherwise, we can accept an array as
3544	 * large as can be addressed (though use large arrays at your peril)!
3545	 */
3546
3547	return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
3548}
3549
3550int
3551i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
3552			   struct drm_file *file)
3553{
3554	struct drm_i915_private *i915 = to_i915(dev);
3555	struct drm_i915_gem_execbuffer2 *args = data;
3556	struct drm_i915_gem_exec_object2 *exec2_list;
3557	const size_t count = args->buffer_count;
3558	int err;
3559
3560	if (!check_buffer_count(count)) {
3561		drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count);
3562		return -EINVAL;
3563	}
3564
3565	err = i915_gem_check_execbuffer(i915, args);
3566	if (err)
3567		return err;
3568
3569	/* Allocate extra slots for use by the command parser */
3570	exec2_list = kvmalloc_array(count + 2, eb_element_size(),
3571				    __GFP_NOWARN | GFP_KERNEL);
3572	if (exec2_list == NULL) {
3573		drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n",
3574			count);
3575		return -ENOMEM;
3576	}
3577	if (copy_from_user(exec2_list,
3578			   u64_to_user_ptr(args->buffers_ptr),
3579			   sizeof(*exec2_list) * count)) {
3580		drm_dbg(&i915->drm, "copy %zd exec entries failed\n", count);
3581		kvfree(exec2_list);
3582		return -EFAULT;
3583	}
3584
3585	err = i915_gem_do_execbuffer(dev, file, args, exec2_list);
3586
3587	/*
3588	 * Now that we have begun execution of the batchbuffer, we ignore
3589	 * any new error after this point. Also given that we have already
3590	 * updated the associated relocations, we try to write out the current
3591	 * object locations irrespective of any error.
3592	 */
3593	if (args->flags & __EXEC_HAS_RELOC) {
3594		struct drm_i915_gem_exec_object2 __user *user_exec_list =
3595			u64_to_user_ptr(args->buffers_ptr);
3596		unsigned int i;
3597
3598		/* Copy the new buffer offsets back to the user's exec list. */
3599		/*
3600		 * Note: count * sizeof(*user_exec_list) does not overflow,
3601		 * because we checked 'count' in check_buffer_count().
3602		 *
3603		 * And this range already got effectively checked earlier
3604		 * when we did the "copy_from_user()" above.
3605		 */
3606		if (!user_write_access_begin(user_exec_list,
3607					     count * sizeof(*user_exec_list)))
3608			goto end;
3609
3610		for (i = 0; i < args->buffer_count; i++) {
3611			if (!(exec2_list[i].offset & UPDATE))
3612				continue;
3613
3614			exec2_list[i].offset =
3615				gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
3616			unsafe_put_user(exec2_list[i].offset,
3617					&user_exec_list[i].offset,
3618					end_user);
3619		}
3620end_user:
3621		user_write_access_end();
3622end:;
3623	}
3624
3625	args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
3626	kvfree(exec2_list);
3627	return err;
3628}