Linux Audio

Check our new training course

Loading...
v6.13.7
   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2020 Intel Corporation
   4 */
   5
   6#include "i915_drv.h"
   7#include "intel_context.h"
   8#include "intel_gpu_commands.h"
   9#include "intel_gt.h"
  10#include "intel_gtt.h"
  11#include "intel_migrate.h"
  12#include "intel_ring.h"
  13#include "gem/i915_gem_lmem.h"
  14
  15struct insert_pte_data {
  16	u64 offset;
  17};
  18
  19#define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */
  20
  21#define GET_CCS_BYTES(i915, size)	(HAS_FLAT_CCS(i915) ? \
  22					 DIV_ROUND_UP(size, NUM_BYTES_PER_CCS_BYTE) : 0)
  23static bool engine_supports_migration(struct intel_engine_cs *engine)
  24{
  25	if (!engine)
  26		return false;
  27
  28	/*
  29	 * We need the ability to prevent aribtration (MI_ARB_ON_OFF),
  30	 * the ability to write PTE using inline data (MI_STORE_DATA)
  31	 * and of course the ability to do the block transfer (blits).
  32	 */
  33	GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS);
  34
  35	return true;
  36}
  37
  38static void xehp_toggle_pdes(struct i915_address_space *vm,
  39			     struct i915_page_table *pt,
  40			     void *data)
  41{
  42	struct insert_pte_data *d = data;
  43
  44	/*
  45	 * Insert a dummy PTE into every PT that will map to LMEM to ensure
  46	 * we have a correctly setup PDE structure for later use.
  47	 */
  48	vm->insert_page(vm, 0, d->offset,
  49			i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE),
  50			PTE_LM);
  51	GEM_BUG_ON(!pt->is_compact);
  52	d->offset += SZ_2M;
  53}
  54
  55static void xehp_insert_pte(struct i915_address_space *vm,
  56			    struct i915_page_table *pt,
  57			    void *data)
  58{
  59	struct insert_pte_data *d = data;
  60
  61	/*
  62	 * We are playing tricks here, since the actual pt, from the hw
  63	 * pov, is only 256bytes with 32 entries, or 4096bytes with 512
  64	 * entries, but we are still guaranteed that the physical
  65	 * alignment is 64K underneath for the pt, and we are careful
  66	 * not to access the space in the void.
  67	 */
  68	vm->insert_page(vm, px_dma(pt), d->offset,
  69			i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE),
  70			PTE_LM);
  71	d->offset += SZ_64K;
  72}
  73
  74static void insert_pte(struct i915_address_space *vm,
  75		       struct i915_page_table *pt,
  76		       void *data)
  77{
  78	struct insert_pte_data *d = data;
  79
  80	vm->insert_page(vm, px_dma(pt), d->offset,
  81			i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE),
  82			i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0);
  83	d->offset += PAGE_SIZE;
  84}
  85
  86static struct i915_address_space *migrate_vm(struct intel_gt *gt)
  87{
  88	struct i915_vm_pt_stash stash = {};
  89	struct i915_ppgtt *vm;
  90	int err;
  91	int i;
  92
  93	/*
  94	 * We construct a very special VM for use by all migration contexts,
  95	 * it is kept pinned so that it can be used at any time. As we need
  96	 * to pre-allocate the page directories for the migration VM, this
  97	 * limits us to only using a small number of prepared vma.
  98	 *
  99	 * To be able to pipeline and reschedule migration operations while
 100	 * avoiding unnecessary contention on the vm itself, the PTE updates
 101	 * are inline with the blits. All the blits use the same fixed
 102	 * addresses, with the backing store redirection being updated on the
 103	 * fly. Only 2 implicit vma are used for all migration operations.
 104	 *
 105	 * We lay the ppGTT out as:
 106	 *
 107	 *	[0, CHUNK_SZ) -> first object
 108	 *	[CHUNK_SZ, 2 * CHUNK_SZ) -> second object
 109	 *	[2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE
 110	 *
 111	 * By exposing the dma addresses of the page directories themselves
 112	 * within the ppGTT, we are then able to rewrite the PTE prior to use.
 113	 * But the PTE update and subsequent migration operation must be atomic,
 114	 * i.e. within the same non-preemptible window so that we do not switch
 115	 * to another migration context that overwrites the PTE.
 116	 *
 117	 * This changes quite a bit on platforms with HAS_64K_PAGES support,
 118	 * where we instead have three windows, each CHUNK_SIZE in size. The
 119	 * first is reserved for mapping system-memory, and that just uses the
 120	 * 512 entry layout using 4K GTT pages. The other two windows just map
 121	 * lmem pages and must use the new compact 32 entry layout using 64K GTT
 122	 * pages, which ensures we can address any lmem object that the user
 123	 * throws at us. We then also use the xehp_toggle_pdes as a way of
 124	 * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the
 125	 * compact layout for each of these page-tables, that fall within the
 126	 * [CHUNK_SIZE, 3 * CHUNK_SIZE) range.
 127	 *
 128	 * We lay the ppGTT out as:
 129	 *
 130	 * [0, CHUNK_SZ) -> first window/object, maps smem
 131	 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src
 132	 * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst
 133	 *
 134	 * For the PTE window it's also quite different, since each PTE must
 135	 * point to some 64K page, one for each PT(since it's in lmem), and yet
 136	 * each is only <= 4096bytes, but since the unused space within that PTE
 137	 * range is never touched, this should be fine.
 138	 *
 139	 * So basically each PT now needs 64K of virtual memory, instead of 4K,
 140	 * which looks like:
 141	 *
 142	 * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE
 143	 */
 144
 145	vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY);
 146	if (IS_ERR(vm))
 147		return ERR_CAST(vm);
 148
 149	if (!vm->vm.allocate_va_range || !vm->vm.foreach) {
 150		err = -ENODEV;
 151		goto err_vm;
 152	}
 153
 154	if (HAS_64K_PAGES(gt->i915))
 155		stash.pt_sz = I915_GTT_PAGE_SIZE_64K;
 156
 157	/*
 158	 * Each engine instance is assigned its own chunk in the VM, so
 159	 * that we can run multiple instances concurrently
 160	 */
 161	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 162		struct intel_engine_cs *engine;
 163		u64 base = (u64)i << 32;
 164		struct insert_pte_data d = {};
 165		struct i915_gem_ww_ctx ww;
 166		u64 sz;
 167
 168		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 169		if (!engine_supports_migration(engine))
 170			continue;
 171
 172		/*
 173		 * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need
 174		 * 4x2 page directories for source/destination.
 175		 */
 176		if (HAS_64K_PAGES(gt->i915))
 177			sz = 3 * CHUNK_SZ;
 178		else
 179			sz = 2 * CHUNK_SZ;
 180		d.offset = base + sz;
 181
 182		/*
 183		 * We need another page directory setup so that we can write
 184		 * the 8x512 PTE in each chunk.
 185		 */
 186		if (HAS_64K_PAGES(gt->i915))
 187			sz += (sz / SZ_2M) * SZ_64K;
 188		else
 189			sz += (sz >> 12) * sizeof(u64);
 190
 191		err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz);
 192		if (err)
 193			goto err_vm;
 194
 195		for_i915_gem_ww(&ww, err, true) {
 196			err = i915_vm_lock_objects(&vm->vm, &ww);
 197			if (err)
 198				continue;
 199			err = i915_vm_map_pt_stash(&vm->vm, &stash);
 200			if (err)
 201				continue;
 202
 203			vm->vm.allocate_va_range(&vm->vm, &stash, base, sz);
 204		}
 205		i915_vm_free_pt_stash(&vm->vm, &stash);
 206		if (err)
 207			goto err_vm;
 208
 209		/* Now allow the GPU to rewrite the PTE via its own ppGTT */
 210		if (HAS_64K_PAGES(gt->i915)) {
 211			vm->vm.foreach(&vm->vm, base, d.offset - base,
 212				       xehp_insert_pte, &d);
 213			d.offset = base + CHUNK_SZ;
 214			vm->vm.foreach(&vm->vm,
 215				       d.offset,
 216				       2 * CHUNK_SZ,
 217				       xehp_toggle_pdes, &d);
 218		} else {
 219			vm->vm.foreach(&vm->vm, base, d.offset - base,
 220				       insert_pte, &d);
 221		}
 222	}
 223
 224	return &vm->vm;
 225
 226err_vm:
 227	i915_vm_put(&vm->vm);
 228	return ERR_PTR(err);
 229}
 230
 231static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt)
 232{
 233	struct intel_engine_cs *engine;
 234	int i;
 235
 236	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 237		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 238		if (engine_supports_migration(engine))
 239			return engine;
 240	}
 241
 242	return NULL;
 243}
 244
 245static struct intel_context *pinned_context(struct intel_gt *gt)
 246{
 247	static struct lock_class_key key;
 248	struct intel_engine_cs *engine;
 249	struct i915_address_space *vm;
 250	struct intel_context *ce;
 251
 252	engine = first_copy_engine(gt);
 253	if (!engine)
 254		return ERR_PTR(-ENODEV);
 255
 256	vm = migrate_vm(gt);
 257	if (IS_ERR(vm))
 258		return ERR_CAST(vm);
 259
 260	ce = intel_engine_create_pinned_context(engine, vm, SZ_512K,
 261						I915_GEM_HWS_MIGRATE,
 262						&key, "migrate");
 263	i915_vm_put(vm);
 264	return ce;
 265}
 266
 267int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt)
 268{
 269	struct intel_context *ce;
 270
 271	memset(m, 0, sizeof(*m));
 272
 273	ce = pinned_context(gt);
 274	if (IS_ERR(ce))
 275		return PTR_ERR(ce);
 276
 277	m->context = ce;
 278	return 0;
 279}
 280
 281static int random_index(unsigned int max)
 282{
 283	return upper_32_bits(mul_u32_u32(get_random_u32(), max));
 284}
 285
 286static struct intel_context *__migrate_engines(struct intel_gt *gt)
 287{
 288	struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE];
 289	struct intel_engine_cs *engine;
 290	unsigned int count, i;
 291
 292	count = 0;
 293	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 294		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 295		if (engine_supports_migration(engine))
 296			engines[count++] = engine;
 297	}
 298
 299	return intel_context_create(engines[random_index(count)]);
 300}
 301
 302struct intel_context *intel_migrate_create_context(struct intel_migrate *m)
 303{
 304	struct intel_context *ce;
 305
 306	/*
 307	 * We randomly distribute contexts across the engines upon constrction,
 308	 * as they all share the same pinned vm, and so in order to allow
 309	 * multiple blits to run in parallel, we must construct each blit
 310	 * to use a different range of the vm for its GTT. This has to be
 311	 * known at construction, so we can not use the late greedy load
 312	 * balancing of the virtual-engine.
 313	 */
 314	ce = __migrate_engines(m->context->engine->gt);
 315	if (IS_ERR(ce))
 316		return ce;
 317
 318	ce->ring = NULL;
 319	ce->ring_size = SZ_256K;
 320
 321	i915_vm_put(ce->vm);
 322	ce->vm = i915_vm_get(m->context->vm);
 323
 324	return ce;
 325}
 326
 327static inline struct sgt_dma sg_sgt(struct scatterlist *sg)
 328{
 329	dma_addr_t addr = sg_dma_address(sg);
 330
 331	return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) };
 332}
 333
 334static int emit_no_arbitration(struct i915_request *rq)
 335{
 336	u32 *cs;
 337
 338	cs = intel_ring_begin(rq, 2);
 339	if (IS_ERR(cs))
 340		return PTR_ERR(cs);
 341
 342	/* Explicitly disable preemption for this request. */
 343	*cs++ = MI_ARB_ON_OFF;
 344	*cs++ = MI_NOOP;
 345	intel_ring_advance(rq, cs);
 346
 347	return 0;
 348}
 349
 350static int max_pte_pkt_size(struct i915_request *rq, int pkt)
 351{
 352	struct intel_ring *ring = rq->ring;
 353
 354	pkt = min_t(int, pkt, (ring->space - rq->reserved_space) / sizeof(u32) + 5);
 355	pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5);
 356
 357	return pkt;
 358}
 359
 360#define I915_EMIT_PTE_NUM_DWORDS 6
 361
 362static int emit_pte(struct i915_request *rq,
 363		    struct sgt_dma *it,
 364		    unsigned int pat_index,
 365		    bool is_lmem,
 366		    u64 offset,
 367		    int length)
 368{
 369	bool has_64K_pages = HAS_64K_PAGES(rq->i915);
 370	const u64 encode = rq->context->vm->pte_encode(0, pat_index,
 371						       is_lmem ? PTE_LM : 0);
 372	struct intel_ring *ring = rq->ring;
 373	int pkt, dword_length;
 374	u32 total = 0;
 375	u32 page_size;
 376	u32 *hdr, *cs;
 377
 378	GEM_BUG_ON(GRAPHICS_VER(rq->i915) < 8);
 379
 380	page_size = I915_GTT_PAGE_SIZE;
 381	dword_length = 0x400;
 382
 383	/* Compute the page directory offset for the target address range */
 384	if (has_64K_pages) {
 385		GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M));
 386
 387		offset /= SZ_2M;
 388		offset *= SZ_64K;
 389		offset += 3 * CHUNK_SZ;
 390
 391		if (is_lmem) {
 392			page_size = I915_GTT_PAGE_SIZE_64K;
 393			dword_length = 0x40;
 394		}
 395	} else {
 396		offset >>= 12;
 397		offset *= sizeof(u64);
 398		offset += 2 * CHUNK_SZ;
 399	}
 400
 401	offset += (u64)rq->engine->instance << 32;
 402
 403	cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS);
 404	if (IS_ERR(cs))
 405		return PTR_ERR(cs);
 406
 407	/* Pack as many PTE updates as possible into a single MI command */
 408	pkt = max_pte_pkt_size(rq, dword_length);
 409
 410	hdr = cs;
 411	*cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */
 412	*cs++ = lower_32_bits(offset);
 413	*cs++ = upper_32_bits(offset);
 414
 415	do {
 416		if (cs - hdr >= pkt) {
 417			int dword_rem;
 418
 419			*hdr += cs - hdr - 2;
 420			*cs++ = MI_NOOP;
 421
 422			ring->emit = (void *)cs - ring->vaddr;
 423			intel_ring_advance(rq, cs);
 424			intel_ring_update_space(ring);
 425
 426			cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS);
 427			if (IS_ERR(cs))
 428				return PTR_ERR(cs);
 429
 430			dword_rem = dword_length;
 431			if (has_64K_pages) {
 432				if (IS_ALIGNED(total, SZ_2M)) {
 433					offset = round_up(offset, SZ_64K);
 434				} else {
 435					dword_rem = SZ_2M - (total & (SZ_2M - 1));
 436					dword_rem /= page_size;
 437					dword_rem *= 2;
 438				}
 439			}
 440
 441			pkt = max_pte_pkt_size(rq, dword_rem);
 442
 443			hdr = cs;
 444			*cs++ = MI_STORE_DATA_IMM | REG_BIT(21);
 445			*cs++ = lower_32_bits(offset);
 446			*cs++ = upper_32_bits(offset);
 447		}
 448
 449		GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size));
 450
 451		*cs++ = lower_32_bits(encode | it->dma);
 452		*cs++ = upper_32_bits(encode | it->dma);
 453
 454		offset += 8;
 455		total += page_size;
 456
 457		it->dma += page_size;
 458		if (it->dma >= it->max) {
 459			it->sg = __sg_next(it->sg);
 460			if (!it->sg || sg_dma_len(it->sg) == 0)
 461				break;
 462
 463			it->dma = sg_dma_address(it->sg);
 464			it->max = it->dma + sg_dma_len(it->sg);
 465		}
 466	} while (total < length);
 467
 468	*hdr += cs - hdr - 2;
 469	*cs++ = MI_NOOP;
 470
 471	ring->emit = (void *)cs - ring->vaddr;
 472	intel_ring_advance(rq, cs);
 473	intel_ring_update_space(ring);
 474
 475	return total;
 476}
 477
 478static bool wa_1209644611_applies(int ver, u32 size)
 479{
 480	u32 height = size >> PAGE_SHIFT;
 481
 482	if (ver != 11)
 483		return false;
 484
 485	return height % 4 == 3 && height <= 8;
 486}
 487
 488/**
 489 * DOC: Flat-CCS - Memory compression for Local memory
 490 *
 491 * On Xe-HP and later devices, we use dedicated compression control state (CCS)
 492 * stored in local memory for each surface, to support the 3D and media
 493 * compression formats.
 494 *
 495 * The memory required for the CCS of the entire local memory is 1/256 of the
 496 * local memory size. So before the kernel boot, the required memory is reserved
 497 * for the CCS data and a secure register will be programmed with the CCS base
 498 * address.
 499 *
 500 * Flat CCS data needs to be cleared when a lmem object is allocated.
 501 * And CCS data can be copied in and out of CCS region through
 502 * XY_CTRL_SURF_COPY_BLT. CPU can't access the CCS data directly.
 503 *
 504 * I915 supports Flat-CCS on lmem only objects. When an objects has smem in
 505 * its preference list, on memory pressure, i915 needs to migrate the lmem
 506 * content into smem. If the lmem object is Flat-CCS compressed by userspace,
 507 * then i915 needs to decompress it. But I915 lack the required information
 508 * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects.
 509 *
 510 * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can
 511 * be temporarily evicted to smem, along with the auxiliary CCS state, where
 512 * it can be potentially swapped-out at a later point, if required.
 513 * If userspace later touches the evicted pages, then we always move
 514 * the backing memory back to lmem, which includes restoring the saved CCS state,
 515 * and potentially performing any required swap-in.
 516 *
 517 * For the migration of the lmem objects with smem in placement list, such as
 518 * {lmem, smem}, objects are treated as non Flat-CCS capable objects.
 519 */
 520
 521static inline u32 *i915_flush_dw(u32 *cmd, u32 flags)
 522{
 523	*cmd++ = MI_FLUSH_DW | flags;
 524	*cmd++ = 0;
 525	*cmd++ = 0;
 526
 527	return cmd;
 528}
 529
 530static int emit_copy_ccs(struct i915_request *rq,
 531			 u32 dst_offset, u8 dst_access,
 532			 u32 src_offset, u8 src_access, int size)
 533{
 534	struct drm_i915_private *i915 = rq->i915;
 535	int mocs = rq->engine->gt->mocs.uc_index << 1;
 536	u32 num_ccs_blks;
 537	u32 *cs;
 538
 539	cs = intel_ring_begin(rq, 12);
 540	if (IS_ERR(cs))
 541		return PTR_ERR(cs);
 542
 543	num_ccs_blks = DIV_ROUND_UP(GET_CCS_BYTES(i915, size),
 544				    NUM_CCS_BYTES_PER_BLOCK);
 545	GEM_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER);
 546	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
 547
 548	/*
 549	 * The XY_CTRL_SURF_COPY_BLT instruction is used to copy the CCS
 550	 * data in and out of the CCS region.
 551	 *
 552	 * We can copy at most 1024 blocks of 256 bytes using one
 553	 * XY_CTRL_SURF_COPY_BLT instruction.
 554	 *
 555	 * In case we need to copy more than 1024 blocks, we need to add
 556	 * another instruction to the same batch buffer.
 557	 *
 558	 * 1024 blocks of 256 bytes of CCS represent a total 256KB of CCS.
 559	 *
 560	 * 256 KB of CCS represents 256 * 256 KB = 64 MB of LMEM.
 561	 */
 562	*cs++ = XY_CTRL_SURF_COPY_BLT |
 563		src_access << SRC_ACCESS_TYPE_SHIFT |
 564		dst_access << DST_ACCESS_TYPE_SHIFT |
 565		((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT;
 566	*cs++ = src_offset;
 567	*cs++ = rq->engine->instance |
 568		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
 569	*cs++ = dst_offset;
 570	*cs++ = rq->engine->instance |
 571		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
 572
 573	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
 574	*cs++ = MI_NOOP;
 575
 576	intel_ring_advance(rq, cs);
 577
 578	return 0;
 579}
 580
 581static int emit_copy(struct i915_request *rq,
 582		     u32 dst_offset, u32 src_offset, int size)
 583{
 584	const int ver = GRAPHICS_VER(rq->i915);
 585	u32 instance = rq->engine->instance;
 586	u32 *cs;
 587
 588	cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6);
 589	if (IS_ERR(cs))
 590		return PTR_ERR(cs);
 591
 592	if (ver >= 9 && !wa_1209644611_applies(ver, size)) {
 593		*cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2);
 594		*cs++ = BLT_DEPTH_32 | PAGE_SIZE;
 595		*cs++ = 0;
 596		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 597		*cs++ = dst_offset;
 598		*cs++ = instance;
 599		*cs++ = 0;
 600		*cs++ = PAGE_SIZE;
 601		*cs++ = src_offset;
 602		*cs++ = instance;
 603	} else if (ver >= 8) {
 604		*cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2);
 605		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
 606		*cs++ = 0;
 607		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 608		*cs++ = dst_offset;
 609		*cs++ = instance;
 610		*cs++ = 0;
 611		*cs++ = PAGE_SIZE;
 612		*cs++ = src_offset;
 613		*cs++ = instance;
 614	} else {
 615		GEM_BUG_ON(instance);
 616		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
 617		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
 618		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE;
 619		*cs++ = dst_offset;
 620		*cs++ = PAGE_SIZE;
 621		*cs++ = src_offset;
 622	}
 623
 624	intel_ring_advance(rq, cs);
 625	return 0;
 626}
 627
 628static u64 scatter_list_length(struct scatterlist *sg)
 629{
 630	u64 len = 0;
 631
 632	while (sg && sg_dma_len(sg)) {
 633		len += sg_dma_len(sg);
 634		sg = sg_next(sg);
 635	}
 636
 637	return len;
 638}
 639
 640static int
 641calculate_chunk_sz(struct drm_i915_private *i915, bool src_is_lmem,
 642		   u64 bytes_to_cpy, u64 ccs_bytes_to_cpy)
 643{
 644	if (ccs_bytes_to_cpy && !src_is_lmem)
 645		/*
 646		 * When CHUNK_SZ is passed all the pages upto CHUNK_SZ
 647		 * will be taken for the blt. in Flat-ccs supported
 648		 * platform Smem obj will have more pages than required
 649		 * for main meory hence limit it to the required size
 650		 * for main memory
 651		 */
 652		return min_t(u64, bytes_to_cpy, CHUNK_SZ);
 653	else
 654		return CHUNK_SZ;
 655}
 656
 657static void get_ccs_sg_sgt(struct sgt_dma *it, u64 bytes_to_cpy)
 658{
 659	u64 len;
 660
 661	do {
 662		GEM_BUG_ON(!it->sg || !sg_dma_len(it->sg));
 663		len = it->max - it->dma;
 664		if (len > bytes_to_cpy) {
 665			it->dma += bytes_to_cpy;
 666			break;
 667		}
 668
 669		bytes_to_cpy -= len;
 670
 671		it->sg = __sg_next(it->sg);
 672		it->dma = sg_dma_address(it->sg);
 673		it->max = it->dma + sg_dma_len(it->sg);
 674	} while (bytes_to_cpy);
 675}
 676
 677int
 678intel_context_migrate_copy(struct intel_context *ce,
 679			   const struct i915_deps *deps,
 680			   struct scatterlist *src,
 681			   unsigned int src_pat_index,
 682			   bool src_is_lmem,
 683			   struct scatterlist *dst,
 684			   unsigned int dst_pat_index,
 685			   bool dst_is_lmem,
 686			   struct i915_request **out)
 687{
 688	struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst), it_ccs;
 689	struct drm_i915_private *i915 = ce->engine->i915;
 690	u64 ccs_bytes_to_cpy = 0, bytes_to_cpy;
 691	unsigned int ccs_pat_index;
 692	u32 src_offset, dst_offset;
 693	u8 src_access, dst_access;
 694	struct i915_request *rq;
 695	u64 src_sz, dst_sz;
 696	bool ccs_is_src, overwrite_ccs;
 697	int err;
 698
 699	GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
 700	GEM_BUG_ON(IS_DGFX(ce->engine->i915) && (!src_is_lmem && !dst_is_lmem));
 701	*out = NULL;
 702
 703	GEM_BUG_ON(ce->ring->size < SZ_64K);
 704
 705	src_sz = scatter_list_length(src);
 706	bytes_to_cpy = src_sz;
 707
 708	if (HAS_FLAT_CCS(i915) && src_is_lmem ^ dst_is_lmem) {
 709		src_access = !src_is_lmem && dst_is_lmem;
 710		dst_access = !src_access;
 711
 712		dst_sz = scatter_list_length(dst);
 713		if (src_is_lmem) {
 714			it_ccs = it_dst;
 715			ccs_pat_index = dst_pat_index;
 716			ccs_is_src = false;
 717		} else if (dst_is_lmem) {
 718			bytes_to_cpy = dst_sz;
 719			it_ccs = it_src;
 720			ccs_pat_index = src_pat_index;
 721			ccs_is_src = true;
 722		}
 723
 724		/*
 725		 * When there is a eviction of ccs needed smem will have the
 726		 * extra pages for the ccs data
 727		 *
 728		 * TO-DO: Want to move the size mismatch check to a WARN_ON,
 729		 * but still we have some requests of smem->lmem with same size.
 730		 * Need to fix it.
 731		 */
 732		ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0;
 733		if (ccs_bytes_to_cpy)
 734			get_ccs_sg_sgt(&it_ccs, bytes_to_cpy);
 735	}
 736
 737	overwrite_ccs = HAS_FLAT_CCS(i915) && !ccs_bytes_to_cpy && dst_is_lmem;
 738
 739	src_offset = 0;
 740	dst_offset = CHUNK_SZ;
 741	if (HAS_64K_PAGES(ce->engine->i915)) {
 742		src_offset = 0;
 743		dst_offset = 0;
 744		if (src_is_lmem)
 745			src_offset = CHUNK_SZ;
 746		if (dst_is_lmem)
 747			dst_offset = 2 * CHUNK_SZ;
 748	}
 749
 750	do {
 751		int len;
 752
 753		rq = i915_request_create(ce);
 754		if (IS_ERR(rq)) {
 755			err = PTR_ERR(rq);
 756			goto out_ce;
 757		}
 758
 759		if (deps) {
 760			err = i915_request_await_deps(rq, deps);
 761			if (err)
 762				goto out_rq;
 763
 764			if (rq->engine->emit_init_breadcrumb) {
 765				err = rq->engine->emit_init_breadcrumb(rq);
 766				if (err)
 767					goto out_rq;
 768			}
 769
 770			deps = NULL;
 771		}
 772
 773		/* The PTE updates + copy must not be interrupted. */
 774		err = emit_no_arbitration(rq);
 775		if (err)
 776			goto out_rq;
 777
 778		src_sz = calculate_chunk_sz(i915, src_is_lmem,
 779					    bytes_to_cpy, ccs_bytes_to_cpy);
 780
 781		len = emit_pte(rq, &it_src, src_pat_index, src_is_lmem,
 782			       src_offset, src_sz);
 783		if (!len) {
 784			err = -EINVAL;
 785			goto out_rq;
 786		}
 787		if (len < 0) {
 788			err = len;
 789			goto out_rq;
 790		}
 791
 792		err = emit_pte(rq, &it_dst, dst_pat_index, dst_is_lmem,
 793			       dst_offset, len);
 794		if (err < 0)
 795			goto out_rq;
 796		if (err < len) {
 797			err = -EINVAL;
 798			goto out_rq;
 799		}
 800
 801		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 802		if (err)
 803			goto out_rq;
 804
 805		err = emit_copy(rq, dst_offset,	src_offset, len);
 806		if (err)
 807			goto out_rq;
 808
 809		bytes_to_cpy -= len;
 810
 811		if (ccs_bytes_to_cpy) {
 812			int ccs_sz;
 813
 814			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 815			if (err)
 816				goto out_rq;
 817
 818			ccs_sz = GET_CCS_BYTES(i915, len);
 819			err = emit_pte(rq, &it_ccs, ccs_pat_index, false,
 820				       ccs_is_src ? src_offset : dst_offset,
 821				       ccs_sz);
 822			if (err < 0)
 823				goto out_rq;
 824			if (err < ccs_sz) {
 825				err = -EINVAL;
 826				goto out_rq;
 827			}
 828
 829			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 830			if (err)
 831				goto out_rq;
 832
 833			err = emit_copy_ccs(rq, dst_offset, dst_access,
 834					    src_offset, src_access, len);
 835			if (err)
 836				goto out_rq;
 837
 838			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 839			if (err)
 840				goto out_rq;
 841			ccs_bytes_to_cpy -= ccs_sz;
 842		} else if (overwrite_ccs) {
 843			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 844			if (err)
 845				goto out_rq;
 846
 847			if (src_is_lmem) {
 848				/*
 849				 * If the src is already in lmem, then we must
 850				 * be doing an lmem -> lmem transfer, and so
 851				 * should be safe to directly copy the CCS
 852				 * state. In this case we have either
 853				 * initialised the CCS aux state when first
 854				 * clearing the pages (since it is already
 855				 * allocated in lmem), or the user has
 856				 * potentially populated it, in which case we
 857				 * need to copy the CCS state as-is.
 858				 */
 859				err = emit_copy_ccs(rq,
 860						    dst_offset, INDIRECT_ACCESS,
 861						    src_offset, INDIRECT_ACCESS,
 862						    len);
 863			} else {
 864				/*
 865				 * While we can't always restore/manage the CCS
 866				 * state, we still need to ensure we don't leak
 867				 * the CCS state from the previous user, so make
 868				 * sure we overwrite it with something.
 869				 */
 870				err = emit_copy_ccs(rq,
 871						    dst_offset, INDIRECT_ACCESS,
 872						    dst_offset, DIRECT_ACCESS,
 873						    len);
 874			}
 875
 876			if (err)
 877				goto out_rq;
 878
 879			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 880			if (err)
 881				goto out_rq;
 882		}
 883
 884		/* Arbitration is re-enabled between requests. */
 885out_rq:
 886		if (*out)
 887			i915_request_put(*out);
 888		*out = i915_request_get(rq);
 889		i915_request_add(rq);
 890
 891		if (err)
 892			break;
 893
 894		if (!bytes_to_cpy && !ccs_bytes_to_cpy) {
 895			if (src_is_lmem)
 896				WARN_ON(it_src.sg && sg_dma_len(it_src.sg));
 897			else
 898				WARN_ON(it_dst.sg && sg_dma_len(it_dst.sg));
 899			break;
 900		}
 901
 902		if (WARN_ON(!it_src.sg || !sg_dma_len(it_src.sg) ||
 903			    !it_dst.sg || !sg_dma_len(it_dst.sg) ||
 904			    (ccs_bytes_to_cpy && (!it_ccs.sg ||
 905						  !sg_dma_len(it_ccs.sg))))) {
 906			err = -EINVAL;
 907			break;
 908		}
 909
 910		cond_resched();
 911	} while (1);
 912
 913out_ce:
 914	return err;
 915}
 916
 917static int emit_clear(struct i915_request *rq, u32 offset, int size,
 918		      u32 value, bool is_lmem)
 919{
 920	struct drm_i915_private *i915 = rq->i915;
 921	int mocs = rq->engine->gt->mocs.uc_index << 1;
 922	const int ver = GRAPHICS_VER(i915);
 923	int ring_sz;
 924	u32 *cs;
 925
 926	GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX);
 927
 928	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 929		ring_sz = XY_FAST_COLOR_BLT_DW;
 930	else if (ver >= 8)
 931		ring_sz = 8;
 932	else
 933		ring_sz = 6;
 934
 935	cs = intel_ring_begin(rq, ring_sz);
 936	if (IS_ERR(cs))
 937		return PTR_ERR(cs);
 938
 939	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
 940		*cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
 941			(XY_FAST_COLOR_BLT_DW - 2);
 942		*cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) |
 943			(PAGE_SIZE - 1);
 944		*cs++ = 0;
 945		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 946		*cs++ = offset;
 947		*cs++ = rq->engine->instance;
 948		*cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
 949		/* BG7 */
 950		*cs++ = value;
 951		*cs++ = 0;
 952		*cs++ = 0;
 953		*cs++ = 0;
 954		/* BG11 */
 955		*cs++ = 0;
 956		*cs++ = 0;
 957		/* BG13 */
 958		*cs++ = 0;
 959		*cs++ = 0;
 960		*cs++ = 0;
 961	} else if (ver >= 8) {
 962		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2);
 963		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
 964		*cs++ = 0;
 965		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 966		*cs++ = offset;
 967		*cs++ = rq->engine->instance;
 968		*cs++ = value;
 969		*cs++ = MI_NOOP;
 970	} else {
 971		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
 972		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
 973		*cs++ = 0;
 974		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 975		*cs++ = offset;
 976		*cs++ = value;
 977	}
 978
 979	intel_ring_advance(rq, cs);
 980	return 0;
 981}
 982
 983int
 984intel_context_migrate_clear(struct intel_context *ce,
 985			    const struct i915_deps *deps,
 986			    struct scatterlist *sg,
 987			    unsigned int pat_index,
 988			    bool is_lmem,
 989			    u32 value,
 990			    struct i915_request **out)
 991{
 992	struct drm_i915_private *i915 = ce->engine->i915;
 993	struct sgt_dma it = sg_sgt(sg);
 994	struct i915_request *rq;
 995	u32 offset;
 996	int err;
 997
 998	GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
 999	*out = NULL;
1000
1001	GEM_BUG_ON(ce->ring->size < SZ_64K);
1002
1003	offset = 0;
1004	if (HAS_64K_PAGES(i915) && is_lmem)
1005		offset = CHUNK_SZ;
1006
1007	do {
1008		int len;
1009
1010		rq = i915_request_create(ce);
1011		if (IS_ERR(rq)) {
1012			err = PTR_ERR(rq);
1013			goto out_ce;
1014		}
1015
1016		if (deps) {
1017			err = i915_request_await_deps(rq, deps);
1018			if (err)
1019				goto out_rq;
1020
1021			if (rq->engine->emit_init_breadcrumb) {
1022				err = rq->engine->emit_init_breadcrumb(rq);
1023				if (err)
1024					goto out_rq;
1025			}
1026
1027			deps = NULL;
1028		}
1029
1030		/* The PTE updates + clear must not be interrupted. */
1031		err = emit_no_arbitration(rq);
1032		if (err)
1033			goto out_rq;
1034
1035		len = emit_pte(rq, &it, pat_index, is_lmem, offset, CHUNK_SZ);
1036		if (len <= 0) {
1037			err = len;
1038			goto out_rq;
1039		}
1040
1041		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1042		if (err)
1043			goto out_rq;
1044
1045		err = emit_clear(rq, offset, len, value, is_lmem);
1046		if (err)
1047			goto out_rq;
1048
1049		if (HAS_FLAT_CCS(i915) && is_lmem && !value) {
1050			/*
1051			 * copy the content of memory into corresponding
1052			 * ccs surface
1053			 */
1054			err = emit_copy_ccs(rq, offset, INDIRECT_ACCESS, offset,
1055					    DIRECT_ACCESS, len);
1056			if (err)
1057				goto out_rq;
1058		}
1059
1060		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1061
1062		/* Arbitration is re-enabled between requests. */
1063out_rq:
1064		if (*out)
1065			i915_request_put(*out);
1066		*out = i915_request_get(rq);
1067		i915_request_add(rq);
1068		if (err || !it.sg || !sg_dma_len(it.sg))
1069			break;
1070
1071		cond_resched();
1072	} while (1);
1073
1074out_ce:
1075	return err;
1076}
1077
1078int intel_migrate_copy(struct intel_migrate *m,
1079		       struct i915_gem_ww_ctx *ww,
1080		       const struct i915_deps *deps,
1081		       struct scatterlist *src,
1082		       unsigned int src_pat_index,
1083		       bool src_is_lmem,
1084		       struct scatterlist *dst,
1085		       unsigned int dst_pat_index,
1086		       bool dst_is_lmem,
1087		       struct i915_request **out)
1088{
1089	struct intel_context *ce;
1090	int err;
1091
1092	*out = NULL;
1093	if (!m->context)
1094		return -ENODEV;
1095
1096	ce = intel_migrate_create_context(m);
1097	if (IS_ERR(ce))
1098		ce = intel_context_get(m->context);
1099	GEM_BUG_ON(IS_ERR(ce));
1100
1101	err = intel_context_pin_ww(ce, ww);
1102	if (err)
1103		goto out;
1104
1105	err = intel_context_migrate_copy(ce, deps,
1106					 src, src_pat_index, src_is_lmem,
1107					 dst, dst_pat_index, dst_is_lmem,
1108					 out);
1109
1110	intel_context_unpin(ce);
1111out:
1112	intel_context_put(ce);
1113	return err;
1114}
1115
1116int
1117intel_migrate_clear(struct intel_migrate *m,
1118		    struct i915_gem_ww_ctx *ww,
1119		    const struct i915_deps *deps,
1120		    struct scatterlist *sg,
1121		    unsigned int pat_index,
1122		    bool is_lmem,
1123		    u32 value,
1124		    struct i915_request **out)
1125{
1126	struct intel_context *ce;
1127	int err;
1128
1129	*out = NULL;
1130	if (!m->context)
1131		return -ENODEV;
1132
1133	ce = intel_migrate_create_context(m);
1134	if (IS_ERR(ce))
1135		ce = intel_context_get(m->context);
1136	GEM_BUG_ON(IS_ERR(ce));
1137
1138	err = intel_context_pin_ww(ce, ww);
1139	if (err)
1140		goto out;
1141
1142	err = intel_context_migrate_clear(ce, deps, sg, pat_index,
1143					  is_lmem, value, out);
1144
1145	intel_context_unpin(ce);
1146out:
1147	intel_context_put(ce);
1148	return err;
1149}
1150
1151void intel_migrate_fini(struct intel_migrate *m)
1152{
1153	struct intel_context *ce;
1154
1155	ce = fetch_and_zero(&m->context);
1156	if (!ce)
1157		return;
1158
1159	intel_engine_destroy_pinned_context(ce);
1160}
1161
1162#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1163#include "selftest_migrate.c"
1164#endif
v6.2
   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2020 Intel Corporation
   4 */
   5
   6#include "i915_drv.h"
   7#include "intel_context.h"
   8#include "intel_gpu_commands.h"
   9#include "intel_gt.h"
  10#include "intel_gtt.h"
  11#include "intel_migrate.h"
  12#include "intel_ring.h"
  13#include "gem/i915_gem_lmem.h"
  14
  15struct insert_pte_data {
  16	u64 offset;
  17};
  18
  19#define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */
  20
  21#define GET_CCS_BYTES(i915, size)	(HAS_FLAT_CCS(i915) ? \
  22					 DIV_ROUND_UP(size, NUM_BYTES_PER_CCS_BYTE) : 0)
  23static bool engine_supports_migration(struct intel_engine_cs *engine)
  24{
  25	if (!engine)
  26		return false;
  27
  28	/*
  29	 * We need the ability to prevent aribtration (MI_ARB_ON_OFF),
  30	 * the ability to write PTE using inline data (MI_STORE_DATA)
  31	 * and of course the ability to do the block transfer (blits).
  32	 */
  33	GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS);
  34
  35	return true;
  36}
  37
  38static void xehpsdv_toggle_pdes(struct i915_address_space *vm,
  39				struct i915_page_table *pt,
  40				void *data)
  41{
  42	struct insert_pte_data *d = data;
  43
  44	/*
  45	 * Insert a dummy PTE into every PT that will map to LMEM to ensure
  46	 * we have a correctly setup PDE structure for later use.
  47	 */
  48	vm->insert_page(vm, 0, d->offset, I915_CACHE_NONE, PTE_LM);
 
 
  49	GEM_BUG_ON(!pt->is_compact);
  50	d->offset += SZ_2M;
  51}
  52
  53static void xehpsdv_insert_pte(struct i915_address_space *vm,
  54			       struct i915_page_table *pt,
  55			       void *data)
  56{
  57	struct insert_pte_data *d = data;
  58
  59	/*
  60	 * We are playing tricks here, since the actual pt, from the hw
  61	 * pov, is only 256bytes with 32 entries, or 4096bytes with 512
  62	 * entries, but we are still guaranteed that the physical
  63	 * alignment is 64K underneath for the pt, and we are careful
  64	 * not to access the space in the void.
  65	 */
  66	vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, PTE_LM);
 
 
  67	d->offset += SZ_64K;
  68}
  69
  70static void insert_pte(struct i915_address_space *vm,
  71		       struct i915_page_table *pt,
  72		       void *data)
  73{
  74	struct insert_pte_data *d = data;
  75
  76	vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE,
 
  77			i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0);
  78	d->offset += PAGE_SIZE;
  79}
  80
  81static struct i915_address_space *migrate_vm(struct intel_gt *gt)
  82{
  83	struct i915_vm_pt_stash stash = {};
  84	struct i915_ppgtt *vm;
  85	int err;
  86	int i;
  87
  88	/*
  89	 * We construct a very special VM for use by all migration contexts,
  90	 * it is kept pinned so that it can be used at any time. As we need
  91	 * to pre-allocate the page directories for the migration VM, this
  92	 * limits us to only using a small number of prepared vma.
  93	 *
  94	 * To be able to pipeline and reschedule migration operations while
  95	 * avoiding unnecessary contention on the vm itself, the PTE updates
  96	 * are inline with the blits. All the blits use the same fixed
  97	 * addresses, with the backing store redirection being updated on the
  98	 * fly. Only 2 implicit vma are used for all migration operations.
  99	 *
 100	 * We lay the ppGTT out as:
 101	 *
 102	 *	[0, CHUNK_SZ) -> first object
 103	 *	[CHUNK_SZ, 2 * CHUNK_SZ) -> second object
 104	 *	[2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE
 105	 *
 106	 * By exposing the dma addresses of the page directories themselves
 107	 * within the ppGTT, we are then able to rewrite the PTE prior to use.
 108	 * But the PTE update and subsequent migration operation must be atomic,
 109	 * i.e. within the same non-preemptible window so that we do not switch
 110	 * to another migration context that overwrites the PTE.
 111	 *
 112	 * This changes quite a bit on platforms with HAS_64K_PAGES support,
 113	 * where we instead have three windows, each CHUNK_SIZE in size. The
 114	 * first is reserved for mapping system-memory, and that just uses the
 115	 * 512 entry layout using 4K GTT pages. The other two windows just map
 116	 * lmem pages and must use the new compact 32 entry layout using 64K GTT
 117	 * pages, which ensures we can address any lmem object that the user
 118	 * throws at us. We then also use the xehpsdv_toggle_pdes as a way of
 119	 * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the
 120	 * compact layout for each of these page-tables, that fall within the
 121	 * [CHUNK_SIZE, 3 * CHUNK_SIZE) range.
 122	 *
 123	 * We lay the ppGTT out as:
 124	 *
 125	 * [0, CHUNK_SZ) -> first window/object, maps smem
 126	 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src
 127	 * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst
 128	 *
 129	 * For the PTE window it's also quite different, since each PTE must
 130	 * point to some 64K page, one for each PT(since it's in lmem), and yet
 131	 * each is only <= 4096bytes, but since the unused space within that PTE
 132	 * range is never touched, this should be fine.
 133	 *
 134	 * So basically each PT now needs 64K of virtual memory, instead of 4K,
 135	 * which looks like:
 136	 *
 137	 * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE
 138	 */
 139
 140	vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY);
 141	if (IS_ERR(vm))
 142		return ERR_CAST(vm);
 143
 144	if (!vm->vm.allocate_va_range || !vm->vm.foreach) {
 145		err = -ENODEV;
 146		goto err_vm;
 147	}
 148
 149	if (HAS_64K_PAGES(gt->i915))
 150		stash.pt_sz = I915_GTT_PAGE_SIZE_64K;
 151
 152	/*
 153	 * Each engine instance is assigned its own chunk in the VM, so
 154	 * that we can run multiple instances concurrently
 155	 */
 156	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 157		struct intel_engine_cs *engine;
 158		u64 base = (u64)i << 32;
 159		struct insert_pte_data d = {};
 160		struct i915_gem_ww_ctx ww;
 161		u64 sz;
 162
 163		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 164		if (!engine_supports_migration(engine))
 165			continue;
 166
 167		/*
 168		 * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need
 169		 * 4x2 page directories for source/destination.
 170		 */
 171		if (HAS_64K_PAGES(gt->i915))
 172			sz = 3 * CHUNK_SZ;
 173		else
 174			sz = 2 * CHUNK_SZ;
 175		d.offset = base + sz;
 176
 177		/*
 178		 * We need another page directory setup so that we can write
 179		 * the 8x512 PTE in each chunk.
 180		 */
 181		if (HAS_64K_PAGES(gt->i915))
 182			sz += (sz / SZ_2M) * SZ_64K;
 183		else
 184			sz += (sz >> 12) * sizeof(u64);
 185
 186		err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz);
 187		if (err)
 188			goto err_vm;
 189
 190		for_i915_gem_ww(&ww, err, true) {
 191			err = i915_vm_lock_objects(&vm->vm, &ww);
 192			if (err)
 193				continue;
 194			err = i915_vm_map_pt_stash(&vm->vm, &stash);
 195			if (err)
 196				continue;
 197
 198			vm->vm.allocate_va_range(&vm->vm, &stash, base, sz);
 199		}
 200		i915_vm_free_pt_stash(&vm->vm, &stash);
 201		if (err)
 202			goto err_vm;
 203
 204		/* Now allow the GPU to rewrite the PTE via its own ppGTT */
 205		if (HAS_64K_PAGES(gt->i915)) {
 206			vm->vm.foreach(&vm->vm, base, d.offset - base,
 207				       xehpsdv_insert_pte, &d);
 208			d.offset = base + CHUNK_SZ;
 209			vm->vm.foreach(&vm->vm,
 210				       d.offset,
 211				       2 * CHUNK_SZ,
 212				       xehpsdv_toggle_pdes, &d);
 213		} else {
 214			vm->vm.foreach(&vm->vm, base, d.offset - base,
 215				       insert_pte, &d);
 216		}
 217	}
 218
 219	return &vm->vm;
 220
 221err_vm:
 222	i915_vm_put(&vm->vm);
 223	return ERR_PTR(err);
 224}
 225
 226static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt)
 227{
 228	struct intel_engine_cs *engine;
 229	int i;
 230
 231	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 232		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 233		if (engine_supports_migration(engine))
 234			return engine;
 235	}
 236
 237	return NULL;
 238}
 239
 240static struct intel_context *pinned_context(struct intel_gt *gt)
 241{
 242	static struct lock_class_key key;
 243	struct intel_engine_cs *engine;
 244	struct i915_address_space *vm;
 245	struct intel_context *ce;
 246
 247	engine = first_copy_engine(gt);
 248	if (!engine)
 249		return ERR_PTR(-ENODEV);
 250
 251	vm = migrate_vm(gt);
 252	if (IS_ERR(vm))
 253		return ERR_CAST(vm);
 254
 255	ce = intel_engine_create_pinned_context(engine, vm, SZ_512K,
 256						I915_GEM_HWS_MIGRATE,
 257						&key, "migrate");
 258	i915_vm_put(vm);
 259	return ce;
 260}
 261
 262int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt)
 263{
 264	struct intel_context *ce;
 265
 266	memset(m, 0, sizeof(*m));
 267
 268	ce = pinned_context(gt);
 269	if (IS_ERR(ce))
 270		return PTR_ERR(ce);
 271
 272	m->context = ce;
 273	return 0;
 274}
 275
 276static int random_index(unsigned int max)
 277{
 278	return upper_32_bits(mul_u32_u32(get_random_u32(), max));
 279}
 280
 281static struct intel_context *__migrate_engines(struct intel_gt *gt)
 282{
 283	struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE];
 284	struct intel_engine_cs *engine;
 285	unsigned int count, i;
 286
 287	count = 0;
 288	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 289		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 290		if (engine_supports_migration(engine))
 291			engines[count++] = engine;
 292	}
 293
 294	return intel_context_create(engines[random_index(count)]);
 295}
 296
 297struct intel_context *intel_migrate_create_context(struct intel_migrate *m)
 298{
 299	struct intel_context *ce;
 300
 301	/*
 302	 * We randomly distribute contexts across the engines upon constrction,
 303	 * as they all share the same pinned vm, and so in order to allow
 304	 * multiple blits to run in parallel, we must construct each blit
 305	 * to use a different range of the vm for its GTT. This has to be
 306	 * known at construction, so we can not use the late greedy load
 307	 * balancing of the virtual-engine.
 308	 */
 309	ce = __migrate_engines(m->context->engine->gt);
 310	if (IS_ERR(ce))
 311		return ce;
 312
 313	ce->ring = NULL;
 314	ce->ring_size = SZ_256K;
 315
 316	i915_vm_put(ce->vm);
 317	ce->vm = i915_vm_get(m->context->vm);
 318
 319	return ce;
 320}
 321
 322static inline struct sgt_dma sg_sgt(struct scatterlist *sg)
 323{
 324	dma_addr_t addr = sg_dma_address(sg);
 325
 326	return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) };
 327}
 328
 329static int emit_no_arbitration(struct i915_request *rq)
 330{
 331	u32 *cs;
 332
 333	cs = intel_ring_begin(rq, 2);
 334	if (IS_ERR(cs))
 335		return PTR_ERR(cs);
 336
 337	/* Explicitly disable preemption for this request. */
 338	*cs++ = MI_ARB_ON_OFF;
 339	*cs++ = MI_NOOP;
 340	intel_ring_advance(rq, cs);
 341
 342	return 0;
 343}
 344
 345static int max_pte_pkt_size(struct i915_request *rq, int pkt)
 346{
 347	struct intel_ring *ring = rq->ring;
 348
 349	pkt = min_t(int, pkt, (ring->space - rq->reserved_space) / sizeof(u32) + 5);
 350	pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5);
 351
 352	return pkt;
 353}
 354
 
 
 355static int emit_pte(struct i915_request *rq,
 356		    struct sgt_dma *it,
 357		    enum i915_cache_level cache_level,
 358		    bool is_lmem,
 359		    u64 offset,
 360		    int length)
 361{
 362	bool has_64K_pages = HAS_64K_PAGES(rq->engine->i915);
 363	const u64 encode = rq->context->vm->pte_encode(0, cache_level,
 364						       is_lmem ? PTE_LM : 0);
 365	struct intel_ring *ring = rq->ring;
 366	int pkt, dword_length;
 367	u32 total = 0;
 368	u32 page_size;
 369	u32 *hdr, *cs;
 370
 371	GEM_BUG_ON(GRAPHICS_VER(rq->engine->i915) < 8);
 372
 373	page_size = I915_GTT_PAGE_SIZE;
 374	dword_length = 0x400;
 375
 376	/* Compute the page directory offset for the target address range */
 377	if (has_64K_pages) {
 378		GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M));
 379
 380		offset /= SZ_2M;
 381		offset *= SZ_64K;
 382		offset += 3 * CHUNK_SZ;
 383
 384		if (is_lmem) {
 385			page_size = I915_GTT_PAGE_SIZE_64K;
 386			dword_length = 0x40;
 387		}
 388	} else {
 389		offset >>= 12;
 390		offset *= sizeof(u64);
 391		offset += 2 * CHUNK_SZ;
 392	}
 393
 394	offset += (u64)rq->engine->instance << 32;
 395
 396	cs = intel_ring_begin(rq, 6);
 397	if (IS_ERR(cs))
 398		return PTR_ERR(cs);
 399
 400	/* Pack as many PTE updates as possible into a single MI command */
 401	pkt = max_pte_pkt_size(rq, dword_length);
 402
 403	hdr = cs;
 404	*cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */
 405	*cs++ = lower_32_bits(offset);
 406	*cs++ = upper_32_bits(offset);
 407
 408	do {
 409		if (cs - hdr >= pkt) {
 410			int dword_rem;
 411
 412			*hdr += cs - hdr - 2;
 413			*cs++ = MI_NOOP;
 414
 415			ring->emit = (void *)cs - ring->vaddr;
 416			intel_ring_advance(rq, cs);
 417			intel_ring_update_space(ring);
 418
 419			cs = intel_ring_begin(rq, 6);
 420			if (IS_ERR(cs))
 421				return PTR_ERR(cs);
 422
 423			dword_rem = dword_length;
 424			if (has_64K_pages) {
 425				if (IS_ALIGNED(total, SZ_2M)) {
 426					offset = round_up(offset, SZ_64K);
 427				} else {
 428					dword_rem = SZ_2M - (total & (SZ_2M - 1));
 429					dword_rem /= page_size;
 430					dword_rem *= 2;
 431				}
 432			}
 433
 434			pkt = max_pte_pkt_size(rq, dword_rem);
 435
 436			hdr = cs;
 437			*cs++ = MI_STORE_DATA_IMM | REG_BIT(21);
 438			*cs++ = lower_32_bits(offset);
 439			*cs++ = upper_32_bits(offset);
 440		}
 441
 442		GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size));
 443
 444		*cs++ = lower_32_bits(encode | it->dma);
 445		*cs++ = upper_32_bits(encode | it->dma);
 446
 447		offset += 8;
 448		total += page_size;
 449
 450		it->dma += page_size;
 451		if (it->dma >= it->max) {
 452			it->sg = __sg_next(it->sg);
 453			if (!it->sg || sg_dma_len(it->sg) == 0)
 454				break;
 455
 456			it->dma = sg_dma_address(it->sg);
 457			it->max = it->dma + sg_dma_len(it->sg);
 458		}
 459	} while (total < length);
 460
 461	*hdr += cs - hdr - 2;
 462	*cs++ = MI_NOOP;
 463
 464	ring->emit = (void *)cs - ring->vaddr;
 465	intel_ring_advance(rq, cs);
 466	intel_ring_update_space(ring);
 467
 468	return total;
 469}
 470
 471static bool wa_1209644611_applies(int ver, u32 size)
 472{
 473	u32 height = size >> PAGE_SHIFT;
 474
 475	if (ver != 11)
 476		return false;
 477
 478	return height % 4 == 3 && height <= 8;
 479}
 480
 481/**
 482 * DOC: Flat-CCS - Memory compression for Local memory
 483 *
 484 * On Xe-HP and later devices, we use dedicated compression control state (CCS)
 485 * stored in local memory for each surface, to support the 3D and media
 486 * compression formats.
 487 *
 488 * The memory required for the CCS of the entire local memory is 1/256 of the
 489 * local memory size. So before the kernel boot, the required memory is reserved
 490 * for the CCS data and a secure register will be programmed with the CCS base
 491 * address.
 492 *
 493 * Flat CCS data needs to be cleared when a lmem object is allocated.
 494 * And CCS data can be copied in and out of CCS region through
 495 * XY_CTRL_SURF_COPY_BLT. CPU can't access the CCS data directly.
 496 *
 497 * I915 supports Flat-CCS on lmem only objects. When an objects has smem in
 498 * its preference list, on memory pressure, i915 needs to migrate the lmem
 499 * content into smem. If the lmem object is Flat-CCS compressed by userspace,
 500 * then i915 needs to decompress it. But I915 lack the required information
 501 * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects.
 502 *
 503 * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can
 504 * be temporarily evicted to smem, along with the auxiliary CCS state, where
 505 * it can be potentially swapped-out at a later point, if required.
 506 * If userspace later touches the evicted pages, then we always move
 507 * the backing memory back to lmem, which includes restoring the saved CCS state,
 508 * and potentially performing any required swap-in.
 509 *
 510 * For the migration of the lmem objects with smem in placement list, such as
 511 * {lmem, smem}, objects are treated as non Flat-CCS capable objects.
 512 */
 513
 514static inline u32 *i915_flush_dw(u32 *cmd, u32 flags)
 515{
 516	*cmd++ = MI_FLUSH_DW | flags;
 517	*cmd++ = 0;
 518	*cmd++ = 0;
 519
 520	return cmd;
 521}
 522
 523static int emit_copy_ccs(struct i915_request *rq,
 524			 u32 dst_offset, u8 dst_access,
 525			 u32 src_offset, u8 src_access, int size)
 526{
 527	struct drm_i915_private *i915 = rq->engine->i915;
 528	int mocs = rq->engine->gt->mocs.uc_index << 1;
 529	u32 num_ccs_blks;
 530	u32 *cs;
 531
 532	cs = intel_ring_begin(rq, 12);
 533	if (IS_ERR(cs))
 534		return PTR_ERR(cs);
 535
 536	num_ccs_blks = DIV_ROUND_UP(GET_CCS_BYTES(i915, size),
 537				    NUM_CCS_BYTES_PER_BLOCK);
 538	GEM_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER);
 539	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
 540
 541	/*
 542	 * The XY_CTRL_SURF_COPY_BLT instruction is used to copy the CCS
 543	 * data in and out of the CCS region.
 544	 *
 545	 * We can copy at most 1024 blocks of 256 bytes using one
 546	 * XY_CTRL_SURF_COPY_BLT instruction.
 547	 *
 548	 * In case we need to copy more than 1024 blocks, we need to add
 549	 * another instruction to the same batch buffer.
 550	 *
 551	 * 1024 blocks of 256 bytes of CCS represent a total 256KB of CCS.
 552	 *
 553	 * 256 KB of CCS represents 256 * 256 KB = 64 MB of LMEM.
 554	 */
 555	*cs++ = XY_CTRL_SURF_COPY_BLT |
 556		src_access << SRC_ACCESS_TYPE_SHIFT |
 557		dst_access << DST_ACCESS_TYPE_SHIFT |
 558		((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT;
 559	*cs++ = src_offset;
 560	*cs++ = rq->engine->instance |
 561		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
 562	*cs++ = dst_offset;
 563	*cs++ = rq->engine->instance |
 564		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
 565
 566	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
 567	*cs++ = MI_NOOP;
 568
 569	intel_ring_advance(rq, cs);
 570
 571	return 0;
 572}
 573
 574static int emit_copy(struct i915_request *rq,
 575		     u32 dst_offset, u32 src_offset, int size)
 576{
 577	const int ver = GRAPHICS_VER(rq->engine->i915);
 578	u32 instance = rq->engine->instance;
 579	u32 *cs;
 580
 581	cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6);
 582	if (IS_ERR(cs))
 583		return PTR_ERR(cs);
 584
 585	if (ver >= 9 && !wa_1209644611_applies(ver, size)) {
 586		*cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2);
 587		*cs++ = BLT_DEPTH_32 | PAGE_SIZE;
 588		*cs++ = 0;
 589		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 590		*cs++ = dst_offset;
 591		*cs++ = instance;
 592		*cs++ = 0;
 593		*cs++ = PAGE_SIZE;
 594		*cs++ = src_offset;
 595		*cs++ = instance;
 596	} else if (ver >= 8) {
 597		*cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2);
 598		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
 599		*cs++ = 0;
 600		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 601		*cs++ = dst_offset;
 602		*cs++ = instance;
 603		*cs++ = 0;
 604		*cs++ = PAGE_SIZE;
 605		*cs++ = src_offset;
 606		*cs++ = instance;
 607	} else {
 608		GEM_BUG_ON(instance);
 609		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
 610		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
 611		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE;
 612		*cs++ = dst_offset;
 613		*cs++ = PAGE_SIZE;
 614		*cs++ = src_offset;
 615	}
 616
 617	intel_ring_advance(rq, cs);
 618	return 0;
 619}
 620
 621static u64 scatter_list_length(struct scatterlist *sg)
 622{
 623	u64 len = 0;
 624
 625	while (sg && sg_dma_len(sg)) {
 626		len += sg_dma_len(sg);
 627		sg = sg_next(sg);
 628	}
 629
 630	return len;
 631}
 632
 633static int
 634calculate_chunk_sz(struct drm_i915_private *i915, bool src_is_lmem,
 635		   u64 bytes_to_cpy, u64 ccs_bytes_to_cpy)
 636{
 637	if (ccs_bytes_to_cpy && !src_is_lmem)
 638		/*
 639		 * When CHUNK_SZ is passed all the pages upto CHUNK_SZ
 640		 * will be taken for the blt. in Flat-ccs supported
 641		 * platform Smem obj will have more pages than required
 642		 * for main meory hence limit it to the required size
 643		 * for main memory
 644		 */
 645		return min_t(u64, bytes_to_cpy, CHUNK_SZ);
 646	else
 647		return CHUNK_SZ;
 648}
 649
 650static void get_ccs_sg_sgt(struct sgt_dma *it, u64 bytes_to_cpy)
 651{
 652	u64 len;
 653
 654	do {
 655		GEM_BUG_ON(!it->sg || !sg_dma_len(it->sg));
 656		len = it->max - it->dma;
 657		if (len > bytes_to_cpy) {
 658			it->dma += bytes_to_cpy;
 659			break;
 660		}
 661
 662		bytes_to_cpy -= len;
 663
 664		it->sg = __sg_next(it->sg);
 665		it->dma = sg_dma_address(it->sg);
 666		it->max = it->dma + sg_dma_len(it->sg);
 667	} while (bytes_to_cpy);
 668}
 669
 670int
 671intel_context_migrate_copy(struct intel_context *ce,
 672			   const struct i915_deps *deps,
 673			   struct scatterlist *src,
 674			   enum i915_cache_level src_cache_level,
 675			   bool src_is_lmem,
 676			   struct scatterlist *dst,
 677			   enum i915_cache_level dst_cache_level,
 678			   bool dst_is_lmem,
 679			   struct i915_request **out)
 680{
 681	struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst), it_ccs;
 682	struct drm_i915_private *i915 = ce->engine->i915;
 683	u64 ccs_bytes_to_cpy = 0, bytes_to_cpy;
 684	enum i915_cache_level ccs_cache_level;
 685	u32 src_offset, dst_offset;
 686	u8 src_access, dst_access;
 687	struct i915_request *rq;
 688	u64 src_sz, dst_sz;
 689	bool ccs_is_src, overwrite_ccs;
 690	int err;
 691
 692	GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
 693	GEM_BUG_ON(IS_DGFX(ce->engine->i915) && (!src_is_lmem && !dst_is_lmem));
 694	*out = NULL;
 695
 696	GEM_BUG_ON(ce->ring->size < SZ_64K);
 697
 698	src_sz = scatter_list_length(src);
 699	bytes_to_cpy = src_sz;
 700
 701	if (HAS_FLAT_CCS(i915) && src_is_lmem ^ dst_is_lmem) {
 702		src_access = !src_is_lmem && dst_is_lmem;
 703		dst_access = !src_access;
 704
 705		dst_sz = scatter_list_length(dst);
 706		if (src_is_lmem) {
 707			it_ccs = it_dst;
 708			ccs_cache_level = dst_cache_level;
 709			ccs_is_src = false;
 710		} else if (dst_is_lmem) {
 711			bytes_to_cpy = dst_sz;
 712			it_ccs = it_src;
 713			ccs_cache_level = src_cache_level;
 714			ccs_is_src = true;
 715		}
 716
 717		/*
 718		 * When there is a eviction of ccs needed smem will have the
 719		 * extra pages for the ccs data
 720		 *
 721		 * TO-DO: Want to move the size mismatch check to a WARN_ON,
 722		 * but still we have some requests of smem->lmem with same size.
 723		 * Need to fix it.
 724		 */
 725		ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0;
 726		if (ccs_bytes_to_cpy)
 727			get_ccs_sg_sgt(&it_ccs, bytes_to_cpy);
 728	}
 729
 730	overwrite_ccs = HAS_FLAT_CCS(i915) && !ccs_bytes_to_cpy && dst_is_lmem;
 731
 732	src_offset = 0;
 733	dst_offset = CHUNK_SZ;
 734	if (HAS_64K_PAGES(ce->engine->i915)) {
 735		src_offset = 0;
 736		dst_offset = 0;
 737		if (src_is_lmem)
 738			src_offset = CHUNK_SZ;
 739		if (dst_is_lmem)
 740			dst_offset = 2 * CHUNK_SZ;
 741	}
 742
 743	do {
 744		int len;
 745
 746		rq = i915_request_create(ce);
 747		if (IS_ERR(rq)) {
 748			err = PTR_ERR(rq);
 749			goto out_ce;
 750		}
 751
 752		if (deps) {
 753			err = i915_request_await_deps(rq, deps);
 754			if (err)
 755				goto out_rq;
 756
 757			if (rq->engine->emit_init_breadcrumb) {
 758				err = rq->engine->emit_init_breadcrumb(rq);
 759				if (err)
 760					goto out_rq;
 761			}
 762
 763			deps = NULL;
 764		}
 765
 766		/* The PTE updates + copy must not be interrupted. */
 767		err = emit_no_arbitration(rq);
 768		if (err)
 769			goto out_rq;
 770
 771		src_sz = calculate_chunk_sz(i915, src_is_lmem,
 772					    bytes_to_cpy, ccs_bytes_to_cpy);
 773
 774		len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem,
 775			       src_offset, src_sz);
 776		if (!len) {
 777			err = -EINVAL;
 778			goto out_rq;
 779		}
 780		if (len < 0) {
 781			err = len;
 782			goto out_rq;
 783		}
 784
 785		err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem,
 786			       dst_offset, len);
 787		if (err < 0)
 788			goto out_rq;
 789		if (err < len) {
 790			err = -EINVAL;
 791			goto out_rq;
 792		}
 793
 794		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 795		if (err)
 796			goto out_rq;
 797
 798		err = emit_copy(rq, dst_offset,	src_offset, len);
 799		if (err)
 800			goto out_rq;
 801
 802		bytes_to_cpy -= len;
 803
 804		if (ccs_bytes_to_cpy) {
 805			int ccs_sz;
 806
 807			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 808			if (err)
 809				goto out_rq;
 810
 811			ccs_sz = GET_CCS_BYTES(i915, len);
 812			err = emit_pte(rq, &it_ccs, ccs_cache_level, false,
 813				       ccs_is_src ? src_offset : dst_offset,
 814				       ccs_sz);
 815			if (err < 0)
 816				goto out_rq;
 817			if (err < ccs_sz) {
 818				err = -EINVAL;
 819				goto out_rq;
 820			}
 821
 822			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 823			if (err)
 824				goto out_rq;
 825
 826			err = emit_copy_ccs(rq, dst_offset, dst_access,
 827					    src_offset, src_access, len);
 828			if (err)
 829				goto out_rq;
 830
 831			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 832			if (err)
 833				goto out_rq;
 834			ccs_bytes_to_cpy -= ccs_sz;
 835		} else if (overwrite_ccs) {
 836			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 837			if (err)
 838				goto out_rq;
 839
 840			if (src_is_lmem) {
 841				/*
 842				 * If the src is already in lmem, then we must
 843				 * be doing an lmem -> lmem transfer, and so
 844				 * should be safe to directly copy the CCS
 845				 * state. In this case we have either
 846				 * initialised the CCS aux state when first
 847				 * clearing the pages (since it is already
 848				 * allocated in lmem), or the user has
 849				 * potentially populated it, in which case we
 850				 * need to copy the CCS state as-is.
 851				 */
 852				err = emit_copy_ccs(rq,
 853						    dst_offset, INDIRECT_ACCESS,
 854						    src_offset, INDIRECT_ACCESS,
 855						    len);
 856			} else {
 857				/*
 858				 * While we can't always restore/manage the CCS
 859				 * state, we still need to ensure we don't leak
 860				 * the CCS state from the previous user, so make
 861				 * sure we overwrite it with something.
 862				 */
 863				err = emit_copy_ccs(rq,
 864						    dst_offset, INDIRECT_ACCESS,
 865						    dst_offset, DIRECT_ACCESS,
 866						    len);
 867			}
 868
 869			if (err)
 870				goto out_rq;
 871
 872			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 873			if (err)
 874				goto out_rq;
 875		}
 876
 877		/* Arbitration is re-enabled between requests. */
 878out_rq:
 879		if (*out)
 880			i915_request_put(*out);
 881		*out = i915_request_get(rq);
 882		i915_request_add(rq);
 883
 884		if (err)
 885			break;
 886
 887		if (!bytes_to_cpy && !ccs_bytes_to_cpy) {
 888			if (src_is_lmem)
 889				WARN_ON(it_src.sg && sg_dma_len(it_src.sg));
 890			else
 891				WARN_ON(it_dst.sg && sg_dma_len(it_dst.sg));
 892			break;
 893		}
 894
 895		if (WARN_ON(!it_src.sg || !sg_dma_len(it_src.sg) ||
 896			    !it_dst.sg || !sg_dma_len(it_dst.sg) ||
 897			    (ccs_bytes_to_cpy && (!it_ccs.sg ||
 898						  !sg_dma_len(it_ccs.sg))))) {
 899			err = -EINVAL;
 900			break;
 901		}
 902
 903		cond_resched();
 904	} while (1);
 905
 906out_ce:
 907	return err;
 908}
 909
 910static int emit_clear(struct i915_request *rq, u32 offset, int size,
 911		      u32 value, bool is_lmem)
 912{
 913	struct drm_i915_private *i915 = rq->engine->i915;
 914	int mocs = rq->engine->gt->mocs.uc_index << 1;
 915	const int ver = GRAPHICS_VER(i915);
 916	int ring_sz;
 917	u32 *cs;
 918
 919	GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX);
 920
 921	if (HAS_FLAT_CCS(i915) && ver >= 12)
 922		ring_sz = XY_FAST_COLOR_BLT_DW;
 923	else if (ver >= 8)
 924		ring_sz = 8;
 925	else
 926		ring_sz = 6;
 927
 928	cs = intel_ring_begin(rq, ring_sz);
 929	if (IS_ERR(cs))
 930		return PTR_ERR(cs);
 931
 932	if (HAS_FLAT_CCS(i915) && ver >= 12) {
 933		*cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
 934			(XY_FAST_COLOR_BLT_DW - 2);
 935		*cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) |
 936			(PAGE_SIZE - 1);
 937		*cs++ = 0;
 938		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 939		*cs++ = offset;
 940		*cs++ = rq->engine->instance;
 941		*cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
 942		/* BG7 */
 943		*cs++ = value;
 944		*cs++ = 0;
 945		*cs++ = 0;
 946		*cs++ = 0;
 947		/* BG11 */
 948		*cs++ = 0;
 949		*cs++ = 0;
 950		/* BG13 */
 951		*cs++ = 0;
 952		*cs++ = 0;
 953		*cs++ = 0;
 954	} else if (ver >= 8) {
 955		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2);
 956		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
 957		*cs++ = 0;
 958		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 959		*cs++ = offset;
 960		*cs++ = rq->engine->instance;
 961		*cs++ = value;
 962		*cs++ = MI_NOOP;
 963	} else {
 964		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
 965		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
 966		*cs++ = 0;
 967		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 968		*cs++ = offset;
 969		*cs++ = value;
 970	}
 971
 972	intel_ring_advance(rq, cs);
 973	return 0;
 974}
 975
 976int
 977intel_context_migrate_clear(struct intel_context *ce,
 978			    const struct i915_deps *deps,
 979			    struct scatterlist *sg,
 980			    enum i915_cache_level cache_level,
 981			    bool is_lmem,
 982			    u32 value,
 983			    struct i915_request **out)
 984{
 985	struct drm_i915_private *i915 = ce->engine->i915;
 986	struct sgt_dma it = sg_sgt(sg);
 987	struct i915_request *rq;
 988	u32 offset;
 989	int err;
 990
 991	GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
 992	*out = NULL;
 993
 994	GEM_BUG_ON(ce->ring->size < SZ_64K);
 995
 996	offset = 0;
 997	if (HAS_64K_PAGES(i915) && is_lmem)
 998		offset = CHUNK_SZ;
 999
1000	do {
1001		int len;
1002
1003		rq = i915_request_create(ce);
1004		if (IS_ERR(rq)) {
1005			err = PTR_ERR(rq);
1006			goto out_ce;
1007		}
1008
1009		if (deps) {
1010			err = i915_request_await_deps(rq, deps);
1011			if (err)
1012				goto out_rq;
1013
1014			if (rq->engine->emit_init_breadcrumb) {
1015				err = rq->engine->emit_init_breadcrumb(rq);
1016				if (err)
1017					goto out_rq;
1018			}
1019
1020			deps = NULL;
1021		}
1022
1023		/* The PTE updates + clear must not be interrupted. */
1024		err = emit_no_arbitration(rq);
1025		if (err)
1026			goto out_rq;
1027
1028		len = emit_pte(rq, &it, cache_level, is_lmem, offset, CHUNK_SZ);
1029		if (len <= 0) {
1030			err = len;
1031			goto out_rq;
1032		}
1033
1034		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1035		if (err)
1036			goto out_rq;
1037
1038		err = emit_clear(rq, offset, len, value, is_lmem);
1039		if (err)
1040			goto out_rq;
1041
1042		if (HAS_FLAT_CCS(i915) && is_lmem && !value) {
1043			/*
1044			 * copy the content of memory into corresponding
1045			 * ccs surface
1046			 */
1047			err = emit_copy_ccs(rq, offset, INDIRECT_ACCESS, offset,
1048					    DIRECT_ACCESS, len);
1049			if (err)
1050				goto out_rq;
1051		}
1052
1053		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1054
1055		/* Arbitration is re-enabled between requests. */
1056out_rq:
1057		if (*out)
1058			i915_request_put(*out);
1059		*out = i915_request_get(rq);
1060		i915_request_add(rq);
1061		if (err || !it.sg || !sg_dma_len(it.sg))
1062			break;
1063
1064		cond_resched();
1065	} while (1);
1066
1067out_ce:
1068	return err;
1069}
1070
1071int intel_migrate_copy(struct intel_migrate *m,
1072		       struct i915_gem_ww_ctx *ww,
1073		       const struct i915_deps *deps,
1074		       struct scatterlist *src,
1075		       enum i915_cache_level src_cache_level,
1076		       bool src_is_lmem,
1077		       struct scatterlist *dst,
1078		       enum i915_cache_level dst_cache_level,
1079		       bool dst_is_lmem,
1080		       struct i915_request **out)
1081{
1082	struct intel_context *ce;
1083	int err;
1084
1085	*out = NULL;
1086	if (!m->context)
1087		return -ENODEV;
1088
1089	ce = intel_migrate_create_context(m);
1090	if (IS_ERR(ce))
1091		ce = intel_context_get(m->context);
1092	GEM_BUG_ON(IS_ERR(ce));
1093
1094	err = intel_context_pin_ww(ce, ww);
1095	if (err)
1096		goto out;
1097
1098	err = intel_context_migrate_copy(ce, deps,
1099					 src, src_cache_level, src_is_lmem,
1100					 dst, dst_cache_level, dst_is_lmem,
1101					 out);
1102
1103	intel_context_unpin(ce);
1104out:
1105	intel_context_put(ce);
1106	return err;
1107}
1108
1109int
1110intel_migrate_clear(struct intel_migrate *m,
1111		    struct i915_gem_ww_ctx *ww,
1112		    const struct i915_deps *deps,
1113		    struct scatterlist *sg,
1114		    enum i915_cache_level cache_level,
1115		    bool is_lmem,
1116		    u32 value,
1117		    struct i915_request **out)
1118{
1119	struct intel_context *ce;
1120	int err;
1121
1122	*out = NULL;
1123	if (!m->context)
1124		return -ENODEV;
1125
1126	ce = intel_migrate_create_context(m);
1127	if (IS_ERR(ce))
1128		ce = intel_context_get(m->context);
1129	GEM_BUG_ON(IS_ERR(ce));
1130
1131	err = intel_context_pin_ww(ce, ww);
1132	if (err)
1133		goto out;
1134
1135	err = intel_context_migrate_clear(ce, deps, sg, cache_level,
1136					  is_lmem, value, out);
1137
1138	intel_context_unpin(ce);
1139out:
1140	intel_context_put(ce);
1141	return err;
1142}
1143
1144void intel_migrate_fini(struct intel_migrate *m)
1145{
1146	struct intel_context *ce;
1147
1148	ce = fetch_and_zero(&m->context);
1149	if (!ce)
1150		return;
1151
1152	intel_engine_destroy_pinned_context(ce);
1153}
1154
1155#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1156#include "selftest_migrate.c"
1157#endif