Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5
6#include "xe_migrate.h"
7
8#include <linux/bitfield.h>
9#include <linux/sizes.h>
10
11#include <drm/drm_managed.h>
12#include <drm/ttm/ttm_tt.h>
13#include <uapi/drm/xe_drm.h>
14
15#include <generated/xe_wa_oob.h>
16
17#include "instructions/xe_gpu_commands.h"
18#include "instructions/xe_mi_commands.h"
19#include "regs/xe_gtt_defs.h"
20#include "tests/xe_test.h"
21#include "xe_assert.h"
22#include "xe_bb.h"
23#include "xe_bo.h"
24#include "xe_exec_queue.h"
25#include "xe_ggtt.h"
26#include "xe_gt.h"
27#include "xe_hw_engine.h"
28#include "xe_lrc.h"
29#include "xe_map.h"
30#include "xe_mocs.h"
31#include "xe_pt.h"
32#include "xe_res_cursor.h"
33#include "xe_sched_job.h"
34#include "xe_sync.h"
35#include "xe_trace_bo.h"
36#include "xe_vm.h"
37
38/**
39 * struct xe_migrate - migrate context.
40 */
41struct xe_migrate {
42 /** @q: Default exec queue used for migration */
43 struct xe_exec_queue *q;
44 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */
45 struct xe_tile *tile;
46 /** @job_mutex: Timeline mutex for @eng. */
47 struct mutex job_mutex;
48 /** @pt_bo: Page-table buffer object. */
49 struct xe_bo *pt_bo;
50 /** @batch_base_ofs: VM offset of the migration batch buffer */
51 u64 batch_base_ofs;
52 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */
53 u64 usm_batch_base_ofs;
54 /** @cleared_mem_ofs: VM offset of @cleared_bo. */
55 u64 cleared_mem_ofs;
56 /**
57 * @fence: dma-fence representing the last migration job batch.
58 * Protected by @job_mutex.
59 */
60 struct dma_fence *fence;
61 /**
62 * @vm_update_sa: For integrated, used to suballocate page-tables
63 * out of the pt_bo.
64 */
65 struct drm_suballoc_manager vm_update_sa;
66 /** @min_chunk_size: For dgfx, Minimum chunk size */
67 u64 min_chunk_size;
68};
69
70#define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */
71#define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */
72#define NUM_KERNEL_PDE 15
73#define NUM_PT_SLOTS 32
74#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M
75#define MAX_NUM_PTE 512
76#define IDENTITY_OFFSET 256ULL
77
78/*
79 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest
80 * legal value accepted. Since that instruction field is always stored in
81 * (val-2) format, this translates to 0x400 dwords for the true maximum length
82 * of the instruction. Subtracting the instruction header (1 dword) and
83 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values.
84 */
85#define MAX_PTE_PER_SDI 0x1FE
86
87/**
88 * xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue.
89 * @tile: The tile.
90 *
91 * Returns the default migrate exec queue of this tile.
92 *
93 * Return: The default migrate exec queue
94 */
95struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile)
96{
97 return tile->migrate->q;
98}
99
100static void xe_migrate_fini(struct drm_device *dev, void *arg)
101{
102 struct xe_migrate *m = arg;
103
104 xe_vm_lock(m->q->vm, false);
105 xe_bo_unpin(m->pt_bo);
106 xe_vm_unlock(m->q->vm);
107
108 dma_fence_put(m->fence);
109 xe_bo_put(m->pt_bo);
110 drm_suballoc_manager_fini(&m->vm_update_sa);
111 mutex_destroy(&m->job_mutex);
112 xe_vm_close_and_put(m->q->vm);
113 xe_exec_queue_put(m->q);
114}
115
116static u64 xe_migrate_vm_addr(u64 slot, u32 level)
117{
118 XE_WARN_ON(slot >= NUM_PT_SLOTS);
119
120 /* First slot is reserved for mapping of PT bo and bb, start from 1 */
121 return (slot + 1ULL) << xe_pt_shift(level + 1);
122}
123
124static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte)
125{
126 /*
127 * Remove the DPA to get a correct offset into identity table for the
128 * migrate offset
129 */
130 u64 identity_offset = IDENTITY_OFFSET;
131
132 if (GRAPHICS_VER(xe) >= 20 && is_comp_pte)
133 identity_offset += DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G);
134
135 addr -= xe->mem.vram.dpa_base;
136 return addr + (identity_offset << xe_pt_shift(2));
137}
138
139static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo,
140 u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs)
141{
142 u64 pos, ofs, flags;
143 u64 entry;
144 /* XXX: Unclear if this should be usable_size? */
145 u64 vram_limit = xe->mem.vram.actual_physical_size +
146 xe->mem.vram.dpa_base;
147 u32 level = 2;
148
149 ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8;
150 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level,
151 true, 0);
152
153 xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M));
154
155 /*
156 * Use 1GB pages when possible, last chunk always use 2M
157 * pages as mixing reserved memory (stolen, WOCPM) with a single
158 * mapping is not allowed on certain platforms.
159 */
160 for (pos = xe->mem.vram.dpa_base; pos < vram_limit;
161 pos += SZ_1G, ofs += 8) {
162 if (pos + SZ_1G >= vram_limit) {
163 entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs,
164 pat_index);
165 xe_map_wr(xe, &bo->vmap, ofs, u64, entry);
166
167 flags = vm->pt_ops->pte_encode_addr(xe, 0,
168 pat_index,
169 level - 1,
170 true, 0);
171
172 for (ofs = pt_2m_ofs; pos < vram_limit;
173 pos += SZ_2M, ofs += 8)
174 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);
175 break; /* Ensure pos == vram_limit assert correct */
176 }
177
178 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);
179 }
180
181 xe_assert(xe, pos == vram_limit);
182}
183
184static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
185 struct xe_vm *vm)
186{
187 struct xe_device *xe = tile_to_xe(tile);
188 u16 pat_index = xe->pat.idx[XE_CACHE_WB];
189 u8 id = tile->id;
190 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;
191#define VRAM_IDENTITY_MAP_COUNT 2
192 u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT;
193#undef VRAM_IDENTITY_MAP_COUNT
194 u32 map_ofs, level, i;
195 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo;
196 u64 entry, pt29_ofs;
197
198 /* Can't bump NUM_PT_SLOTS too high */
199 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);
200 /* Must be a multiple of 64K to support all platforms */
201 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K);
202 /* And one slot reserved for the 4KiB page table updates */
203 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1));
204
205 /* Need to be sure everything fits in the first PT, or create more */
206 xe_tile_assert(tile, m->batch_base_ofs + batch->size < SZ_2M);
207
208 bo = xe_bo_create_pin_map(vm->xe, tile, vm,
209 num_entries * XE_PAGE_SIZE,
210 ttm_bo_type_kernel,
211 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
212 XE_BO_FLAG_PINNED |
213 XE_BO_FLAG_PAGETABLE);
214 if (IS_ERR(bo))
215 return PTR_ERR(bo);
216
217 /* PT30 & PT31 reserved for 2M identity map */
218 pt29_ofs = bo->size - 3 * XE_PAGE_SIZE;
219 entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs, pat_index);
220 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry);
221
222 map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE;
223
224 /* Map the entire BO in our level 0 pt */
225 for (i = 0, level = 0; i < num_entries; level++) {
226 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE,
227 pat_index, 0);
228
229 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry);
230
231 if (vm->flags & XE_VM_FLAG_64K)
232 i += 16;
233 else
234 i += 1;
235 }
236
237 if (!IS_DGFX(xe)) {
238 /* Write out batch too */
239 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE;
240 for (i = 0; i < batch->size;
241 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE :
242 XE_PAGE_SIZE) {
243 entry = vm->pt_ops->pte_encode_bo(batch, i,
244 pat_index, 0);
245
246 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,
247 entry);
248 level++;
249 }
250 if (xe->info.has_usm) {
251 xe_tile_assert(tile, batch->size == SZ_1M);
252
253 batch = tile->primary_gt->usm.bb_pool->bo;
254 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M;
255 xe_tile_assert(tile, batch->size == SZ_512K);
256
257 for (i = 0; i < batch->size;
258 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE :
259 XE_PAGE_SIZE) {
260 entry = vm->pt_ops->pte_encode_bo(batch, i,
261 pat_index, 0);
262
263 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,
264 entry);
265 level++;
266 }
267 }
268 } else {
269 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);
270
271 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false);
272
273 if (xe->info.has_usm) {
274 batch = tile->primary_gt->usm.bb_pool->bo;
275 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);
276 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false);
277 }
278 }
279
280 for (level = 1; level < num_level; level++) {
281 u32 flags = 0;
282
283 if (vm->flags & XE_VM_FLAG_64K && level == 1)
284 flags = XE_PDE_64K;
285
286 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) *
287 XE_PAGE_SIZE, pat_index);
288 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64,
289 entry | flags);
290 }
291
292 /* Write PDE's that point to our BO. */
293 for (i = 0; i < map_ofs / PAGE_SIZE; i++) {
294 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE,
295 pat_index);
296
297 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE +
298 (i + 1) * 8, u64, entry);
299 }
300
301 /* Set up a 1GiB NULL mapping at 255GiB offset. */
302 level = 2;
303 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64,
304 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0)
305 | XE_PTE_NULL);
306 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level));
307
308 /* Identity map the entire vram at 256GiB offset */
309 if (IS_DGFX(xe)) {
310 u64 pt30_ofs = bo->size - 2 * XE_PAGE_SIZE;
311
312 xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET,
313 pat_index, pt30_ofs);
314 xe_assert(xe, xe->mem.vram.actual_physical_size <=
315 (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G);
316
317 /*
318 * Identity map the entire vram for compressed pat_index for xe2+
319 * if flat ccs is enabled.
320 */
321 if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) {
322 u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION];
323 u64 vram_offset = IDENTITY_OFFSET +
324 DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G);
325 u64 pt31_ofs = bo->size - XE_PAGE_SIZE;
326
327 xe_assert(xe, xe->mem.vram.actual_physical_size <= (MAX_NUM_PTE -
328 IDENTITY_OFFSET - IDENTITY_OFFSET / 2) * SZ_1G);
329 xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset,
330 comp_pat_index, pt31_ofs);
331 }
332 }
333
334 /*
335 * Example layout created above, with root level = 3:
336 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's
337 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's
338 * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's
339 * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map]
340 *
341 * This makes the lowest part of the VM point to the pagetables.
342 * Hence the lowest 2M in the vm should point to itself, with a few writes
343 * and flushes, other parts of the VM can be used either for copying and
344 * clearing.
345 *
346 * For performance, the kernel reserves PDE's, so about 20 are left
347 * for async VM updates.
348 *
349 * To make it easier to work, each scratch PT is put in slot (1 + PT #)
350 * everywhere, this allows lockless updates to scratch pages by using
351 * the different addresses in VM.
352 */
353#define NUM_VMUSA_UNIT_PER_PAGE 32
354#define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE)
355#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64))
356 drm_suballoc_manager_init(&m->vm_update_sa,
357 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) *
358 NUM_VMUSA_UNIT_PER_PAGE, 0);
359
360 m->pt_bo = bo;
361 return 0;
362}
363
364/*
365 * Including the reserved copy engine is required to avoid deadlocks due to
366 * migrate jobs servicing the faults gets stuck behind the job that faulted.
367 */
368static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt)
369{
370 u32 logical_mask = 0;
371 struct xe_hw_engine *hwe;
372 enum xe_hw_engine_id id;
373
374 for_each_hw_engine(hwe, gt, id) {
375 if (hwe->class != XE_ENGINE_CLASS_COPY)
376 continue;
377
378 if (xe_gt_is_usm_hwe(gt, hwe))
379 logical_mask |= BIT(hwe->logical_instance);
380 }
381
382 return logical_mask;
383}
384
385static bool xe_migrate_needs_ccs_emit(struct xe_device *xe)
386{
387 return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe));
388}
389
390/**
391 * xe_migrate_init() - Initialize a migrate context
392 * @tile: Back-pointer to the tile we're initializing for.
393 *
394 * Return: Pointer to a migrate context on success. Error pointer on error.
395 */
396struct xe_migrate *xe_migrate_init(struct xe_tile *tile)
397{
398 struct xe_device *xe = tile_to_xe(tile);
399 struct xe_gt *primary_gt = tile->primary_gt;
400 struct xe_migrate *m;
401 struct xe_vm *vm;
402 int err;
403
404 m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL);
405 if (!m)
406 return ERR_PTR(-ENOMEM);
407
408 m->tile = tile;
409
410 /* Special layout, prepared below.. */
411 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION |
412 XE_VM_FLAG_SET_TILE_ID(tile));
413 if (IS_ERR(vm))
414 return ERR_CAST(vm);
415
416 xe_vm_lock(vm, false);
417 err = xe_migrate_prepare_vm(tile, m, vm);
418 xe_vm_unlock(vm);
419 if (err) {
420 xe_vm_close_and_put(vm);
421 return ERR_PTR(err);
422 }
423
424 if (xe->info.has_usm) {
425 struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt,
426 XE_ENGINE_CLASS_COPY,
427 primary_gt->usm.reserved_bcs_instance,
428 false);
429 u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt);
430
431 if (!hwe || !logical_mask)
432 return ERR_PTR(-EINVAL);
433
434 /*
435 * XXX: Currently only reserving 1 (likely slow) BCS instance on
436 * PVC, may want to revisit if performance is needed.
437 */
438 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,
439 EXEC_QUEUE_FLAG_KERNEL |
440 EXEC_QUEUE_FLAG_PERMANENT |
441 EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0);
442 } else {
443 m->q = xe_exec_queue_create_class(xe, primary_gt, vm,
444 XE_ENGINE_CLASS_COPY,
445 EXEC_QUEUE_FLAG_KERNEL |
446 EXEC_QUEUE_FLAG_PERMANENT, 0);
447 }
448 if (IS_ERR(m->q)) {
449 xe_vm_close_and_put(vm);
450 return ERR_CAST(m->q);
451 }
452
453 mutex_init(&m->job_mutex);
454 fs_reclaim_acquire(GFP_KERNEL);
455 might_lock(&m->job_mutex);
456 fs_reclaim_release(GFP_KERNEL);
457
458 err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m);
459 if (err)
460 return ERR_PTR(err);
461
462 if (IS_DGFX(xe)) {
463 if (xe_migrate_needs_ccs_emit(xe))
464 /* min chunk size corresponds to 4K of CCS Metadata */
465 m->min_chunk_size = SZ_4K * SZ_64K /
466 xe_device_ccs_bytes(xe, SZ_64K);
467 else
468 /* Somewhat arbitrary to avoid a huge amount of blits */
469 m->min_chunk_size = SZ_64K;
470 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size);
471 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n",
472 (unsigned long long)m->min_chunk_size);
473 }
474
475 return m;
476}
477
478static u64 max_mem_transfer_per_pass(struct xe_device *xe)
479{
480 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe))
481 return MAX_CCS_LIMITED_TRANSFER;
482
483 return MAX_PREEMPTDISABLE_TRANSFER;
484}
485
486static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur)
487{
488 struct xe_device *xe = tile_to_xe(m->tile);
489 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining);
490
491 if (mem_type_is_vram(cur->mem_type)) {
492 /*
493 * VRAM we want to blit in chunks with sizes aligned to
494 * min_chunk_size in order for the offset to CCS metadata to be
495 * page-aligned. If it's the last chunk it may be smaller.
496 *
497 * Another constraint is that we need to limit the blit to
498 * the VRAM block size, unless size is smaller than
499 * min_chunk_size.
500 */
501 u64 chunk = max_t(u64, cur->size, m->min_chunk_size);
502
503 size = min_t(u64, size, chunk);
504 if (size > m->min_chunk_size)
505 size = round_down(size, m->min_chunk_size);
506 }
507
508 return size;
509}
510
511static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur)
512{
513 /* If the chunk is not fragmented, allow identity map. */
514 return cur->size >= size;
515}
516
517#define PTE_UPDATE_FLAG_IS_VRAM BIT(0)
518#define PTE_UPDATE_FLAG_IS_COMP_PTE BIT(1)
519
520static u32 pte_update_size(struct xe_migrate *m,
521 u32 flags,
522 struct ttm_resource *res,
523 struct xe_res_cursor *cur,
524 u64 *L0, u64 *L0_ofs, u32 *L0_pt,
525 u32 cmd_size, u32 pt_ofs, u32 avail_pts)
526{
527 u32 cmds = 0;
528 bool is_vram = PTE_UPDATE_FLAG_IS_VRAM & flags;
529 bool is_comp_pte = PTE_UPDATE_FLAG_IS_COMP_PTE & flags;
530
531 *L0_pt = pt_ofs;
532 if (is_vram && xe_migrate_allow_identity(*L0, cur)) {
533 /* Offset into identity map. */
534 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile),
535 cur->start + vram_region_gpu_offset(res),
536 is_comp_pte);
537 cmds += cmd_size;
538 } else {
539 /* Clip L0 to available size */
540 u64 size = min(*L0, (u64)avail_pts * SZ_2M);
541 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT;
542
543 *L0 = size;
544 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0);
545
546 /* MI_STORE_DATA_IMM */
547 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI);
548
549 /* PDE qwords */
550 cmds += num_4k_pages * 2;
551
552 /* Each chunk has a single blit command */
553 cmds += cmd_size;
554 }
555
556 return cmds;
557}
558
559static void emit_pte(struct xe_migrate *m,
560 struct xe_bb *bb, u32 at_pt,
561 bool is_vram, bool is_comp_pte,
562 struct xe_res_cursor *cur,
563 u32 size, struct ttm_resource *res)
564{
565 struct xe_device *xe = tile_to_xe(m->tile);
566 struct xe_vm *vm = m->q->vm;
567 u16 pat_index;
568 u32 ptes;
569 u64 ofs = (u64)at_pt * XE_PAGE_SIZE;
570 u64 cur_ofs;
571
572 /* Indirect access needs compression enabled uncached PAT index */
573 if (GRAPHICS_VERx100(xe) >= 2000)
574 pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] :
575 xe->pat.idx[XE_CACHE_WB];
576 else
577 pat_index = xe->pat.idx[XE_CACHE_WB];
578
579 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE);
580
581 while (ptes) {
582 u32 chunk = min(MAX_PTE_PER_SDI, ptes);
583
584 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
585 bb->cs[bb->len++] = ofs;
586 bb->cs[bb->len++] = 0;
587
588 cur_ofs = ofs;
589 ofs += chunk * 8;
590 ptes -= chunk;
591
592 while (chunk--) {
593 u64 addr, flags = 0;
594 bool devmem = false;
595
596 addr = xe_res_dma(cur) & PAGE_MASK;
597 if (is_vram) {
598 if (vm->flags & XE_VM_FLAG_64K) {
599 u64 va = cur_ofs * XE_PAGE_SIZE / 8;
600
601 xe_assert(xe, (va & (SZ_64K - 1)) ==
602 (addr & (SZ_64K - 1)));
603
604 flags |= XE_PTE_PS64;
605 }
606
607 addr += vram_region_gpu_offset(res);
608 devmem = true;
609 }
610
611 addr = vm->pt_ops->pte_encode_addr(m->tile->xe,
612 addr, pat_index,
613 0, devmem, flags);
614 bb->cs[bb->len++] = lower_32_bits(addr);
615 bb->cs[bb->len++] = upper_32_bits(addr);
616
617 xe_res_next(cur, min_t(u32, size, PAGE_SIZE));
618 cur_ofs += 8;
619 }
620 }
621}
622
623#define EMIT_COPY_CCS_DW 5
624static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb,
625 u64 dst_ofs, bool dst_is_indirect,
626 u64 src_ofs, bool src_is_indirect,
627 u32 size)
628{
629 struct xe_device *xe = gt_to_xe(gt);
630 u32 *cs = bb->cs + bb->len;
631 u32 num_ccs_blks;
632 u32 num_pages;
633 u32 ccs_copy_size;
634 u32 mocs;
635
636 if (GRAPHICS_VERx100(xe) >= 2000) {
637 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE);
638 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1));
639
640 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1);
641 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index);
642
643 } else {
644 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size),
645 NUM_CCS_BYTES_PER_BLOCK);
646 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1));
647
648 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1);
649 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index);
650 }
651
652 *cs++ = XY_CTRL_SURF_COPY_BLT |
653 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT |
654 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT |
655 ccs_copy_size;
656 *cs++ = lower_32_bits(src_ofs);
657 *cs++ = upper_32_bits(src_ofs) | mocs;
658 *cs++ = lower_32_bits(dst_ofs);
659 *cs++ = upper_32_bits(dst_ofs) | mocs;
660
661 bb->len = cs - bb->cs;
662}
663
664#define EMIT_COPY_DW 10
665static void emit_copy(struct xe_gt *gt, struct xe_bb *bb,
666 u64 src_ofs, u64 dst_ofs, unsigned int size,
667 unsigned int pitch)
668{
669 struct xe_device *xe = gt_to_xe(gt);
670 u32 mocs = 0;
671 u32 tile_y = 0;
672
673 xe_gt_assert(gt, size / pitch <= S16_MAX);
674 xe_gt_assert(gt, pitch / 4 <= S16_MAX);
675 xe_gt_assert(gt, pitch <= U16_MAX);
676
677 if (GRAPHICS_VER(xe) >= 20)
678 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index);
679
680 if (GRAPHICS_VERx100(xe) >= 1250)
681 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4;
682
683 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2);
684 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs;
685 bb->cs[bb->len++] = 0;
686 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4;
687 bb->cs[bb->len++] = lower_32_bits(dst_ofs);
688 bb->cs[bb->len++] = upper_32_bits(dst_ofs);
689 bb->cs[bb->len++] = 0;
690 bb->cs[bb->len++] = pitch | mocs;
691 bb->cs[bb->len++] = lower_32_bits(src_ofs);
692 bb->cs[bb->len++] = upper_32_bits(src_ofs);
693}
694
695static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm)
696{
697 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs;
698}
699
700static u32 xe_migrate_ccs_copy(struct xe_migrate *m,
701 struct xe_bb *bb,
702 u64 src_ofs, bool src_is_indirect,
703 u64 dst_ofs, bool dst_is_indirect, u32 dst_size,
704 u64 ccs_ofs, bool copy_ccs)
705{
706 struct xe_gt *gt = m->tile->primary_gt;
707 u32 flush_flags = 0;
708
709 if (!copy_ccs && dst_is_indirect) {
710 /*
711 * If the src is already in vram, then it should already
712 * have been cleared by us, or has been populated by the
713 * user. Make sure we copy the CCS aux state as-is.
714 *
715 * Otherwise if the bo doesn't have any CCS metadata attached,
716 * we still need to clear it for security reasons.
717 */
718 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs;
719
720 emit_copy_ccs(gt, bb,
721 dst_ofs, true,
722 ccs_src_ofs, src_is_indirect, dst_size);
723
724 flush_flags = MI_FLUSH_DW_CCS;
725 } else if (copy_ccs) {
726 if (!src_is_indirect)
727 src_ofs = ccs_ofs;
728 else if (!dst_is_indirect)
729 dst_ofs = ccs_ofs;
730
731 xe_gt_assert(gt, src_is_indirect || dst_is_indirect);
732
733 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs,
734 src_is_indirect, dst_size);
735 if (dst_is_indirect)
736 flush_flags = MI_FLUSH_DW_CCS;
737 }
738
739 return flush_flags;
740}
741
742/**
743 * xe_migrate_copy() - Copy content of TTM resources.
744 * @m: The migration context.
745 * @src_bo: The buffer object @src is currently bound to.
746 * @dst_bo: If copying between resources created for the same bo, set this to
747 * the same value as @src_bo. If copying between buffer objects, set it to
748 * the buffer object @dst is currently bound to.
749 * @src: The source TTM resource.
750 * @dst: The dst TTM resource.
751 * @copy_only_ccs: If true copy only CCS metadata
752 *
753 * Copies the contents of @src to @dst: On flat CCS devices,
754 * the CCS metadata is copied as well if needed, or if not present,
755 * the CCS metadata of @dst is cleared for security reasons.
756 *
757 * Return: Pointer to a dma_fence representing the last copy batch, or
758 * an error pointer on failure. If there is a failure, any copy operation
759 * started by the function call has been synced.
760 */
761struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
762 struct xe_bo *src_bo,
763 struct xe_bo *dst_bo,
764 struct ttm_resource *src,
765 struct ttm_resource *dst,
766 bool copy_only_ccs)
767{
768 struct xe_gt *gt = m->tile->primary_gt;
769 struct xe_device *xe = gt_to_xe(gt);
770 struct dma_fence *fence = NULL;
771 u64 size = src_bo->size;
772 struct xe_res_cursor src_it, dst_it, ccs_it;
773 u64 src_L0_ofs, dst_L0_ofs;
774 u32 src_L0_pt, dst_L0_pt;
775 u64 src_L0, dst_L0;
776 int pass = 0;
777 int err;
778 bool src_is_pltt = src->mem_type == XE_PL_TT;
779 bool dst_is_pltt = dst->mem_type == XE_PL_TT;
780 bool src_is_vram = mem_type_is_vram(src->mem_type);
781 bool dst_is_vram = mem_type_is_vram(dst->mem_type);
782 bool copy_ccs = xe_device_has_flat_ccs(xe) &&
783 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo);
784 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram);
785 bool use_comp_pat = xe_device_has_flat_ccs(xe) &&
786 GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram;
787
788 /* Copying CCS between two different BOs is not supported yet. */
789 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo))
790 return ERR_PTR(-EINVAL);
791
792 if (src_bo != dst_bo && XE_WARN_ON(src_bo->size != dst_bo->size))
793 return ERR_PTR(-EINVAL);
794
795 if (!src_is_vram)
796 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
797 else
798 xe_res_first(src, 0, size, &src_it);
799 if (!dst_is_vram)
800 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it);
801 else
802 xe_res_first(dst, 0, size, &dst_it);
803
804 if (copy_system_ccs)
805 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),
806 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)),
807 &ccs_it);
808
809 while (size) {
810 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */
811 struct xe_sched_job *job;
812 struct xe_bb *bb;
813 u32 flush_flags = 0;
814 u32 update_idx;
815 u64 ccs_ofs, ccs_size;
816 u32 ccs_pt;
817 u32 pte_flags;
818
819 bool usm = xe->info.has_usm;
820 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
821
822 src_L0 = xe_migrate_res_sizes(m, &src_it);
823 dst_L0 = xe_migrate_res_sizes(m, &dst_it);
824
825 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n",
826 pass++, src_L0, dst_L0);
827
828 src_L0 = min(src_L0, dst_L0);
829
830 pte_flags = src_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
831 pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0;
832 batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0,
833 &src_L0_ofs, &src_L0_pt, 0, 0,
834 avail_pts);
835
836 pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
837 batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0,
838 &dst_L0_ofs, &dst_L0_pt, 0,
839 avail_pts, avail_pts);
840
841 if (copy_system_ccs) {
842 ccs_size = xe_device_ccs_bytes(xe, src_L0);
843 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size,
844 &ccs_ofs, &ccs_pt, 0,
845 2 * avail_pts,
846 avail_pts);
847 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
848 }
849
850 /* Add copy commands size here */
851 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +
852 ((xe_migrate_needs_ccs_emit(xe) ? EMIT_COPY_CCS_DW : 0));
853
854 bb = xe_bb_new(gt, batch_size, usm);
855 if (IS_ERR(bb)) {
856 err = PTR_ERR(bb);
857 goto err_sync;
858 }
859
860 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it))
861 xe_res_next(&src_it, src_L0);
862 else
863 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs,
864 &src_it, src_L0, src);
865
866 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it))
867 xe_res_next(&dst_it, src_L0);
868 else
869 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs,
870 &dst_it, src_L0, dst);
871
872 if (copy_system_ccs)
873 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src);
874
875 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
876 update_idx = bb->len;
877
878 if (!copy_only_ccs)
879 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);
880
881 if (xe_migrate_needs_ccs_emit(xe))
882 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,
883 IS_DGFX(xe) ? src_is_vram : src_is_pltt,
884 dst_L0_ofs,
885 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt,
886 src_L0, ccs_ofs, copy_ccs);
887
888 job = xe_bb_create_migration_job(m->q, bb,
889 xe_migrate_batch_base(m, usm),
890 update_idx);
891 if (IS_ERR(job)) {
892 err = PTR_ERR(job);
893 goto err;
894 }
895
896 xe_sched_job_add_migrate_flush(job, flush_flags);
897 if (!fence) {
898 err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv,
899 DMA_RESV_USAGE_BOOKKEEP);
900 if (!err && src_bo != dst_bo)
901 err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv,
902 DMA_RESV_USAGE_BOOKKEEP);
903 if (err)
904 goto err_job;
905 }
906
907 mutex_lock(&m->job_mutex);
908 xe_sched_job_arm(job);
909 dma_fence_put(fence);
910 fence = dma_fence_get(&job->drm.s_fence->finished);
911 xe_sched_job_push(job);
912
913 dma_fence_put(m->fence);
914 m->fence = dma_fence_get(fence);
915
916 mutex_unlock(&m->job_mutex);
917
918 xe_bb_free(bb, fence);
919 size -= src_L0;
920 continue;
921
922err_job:
923 xe_sched_job_put(job);
924err:
925 xe_bb_free(bb, NULL);
926
927err_sync:
928 /* Sync partial copy if any. FIXME: under job_mutex? */
929 if (fence) {
930 dma_fence_wait(fence, false);
931 dma_fence_put(fence);
932 }
933
934 return ERR_PTR(err);
935 }
936
937 return fence;
938}
939
940static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
941 u32 size, u32 pitch)
942{
943 struct xe_device *xe = gt_to_xe(gt);
944 u32 *cs = bb->cs + bb->len;
945 u32 len = PVC_MEM_SET_CMD_LEN_DW;
946
947 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2);
948 *cs++ = pitch - 1;
949 *cs++ = (size / pitch) - 1;
950 *cs++ = pitch - 1;
951 *cs++ = lower_32_bits(src_ofs);
952 *cs++ = upper_32_bits(src_ofs);
953 if (GRAPHICS_VERx100(xe) >= 2000)
954 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index);
955 else
956 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index);
957
958 xe_gt_assert(gt, cs - bb->cs == len + bb->len);
959
960 bb->len += len;
961}
962
963static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb,
964 u64 src_ofs, u32 size, u32 pitch, bool is_vram)
965{
966 struct xe_device *xe = gt_to_xe(gt);
967 u32 *cs = bb->cs + bb->len;
968 u32 len = XY_FAST_COLOR_BLT_DW;
969
970 if (GRAPHICS_VERx100(xe) < 1250)
971 len = 11;
972
973 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
974 (len - 2);
975 if (GRAPHICS_VERx100(xe) >= 2000)
976 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) |
977 (pitch - 1);
978 else
979 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) |
980 (pitch - 1);
981 *cs++ = 0;
982 *cs++ = (size / pitch) << 16 | pitch / 4;
983 *cs++ = lower_32_bits(src_ofs);
984 *cs++ = upper_32_bits(src_ofs);
985 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
986 *cs++ = 0;
987 *cs++ = 0;
988 *cs++ = 0;
989 *cs++ = 0;
990
991 if (len > 11) {
992 *cs++ = 0;
993 *cs++ = 0;
994 *cs++ = 0;
995 *cs++ = 0;
996 *cs++ = 0;
997 }
998
999 xe_gt_assert(gt, cs - bb->cs == len + bb->len);
1000
1001 bb->len += len;
1002}
1003
1004static bool has_service_copy_support(struct xe_gt *gt)
1005{
1006 /*
1007 * What we care about is whether the architecture was designed with
1008 * service copy functionality (specifically the new MEM_SET / MEM_COPY
1009 * instructions) so check the architectural engine list rather than the
1010 * actual list since these instructions are usable on BCS0 even if
1011 * all of the actual service copy engines (BCS1-BCS8) have been fused
1012 * off.
1013 */
1014 return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8,
1015 XE_HW_ENGINE_BCS1);
1016}
1017
1018static u32 emit_clear_cmd_len(struct xe_gt *gt)
1019{
1020 if (has_service_copy_support(gt))
1021 return PVC_MEM_SET_CMD_LEN_DW;
1022 else
1023 return XY_FAST_COLOR_BLT_DW;
1024}
1025
1026static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
1027 u32 size, u32 pitch, bool is_vram)
1028{
1029 if (has_service_copy_support(gt))
1030 emit_clear_link_copy(gt, bb, src_ofs, size, pitch);
1031 else
1032 emit_clear_main_copy(gt, bb, src_ofs, size, pitch,
1033 is_vram);
1034}
1035
1036/**
1037 * xe_migrate_clear() - Copy content of TTM resources.
1038 * @m: The migration context.
1039 * @bo: The buffer object @dst is currently bound to.
1040 * @dst: The dst TTM resource to be cleared.
1041 * @clear_flags: flags to specify which data to clear: CCS, BO, or both.
1042 *
1043 * Clear the contents of @dst to zero when XE_MIGRATE_CLEAR_FLAG_BO_DATA is set.
1044 * On flat CCS devices, the CCS metadata is cleared to zero with XE_MIGRATE_CLEAR_FLAG_CCS_DATA.
1045 * Set XE_MIGRATE_CLEAR_FLAG_FULL to clear bo as well as CCS metadata.
1046 * TODO: Eliminate the @bo argument.
1047 *
1048 * Return: Pointer to a dma_fence representing the last clear batch, or
1049 * an error pointer on failure. If there is a failure, any clear operation
1050 * started by the function call has been synced.
1051 */
1052struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
1053 struct xe_bo *bo,
1054 struct ttm_resource *dst,
1055 u32 clear_flags)
1056{
1057 bool clear_vram = mem_type_is_vram(dst->mem_type);
1058 bool clear_bo_data = XE_MIGRATE_CLEAR_FLAG_BO_DATA & clear_flags;
1059 bool clear_ccs = XE_MIGRATE_CLEAR_FLAG_CCS_DATA & clear_flags;
1060 struct xe_gt *gt = m->tile->primary_gt;
1061 struct xe_device *xe = gt_to_xe(gt);
1062 bool clear_only_system_ccs = false;
1063 struct dma_fence *fence = NULL;
1064 u64 size = bo->size;
1065 struct xe_res_cursor src_it;
1066 struct ttm_resource *src = dst;
1067 int err;
1068
1069 if (WARN_ON(!clear_bo_data && !clear_ccs))
1070 return NULL;
1071
1072 if (!clear_bo_data && clear_ccs && !IS_DGFX(xe))
1073 clear_only_system_ccs = true;
1074
1075 if (!clear_vram)
1076 xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it);
1077 else
1078 xe_res_first(src, 0, bo->size, &src_it);
1079
1080 while (size) {
1081 u64 clear_L0_ofs;
1082 u32 clear_L0_pt;
1083 u32 flush_flags = 0;
1084 u64 clear_L0;
1085 struct xe_sched_job *job;
1086 struct xe_bb *bb;
1087 u32 batch_size, update_idx;
1088 u32 pte_flags;
1089
1090 bool usm = xe->info.has_usm;
1091 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
1092
1093 clear_L0 = xe_migrate_res_sizes(m, &src_it);
1094
1095 /* Calculate final sizes and batch size.. */
1096 pte_flags = clear_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
1097 batch_size = 2 +
1098 pte_update_size(m, pte_flags, src, &src_it,
1099 &clear_L0, &clear_L0_ofs, &clear_L0_pt,
1100 clear_bo_data ? emit_clear_cmd_len(gt) : 0, 0,
1101 avail_pts);
1102
1103 if (xe_migrate_needs_ccs_emit(xe))
1104 batch_size += EMIT_COPY_CCS_DW;
1105
1106 /* Clear commands */
1107
1108 if (WARN_ON_ONCE(!clear_L0))
1109 break;
1110
1111 bb = xe_bb_new(gt, batch_size, usm);
1112 if (IS_ERR(bb)) {
1113 err = PTR_ERR(bb);
1114 goto err_sync;
1115 }
1116
1117 size -= clear_L0;
1118 /* Preemption is enabled again by the ring ops. */
1119 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it))
1120 xe_res_next(&src_it, clear_L0);
1121 else
1122 emit_pte(m, bb, clear_L0_pt, clear_vram, clear_only_system_ccs,
1123 &src_it, clear_L0, dst);
1124
1125 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1126 update_idx = bb->len;
1127
1128 if (clear_bo_data)
1129 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram);
1130
1131 if (xe_migrate_needs_ccs_emit(xe)) {
1132 emit_copy_ccs(gt, bb, clear_L0_ofs, true,
1133 m->cleared_mem_ofs, false, clear_L0);
1134 flush_flags = MI_FLUSH_DW_CCS;
1135 }
1136
1137 job = xe_bb_create_migration_job(m->q, bb,
1138 xe_migrate_batch_base(m, usm),
1139 update_idx);
1140 if (IS_ERR(job)) {
1141 err = PTR_ERR(job);
1142 goto err;
1143 }
1144
1145 xe_sched_job_add_migrate_flush(job, flush_flags);
1146 if (!fence) {
1147 /*
1148 * There can't be anything userspace related at this
1149 * point, so we just need to respect any potential move
1150 * fences, which are always tracked as
1151 * DMA_RESV_USAGE_KERNEL.
1152 */
1153 err = xe_sched_job_add_deps(job, bo->ttm.base.resv,
1154 DMA_RESV_USAGE_KERNEL);
1155 if (err)
1156 goto err_job;
1157 }
1158
1159 mutex_lock(&m->job_mutex);
1160 xe_sched_job_arm(job);
1161 dma_fence_put(fence);
1162 fence = dma_fence_get(&job->drm.s_fence->finished);
1163 xe_sched_job_push(job);
1164
1165 dma_fence_put(m->fence);
1166 m->fence = dma_fence_get(fence);
1167
1168 mutex_unlock(&m->job_mutex);
1169
1170 xe_bb_free(bb, fence);
1171 continue;
1172
1173err_job:
1174 xe_sched_job_put(job);
1175err:
1176 xe_bb_free(bb, NULL);
1177err_sync:
1178 /* Sync partial copies if any. FIXME: job_mutex? */
1179 if (fence) {
1180 dma_fence_wait(m->fence, false);
1181 dma_fence_put(fence);
1182 }
1183
1184 return ERR_PTR(err);
1185 }
1186
1187 if (clear_ccs)
1188 bo->ccs_cleared = true;
1189
1190 return fence;
1191}
1192
1193static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
1194 const struct xe_vm_pgtable_update_op *pt_op,
1195 const struct xe_vm_pgtable_update *update,
1196 struct xe_migrate_pt_update *pt_update)
1197{
1198 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1199 u32 chunk;
1200 u32 ofs = update->ofs, size = update->qwords;
1201
1202 /*
1203 * If we have 512 entries (max), we would populate it ourselves,
1204 * and update the PDE above it to the new pointer.
1205 * The only time this can only happen if we have to update the top
1206 * PDE. This requires a BO that is almost vm->size big.
1207 *
1208 * This shouldn't be possible in practice.. might change when 16K
1209 * pages are used. Hence the assert.
1210 */
1211 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE);
1212 if (!ppgtt_ofs)
1213 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),
1214 xe_bo_addr(update->pt_bo, 0,
1215 XE_PAGE_SIZE), false);
1216
1217 do {
1218 u64 addr = ppgtt_ofs + ofs * 8;
1219
1220 chunk = min(size, MAX_PTE_PER_SDI);
1221
1222 /* Ensure populatefn can do memset64 by aligning bb->cs */
1223 if (!(bb->len & 1))
1224 bb->cs[bb->len++] = MI_NOOP;
1225
1226 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
1227 bb->cs[bb->len++] = lower_32_bits(addr);
1228 bb->cs[bb->len++] = upper_32_bits(addr);
1229 if (pt_op->bind)
1230 ops->populate(pt_update, tile, NULL, bb->cs + bb->len,
1231 ofs, chunk, update);
1232 else
1233 ops->clear(pt_update, tile, NULL, bb->cs + bb->len,
1234 ofs, chunk, update);
1235
1236 bb->len += chunk * 2;
1237 ofs += chunk;
1238 size -= chunk;
1239 } while (size);
1240}
1241
1242struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m)
1243{
1244 return xe_vm_get(m->q->vm);
1245}
1246
1247#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
1248struct migrate_test_params {
1249 struct xe_test_priv base;
1250 bool force_gpu;
1251};
1252
1253#define to_migrate_test_params(_priv) \
1254 container_of(_priv, struct migrate_test_params, base)
1255#endif
1256
1257static struct dma_fence *
1258xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
1259 struct xe_migrate_pt_update *pt_update)
1260{
1261 XE_TEST_DECLARE(struct migrate_test_params *test =
1262 to_migrate_test_params
1263 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)
1264 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1265 struct xe_vm *vm = pt_update->vops->vm;
1266 struct xe_vm_pgtable_update_ops *pt_update_ops =
1267 &pt_update->vops->pt_update_ops[pt_update->tile_id];
1268 int err;
1269 u32 i, j;
1270
1271 if (XE_TEST_ONLY(test && test->force_gpu))
1272 return ERR_PTR(-ETIME);
1273
1274 if (ops->pre_commit) {
1275 pt_update->job = NULL;
1276 err = ops->pre_commit(pt_update);
1277 if (err)
1278 return ERR_PTR(err);
1279 }
1280
1281 for (i = 0; i < pt_update_ops->num_ops; ++i) {
1282 const struct xe_vm_pgtable_update_op *pt_op =
1283 &pt_update_ops->ops[i];
1284
1285 for (j = 0; j < pt_op->num_entries; j++) {
1286 const struct xe_vm_pgtable_update *update =
1287 &pt_op->entries[j];
1288
1289 if (pt_op->bind)
1290 ops->populate(pt_update, m->tile,
1291 &update->pt_bo->vmap, NULL,
1292 update->ofs, update->qwords,
1293 update);
1294 else
1295 ops->clear(pt_update, m->tile,
1296 &update->pt_bo->vmap, NULL,
1297 update->ofs, update->qwords, update);
1298 }
1299 }
1300
1301 trace_xe_vm_cpu_bind(vm);
1302 xe_device_wmb(vm->xe);
1303
1304 return dma_fence_get_stub();
1305}
1306
1307static struct dma_fence *
1308__xe_migrate_update_pgtables(struct xe_migrate *m,
1309 struct xe_migrate_pt_update *pt_update,
1310 struct xe_vm_pgtable_update_ops *pt_update_ops)
1311{
1312 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1313 struct xe_tile *tile = m->tile;
1314 struct xe_gt *gt = tile->primary_gt;
1315 struct xe_device *xe = tile_to_xe(tile);
1316 struct xe_sched_job *job;
1317 struct dma_fence *fence;
1318 struct drm_suballoc *sa_bo = NULL;
1319 struct xe_bb *bb;
1320 u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0;
1321 u32 num_updates = 0, current_update = 0;
1322 u64 addr;
1323 int err = 0;
1324 bool is_migrate = pt_update_ops->q == m->q;
1325 bool usm = is_migrate && xe->info.has_usm;
1326
1327 for (i = 0; i < pt_update_ops->num_ops; ++i) {
1328 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
1329 struct xe_vm_pgtable_update *updates = pt_op->entries;
1330
1331 num_updates += pt_op->num_entries;
1332 for (j = 0; j < pt_op->num_entries; ++j) {
1333 u32 num_cmds = DIV_ROUND_UP(updates[j].qwords,
1334 MAX_PTE_PER_SDI);
1335
1336 /* align noop + MI_STORE_DATA_IMM cmd prefix */
1337 batch_size += 4 * num_cmds + updates[j].qwords * 2;
1338 }
1339 }
1340
1341 /* fixed + PTE entries */
1342 if (IS_DGFX(xe))
1343 batch_size += 2;
1344 else
1345 batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) +
1346 num_updates * 2;
1347
1348 bb = xe_bb_new(gt, batch_size, usm);
1349 if (IS_ERR(bb))
1350 return ERR_CAST(bb);
1351
1352 /* For sysmem PTE's, need to map them in our hole.. */
1353 if (!IS_DGFX(xe)) {
1354 u16 pat_index = xe->pat.idx[XE_CACHE_WB];
1355 u32 ptes, ofs;
1356
1357 ppgtt_ofs = NUM_KERNEL_PDE - 1;
1358 if (!is_migrate) {
1359 u32 num_units = DIV_ROUND_UP(num_updates,
1360 NUM_VMUSA_WRITES_PER_UNIT);
1361
1362 if (num_units > m->vm_update_sa.size) {
1363 err = -ENOBUFS;
1364 goto err_bb;
1365 }
1366 sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units,
1367 GFP_KERNEL, true, 0);
1368 if (IS_ERR(sa_bo)) {
1369 err = PTR_ERR(sa_bo);
1370 goto err_bb;
1371 }
1372
1373 ppgtt_ofs = NUM_KERNEL_PDE +
1374 (drm_suballoc_soffset(sa_bo) /
1375 NUM_VMUSA_UNIT_PER_PAGE);
1376 page_ofs = (drm_suballoc_soffset(sa_bo) %
1377 NUM_VMUSA_UNIT_PER_PAGE) *
1378 VM_SA_UPDATE_UNIT_SIZE;
1379 }
1380
1381 /* Map our PT's to gtt */
1382 i = 0;
1383 j = 0;
1384 ptes = num_updates;
1385 ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;
1386 while (ptes) {
1387 u32 chunk = min(MAX_PTE_PER_SDI, ptes);
1388 u32 idx = 0;
1389
1390 bb->cs[bb->len++] = MI_STORE_DATA_IMM |
1391 MI_SDI_NUM_QW(chunk);
1392 bb->cs[bb->len++] = ofs;
1393 bb->cs[bb->len++] = 0; /* upper_32_bits */
1394
1395 for (; i < pt_update_ops->num_ops; ++i) {
1396 struct xe_vm_pgtable_update_op *pt_op =
1397 &pt_update_ops->ops[i];
1398 struct xe_vm_pgtable_update *updates = pt_op->entries;
1399
1400 for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) {
1401 struct xe_vm *vm = pt_update->vops->vm;
1402 struct xe_bo *pt_bo = updates[j].pt_bo;
1403
1404 if (idx == chunk)
1405 goto next_cmd;
1406
1407 xe_tile_assert(tile, pt_bo->size == SZ_4K);
1408
1409 /* Map a PT at most once */
1410 if (pt_bo->update_index < 0)
1411 pt_bo->update_index = current_update;
1412
1413 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0,
1414 pat_index, 0);
1415 bb->cs[bb->len++] = lower_32_bits(addr);
1416 bb->cs[bb->len++] = upper_32_bits(addr);
1417 }
1418
1419 j = 0;
1420 }
1421
1422next_cmd:
1423 ptes -= chunk;
1424 ofs += chunk * sizeof(u64);
1425 }
1426
1427 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1428 update_idx = bb->len;
1429
1430 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
1431 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
1432 for (i = 0; i < pt_update_ops->num_ops; ++i) {
1433 struct xe_vm_pgtable_update_op *pt_op =
1434 &pt_update_ops->ops[i];
1435 struct xe_vm_pgtable_update *updates = pt_op->entries;
1436
1437 for (j = 0; j < pt_op->num_entries; ++j) {
1438 struct xe_bo *pt_bo = updates[j].pt_bo;
1439
1440 write_pgtable(tile, bb, addr +
1441 pt_bo->update_index * XE_PAGE_SIZE,
1442 pt_op, &updates[j], pt_update);
1443 }
1444 }
1445 } else {
1446 /* phys pages, no preamble required */
1447 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1448 update_idx = bb->len;
1449
1450 for (i = 0; i < pt_update_ops->num_ops; ++i) {
1451 struct xe_vm_pgtable_update_op *pt_op =
1452 &pt_update_ops->ops[i];
1453 struct xe_vm_pgtable_update *updates = pt_op->entries;
1454
1455 for (j = 0; j < pt_op->num_entries; ++j)
1456 write_pgtable(tile, bb, 0, pt_op, &updates[j],
1457 pt_update);
1458 }
1459 }
1460
1461 job = xe_bb_create_migration_job(pt_update_ops->q, bb,
1462 xe_migrate_batch_base(m, usm),
1463 update_idx);
1464 if (IS_ERR(job)) {
1465 err = PTR_ERR(job);
1466 goto err_sa;
1467 }
1468
1469 if (ops->pre_commit) {
1470 pt_update->job = job;
1471 err = ops->pre_commit(pt_update);
1472 if (err)
1473 goto err_job;
1474 }
1475 if (is_migrate)
1476 mutex_lock(&m->job_mutex);
1477
1478 xe_sched_job_arm(job);
1479 fence = dma_fence_get(&job->drm.s_fence->finished);
1480 xe_sched_job_push(job);
1481
1482 if (is_migrate)
1483 mutex_unlock(&m->job_mutex);
1484
1485 xe_bb_free(bb, fence);
1486 drm_suballoc_free(sa_bo, fence);
1487
1488 return fence;
1489
1490err_job:
1491 xe_sched_job_put(job);
1492err_sa:
1493 drm_suballoc_free(sa_bo, NULL);
1494err_bb:
1495 xe_bb_free(bb, NULL);
1496 return ERR_PTR(err);
1497}
1498
1499/**
1500 * xe_migrate_update_pgtables() - Pipelined page-table update
1501 * @m: The migrate context.
1502 * @pt_update: PT update arguments
1503 *
1504 * Perform a pipelined page-table update. The update descriptors are typically
1505 * built under the same lock critical section as a call to this function. If
1506 * using the default engine for the updates, they will be performed in the
1507 * order they grab the job_mutex. If different engines are used, external
1508 * synchronization is needed for overlapping updates to maintain page-table
1509 * consistency. Note that the meaing of "overlapping" is that the updates
1510 * touch the same page-table, which might be a higher-level page-directory.
1511 * If no pipelining is needed, then updates may be performed by the cpu.
1512 *
1513 * Return: A dma_fence that, when signaled, indicates the update completion.
1514 */
1515struct dma_fence *
1516xe_migrate_update_pgtables(struct xe_migrate *m,
1517 struct xe_migrate_pt_update *pt_update)
1518
1519{
1520 struct xe_vm_pgtable_update_ops *pt_update_ops =
1521 &pt_update->vops->pt_update_ops[pt_update->tile_id];
1522 struct dma_fence *fence;
1523
1524 fence = xe_migrate_update_pgtables_cpu(m, pt_update);
1525
1526 /* -ETIME indicates a job is needed, anything else is legit error */
1527 if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)
1528 return fence;
1529
1530 return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops);
1531}
1532
1533/**
1534 * xe_migrate_wait() - Complete all operations using the xe_migrate context
1535 * @m: Migrate context to wait for.
1536 *
1537 * Waits until the GPU no longer uses the migrate context's default engine
1538 * or its page-table objects. FIXME: What about separate page-table update
1539 * engines?
1540 */
1541void xe_migrate_wait(struct xe_migrate *m)
1542{
1543 if (m->fence)
1544 dma_fence_wait(m->fence, false);
1545}
1546
1547#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
1548#include "tests/xe_migrate.c"
1549#endif