Loading...
Note: File does not exist in v5.14.15.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include <linux/dma-fence-array.h>
7
8#include "xe_pt.h"
9
10#include "regs/xe_gtt_defs.h"
11#include "xe_bo.h"
12#include "xe_device.h"
13#include "xe_drm_client.h"
14#include "xe_exec_queue.h"
15#include "xe_gt.h"
16#include "xe_gt_tlb_invalidation.h"
17#include "xe_migrate.h"
18#include "xe_pt_types.h"
19#include "xe_pt_walk.h"
20#include "xe_res_cursor.h"
21#include "xe_sched_job.h"
22#include "xe_sync.h"
23#include "xe_trace.h"
24#include "xe_ttm_stolen_mgr.h"
25#include "xe_vm.h"
26
27struct xe_pt_dir {
28 struct xe_pt pt;
29 /** @children: Array of page-table child nodes */
30 struct xe_ptw *children[XE_PDES];
31 /** @staging: Array of page-table staging nodes */
32 struct xe_ptw *staging[XE_PDES];
33};
34
35#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
36#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
37#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
38#else
39#define xe_pt_set_addr(__xe_pt, __addr)
40#define xe_pt_addr(__xe_pt) 0ull
41#endif
42
43static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48};
44static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48};
45
46#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
47
48static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt)
49{
50 return container_of(pt, struct xe_pt_dir, pt);
51}
52
53static struct xe_pt *
54xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index)
55{
56 return container_of(pt_dir->staging[index], struct xe_pt, base);
57}
58
59static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
60 unsigned int level)
61{
62 struct xe_device *xe = tile_to_xe(tile);
63 u16 pat_index = xe->pat.idx[XE_CACHE_WB];
64 u8 id = tile->id;
65
66 if (!xe_vm_has_scratch(vm))
67 return 0;
68
69 if (level > MAX_HUGEPTE_LEVEL)
70 return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo,
71 0, pat_index);
72
73 return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) |
74 XE_PTE_NULL;
75}
76
77static void xe_pt_free(struct xe_pt *pt)
78{
79 if (pt->level)
80 kfree(as_xe_pt_dir(pt));
81 else
82 kfree(pt);
83}
84
85/**
86 * xe_pt_create() - Create a page-table.
87 * @vm: The vm to create for.
88 * @tile: The tile to create for.
89 * @level: The page-table level.
90 *
91 * Allocate and initialize a single struct xe_pt metadata structure. Also
92 * create the corresponding page-table bo, but don't initialize it. If the
93 * level is grater than zero, then it's assumed to be a directory page-
94 * table and the directory structure is also allocated and initialized to
95 * NULL pointers.
96 *
97 * Return: A valid struct xe_pt pointer on success, Pointer error code on
98 * error.
99 */
100struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
101 unsigned int level)
102{
103 struct xe_pt *pt;
104 struct xe_bo *bo;
105 int err;
106
107 if (level) {
108 struct xe_pt_dir *dir = kzalloc(sizeof(*dir), GFP_KERNEL);
109
110 pt = (dir) ? &dir->pt : NULL;
111 } else {
112 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
113 }
114 if (!pt)
115 return ERR_PTR(-ENOMEM);
116
117 pt->level = level;
118 bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,
119 ttm_bo_type_kernel,
120 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
121 XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE |
122 XE_BO_FLAG_PINNED |
123 XE_BO_FLAG_NO_RESV_EVICT |
124 XE_BO_FLAG_PAGETABLE);
125 if (IS_ERR(bo)) {
126 err = PTR_ERR(bo);
127 goto err_kfree;
128 }
129 pt->bo = bo;
130 pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL;
131 pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL;
132
133 if (vm->xef)
134 xe_drm_client_add_bo(vm->xef->client, pt->bo);
135 xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);
136
137 return pt;
138
139err_kfree:
140 xe_pt_free(pt);
141 return ERR_PTR(err);
142}
143
144/**
145 * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero
146 * entries.
147 * @tile: The tile the scratch pagetable of which to use.
148 * @vm: The vm we populate for.
149 * @pt: The pagetable the bo of which to initialize.
150 *
151 * Populate the page-table bo of @pt with entries pointing into the tile's
152 * scratch page-table tree if any. Otherwise populate with zeros.
153 */
154void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
155 struct xe_pt *pt)
156{
157 struct iosys_map *map = &pt->bo->vmap;
158 u64 empty;
159 int i;
160
161 if (!xe_vm_has_scratch(vm)) {
162 /*
163 * FIXME: Some memory is allocated already allocated to zero?
164 * Find out which memory that is and avoid this memset...
165 */
166 xe_map_memset(vm->xe, map, 0, 0, SZ_4K);
167 } else {
168 empty = __xe_pt_empty_pte(tile, vm, pt->level);
169 for (i = 0; i < XE_PDES; i++)
170 xe_pt_write(vm->xe, map, i, empty);
171 }
172}
173
174/**
175 * xe_pt_shift() - Return the ilog2 value of the size of the address range of
176 * a page-table at a certain level.
177 * @level: The level.
178 *
179 * Return: The ilog2 value of the size of the address range of a page-table
180 * at level @level.
181 */
182unsigned int xe_pt_shift(unsigned int level)
183{
184 return XE_PTE_SHIFT + XE_PDE_SHIFT * level;
185}
186
187/**
188 * xe_pt_destroy() - Destroy a page-table tree.
189 * @pt: The root of the page-table tree to destroy.
190 * @flags: vm flags. Currently unused.
191 * @deferred: List head of lockless list for deferred putting. NULL for
192 * immediate putting.
193 *
194 * Puts the page-table bo, recursively calls xe_pt_destroy on all children
195 * and finally frees @pt. TODO: Can we remove the @flags argument?
196 */
197void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
198{
199 int i;
200
201 if (!pt)
202 return;
203
204 XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list));
205 xe_bo_unpin(pt->bo);
206 xe_bo_put_deferred(pt->bo, deferred);
207
208 if (pt->level > 0 && pt->num_live) {
209 struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
210
211 for (i = 0; i < XE_PDES; i++) {
212 if (xe_pt_entry_staging(pt_dir, i))
213 xe_pt_destroy(xe_pt_entry_staging(pt_dir, i), flags,
214 deferred);
215 }
216 }
217 xe_pt_free(pt);
218}
219
220/**
221 * DOC: Pagetable building
222 *
223 * Below we use the term "page-table" for both page-directories, containing
224 * pointers to lower level page-directories or page-tables, and level 0
225 * page-tables that contain only page-table-entries pointing to memory pages.
226 *
227 * When inserting an address range in an already existing page-table tree
228 * there will typically be a set of page-tables that are shared with other
229 * address ranges, and a set that are private to this address range.
230 * The set of shared page-tables can be at most two per level,
231 * and those can't be updated immediately because the entries of those
232 * page-tables may still be in use by the gpu for other mappings. Therefore
233 * when inserting entries into those, we instead stage those insertions by
234 * adding insertion data into struct xe_vm_pgtable_update structures. This
235 * data, (subtrees for the cpu and page-table-entries for the gpu) is then
236 * added in a separate commit step. CPU-data is committed while still under the
237 * vm lock, the object lock and for userptr, the notifier lock in read mode.
238 * The GPU async data is committed either by the GPU or CPU after fulfilling
239 * relevant dependencies.
240 * For non-shared page-tables (and, in fact, for shared ones that aren't
241 * existing at the time of staging), we add the data in-place without the
242 * special update structures. This private part of the page-table tree will
243 * remain disconnected from the vm page-table tree until data is committed to
244 * the shared page tables of the vm tree in the commit phase.
245 */
246
247struct xe_pt_update {
248 /** @update: The update structure we're building for this parent. */
249 struct xe_vm_pgtable_update *update;
250 /** @parent: The parent. Used to detect a parent change. */
251 struct xe_pt *parent;
252 /** @preexisting: Whether the parent was pre-existing or allocated */
253 bool preexisting;
254};
255
256struct xe_pt_stage_bind_walk {
257 /** base: The base class. */
258 struct xe_pt_walk base;
259
260 /* Input parameters for the walk */
261 /** @vm: The vm we're building for. */
262 struct xe_vm *vm;
263 /** @tile: The tile we're building for. */
264 struct xe_tile *tile;
265 /** @default_pte: PTE flag only template. No address is associated */
266 u64 default_pte;
267 /** @dma_offset: DMA offset to add to the PTE. */
268 u64 dma_offset;
269 /**
270 * @needs_64k: This address range enforces 64K alignment and
271 * granularity.
272 */
273 bool needs_64K;
274 /**
275 * @vma: VMA being mapped
276 */
277 struct xe_vma *vma;
278
279 /* Also input, but is updated during the walk*/
280 /** @curs: The DMA address cursor. */
281 struct xe_res_cursor *curs;
282 /** @va_curs_start: The Virtual address coresponding to @curs->start */
283 u64 va_curs_start;
284
285 /* Output */
286 struct xe_walk_update {
287 /** @wupd.entries: Caller provided storage. */
288 struct xe_vm_pgtable_update *entries;
289 /** @wupd.num_used_entries: Number of update @entries used. */
290 unsigned int num_used_entries;
291 /** @wupd.updates: Tracks the update entry at a given level */
292 struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1];
293 } wupd;
294
295 /* Walk state */
296 /**
297 * @l0_end_addr: The end address of the current l0 leaf. Used for
298 * 64K granularity detection.
299 */
300 u64 l0_end_addr;
301 /** @addr_64K: The start address of the current 64K chunk. */
302 u64 addr_64K;
303 /** @found_64: Whether @add_64K actually points to a 64K chunk. */
304 bool found_64K;
305};
306
307static int
308xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
309 pgoff_t offset, bool alloc_entries)
310{
311 struct xe_pt_update *upd = &wupd->updates[parent->level];
312 struct xe_vm_pgtable_update *entry;
313
314 /*
315 * For *each level*, we could only have one active
316 * struct xt_pt_update at any one time. Once we move on to a
317 * new parent and page-directory, the old one is complete, and
318 * updates are either already stored in the build tree or in
319 * @wupd->entries
320 */
321 if (likely(upd->parent == parent))
322 return 0;
323
324 upd->parent = parent;
325 upd->preexisting = true;
326
327 if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1)
328 return -EINVAL;
329
330 entry = wupd->entries + wupd->num_used_entries++;
331 upd->update = entry;
332 entry->ofs = offset;
333 entry->pt_bo = parent->bo;
334 entry->pt = parent;
335 entry->flags = 0;
336 entry->qwords = 0;
337 entry->pt_bo->update_index = -1;
338
339 if (alloc_entries) {
340 entry->pt_entries = kmalloc_array(XE_PDES,
341 sizeof(*entry->pt_entries),
342 GFP_KERNEL);
343 if (!entry->pt_entries)
344 return -ENOMEM;
345 }
346
347 return 0;
348}
349
350/*
351 * NOTE: This is a very frequently called function so we allow ourselves
352 * to annotate (using branch prediction hints) the fastpath of updating a
353 * non-pre-existing pagetable with leaf ptes.
354 */
355static int
356xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent,
357 pgoff_t offset, struct xe_pt *xe_child, u64 pte)
358{
359 struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level];
360 struct xe_pt_update *child_upd = xe_child ?
361 &xe_walk->wupd.updates[xe_child->level] : NULL;
362 int ret;
363
364 ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true);
365 if (unlikely(ret))
366 return ret;
367
368 /*
369 * Register this new pagetable so that it won't be recognized as
370 * a shared pagetable by a subsequent insertion.
371 */
372 if (unlikely(child_upd)) {
373 child_upd->update = NULL;
374 child_upd->parent = xe_child;
375 child_upd->preexisting = false;
376 }
377
378 if (likely(!upd->preexisting)) {
379 /* Continue building a non-connected subtree. */
380 struct iosys_map *map = &parent->bo->vmap;
381
382 if (unlikely(xe_child)) {
383 parent->base.children[offset] = &xe_child->base;
384 parent->base.staging[offset] = &xe_child->base;
385 }
386
387 xe_pt_write(xe_walk->vm->xe, map, offset, pte);
388 parent->num_live++;
389 } else {
390 /* Shared pt. Stage update. */
391 unsigned int idx;
392 struct xe_vm_pgtable_update *entry = upd->update;
393
394 idx = offset - entry->ofs;
395 entry->pt_entries[idx].pt = xe_child;
396 entry->pt_entries[idx].pte = pte;
397 entry->qwords++;
398 }
399
400 return 0;
401}
402
403static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level,
404 struct xe_pt_stage_bind_walk *xe_walk)
405{
406 u64 size, dma;
407
408 if (level > MAX_HUGEPTE_LEVEL)
409 return false;
410
411 /* Does the virtual range requested cover a huge pte? */
412 if (!xe_pt_covers(addr, next, level, &xe_walk->base))
413 return false;
414
415 /* Does the DMA segment cover the whole pte? */
416 if (next - xe_walk->va_curs_start > xe_walk->curs->size)
417 return false;
418
419 /* null VMA's do not have dma addresses */
420 if (xe_vma_is_null(xe_walk->vma))
421 return true;
422
423 /* Is the DMA address huge PTE size aligned? */
424 size = next - addr;
425 dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs);
426
427 return IS_ALIGNED(dma, size);
428}
429
430/*
431 * Scan the requested mapping to check whether it can be done entirely
432 * with 64K PTEs.
433 */
434static bool
435xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
436{
437 struct xe_res_cursor curs = *xe_walk->curs;
438
439 if (!IS_ALIGNED(addr, SZ_64K))
440 return false;
441
442 if (next > xe_walk->l0_end_addr)
443 return false;
444
445 /* null VMA's do not have dma addresses */
446 if (xe_vma_is_null(xe_walk->vma))
447 return true;
448
449 xe_res_next(&curs, addr - xe_walk->va_curs_start);
450 for (; addr < next; addr += SZ_64K) {
451 if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K)
452 return false;
453
454 xe_res_next(&curs, SZ_64K);
455 }
456
457 return addr == next;
458}
459
460/*
461 * For non-compact "normal" 4K level-0 pagetables, we want to try to group
462 * addresses together in 64K-contigous regions to add a 64K TLB hint for the
463 * device to the PTE.
464 * This function determines whether the address is part of such a
465 * segment. For VRAM in normal pagetables, this is strictly necessary on
466 * some devices.
467 */
468static bool
469xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
470{
471 /* Address is within an already found 64k region */
472 if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K)
473 return true;
474
475 xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk);
476 xe_walk->addr_64K = addr;
477
478 return xe_walk->found_64K;
479}
480
481static int
482xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset,
483 unsigned int level, u64 addr, u64 next,
484 struct xe_ptw **child,
485 enum page_walk_action *action,
486 struct xe_pt_walk *walk)
487{
488 struct xe_pt_stage_bind_walk *xe_walk =
489 container_of(walk, typeof(*xe_walk), base);
490 u16 pat_index = xe_walk->vma->pat_index;
491 struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base);
492 struct xe_vm *vm = xe_walk->vm;
493 struct xe_pt *xe_child;
494 bool covers;
495 int ret = 0;
496 u64 pte;
497
498 /* Is this a leaf entry ?*/
499 if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) {
500 struct xe_res_cursor *curs = xe_walk->curs;
501 bool is_null = xe_vma_is_null(xe_walk->vma);
502
503 XE_WARN_ON(xe_walk->va_curs_start != addr);
504
505 pte = vm->pt_ops->pte_encode_vma(is_null ? 0 :
506 xe_res_dma(curs) + xe_walk->dma_offset,
507 xe_walk->vma, pat_index, level);
508 pte |= xe_walk->default_pte;
509
510 /*
511 * Set the XE_PTE_PS64 hint if possible, otherwise if
512 * this device *requires* 64K PTE size for VRAM, fail.
513 */
514 if (level == 0 && !xe_parent->is_compact) {
515 if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) {
516 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_64K;
517 pte |= XE_PTE_PS64;
518 } else if (XE_WARN_ON(xe_walk->needs_64K)) {
519 return -EINVAL;
520 }
521 }
522
523 ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte);
524 if (unlikely(ret))
525 return ret;
526
527 if (!is_null)
528 xe_res_next(curs, next - addr);
529 xe_walk->va_curs_start = next;
530 xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level);
531 *action = ACTION_CONTINUE;
532
533 return ret;
534 }
535
536 /*
537 * Descending to lower level. Determine if we need to allocate a
538 * new page table or -directory, which we do if there is no
539 * previous one or there is one we can completely replace.
540 */
541 if (level == 1) {
542 walk->shifts = xe_normal_pt_shifts;
543 xe_walk->l0_end_addr = next;
544 }
545
546 covers = xe_pt_covers(addr, next, level, &xe_walk->base);
547 if (covers || !*child) {
548 u64 flags = 0;
549
550 xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1);
551 if (IS_ERR(xe_child))
552 return PTR_ERR(xe_child);
553
554 xe_pt_set_addr(xe_child,
555 round_down(addr, 1ull << walk->shifts[level]));
556
557 if (!covers)
558 xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child);
559
560 *child = &xe_child->base;
561
562 /*
563 * Prefer the compact pagetable layout for L0 if possible. Only
564 * possible if VMA covers entire 2MB region as compact 64k and
565 * 4k pages cannot be mixed within a 2MB region.
566 * TODO: Suballocate the pt bo to avoid wasting a lot of
567 * memory.
568 */
569 if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&
570 covers && xe_pt_scan_64K(addr, next, xe_walk)) {
571 walk->shifts = xe_compact_pt_shifts;
572 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT;
573 flags |= XE_PDE_64K;
574 xe_child->is_compact = true;
575 }
576
577 pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags;
578 ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child,
579 pte);
580 }
581
582 *action = ACTION_SUBTREE;
583 return ret;
584}
585
586static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = {
587 .pt_entry = xe_pt_stage_bind_entry,
588};
589
590/**
591 * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
592 * range.
593 * @tile: The tile we're building for.
594 * @vma: The vma indicating the address range.
595 * @entries: Storage for the update entries used for connecting the tree to
596 * the main tree at commit time.
597 * @num_entries: On output contains the number of @entries used.
598 *
599 * This function builds a disconnected page-table tree for a given address
600 * range. The tree is connected to the main vm tree for the gpu using
601 * xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind().
602 * The function builds xe_vm_pgtable_update structures for already existing
603 * shared page-tables, and non-existing shared and non-shared page-tables
604 * are built and populated directly.
605 *
606 * Return 0 on success, negative error code on error.
607 */
608static int
609xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
610 struct xe_vm_pgtable_update *entries, u32 *num_entries)
611{
612 struct xe_device *xe = tile_to_xe(tile);
613 struct xe_bo *bo = xe_vma_bo(vma);
614 bool is_devmem = !xe_vma_is_userptr(vma) && bo &&
615 (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo));
616 struct xe_res_cursor curs;
617 struct xe_pt_stage_bind_walk xe_walk = {
618 .base = {
619 .ops = &xe_pt_stage_bind_ops,
620 .shifts = xe_normal_pt_shifts,
621 .max_level = XE_PT_HIGHEST_LEVEL,
622 .staging = true,
623 },
624 .vm = xe_vma_vm(vma),
625 .tile = tile,
626 .curs = &curs,
627 .va_curs_start = xe_vma_start(vma),
628 .vma = vma,
629 .wupd.entries = entries,
630 .needs_64K = (xe_vma_vm(vma)->flags & XE_VM_FLAG_64K) && is_devmem,
631 };
632 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
633 int ret;
634
635 /**
636 * Default atomic expectations for different allocation scenarios are as follows:
637 *
638 * 1. Traditional API: When the VM is not in LR mode:
639 * - Device atomics are expected to function with all allocations.
640 *
641 * 2. Compute/SVM API: When the VM is in LR mode:
642 * - Device atomics are the default behavior when the bo is placed in a single region.
643 * - In all other cases device atomics will be disabled with AE=0 until an application
644 * request differently using a ioctl like madvise.
645 */
646 if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) {
647 if (xe_vm_in_lr_mode(xe_vma_vm(vma))) {
648 if (bo && xe_bo_has_single_placement(bo))
649 xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE;
650 /**
651 * If a SMEM+LMEM allocation is backed by SMEM, a device
652 * atomics will cause a gpu page fault and which then
653 * gets migrated to LMEM, bind such allocations with
654 * device atomics enabled.
655 */
656 else if (is_devmem && !xe_bo_has_single_placement(bo))
657 xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE;
658 } else {
659 xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE;
660 }
661
662 /**
663 * Unset AE if the platform(PVC) doesn't support it on an
664 * allocation
665 */
666 if (!xe->info.has_device_atomics_on_smem && !is_devmem)
667 xe_walk.default_pte &= ~XE_USM_PPGTT_PTE_AE;
668 }
669
670 if (is_devmem) {
671 xe_walk.default_pte |= XE_PPGTT_PTE_DM;
672 xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource);
673 }
674
675 if (!xe_vma_has_no_bo(vma) && xe_bo_is_stolen(bo))
676 xe_walk.dma_offset = xe_ttm_stolen_gpu_offset(xe_bo_device(bo));
677
678 xe_bo_assert_held(bo);
679
680 if (!xe_vma_is_null(vma)) {
681 if (xe_vma_is_userptr(vma))
682 xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0,
683 xe_vma_size(vma), &curs);
684 else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
685 xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
686 xe_vma_size(vma), &curs);
687 else
688 xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma),
689 xe_vma_size(vma), &curs);
690 } else {
691 curs.size = xe_vma_size(vma);
692 }
693
694 ret = xe_pt_walk_range(&pt->base, pt->level, xe_vma_start(vma),
695 xe_vma_end(vma), &xe_walk.base);
696
697 *num_entries = xe_walk.wupd.num_used_entries;
698 return ret;
699}
700
701/**
702 * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a
703 * shared pagetable.
704 * @addr: The start address within the non-shared pagetable.
705 * @end: The end address within the non-shared pagetable.
706 * @level: The level of the non-shared pagetable.
707 * @walk: Walk info. The function adjusts the walk action.
708 * @action: next action to perform (see enum page_walk_action)
709 * @offset: Ignored on input, First non-shared entry on output.
710 * @end_offset: Ignored on input, Last non-shared entry + 1 on output.
711 *
712 * A non-shared page-table has some entries that belong to the address range
713 * and others that don't. This function determines the entries that belong
714 * fully to the address range. Depending on level, some entries may
715 * partially belong to the address range (that can't happen at level 0).
716 * The function detects that and adjust those offsets to not include those
717 * partial entries. Iff it does detect partial entries, we know that there must
718 * be shared page tables also at lower levels, so it adjusts the walk action
719 * accordingly.
720 *
721 * Return: true if there were non-shared entries, false otherwise.
722 */
723static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level,
724 struct xe_pt_walk *walk,
725 enum page_walk_action *action,
726 pgoff_t *offset, pgoff_t *end_offset)
727{
728 u64 size = 1ull << walk->shifts[level];
729
730 *offset = xe_pt_offset(addr, level, walk);
731 *end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset;
732
733 if (!level)
734 return true;
735
736 /*
737 * If addr or next are not size aligned, there are shared pts at lower
738 * level, so in that case traverse down the subtree
739 */
740 *action = ACTION_CONTINUE;
741 if (!IS_ALIGNED(addr, size)) {
742 *action = ACTION_SUBTREE;
743 (*offset)++;
744 }
745
746 if (!IS_ALIGNED(end, size)) {
747 *action = ACTION_SUBTREE;
748 (*end_offset)--;
749 }
750
751 return *end_offset > *offset;
752}
753
754struct xe_pt_zap_ptes_walk {
755 /** @base: The walk base-class */
756 struct xe_pt_walk base;
757
758 /* Input parameters for the walk */
759 /** @tile: The tile we're building for */
760 struct xe_tile *tile;
761
762 /* Output */
763 /** @needs_invalidate: Whether we need to invalidate TLB*/
764 bool needs_invalidate;
765};
766
767static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset,
768 unsigned int level, u64 addr, u64 next,
769 struct xe_ptw **child,
770 enum page_walk_action *action,
771 struct xe_pt_walk *walk)
772{
773 struct xe_pt_zap_ptes_walk *xe_walk =
774 container_of(walk, typeof(*xe_walk), base);
775 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
776 pgoff_t end_offset;
777
778 XE_WARN_ON(!*child);
779 XE_WARN_ON(!level);
780
781 /*
782 * Note that we're called from an entry callback, and we're dealing
783 * with the child of that entry rather than the parent, so need to
784 * adjust level down.
785 */
786 if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset,
787 &end_offset)) {
788 xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap,
789 offset * sizeof(u64), 0,
790 (end_offset - offset) * sizeof(u64));
791 xe_walk->needs_invalidate = true;
792 }
793
794 return 0;
795}
796
797static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
798 .pt_entry = xe_pt_zap_ptes_entry,
799};
800
801/**
802 * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
803 * @tile: The tile we're zapping for.
804 * @vma: GPU VMA detailing address range.
805 *
806 * Eviction and Userptr invalidation needs to be able to zap the
807 * gpu ptes of a given address range in pagefaulting mode.
808 * In order to be able to do that, that function needs access to the shared
809 * page-table entrieaso it can either clear the leaf PTEs or
810 * clear the pointers to lower-level page-tables. The caller is required
811 * to hold the necessary locks to ensure neither the page-table connectivity
812 * nor the page-table entries of the range is updated from under us.
813 *
814 * Return: Whether ptes were actually updated and a TLB invalidation is
815 * required.
816 */
817bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
818{
819 struct xe_pt_zap_ptes_walk xe_walk = {
820 .base = {
821 .ops = &xe_pt_zap_ptes_ops,
822 .shifts = xe_normal_pt_shifts,
823 .max_level = XE_PT_HIGHEST_LEVEL,
824 },
825 .tile = tile,
826 };
827 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
828 u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated);
829
830 if (!(pt_mask & BIT(tile->id)))
831 return false;
832
833 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
834 xe_vma_end(vma), &xe_walk.base);
835
836 return xe_walk.needs_invalidate;
837}
838
839static void
840xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,
841 struct iosys_map *map, void *data,
842 u32 qword_ofs, u32 num_qwords,
843 const struct xe_vm_pgtable_update *update)
844{
845 struct xe_pt_entry *ptes = update->pt_entries;
846 u64 *ptr = data;
847 u32 i;
848
849 for (i = 0; i < num_qwords; i++) {
850 if (map)
851 xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
852 sizeof(u64), u64, ptes[i].pte);
853 else
854 ptr[i] = ptes[i].pte;
855 }
856}
857
858static void xe_pt_cancel_bind(struct xe_vma *vma,
859 struct xe_vm_pgtable_update *entries,
860 u32 num_entries)
861{
862 u32 i, j;
863
864 for (i = 0; i < num_entries; i++) {
865 struct xe_pt *pt = entries[i].pt;
866
867 if (!pt)
868 continue;
869
870 if (pt->level) {
871 for (j = 0; j < entries[i].qwords; j++)
872 xe_pt_destroy(entries[i].pt_entries[j].pt,
873 xe_vma_vm(vma)->flags, NULL);
874 }
875
876 kfree(entries[i].pt_entries);
877 entries[i].pt_entries = NULL;
878 entries[i].qwords = 0;
879 }
880}
881
882static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma)
883{
884 struct xe_vm *vm = xe_vma_vm(vma);
885
886 lockdep_assert_held(&vm->lock);
887
888 if (!xe_vma_is_userptr(vma) && !xe_vma_is_null(vma))
889 dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
890
891 xe_vm_assert_held(vm);
892}
893
894static void xe_pt_commit_locks_assert(struct xe_vma *vma)
895{
896 struct xe_vm *vm = xe_vma_vm(vma);
897
898 xe_pt_commit_prepare_locks_assert(vma);
899
900 if (xe_vma_is_userptr(vma))
901 lockdep_assert_held_read(&vm->userptr.notifier_lock);
902}
903
904static void xe_pt_commit(struct xe_vma *vma,
905 struct xe_vm_pgtable_update *entries,
906 u32 num_entries, struct llist_head *deferred)
907{
908 u32 i, j;
909
910 xe_pt_commit_locks_assert(vma);
911
912 for (i = 0; i < num_entries; i++) {
913 struct xe_pt *pt = entries[i].pt;
914 struct xe_pt_dir *pt_dir;
915
916 if (!pt->level)
917 continue;
918
919 pt_dir = as_xe_pt_dir(pt);
920 for (j = 0; j < entries[i].qwords; j++) {
921 struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
922 int j_ = j + entries[i].ofs;
923
924 pt_dir->children[j_] = pt_dir->staging[j_];
925 xe_pt_destroy(oldpte, xe_vma_vm(vma)->flags, deferred);
926 }
927 }
928}
929
930static void xe_pt_abort_bind(struct xe_vma *vma,
931 struct xe_vm_pgtable_update *entries,
932 u32 num_entries, bool rebind)
933{
934 int i, j;
935
936 xe_pt_commit_prepare_locks_assert(vma);
937
938 for (i = num_entries - 1; i >= 0; --i) {
939 struct xe_pt *pt = entries[i].pt;
940 struct xe_pt_dir *pt_dir;
941
942 if (!rebind)
943 pt->num_live -= entries[i].qwords;
944
945 if (!pt->level)
946 continue;
947
948 pt_dir = as_xe_pt_dir(pt);
949 for (j = 0; j < entries[i].qwords; j++) {
950 u32 j_ = j + entries[i].ofs;
951 struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, j_);
952 struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
953
954 pt_dir->staging[j_] = oldpte ? &oldpte->base : 0;
955 xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);
956 }
957 }
958}
959
960static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
961 struct xe_vm_pgtable_update *entries,
962 u32 num_entries, bool rebind)
963{
964 u32 i, j;
965
966 xe_pt_commit_prepare_locks_assert(vma);
967
968 for (i = 0; i < num_entries; i++) {
969 struct xe_pt *pt = entries[i].pt;
970 struct xe_pt_dir *pt_dir;
971
972 if (!rebind)
973 pt->num_live += entries[i].qwords;
974
975 if (!pt->level)
976 continue;
977
978 pt_dir = as_xe_pt_dir(pt);
979 for (j = 0; j < entries[i].qwords; j++) {
980 u32 j_ = j + entries[i].ofs;
981 struct xe_pt *newpte = entries[i].pt_entries[j].pt;
982 struct xe_pt *oldpte = NULL;
983
984 if (xe_pt_entry_staging(pt_dir, j_))
985 oldpte = xe_pt_entry_staging(pt_dir, j_);
986
987 pt_dir->staging[j_] = &newpte->base;
988 entries[i].pt_entries[j].pt = oldpte;
989 }
990 }
991}
992
993static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries,
994 u32 num_entries)
995{
996 u32 i;
997
998 for (i = 0; i < num_entries; i++)
999 kfree(entries[i].pt_entries);
1000}
1001
1002static int
1003xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
1004 struct xe_vm_pgtable_update *entries, u32 *num_entries)
1005{
1006 int err;
1007
1008 *num_entries = 0;
1009 err = xe_pt_stage_bind(tile, vma, entries, num_entries);
1010 if (!err)
1011 xe_tile_assert(tile, *num_entries);
1012
1013 return err;
1014}
1015
1016static void xe_vm_dbg_print_entries(struct xe_device *xe,
1017 const struct xe_vm_pgtable_update *entries,
1018 unsigned int num_entries, bool bind)
1019#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM))
1020{
1021 unsigned int i;
1022
1023 vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind",
1024 num_entries);
1025 for (i = 0; i < num_entries; i++) {
1026 const struct xe_vm_pgtable_update *entry = &entries[i];
1027 struct xe_pt *xe_pt = entry->pt;
1028 u64 page_size = 1ull << xe_pt_shift(xe_pt->level);
1029 u64 end;
1030 u64 start;
1031
1032 xe_assert(xe, !entry->pt->is_compact);
1033 start = entry->ofs * page_size;
1034 end = start + page_size * entry->qwords;
1035 vm_dbg(&xe->drm,
1036 "\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n",
1037 i, xe_pt->level, entry->ofs, entry->qwords,
1038 xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0);
1039 }
1040}
1041#else
1042{}
1043#endif
1044
1045static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs)
1046{
1047 int i;
1048
1049 for (i = 0; i < num_syncs; i++) {
1050 struct dma_fence *fence = syncs[i].fence;
1051
1052 if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1053 &fence->flags))
1054 return false;
1055 }
1056
1057 return true;
1058}
1059
1060static int job_test_add_deps(struct xe_sched_job *job,
1061 struct dma_resv *resv,
1062 enum dma_resv_usage usage)
1063{
1064 if (!job) {
1065 if (!dma_resv_test_signaled(resv, usage))
1066 return -ETIME;
1067
1068 return 0;
1069 }
1070
1071 return xe_sched_job_add_deps(job, resv, usage);
1072}
1073
1074static int vma_add_deps(struct xe_vma *vma, struct xe_sched_job *job)
1075{
1076 struct xe_bo *bo = xe_vma_bo(vma);
1077
1078 xe_bo_assert_held(bo);
1079
1080 if (bo && !bo->vm)
1081 return job_test_add_deps(job, bo->ttm.base.resv,
1082 DMA_RESV_USAGE_KERNEL);
1083
1084 return 0;
1085}
1086
1087static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op,
1088 struct xe_sched_job *job)
1089{
1090 int err = 0;
1091
1092 switch (op->base.op) {
1093 case DRM_GPUVA_OP_MAP:
1094 if (!op->map.immediate && xe_vm_in_fault_mode(vm))
1095 break;
1096
1097 err = vma_add_deps(op->map.vma, job);
1098 break;
1099 case DRM_GPUVA_OP_REMAP:
1100 if (op->remap.prev)
1101 err = vma_add_deps(op->remap.prev, job);
1102 if (!err && op->remap.next)
1103 err = vma_add_deps(op->remap.next, job);
1104 break;
1105 case DRM_GPUVA_OP_UNMAP:
1106 break;
1107 case DRM_GPUVA_OP_PREFETCH:
1108 err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job);
1109 break;
1110 default:
1111 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1112 }
1113
1114 return err;
1115}
1116
1117static int xe_pt_vm_dependencies(struct xe_sched_job *job,
1118 struct xe_vm *vm,
1119 struct xe_vma_ops *vops,
1120 struct xe_vm_pgtable_update_ops *pt_update_ops,
1121 struct xe_range_fence_tree *rftree)
1122{
1123 struct xe_range_fence *rtfence;
1124 struct dma_fence *fence;
1125 struct xe_vma_op *op;
1126 int err = 0, i;
1127
1128 xe_vm_assert_held(vm);
1129
1130 if (!job && !no_in_syncs(vops->syncs, vops->num_syncs))
1131 return -ETIME;
1132
1133 if (!job && !xe_exec_queue_is_idle(pt_update_ops->q))
1134 return -ETIME;
1135
1136 if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) {
1137 err = job_test_add_deps(job, xe_vm_resv(vm),
1138 pt_update_ops->wait_vm_bookkeep ?
1139 DMA_RESV_USAGE_BOOKKEEP :
1140 DMA_RESV_USAGE_KERNEL);
1141 if (err)
1142 return err;
1143 }
1144
1145 rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start,
1146 pt_update_ops->last);
1147 while (rtfence) {
1148 fence = rtfence->fence;
1149
1150 if (!dma_fence_is_signaled(fence)) {
1151 /*
1152 * Is this a CPU update? GPU is busy updating, so return
1153 * an error
1154 */
1155 if (!job)
1156 return -ETIME;
1157
1158 dma_fence_get(fence);
1159 err = drm_sched_job_add_dependency(&job->drm, fence);
1160 if (err)
1161 return err;
1162 }
1163
1164 rtfence = xe_range_fence_tree_next(rtfence,
1165 pt_update_ops->start,
1166 pt_update_ops->last);
1167 }
1168
1169 list_for_each_entry(op, &vops->list, link) {
1170 err = op_add_deps(vm, op, job);
1171 if (err)
1172 return err;
1173 }
1174
1175 if (!(pt_update_ops->q->flags & EXEC_QUEUE_FLAG_KERNEL)) {
1176 if (job)
1177 err = xe_sched_job_last_fence_add_dep(job, vm);
1178 else
1179 err = xe_exec_queue_last_fence_test_dep(pt_update_ops->q, vm);
1180 }
1181
1182 for (i = 0; job && !err && i < vops->num_syncs; i++)
1183 err = xe_sync_entry_add_deps(&vops->syncs[i], job);
1184
1185 return err;
1186}
1187
1188static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)
1189{
1190 struct xe_vma_ops *vops = pt_update->vops;
1191 struct xe_vm *vm = vops->vm;
1192 struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id];
1193 struct xe_vm_pgtable_update_ops *pt_update_ops =
1194 &vops->pt_update_ops[pt_update->tile_id];
1195
1196 return xe_pt_vm_dependencies(pt_update->job, vm, pt_update->vops,
1197 pt_update_ops, rftree);
1198}
1199
1200#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
1201
1202static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
1203{
1204 u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2;
1205 static u32 count;
1206
1207 if (count++ % divisor == divisor - 1) {
1208 uvma->userptr.divisor = divisor << 1;
1209 return true;
1210 }
1211
1212 return false;
1213}
1214
1215#else
1216
1217static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
1218{
1219 return false;
1220}
1221
1222#endif
1223
1224static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
1225 struct xe_vm_pgtable_update_ops *pt_update)
1226{
1227 struct xe_userptr_vma *uvma;
1228 unsigned long notifier_seq;
1229
1230 lockdep_assert_held_read(&vm->userptr.notifier_lock);
1231
1232 if (!xe_vma_is_userptr(vma))
1233 return 0;
1234
1235 uvma = to_userptr_vma(vma);
1236 if (xe_pt_userptr_inject_eagain(uvma))
1237 xe_vma_userptr_force_invalidate(uvma);
1238
1239 notifier_seq = uvma->userptr.notifier_seq;
1240
1241 if (!mmu_interval_read_retry(&uvma->userptr.notifier,
1242 notifier_seq))
1243 return 0;
1244
1245 if (xe_vm_in_fault_mode(vm))
1246 return -EAGAIN;
1247
1248 /*
1249 * Just continue the operation since exec or rebind worker
1250 * will take care of rebinding.
1251 */
1252 return 0;
1253}
1254
1255static int op_check_userptr(struct xe_vm *vm, struct xe_vma_op *op,
1256 struct xe_vm_pgtable_update_ops *pt_update)
1257{
1258 int err = 0;
1259
1260 lockdep_assert_held_read(&vm->userptr.notifier_lock);
1261
1262 switch (op->base.op) {
1263 case DRM_GPUVA_OP_MAP:
1264 if (!op->map.immediate && xe_vm_in_fault_mode(vm))
1265 break;
1266
1267 err = vma_check_userptr(vm, op->map.vma, pt_update);
1268 break;
1269 case DRM_GPUVA_OP_REMAP:
1270 if (op->remap.prev)
1271 err = vma_check_userptr(vm, op->remap.prev, pt_update);
1272 if (!err && op->remap.next)
1273 err = vma_check_userptr(vm, op->remap.next, pt_update);
1274 break;
1275 case DRM_GPUVA_OP_UNMAP:
1276 break;
1277 case DRM_GPUVA_OP_PREFETCH:
1278 err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va),
1279 pt_update);
1280 break;
1281 default:
1282 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1283 }
1284
1285 return err;
1286}
1287
1288static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
1289{
1290 struct xe_vm *vm = pt_update->vops->vm;
1291 struct xe_vma_ops *vops = pt_update->vops;
1292 struct xe_vm_pgtable_update_ops *pt_update_ops =
1293 &vops->pt_update_ops[pt_update->tile_id];
1294 struct xe_vma_op *op;
1295 int err;
1296
1297 err = xe_pt_pre_commit(pt_update);
1298 if (err)
1299 return err;
1300
1301 down_read(&vm->userptr.notifier_lock);
1302
1303 list_for_each_entry(op, &vops->list, link) {
1304 err = op_check_userptr(vm, op, pt_update_ops);
1305 if (err) {
1306 up_read(&vm->userptr.notifier_lock);
1307 break;
1308 }
1309 }
1310
1311 return err;
1312}
1313
1314struct invalidation_fence {
1315 struct xe_gt_tlb_invalidation_fence base;
1316 struct xe_gt *gt;
1317 struct dma_fence *fence;
1318 struct dma_fence_cb cb;
1319 struct work_struct work;
1320 u64 start;
1321 u64 end;
1322 u32 asid;
1323};
1324
1325static void invalidation_fence_cb(struct dma_fence *fence,
1326 struct dma_fence_cb *cb)
1327{
1328 struct invalidation_fence *ifence =
1329 container_of(cb, struct invalidation_fence, cb);
1330 struct xe_device *xe = gt_to_xe(ifence->gt);
1331
1332 trace_xe_gt_tlb_invalidation_fence_cb(xe, &ifence->base);
1333 if (!ifence->fence->error) {
1334 queue_work(system_wq, &ifence->work);
1335 } else {
1336 ifence->base.base.error = ifence->fence->error;
1337 xe_gt_tlb_invalidation_fence_signal(&ifence->base);
1338 }
1339 dma_fence_put(ifence->fence);
1340}
1341
1342static void invalidation_fence_work_func(struct work_struct *w)
1343{
1344 struct invalidation_fence *ifence =
1345 container_of(w, struct invalidation_fence, work);
1346 struct xe_device *xe = gt_to_xe(ifence->gt);
1347
1348 trace_xe_gt_tlb_invalidation_fence_work_func(xe, &ifence->base);
1349 xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start,
1350 ifence->end, ifence->asid);
1351}
1352
1353static void invalidation_fence_init(struct xe_gt *gt,
1354 struct invalidation_fence *ifence,
1355 struct dma_fence *fence,
1356 u64 start, u64 end, u32 asid)
1357{
1358 int ret;
1359
1360 trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base);
1361
1362 xe_gt_tlb_invalidation_fence_init(gt, &ifence->base, false);
1363
1364 ifence->fence = fence;
1365 ifence->gt = gt;
1366 ifence->start = start;
1367 ifence->end = end;
1368 ifence->asid = asid;
1369
1370 INIT_WORK(&ifence->work, invalidation_fence_work_func);
1371 ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb);
1372 if (ret == -ENOENT) {
1373 dma_fence_put(ifence->fence); /* Usually dropped in CB */
1374 invalidation_fence_work_func(&ifence->work);
1375 } else if (ret) {
1376 dma_fence_put(&ifence->base.base); /* Caller ref */
1377 dma_fence_put(&ifence->base.base); /* Creation ref */
1378 }
1379
1380 xe_gt_assert(gt, !ret || ret == -ENOENT);
1381}
1382
1383struct xe_pt_stage_unbind_walk {
1384 /** @base: The pagewalk base-class. */
1385 struct xe_pt_walk base;
1386
1387 /* Input parameters for the walk */
1388 /** @tile: The tile we're unbinding from. */
1389 struct xe_tile *tile;
1390
1391 /**
1392 * @modified_start: Walk range start, modified to include any
1393 * shared pagetables that we're the only user of and can thus
1394 * treat as private.
1395 */
1396 u64 modified_start;
1397 /** @modified_end: Walk range start, modified like @modified_start. */
1398 u64 modified_end;
1399
1400 /* Output */
1401 /* @wupd: Structure to track the page-table updates we're building */
1402 struct xe_walk_update wupd;
1403};
1404
1405/*
1406 * Check whether this range is the only one populating this pagetable,
1407 * and in that case, update the walk range checks so that higher levels don't
1408 * view us as a shared pagetable.
1409 */
1410static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level,
1411 const struct xe_pt *child,
1412 enum page_walk_action *action,
1413 struct xe_pt_walk *walk)
1414{
1415 struct xe_pt_stage_unbind_walk *xe_walk =
1416 container_of(walk, typeof(*xe_walk), base);
1417 unsigned int shift = walk->shifts[level];
1418 u64 size = 1ull << shift;
1419
1420 if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) &&
1421 ((next - addr) >> shift) == child->num_live) {
1422 u64 size = 1ull << walk->shifts[level + 1];
1423
1424 *action = ACTION_CONTINUE;
1425
1426 if (xe_walk->modified_start >= addr)
1427 xe_walk->modified_start = round_down(addr, size);
1428 if (xe_walk->modified_end <= next)
1429 xe_walk->modified_end = round_up(next, size);
1430
1431 return true;
1432 }
1433
1434 return false;
1435}
1436
1437static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
1438 unsigned int level, u64 addr, u64 next,
1439 struct xe_ptw **child,
1440 enum page_walk_action *action,
1441 struct xe_pt_walk *walk)
1442{
1443 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
1444
1445 XE_WARN_ON(!*child);
1446 XE_WARN_ON(!level);
1447
1448 xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
1449
1450 return 0;
1451}
1452
1453static int
1454xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset,
1455 unsigned int level, u64 addr, u64 next,
1456 struct xe_ptw **child,
1457 enum page_walk_action *action,
1458 struct xe_pt_walk *walk)
1459{
1460 struct xe_pt_stage_unbind_walk *xe_walk =
1461 container_of(walk, typeof(*xe_walk), base);
1462 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
1463 pgoff_t end_offset;
1464 u64 size = 1ull << walk->shifts[--level];
1465 int err;
1466
1467 if (!IS_ALIGNED(addr, size))
1468 addr = xe_walk->modified_start;
1469 if (!IS_ALIGNED(next, size))
1470 next = xe_walk->modified_end;
1471
1472 /* Parent == *child is the root pt. Don't kill it. */
1473 if (parent != *child &&
1474 xe_pt_check_kill(addr, next, level, xe_child, action, walk))
1475 return 0;
1476
1477 if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset,
1478 &end_offset))
1479 return 0;
1480
1481 err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true);
1482 if (err)
1483 return err;
1484
1485 xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
1486
1487 return 0;
1488}
1489
1490static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = {
1491 .pt_entry = xe_pt_stage_unbind_entry,
1492 .pt_post_descend = xe_pt_stage_unbind_post_descend,
1493};
1494
1495/**
1496 * xe_pt_stage_unbind() - Build page-table update structures for an unbind
1497 * operation
1498 * @tile: The tile we're unbinding for.
1499 * @vma: The vma we're unbinding.
1500 * @entries: Caller-provided storage for the update structures.
1501 *
1502 * Builds page-table update structures for an unbind operation. The function
1503 * will attempt to remove all page-tables that we're the only user
1504 * of, and for that to work, the unbind operation must be committed in the
1505 * same critical section that blocks racing binds to the same page-table tree.
1506 *
1507 * Return: The number of entries used.
1508 */
1509static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma,
1510 struct xe_vm_pgtable_update *entries)
1511{
1512 struct xe_pt_stage_unbind_walk xe_walk = {
1513 .base = {
1514 .ops = &xe_pt_stage_unbind_ops,
1515 .shifts = xe_normal_pt_shifts,
1516 .max_level = XE_PT_HIGHEST_LEVEL,
1517 .staging = true,
1518 },
1519 .tile = tile,
1520 .modified_start = xe_vma_start(vma),
1521 .modified_end = xe_vma_end(vma),
1522 .wupd.entries = entries,
1523 };
1524 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
1525
1526 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
1527 xe_vma_end(vma), &xe_walk.base);
1528
1529 return xe_walk.wupd.num_used_entries;
1530}
1531
1532static void
1533xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
1534 struct xe_tile *tile, struct iosys_map *map,
1535 void *ptr, u32 qword_ofs, u32 num_qwords,
1536 const struct xe_vm_pgtable_update *update)
1537{
1538 struct xe_vm *vm = pt_update->vops->vm;
1539 u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
1540 int i;
1541
1542 if (map && map->is_iomem)
1543 for (i = 0; i < num_qwords; ++i)
1544 xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
1545 sizeof(u64), u64, empty);
1546 else if (map)
1547 memset64(map->vaddr + qword_ofs * sizeof(u64), empty,
1548 num_qwords);
1549 else
1550 memset64(ptr, empty, num_qwords);
1551}
1552
1553static void xe_pt_abort_unbind(struct xe_vma *vma,
1554 struct xe_vm_pgtable_update *entries,
1555 u32 num_entries)
1556{
1557 int i, j;
1558
1559 xe_pt_commit_prepare_locks_assert(vma);
1560
1561 for (i = num_entries - 1; i >= 0; --i) {
1562 struct xe_vm_pgtable_update *entry = &entries[i];
1563 struct xe_pt *pt = entry->pt;
1564 struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
1565
1566 pt->num_live += entry->qwords;
1567
1568 if (!pt->level)
1569 continue;
1570
1571 for (j = entry->ofs; j < entry->ofs + entry->qwords; j++)
1572 pt_dir->staging[j] =
1573 entries[i].pt_entries[j - entry->ofs].pt ?
1574 &entries[i].pt_entries[j - entry->ofs].pt->base : NULL;
1575 }
1576}
1577
1578static void
1579xe_pt_commit_prepare_unbind(struct xe_vma *vma,
1580 struct xe_vm_pgtable_update *entries,
1581 u32 num_entries)
1582{
1583 int i, j;
1584
1585 xe_pt_commit_prepare_locks_assert(vma);
1586
1587 for (i = 0; i < num_entries; ++i) {
1588 struct xe_vm_pgtable_update *entry = &entries[i];
1589 struct xe_pt *pt = entry->pt;
1590 struct xe_pt_dir *pt_dir;
1591
1592 pt->num_live -= entry->qwords;
1593 if (!pt->level)
1594 continue;
1595
1596 pt_dir = as_xe_pt_dir(pt);
1597 for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) {
1598 entry->pt_entries[j - entry->ofs].pt =
1599 xe_pt_entry_staging(pt_dir, j);
1600 pt_dir->staging[j] = NULL;
1601 }
1602 }
1603}
1604
1605static void
1606xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops,
1607 struct xe_vma *vma)
1608{
1609 u32 current_op = pt_update_ops->current_op;
1610 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1611 int i, level = 0;
1612 u64 start, last;
1613
1614 for (i = 0; i < pt_op->num_entries; i++) {
1615 const struct xe_vm_pgtable_update *entry = &pt_op->entries[i];
1616
1617 if (entry->pt->level > level)
1618 level = entry->pt->level;
1619 }
1620
1621 /* Greedy (non-optimal) calculation but simple */
1622 start = ALIGN_DOWN(xe_vma_start(vma), 0x1ull << xe_pt_shift(level));
1623 last = ALIGN(xe_vma_end(vma), 0x1ull << xe_pt_shift(level)) - 1;
1624
1625 if (start < pt_update_ops->start)
1626 pt_update_ops->start = start;
1627 if (last > pt_update_ops->last)
1628 pt_update_ops->last = last;
1629}
1630
1631static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma)
1632{
1633 int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0;
1634
1635 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
1636 return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv,
1637 xe->info.tile_count << shift);
1638
1639 return 0;
1640}
1641
1642static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
1643 struct xe_vm_pgtable_update_ops *pt_update_ops,
1644 struct xe_vma *vma)
1645{
1646 u32 current_op = pt_update_ops->current_op;
1647 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1648 int err;
1649
1650 xe_bo_assert_held(xe_vma_bo(vma));
1651
1652 vm_dbg(&xe_vma_vm(vma)->xe->drm,
1653 "Preparing bind, with range [%llx...%llx)\n",
1654 xe_vma_start(vma), xe_vma_end(vma) - 1);
1655
1656 pt_op->vma = NULL;
1657 pt_op->bind = true;
1658 pt_op->rebind = BIT(tile->id) & vma->tile_present;
1659
1660 err = vma_reserve_fences(tile_to_xe(tile), vma);
1661 if (err)
1662 return err;
1663
1664 err = xe_pt_prepare_bind(tile, vma, pt_op->entries,
1665 &pt_op->num_entries);
1666 if (!err) {
1667 xe_tile_assert(tile, pt_op->num_entries <=
1668 ARRAY_SIZE(pt_op->entries));
1669 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
1670 pt_op->num_entries, true);
1671
1672 xe_pt_update_ops_rfence_interval(pt_update_ops, vma);
1673 ++pt_update_ops->current_op;
1674 pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma);
1675
1676 /*
1677 * If rebind, we have to invalidate TLB on !LR vms to invalidate
1678 * cached PTEs point to freed memory. On LR vms this is done
1679 * automatically when the context is re-enabled by the rebind worker,
1680 * or in fault mode it was invalidated on PTE zapping.
1681 *
1682 * If !rebind, and scratch enabled VMs, there is a chance the scratch
1683 * PTE is already cached in the TLB so it needs to be invalidated.
1684 * On !LR VMs this is done in the ring ops preceding a batch, but on
1685 * non-faulting LR, in particular on user-space batch buffer chaining,
1686 * it needs to be done here.
1687 */
1688 if ((!pt_op->rebind && xe_vm_has_scratch(vm) &&
1689 xe_vm_in_preempt_fence_mode(vm)))
1690 pt_update_ops->needs_invalidation = true;
1691 else if (pt_op->rebind && !xe_vm_in_lr_mode(vm))
1692 /* We bump also if batch_invalidate_tlb is true */
1693 vm->tlb_flush_seqno++;
1694
1695 vma->tile_staged |= BIT(tile->id);
1696 pt_op->vma = vma;
1697 xe_pt_commit_prepare_bind(vma, pt_op->entries,
1698 pt_op->num_entries, pt_op->rebind);
1699 } else {
1700 xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
1701 }
1702
1703 return err;
1704}
1705
1706static int unbind_op_prepare(struct xe_tile *tile,
1707 struct xe_vm_pgtable_update_ops *pt_update_ops,
1708 struct xe_vma *vma)
1709{
1710 u32 current_op = pt_update_ops->current_op;
1711 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1712 int err;
1713
1714 if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))
1715 return 0;
1716
1717 xe_bo_assert_held(xe_vma_bo(vma));
1718
1719 vm_dbg(&xe_vma_vm(vma)->xe->drm,
1720 "Preparing unbind, with range [%llx...%llx)\n",
1721 xe_vma_start(vma), xe_vma_end(vma) - 1);
1722
1723 /*
1724 * Wait for invalidation to complete. Can corrupt internal page table
1725 * state if an invalidation is running while preparing an unbind.
1726 */
1727 if (xe_vma_is_userptr(vma) && xe_vm_in_fault_mode(xe_vma_vm(vma)))
1728 mmu_interval_read_begin(&to_userptr_vma(vma)->userptr.notifier);
1729
1730 pt_op->vma = vma;
1731 pt_op->bind = false;
1732 pt_op->rebind = false;
1733
1734 err = vma_reserve_fences(tile_to_xe(tile), vma);
1735 if (err)
1736 return err;
1737
1738 pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries);
1739
1740 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
1741 pt_op->num_entries, false);
1742 xe_pt_update_ops_rfence_interval(pt_update_ops, vma);
1743 ++pt_update_ops->current_op;
1744 pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma);
1745 pt_update_ops->needs_invalidation = true;
1746
1747 xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries);
1748
1749 return 0;
1750}
1751
1752static int op_prepare(struct xe_vm *vm,
1753 struct xe_tile *tile,
1754 struct xe_vm_pgtable_update_ops *pt_update_ops,
1755 struct xe_vma_op *op)
1756{
1757 int err = 0;
1758
1759 xe_vm_assert_held(vm);
1760
1761 switch (op->base.op) {
1762 case DRM_GPUVA_OP_MAP:
1763 if (!op->map.immediate && xe_vm_in_fault_mode(vm))
1764 break;
1765
1766 err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma);
1767 pt_update_ops->wait_vm_kernel = true;
1768 break;
1769 case DRM_GPUVA_OP_REMAP:
1770 err = unbind_op_prepare(tile, pt_update_ops,
1771 gpuva_to_vma(op->base.remap.unmap->va));
1772
1773 if (!err && op->remap.prev) {
1774 err = bind_op_prepare(vm, tile, pt_update_ops,
1775 op->remap.prev);
1776 pt_update_ops->wait_vm_bookkeep = true;
1777 }
1778 if (!err && op->remap.next) {
1779 err = bind_op_prepare(vm, tile, pt_update_ops,
1780 op->remap.next);
1781 pt_update_ops->wait_vm_bookkeep = true;
1782 }
1783 break;
1784 case DRM_GPUVA_OP_UNMAP:
1785 err = unbind_op_prepare(tile, pt_update_ops,
1786 gpuva_to_vma(op->base.unmap.va));
1787 break;
1788 case DRM_GPUVA_OP_PREFETCH:
1789 err = bind_op_prepare(vm, tile, pt_update_ops,
1790 gpuva_to_vma(op->base.prefetch.va));
1791 pt_update_ops->wait_vm_kernel = true;
1792 break;
1793 default:
1794 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1795 }
1796
1797 return err;
1798}
1799
1800static void
1801xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
1802{
1803 init_llist_head(&pt_update_ops->deferred);
1804 pt_update_ops->start = ~0x0ull;
1805 pt_update_ops->last = 0x0ull;
1806}
1807
1808/**
1809 * xe_pt_update_ops_prepare() - Prepare PT update operations
1810 * @tile: Tile of PT update operations
1811 * @vops: VMA operationa
1812 *
1813 * Prepare PT update operations which includes updating internal PT state,
1814 * allocate memory for page tables, populate page table being pruned in, and
1815 * create PT update operations for leaf insertion / removal.
1816 *
1817 * Return: 0 on success, negative error code on error.
1818 */
1819int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
1820{
1821 struct xe_vm_pgtable_update_ops *pt_update_ops =
1822 &vops->pt_update_ops[tile->id];
1823 struct xe_vma_op *op;
1824 int shift = tile->media_gt ? 1 : 0;
1825 int err;
1826
1827 lockdep_assert_held(&vops->vm->lock);
1828 xe_vm_assert_held(vops->vm);
1829
1830 xe_pt_update_ops_init(pt_update_ops);
1831
1832 err = dma_resv_reserve_fences(xe_vm_resv(vops->vm),
1833 tile_to_xe(tile)->info.tile_count << shift);
1834 if (err)
1835 return err;
1836
1837 list_for_each_entry(op, &vops->list, link) {
1838 err = op_prepare(vops->vm, tile, pt_update_ops, op);
1839
1840 if (err)
1841 return err;
1842 }
1843
1844 xe_tile_assert(tile, pt_update_ops->current_op <=
1845 pt_update_ops->num_ops);
1846
1847#ifdef TEST_VM_OPS_ERROR
1848 if (vops->inject_error &&
1849 vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE)
1850 return -ENOSPC;
1851#endif
1852
1853 return 0;
1854}
1855
1856static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
1857 struct xe_vm_pgtable_update_ops *pt_update_ops,
1858 struct xe_vma *vma, struct dma_fence *fence,
1859 struct dma_fence *fence2)
1860{
1861 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
1862 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
1863 pt_update_ops->wait_vm_bookkeep ?
1864 DMA_RESV_USAGE_KERNEL :
1865 DMA_RESV_USAGE_BOOKKEEP);
1866 if (fence2)
1867 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,
1868 pt_update_ops->wait_vm_bookkeep ?
1869 DMA_RESV_USAGE_KERNEL :
1870 DMA_RESV_USAGE_BOOKKEEP);
1871 }
1872 vma->tile_present |= BIT(tile->id);
1873 vma->tile_staged &= ~BIT(tile->id);
1874 if (xe_vma_is_userptr(vma)) {
1875 lockdep_assert_held_read(&vm->userptr.notifier_lock);
1876 to_userptr_vma(vma)->userptr.initial_bind = true;
1877 }
1878
1879 /*
1880 * Kick rebind worker if this bind triggers preempt fences and not in
1881 * the rebind worker
1882 */
1883 if (pt_update_ops->wait_vm_bookkeep &&
1884 xe_vm_in_preempt_fence_mode(vm) &&
1885 !current->mm)
1886 xe_vm_queue_rebind_worker(vm);
1887}
1888
1889static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
1890 struct xe_vm_pgtable_update_ops *pt_update_ops,
1891 struct xe_vma *vma, struct dma_fence *fence,
1892 struct dma_fence *fence2)
1893{
1894 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
1895 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
1896 pt_update_ops->wait_vm_bookkeep ?
1897 DMA_RESV_USAGE_KERNEL :
1898 DMA_RESV_USAGE_BOOKKEEP);
1899 if (fence2)
1900 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,
1901 pt_update_ops->wait_vm_bookkeep ?
1902 DMA_RESV_USAGE_KERNEL :
1903 DMA_RESV_USAGE_BOOKKEEP);
1904 }
1905 vma->tile_present &= ~BIT(tile->id);
1906 if (!vma->tile_present) {
1907 list_del_init(&vma->combined_links.rebind);
1908 if (xe_vma_is_userptr(vma)) {
1909 lockdep_assert_held_read(&vm->userptr.notifier_lock);
1910
1911 spin_lock(&vm->userptr.invalidated_lock);
1912 list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link);
1913 spin_unlock(&vm->userptr.invalidated_lock);
1914 }
1915 }
1916}
1917
1918static void op_commit(struct xe_vm *vm,
1919 struct xe_tile *tile,
1920 struct xe_vm_pgtable_update_ops *pt_update_ops,
1921 struct xe_vma_op *op, struct dma_fence *fence,
1922 struct dma_fence *fence2)
1923{
1924 xe_vm_assert_held(vm);
1925
1926 switch (op->base.op) {
1927 case DRM_GPUVA_OP_MAP:
1928 if (!op->map.immediate && xe_vm_in_fault_mode(vm))
1929 break;
1930
1931 bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence,
1932 fence2);
1933 break;
1934 case DRM_GPUVA_OP_REMAP:
1935 unbind_op_commit(vm, tile, pt_update_ops,
1936 gpuva_to_vma(op->base.remap.unmap->va), fence,
1937 fence2);
1938
1939 if (op->remap.prev)
1940 bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
1941 fence, fence2);
1942 if (op->remap.next)
1943 bind_op_commit(vm, tile, pt_update_ops, op->remap.next,
1944 fence, fence2);
1945 break;
1946 case DRM_GPUVA_OP_UNMAP:
1947 unbind_op_commit(vm, tile, pt_update_ops,
1948 gpuva_to_vma(op->base.unmap.va), fence, fence2);
1949 break;
1950 case DRM_GPUVA_OP_PREFETCH:
1951 bind_op_commit(vm, tile, pt_update_ops,
1952 gpuva_to_vma(op->base.prefetch.va), fence, fence2);
1953 break;
1954 default:
1955 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1956 }
1957}
1958
1959static const struct xe_migrate_pt_update_ops migrate_ops = {
1960 .populate = xe_vm_populate_pgtable,
1961 .clear = xe_migrate_clear_pgtable_callback,
1962 .pre_commit = xe_pt_pre_commit,
1963};
1964
1965static const struct xe_migrate_pt_update_ops userptr_migrate_ops = {
1966 .populate = xe_vm_populate_pgtable,
1967 .clear = xe_migrate_clear_pgtable_callback,
1968 .pre_commit = xe_pt_userptr_pre_commit,
1969};
1970
1971/**
1972 * xe_pt_update_ops_run() - Run PT update operations
1973 * @tile: Tile of PT update operations
1974 * @vops: VMA operationa
1975 *
1976 * Run PT update operations which includes committing internal PT state changes,
1977 * creating job for PT update operations for leaf insertion / removal, and
1978 * installing job fence in various places.
1979 *
1980 * Return: fence on success, negative ERR_PTR on error.
1981 */
1982struct dma_fence *
1983xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
1984{
1985 struct xe_vm *vm = vops->vm;
1986 struct xe_vm_pgtable_update_ops *pt_update_ops =
1987 &vops->pt_update_ops[tile->id];
1988 struct dma_fence *fence;
1989 struct invalidation_fence *ifence = NULL, *mfence = NULL;
1990 struct dma_fence **fences = NULL;
1991 struct dma_fence_array *cf = NULL;
1992 struct xe_range_fence *rfence;
1993 struct xe_vma_op *op;
1994 int err = 0, i;
1995 struct xe_migrate_pt_update update = {
1996 .ops = pt_update_ops->needs_userptr_lock ?
1997 &userptr_migrate_ops :
1998 &migrate_ops,
1999 .vops = vops,
2000 .tile_id = tile->id,
2001 };
2002
2003 lockdep_assert_held(&vm->lock);
2004 xe_vm_assert_held(vm);
2005
2006 if (!pt_update_ops->current_op) {
2007 xe_tile_assert(tile, xe_vm_in_fault_mode(vm));
2008
2009 return dma_fence_get_stub();
2010 }
2011
2012#ifdef TEST_VM_OPS_ERROR
2013 if (vops->inject_error &&
2014 vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
2015 return ERR_PTR(-ENOSPC);
2016#endif
2017
2018 if (pt_update_ops->needs_invalidation) {
2019 ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
2020 if (!ifence) {
2021 err = -ENOMEM;
2022 goto kill_vm_tile1;
2023 }
2024 if (tile->media_gt) {
2025 mfence = kzalloc(sizeof(*ifence), GFP_KERNEL);
2026 if (!mfence) {
2027 err = -ENOMEM;
2028 goto free_ifence;
2029 }
2030 fences = kmalloc_array(2, sizeof(*fences), GFP_KERNEL);
2031 if (!fences) {
2032 err = -ENOMEM;
2033 goto free_ifence;
2034 }
2035 cf = dma_fence_array_alloc(2);
2036 if (!cf) {
2037 err = -ENOMEM;
2038 goto free_ifence;
2039 }
2040 }
2041 }
2042
2043 rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
2044 if (!rfence) {
2045 err = -ENOMEM;
2046 goto free_ifence;
2047 }
2048
2049 fence = xe_migrate_update_pgtables(tile->migrate, &update);
2050 if (IS_ERR(fence)) {
2051 err = PTR_ERR(fence);
2052 goto free_rfence;
2053 }
2054
2055 /* Point of no return - VM killed if failure after this */
2056 for (i = 0; i < pt_update_ops->current_op; ++i) {
2057 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
2058
2059 xe_pt_commit(pt_op->vma, pt_op->entries,
2060 pt_op->num_entries, &pt_update_ops->deferred);
2061 pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */
2062 }
2063
2064 if (xe_range_fence_insert(&vm->rftree[tile->id], rfence,
2065 &xe_range_fence_kfree_ops,
2066 pt_update_ops->start,
2067 pt_update_ops->last, fence))
2068 dma_fence_wait(fence, false);
2069
2070 /* tlb invalidation must be done before signaling rebind */
2071 if (ifence) {
2072 if (mfence)
2073 dma_fence_get(fence);
2074 invalidation_fence_init(tile->primary_gt, ifence, fence,
2075 pt_update_ops->start,
2076 pt_update_ops->last, vm->usm.asid);
2077 if (mfence) {
2078 invalidation_fence_init(tile->media_gt, mfence, fence,
2079 pt_update_ops->start,
2080 pt_update_ops->last, vm->usm.asid);
2081 fences[0] = &ifence->base.base;
2082 fences[1] = &mfence->base.base;
2083 dma_fence_array_init(cf, 2, fences,
2084 vm->composite_fence_ctx,
2085 vm->composite_fence_seqno++,
2086 false);
2087 fence = &cf->base;
2088 } else {
2089 fence = &ifence->base.base;
2090 }
2091 }
2092
2093 if (!mfence) {
2094 dma_resv_add_fence(xe_vm_resv(vm), fence,
2095 pt_update_ops->wait_vm_bookkeep ?
2096 DMA_RESV_USAGE_KERNEL :
2097 DMA_RESV_USAGE_BOOKKEEP);
2098
2099 list_for_each_entry(op, &vops->list, link)
2100 op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL);
2101 } else {
2102 dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base,
2103 pt_update_ops->wait_vm_bookkeep ?
2104 DMA_RESV_USAGE_KERNEL :
2105 DMA_RESV_USAGE_BOOKKEEP);
2106
2107 dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base,
2108 pt_update_ops->wait_vm_bookkeep ?
2109 DMA_RESV_USAGE_KERNEL :
2110 DMA_RESV_USAGE_BOOKKEEP);
2111
2112 list_for_each_entry(op, &vops->list, link)
2113 op_commit(vops->vm, tile, pt_update_ops, op,
2114 &ifence->base.base, &mfence->base.base);
2115 }
2116
2117 if (pt_update_ops->needs_userptr_lock)
2118 up_read(&vm->userptr.notifier_lock);
2119
2120 return fence;
2121
2122free_rfence:
2123 kfree(rfence);
2124free_ifence:
2125 kfree(cf);
2126 kfree(fences);
2127 kfree(mfence);
2128 kfree(ifence);
2129kill_vm_tile1:
2130 if (err != -EAGAIN && tile->id)
2131 xe_vm_kill(vops->vm, false);
2132
2133 return ERR_PTR(err);
2134}
2135
2136/**
2137 * xe_pt_update_ops_fini() - Finish PT update operations
2138 * @tile: Tile of PT update operations
2139 * @vops: VMA operations
2140 *
2141 * Finish PT update operations by committing to destroy page table memory
2142 */
2143void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
2144{
2145 struct xe_vm_pgtable_update_ops *pt_update_ops =
2146 &vops->pt_update_ops[tile->id];
2147 int i;
2148
2149 lockdep_assert_held(&vops->vm->lock);
2150 xe_vm_assert_held(vops->vm);
2151
2152 for (i = 0; i < pt_update_ops->current_op; ++i) {
2153 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
2154
2155 xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
2156 }
2157 xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
2158}
2159
2160/**
2161 * xe_pt_update_ops_abort() - Abort PT update operations
2162 * @tile: Tile of PT update operations
2163 * @vops: VMA operationa
2164 *
2165 * Abort PT update operations by unwinding internal PT state
2166 */
2167void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
2168{
2169 struct xe_vm_pgtable_update_ops *pt_update_ops =
2170 &vops->pt_update_ops[tile->id];
2171 int i;
2172
2173 lockdep_assert_held(&vops->vm->lock);
2174 xe_vm_assert_held(vops->vm);
2175
2176 for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
2177 struct xe_vm_pgtable_update_op *pt_op =
2178 &pt_update_ops->ops[i];
2179
2180 if (!pt_op->vma || i >= pt_update_ops->current_op)
2181 continue;
2182
2183 if (pt_op->bind)
2184 xe_pt_abort_bind(pt_op->vma, pt_op->entries,
2185 pt_op->num_entries,
2186 pt_op->rebind);
2187 else
2188 xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
2189 pt_op->num_entries);
2190 }
2191
2192 xe_pt_update_ops_fini(tile, vops);
2193}