Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include "xe_bo.h"
7
8#include <linux/dma-buf.h>
9
10#include <drm/drm_drv.h>
11#include <drm/drm_gem_ttm_helper.h>
12#include <drm/drm_managed.h>
13#include <drm/ttm/ttm_device.h>
14#include <drm/ttm/ttm_placement.h>
15#include <drm/ttm/ttm_tt.h>
16#include <drm/xe_drm.h>
17
18#include "xe_device.h"
19#include "xe_dma_buf.h"
20#include "xe_drm_client.h"
21#include "xe_ggtt.h"
22#include "xe_gt.h"
23#include "xe_map.h"
24#include "xe_migrate.h"
25#include "xe_preempt_fence.h"
26#include "xe_res_cursor.h"
27#include "xe_trace.h"
28#include "xe_ttm_stolen_mgr.h"
29#include "xe_vm.h"
30
31const char *const xe_mem_type_to_name[TTM_NUM_MEM_TYPES] = {
32 [XE_PL_SYSTEM] = "system",
33 [XE_PL_TT] = "gtt",
34 [XE_PL_VRAM0] = "vram0",
35 [XE_PL_VRAM1] = "vram1",
36 [XE_PL_STOLEN] = "stolen"
37};
38
39static const struct ttm_place sys_placement_flags = {
40 .fpfn = 0,
41 .lpfn = 0,
42 .mem_type = XE_PL_SYSTEM,
43 .flags = 0,
44};
45
46static struct ttm_placement sys_placement = {
47 .num_placement = 1,
48 .placement = &sys_placement_flags,
49};
50
51static const struct ttm_place tt_placement_flags[] = {
52 {
53 .fpfn = 0,
54 .lpfn = 0,
55 .mem_type = XE_PL_TT,
56 .flags = TTM_PL_FLAG_DESIRED,
57 },
58 {
59 .fpfn = 0,
60 .lpfn = 0,
61 .mem_type = XE_PL_SYSTEM,
62 .flags = TTM_PL_FLAG_FALLBACK,
63 }
64};
65
66static struct ttm_placement tt_placement = {
67 .num_placement = 2,
68 .placement = tt_placement_flags,
69};
70
71bool mem_type_is_vram(u32 mem_type)
72{
73 return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN;
74}
75
76static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res)
77{
78 return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe);
79}
80
81static bool resource_is_vram(struct ttm_resource *res)
82{
83 return mem_type_is_vram(res->mem_type);
84}
85
86bool xe_bo_is_vram(struct xe_bo *bo)
87{
88 return resource_is_vram(bo->ttm.resource) ||
89 resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource);
90}
91
92bool xe_bo_is_stolen(struct xe_bo *bo)
93{
94 return bo->ttm.resource->mem_type == XE_PL_STOLEN;
95}
96
97/**
98 * xe_bo_is_stolen_devmem - check if BO is of stolen type accessed via PCI BAR
99 * @bo: The BO
100 *
101 * The stolen memory is accessed through the PCI BAR for both DGFX and some
102 * integrated platforms that have a dedicated bit in the PTE for devmem (DM).
103 *
104 * Returns: true if it's stolen memory accessed via PCI BAR, false otherwise.
105 */
106bool xe_bo_is_stolen_devmem(struct xe_bo *bo)
107{
108 return xe_bo_is_stolen(bo) &&
109 GRAPHICS_VERx100(xe_bo_device(bo)) >= 1270;
110}
111
112static bool xe_bo_is_user(struct xe_bo *bo)
113{
114 return bo->flags & XE_BO_CREATE_USER_BIT;
115}
116
117static struct xe_migrate *
118mem_type_to_migrate(struct xe_device *xe, u32 mem_type)
119{
120 struct xe_tile *tile;
121
122 xe_assert(xe, mem_type == XE_PL_STOLEN || mem_type_is_vram(mem_type));
123 tile = &xe->tiles[mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0)];
124 return tile->migrate;
125}
126
127static struct xe_mem_region *res_to_mem_region(struct ttm_resource *res)
128{
129 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
130 struct ttm_resource_manager *mgr;
131
132 xe_assert(xe, resource_is_vram(res));
133 mgr = ttm_manager_type(&xe->ttm, res->mem_type);
134 return to_xe_ttm_vram_mgr(mgr)->vram;
135}
136
137static void try_add_system(struct xe_device *xe, struct xe_bo *bo,
138 u32 bo_flags, u32 *c)
139{
140 if (bo_flags & XE_BO_CREATE_SYSTEM_BIT) {
141 xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
142
143 bo->placements[*c] = (struct ttm_place) {
144 .mem_type = XE_PL_TT,
145 };
146 *c += 1;
147 }
148}
149
150static void add_vram(struct xe_device *xe, struct xe_bo *bo,
151 struct ttm_place *places, u32 bo_flags, u32 mem_type, u32 *c)
152{
153 struct ttm_place place = { .mem_type = mem_type };
154 struct xe_mem_region *vram;
155 u64 io_size;
156
157 xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
158
159 vram = to_xe_ttm_vram_mgr(ttm_manager_type(&xe->ttm, mem_type))->vram;
160 xe_assert(xe, vram && vram->usable_size);
161 io_size = vram->io_size;
162
163 /*
164 * For eviction / restore on suspend / resume objects
165 * pinned in VRAM must be contiguous
166 */
167 if (bo_flags & (XE_BO_CREATE_PINNED_BIT |
168 XE_BO_CREATE_GGTT_BIT))
169 place.flags |= TTM_PL_FLAG_CONTIGUOUS;
170
171 if (io_size < vram->usable_size) {
172 if (bo_flags & XE_BO_NEEDS_CPU_ACCESS) {
173 place.fpfn = 0;
174 place.lpfn = io_size >> PAGE_SHIFT;
175 } else {
176 place.flags |= TTM_PL_FLAG_TOPDOWN;
177 }
178 }
179 places[*c] = place;
180 *c += 1;
181}
182
183static void try_add_vram(struct xe_device *xe, struct xe_bo *bo,
184 u32 bo_flags, u32 *c)
185{
186 if (bo_flags & XE_BO_CREATE_VRAM0_BIT)
187 add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM0, c);
188 if (bo_flags & XE_BO_CREATE_VRAM1_BIT)
189 add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM1, c);
190}
191
192static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo,
193 u32 bo_flags, u32 *c)
194{
195 if (bo_flags & XE_BO_CREATE_STOLEN_BIT) {
196 xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
197
198 bo->placements[*c] = (struct ttm_place) {
199 .mem_type = XE_PL_STOLEN,
200 .flags = bo_flags & (XE_BO_CREATE_PINNED_BIT |
201 XE_BO_CREATE_GGTT_BIT) ?
202 TTM_PL_FLAG_CONTIGUOUS : 0,
203 };
204 *c += 1;
205 }
206}
207
208static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,
209 u32 bo_flags)
210{
211 u32 c = 0;
212
213 try_add_vram(xe, bo, bo_flags, &c);
214 try_add_system(xe, bo, bo_flags, &c);
215 try_add_stolen(xe, bo, bo_flags, &c);
216
217 if (!c)
218 return -EINVAL;
219
220 bo->placement = (struct ttm_placement) {
221 .num_placement = c,
222 .placement = bo->placements,
223 };
224
225 return 0;
226}
227
228int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,
229 u32 bo_flags)
230{
231 xe_bo_assert_held(bo);
232 return __xe_bo_placement_for_flags(xe, bo, bo_flags);
233}
234
235static void xe_evict_flags(struct ttm_buffer_object *tbo,
236 struct ttm_placement *placement)
237{
238 if (!xe_bo_is_xe_bo(tbo)) {
239 /* Don't handle scatter gather BOs */
240 if (tbo->type == ttm_bo_type_sg) {
241 placement->num_placement = 0;
242 return;
243 }
244
245 *placement = sys_placement;
246 return;
247 }
248
249 /*
250 * For xe, sg bos that are evicted to system just triggers a
251 * rebind of the sg list upon subsequent validation to XE_PL_TT.
252 */
253 switch (tbo->resource->mem_type) {
254 case XE_PL_VRAM0:
255 case XE_PL_VRAM1:
256 case XE_PL_STOLEN:
257 *placement = tt_placement;
258 break;
259 case XE_PL_TT:
260 default:
261 *placement = sys_placement;
262 break;
263 }
264}
265
266struct xe_ttm_tt {
267 struct ttm_tt ttm;
268 struct device *dev;
269 struct sg_table sgt;
270 struct sg_table *sg;
271};
272
273static int xe_tt_map_sg(struct ttm_tt *tt)
274{
275 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
276 unsigned long num_pages = tt->num_pages;
277 int ret;
278
279 XE_WARN_ON(tt->page_flags & TTM_TT_FLAG_EXTERNAL);
280
281 if (xe_tt->sg)
282 return 0;
283
284 ret = sg_alloc_table_from_pages_segment(&xe_tt->sgt, tt->pages,
285 num_pages, 0,
286 (u64)num_pages << PAGE_SHIFT,
287 xe_sg_segment_size(xe_tt->dev),
288 GFP_KERNEL);
289 if (ret)
290 return ret;
291
292 xe_tt->sg = &xe_tt->sgt;
293 ret = dma_map_sgtable(xe_tt->dev, xe_tt->sg, DMA_BIDIRECTIONAL,
294 DMA_ATTR_SKIP_CPU_SYNC);
295 if (ret) {
296 sg_free_table(xe_tt->sg);
297 xe_tt->sg = NULL;
298 return ret;
299 }
300
301 return 0;
302}
303
304struct sg_table *xe_bo_sg(struct xe_bo *bo)
305{
306 struct ttm_tt *tt = bo->ttm.ttm;
307 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
308
309 return xe_tt->sg;
310}
311
312static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
313 u32 page_flags)
314{
315 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
316 struct xe_device *xe = xe_bo_device(bo);
317 struct xe_ttm_tt *tt;
318 unsigned long extra_pages;
319 enum ttm_caching caching;
320 int err;
321
322 tt = kzalloc(sizeof(*tt), GFP_KERNEL);
323 if (!tt)
324 return NULL;
325
326 tt->dev = xe->drm.dev;
327
328 extra_pages = 0;
329 if (xe_bo_needs_ccs_pages(bo))
330 extra_pages = DIV_ROUND_UP(xe_device_ccs_bytes(xe, bo->size),
331 PAGE_SIZE);
332
333 switch (bo->cpu_caching) {
334 case DRM_XE_GEM_CPU_CACHING_WC:
335 caching = ttm_write_combined;
336 break;
337 default:
338 caching = ttm_cached;
339 break;
340 }
341
342 WARN_ON((bo->flags & XE_BO_CREATE_USER_BIT) && !bo->cpu_caching);
343
344 /*
345 * Display scanout is always non-coherent with the CPU cache.
346 *
347 * For Xe_LPG and beyond, PPGTT PTE lookups are also non-coherent and
348 * require a CPU:WC mapping.
349 */
350 if ((!bo->cpu_caching && bo->flags & XE_BO_SCANOUT_BIT) ||
351 (xe->info.graphics_verx100 >= 1270 && bo->flags & XE_BO_PAGETABLE))
352 caching = ttm_write_combined;
353
354 err = ttm_tt_init(&tt->ttm, &bo->ttm, page_flags, caching, extra_pages);
355 if (err) {
356 kfree(tt);
357 return NULL;
358 }
359
360 return &tt->ttm;
361}
362
363static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt,
364 struct ttm_operation_ctx *ctx)
365{
366 int err;
367
368 /*
369 * dma-bufs are not populated with pages, and the dma-
370 * addresses are set up when moved to XE_PL_TT.
371 */
372 if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)
373 return 0;
374
375 err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx);
376 if (err)
377 return err;
378
379 /* A follow up may move this xe_bo_move when BO is moved to XE_PL_TT */
380 err = xe_tt_map_sg(tt);
381 if (err)
382 ttm_pool_free(&ttm_dev->pool, tt);
383
384 return err;
385}
386
387static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt)
388{
389 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
390
391 if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)
392 return;
393
394 if (xe_tt->sg) {
395 dma_unmap_sgtable(xe_tt->dev, xe_tt->sg,
396 DMA_BIDIRECTIONAL, 0);
397 sg_free_table(xe_tt->sg);
398 xe_tt->sg = NULL;
399 }
400
401 return ttm_pool_free(&ttm_dev->pool, tt);
402}
403
404static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt)
405{
406 ttm_tt_fini(tt);
407 kfree(tt);
408}
409
410static int xe_ttm_io_mem_reserve(struct ttm_device *bdev,
411 struct ttm_resource *mem)
412{
413 struct xe_device *xe = ttm_to_xe_device(bdev);
414
415 switch (mem->mem_type) {
416 case XE_PL_SYSTEM:
417 case XE_PL_TT:
418 return 0;
419 case XE_PL_VRAM0:
420 case XE_PL_VRAM1: {
421 struct xe_ttm_vram_mgr_resource *vres =
422 to_xe_ttm_vram_mgr_resource(mem);
423 struct xe_mem_region *vram = res_to_mem_region(mem);
424
425 if (vres->used_visible_size < mem->size)
426 return -EINVAL;
427
428 mem->bus.offset = mem->start << PAGE_SHIFT;
429
430 if (vram->mapping &&
431 mem->placement & TTM_PL_FLAG_CONTIGUOUS)
432 mem->bus.addr = (u8 __force *)vram->mapping +
433 mem->bus.offset;
434
435 mem->bus.offset += vram->io_start;
436 mem->bus.is_iomem = true;
437
438#if !defined(CONFIG_X86)
439 mem->bus.caching = ttm_write_combined;
440#endif
441 return 0;
442 } case XE_PL_STOLEN:
443 return xe_ttm_stolen_io_mem_reserve(xe, mem);
444 default:
445 return -EINVAL;
446 }
447}
448
449static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
450 const struct ttm_operation_ctx *ctx)
451{
452 struct dma_resv_iter cursor;
453 struct dma_fence *fence;
454 struct drm_gem_object *obj = &bo->ttm.base;
455 struct drm_gpuvm_bo *vm_bo;
456 bool idle = false;
457 int ret = 0;
458
459 dma_resv_assert_held(bo->ttm.base.resv);
460
461 if (!list_empty(&bo->ttm.base.gpuva.list)) {
462 dma_resv_iter_begin(&cursor, bo->ttm.base.resv,
463 DMA_RESV_USAGE_BOOKKEEP);
464 dma_resv_for_each_fence_unlocked(&cursor, fence)
465 dma_fence_enable_sw_signaling(fence);
466 dma_resv_iter_end(&cursor);
467 }
468
469 drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
470 struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
471 struct drm_gpuva *gpuva;
472
473 if (!xe_vm_in_fault_mode(vm)) {
474 drm_gpuvm_bo_evict(vm_bo, true);
475 continue;
476 }
477
478 if (!idle) {
479 long timeout;
480
481 if (ctx->no_wait_gpu &&
482 !dma_resv_test_signaled(bo->ttm.base.resv,
483 DMA_RESV_USAGE_BOOKKEEP))
484 return -EBUSY;
485
486 timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
487 DMA_RESV_USAGE_BOOKKEEP,
488 ctx->interruptible,
489 MAX_SCHEDULE_TIMEOUT);
490 if (!timeout)
491 return -ETIME;
492 if (timeout < 0)
493 return timeout;
494
495 idle = true;
496 }
497
498 drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
499 struct xe_vma *vma = gpuva_to_vma(gpuva);
500
501 trace_xe_vma_evict(vma);
502 ret = xe_vm_invalidate_vma(vma);
503 if (XE_WARN_ON(ret))
504 return ret;
505 }
506 }
507
508 return ret;
509}
510
511/*
512 * The dma-buf map_attachment() / unmap_attachment() is hooked up here.
513 * Note that unmapping the attachment is deferred to the next
514 * map_attachment time, or to bo destroy (after idling) whichever comes first.
515 * This is to avoid syncing before unmap_attachment(), assuming that the
516 * caller relies on idling the reservation object before moving the
517 * backing store out. Should that assumption not hold, then we will be able
518 * to unconditionally call unmap_attachment() when moving out to system.
519 */
520static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
521 struct ttm_resource *new_res)
522{
523 struct dma_buf_attachment *attach = ttm_bo->base.import_attach;
524 struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt,
525 ttm);
526 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
527 struct sg_table *sg;
528
529 xe_assert(xe, attach);
530 xe_assert(xe, ttm_bo->ttm);
531
532 if (new_res->mem_type == XE_PL_SYSTEM)
533 goto out;
534
535 if (ttm_bo->sg) {
536 dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
537 ttm_bo->sg = NULL;
538 }
539
540 sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
541 if (IS_ERR(sg))
542 return PTR_ERR(sg);
543
544 ttm_bo->sg = sg;
545 xe_tt->sg = sg;
546
547out:
548 ttm_bo_move_null(ttm_bo, new_res);
549
550 return 0;
551}
552
553/**
554 * xe_bo_move_notify - Notify subsystems of a pending move
555 * @bo: The buffer object
556 * @ctx: The struct ttm_operation_ctx controlling locking and waits.
557 *
558 * This function notifies subsystems of an upcoming buffer move.
559 * Upon receiving such a notification, subsystems should schedule
560 * halting access to the underlying pages and optionally add a fence
561 * to the buffer object's dma_resv object, that signals when access is
562 * stopped. The caller will wait on all dma_resv fences before
563 * starting the move.
564 *
565 * A subsystem may commence access to the object after obtaining
566 * bindings to the new backing memory under the object lock.
567 *
568 * Return: 0 on success, -EINTR or -ERESTARTSYS if interrupted in fault mode,
569 * negative error code on error.
570 */
571static int xe_bo_move_notify(struct xe_bo *bo,
572 const struct ttm_operation_ctx *ctx)
573{
574 struct ttm_buffer_object *ttm_bo = &bo->ttm;
575 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
576 struct ttm_resource *old_mem = ttm_bo->resource;
577 u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM;
578 int ret;
579
580 /*
581 * If this starts to call into many components, consider
582 * using a notification chain here.
583 */
584
585 if (xe_bo_is_pinned(bo))
586 return -EINVAL;
587
588 xe_bo_vunmap(bo);
589 ret = xe_bo_trigger_rebind(xe, bo, ctx);
590 if (ret)
591 return ret;
592
593 /* Don't call move_notify() for imported dma-bufs. */
594 if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach)
595 dma_buf_move_notify(ttm_bo->base.dma_buf);
596
597 /*
598 * TTM has already nuked the mmap for us (see ttm_bo_unmap_virtual),
599 * so if we moved from VRAM make sure to unlink this from the userfault
600 * tracking.
601 */
602 if (mem_type_is_vram(old_mem_type)) {
603 mutex_lock(&xe->mem_access.vram_userfault.lock);
604 if (!list_empty(&bo->vram_userfault_link))
605 list_del_init(&bo->vram_userfault_link);
606 mutex_unlock(&xe->mem_access.vram_userfault.lock);
607 }
608
609 return 0;
610}
611
612static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
613 struct ttm_operation_ctx *ctx,
614 struct ttm_resource *new_mem,
615 struct ttm_place *hop)
616{
617 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
618 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
619 struct ttm_resource *old_mem = ttm_bo->resource;
620 u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM;
621 struct ttm_tt *ttm = ttm_bo->ttm;
622 struct xe_migrate *migrate = NULL;
623 struct dma_fence *fence;
624 bool move_lacks_source;
625 bool tt_has_data;
626 bool needs_clear;
627 bool handle_system_ccs = (!IS_DGFX(xe) && xe_bo_needs_ccs_pages(bo) &&
628 ttm && ttm_tt_is_populated(ttm)) ? true : false;
629 int ret = 0;
630 /* Bo creation path, moving to system or TT. */
631 if ((!old_mem && ttm) && !handle_system_ccs) {
632 ttm_bo_move_null(ttm_bo, new_mem);
633 return 0;
634 }
635
636 if (ttm_bo->type == ttm_bo_type_sg) {
637 ret = xe_bo_move_notify(bo, ctx);
638 if (!ret)
639 ret = xe_bo_move_dmabuf(ttm_bo, new_mem);
640 goto out;
641 }
642
643 tt_has_data = ttm && (ttm_tt_is_populated(ttm) ||
644 (ttm->page_flags & TTM_TT_FLAG_SWAPPED));
645
646 move_lacks_source = handle_system_ccs ? (!bo->ccs_cleared) :
647 (!mem_type_is_vram(old_mem_type) && !tt_has_data);
648
649 needs_clear = (ttm && ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) ||
650 (!ttm && ttm_bo->type == ttm_bo_type_device);
651
652 if ((move_lacks_source && !needs_clear)) {
653 ttm_bo_move_null(ttm_bo, new_mem);
654 goto out;
655 }
656
657 if (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT && !handle_system_ccs) {
658 ttm_bo_move_null(ttm_bo, new_mem);
659 goto out;
660 }
661
662 /*
663 * Failed multi-hop where the old_mem is still marked as
664 * TTM_PL_FLAG_TEMPORARY, should just be a dummy move.
665 */
666 if (old_mem_type == XE_PL_TT &&
667 new_mem->mem_type == XE_PL_TT) {
668 ttm_bo_move_null(ttm_bo, new_mem);
669 goto out;
670 }
671
672 if (!move_lacks_source && !xe_bo_is_pinned(bo)) {
673 ret = xe_bo_move_notify(bo, ctx);
674 if (ret)
675 goto out;
676 }
677
678 if (old_mem_type == XE_PL_TT &&
679 new_mem->mem_type == XE_PL_SYSTEM) {
680 long timeout = dma_resv_wait_timeout(ttm_bo->base.resv,
681 DMA_RESV_USAGE_BOOKKEEP,
682 true,
683 MAX_SCHEDULE_TIMEOUT);
684 if (timeout < 0) {
685 ret = timeout;
686 goto out;
687 }
688
689 if (!handle_system_ccs) {
690 ttm_bo_move_null(ttm_bo, new_mem);
691 goto out;
692 }
693 }
694
695 if (!move_lacks_source &&
696 ((old_mem_type == XE_PL_SYSTEM && resource_is_vram(new_mem)) ||
697 (mem_type_is_vram(old_mem_type) &&
698 new_mem->mem_type == XE_PL_SYSTEM))) {
699 hop->fpfn = 0;
700 hop->lpfn = 0;
701 hop->mem_type = XE_PL_TT;
702 hop->flags = TTM_PL_FLAG_TEMPORARY;
703 ret = -EMULTIHOP;
704 goto out;
705 }
706
707 if (bo->tile)
708 migrate = bo->tile->migrate;
709 else if (resource_is_vram(new_mem))
710 migrate = mem_type_to_migrate(xe, new_mem->mem_type);
711 else if (mem_type_is_vram(old_mem_type))
712 migrate = mem_type_to_migrate(xe, old_mem_type);
713 else
714 migrate = xe->tiles[0].migrate;
715
716 xe_assert(xe, migrate);
717 trace_xe_bo_move(bo, new_mem->mem_type, old_mem_type, move_lacks_source);
718 xe_device_mem_access_get(xe);
719
720 if (xe_bo_is_pinned(bo) && !xe_bo_is_user(bo)) {
721 /*
722 * Kernel memory that is pinned should only be moved on suspend
723 * / resume, some of the pinned memory is required for the
724 * device to resume / use the GPU to move other evicted memory
725 * (user memory) around. This likely could be optimized a bit
726 * futher where we find the minimum set of pinned memory
727 * required for resume but for simplity doing a memcpy for all
728 * pinned memory.
729 */
730 ret = xe_bo_vmap(bo);
731 if (!ret) {
732 ret = ttm_bo_move_memcpy(ttm_bo, ctx, new_mem);
733
734 /* Create a new VMAP once kernel BO back in VRAM */
735 if (!ret && resource_is_vram(new_mem)) {
736 struct xe_mem_region *vram = res_to_mem_region(new_mem);
737 void __iomem *new_addr = vram->mapping +
738 (new_mem->start << PAGE_SHIFT);
739
740 if (XE_WARN_ON(new_mem->start == XE_BO_INVALID_OFFSET)) {
741 ret = -EINVAL;
742 xe_device_mem_access_put(xe);
743 goto out;
744 }
745
746 xe_assert(xe, new_mem->start ==
747 bo->placements->fpfn);
748
749 iosys_map_set_vaddr_iomem(&bo->vmap, new_addr);
750 }
751 }
752 } else {
753 if (move_lacks_source)
754 fence = xe_migrate_clear(migrate, bo, new_mem);
755 else
756 fence = xe_migrate_copy(migrate, bo, bo, old_mem,
757 new_mem, handle_system_ccs);
758 if (IS_ERR(fence)) {
759 ret = PTR_ERR(fence);
760 xe_device_mem_access_put(xe);
761 goto out;
762 }
763 if (!move_lacks_source) {
764 ret = ttm_bo_move_accel_cleanup(ttm_bo, fence, evict,
765 true, new_mem);
766 if (ret) {
767 dma_fence_wait(fence, false);
768 ttm_bo_move_null(ttm_bo, new_mem);
769 ret = 0;
770 }
771 } else {
772 /*
773 * ttm_bo_move_accel_cleanup() may blow up if
774 * bo->resource == NULL, so just attach the
775 * fence and set the new resource.
776 */
777 dma_resv_add_fence(ttm_bo->base.resv, fence,
778 DMA_RESV_USAGE_KERNEL);
779 ttm_bo_move_null(ttm_bo, new_mem);
780 }
781
782 dma_fence_put(fence);
783 }
784
785 xe_device_mem_access_put(xe);
786
787out:
788 return ret;
789
790}
791
792/**
793 * xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory
794 * @bo: The buffer object to move.
795 *
796 * On successful completion, the object memory will be moved to sytem memory.
797 * This function blocks until the object has been fully moved.
798 *
799 * This is needed to for special handling of pinned VRAM object during
800 * suspend-resume.
801 *
802 * Return: 0 on success. Negative error code on failure.
803 */
804int xe_bo_evict_pinned(struct xe_bo *bo)
805{
806 struct ttm_place place = {
807 .mem_type = XE_PL_TT,
808 };
809 struct ttm_placement placement = {
810 .placement = &place,
811 .num_placement = 1,
812 };
813 struct ttm_operation_ctx ctx = {
814 .interruptible = false,
815 };
816 struct ttm_resource *new_mem;
817 int ret;
818
819 xe_bo_assert_held(bo);
820
821 if (WARN_ON(!bo->ttm.resource))
822 return -EINVAL;
823
824 if (WARN_ON(!xe_bo_is_pinned(bo)))
825 return -EINVAL;
826
827 if (WARN_ON(!xe_bo_is_vram(bo)))
828 return -EINVAL;
829
830 ret = ttm_bo_mem_space(&bo->ttm, &placement, &new_mem, &ctx);
831 if (ret)
832 return ret;
833
834 if (!bo->ttm.ttm) {
835 bo->ttm.ttm = xe_ttm_tt_create(&bo->ttm, 0);
836 if (!bo->ttm.ttm) {
837 ret = -ENOMEM;
838 goto err_res_free;
839 }
840 }
841
842 ret = ttm_tt_populate(bo->ttm.bdev, bo->ttm.ttm, &ctx);
843 if (ret)
844 goto err_res_free;
845
846 ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1);
847 if (ret)
848 goto err_res_free;
849
850 ret = xe_bo_move(&bo->ttm, false, &ctx, new_mem, NULL);
851 if (ret)
852 goto err_res_free;
853
854 dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
855 false, MAX_SCHEDULE_TIMEOUT);
856
857 return 0;
858
859err_res_free:
860 ttm_resource_free(&bo->ttm, &new_mem);
861 return ret;
862}
863
864/**
865 * xe_bo_restore_pinned() - Restore a pinned VRAM object
866 * @bo: The buffer object to move.
867 *
868 * On successful completion, the object memory will be moved back to VRAM.
869 * This function blocks until the object has been fully moved.
870 *
871 * This is needed to for special handling of pinned VRAM object during
872 * suspend-resume.
873 *
874 * Return: 0 on success. Negative error code on failure.
875 */
876int xe_bo_restore_pinned(struct xe_bo *bo)
877{
878 struct ttm_operation_ctx ctx = {
879 .interruptible = false,
880 };
881 struct ttm_resource *new_mem;
882 int ret;
883
884 xe_bo_assert_held(bo);
885
886 if (WARN_ON(!bo->ttm.resource))
887 return -EINVAL;
888
889 if (WARN_ON(!xe_bo_is_pinned(bo)))
890 return -EINVAL;
891
892 if (WARN_ON(xe_bo_is_vram(bo) || !bo->ttm.ttm))
893 return -EINVAL;
894
895 ret = ttm_bo_mem_space(&bo->ttm, &bo->placement, &new_mem, &ctx);
896 if (ret)
897 return ret;
898
899 ret = ttm_tt_populate(bo->ttm.bdev, bo->ttm.ttm, &ctx);
900 if (ret)
901 goto err_res_free;
902
903 ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1);
904 if (ret)
905 goto err_res_free;
906
907 ret = xe_bo_move(&bo->ttm, false, &ctx, new_mem, NULL);
908 if (ret)
909 goto err_res_free;
910
911 dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
912 false, MAX_SCHEDULE_TIMEOUT);
913
914 return 0;
915
916err_res_free:
917 ttm_resource_free(&bo->ttm, &new_mem);
918 return ret;
919}
920
921static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo,
922 unsigned long page_offset)
923{
924 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
925 struct xe_res_cursor cursor;
926 struct xe_mem_region *vram;
927
928 if (ttm_bo->resource->mem_type == XE_PL_STOLEN)
929 return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT;
930
931 vram = res_to_mem_region(ttm_bo->resource);
932 xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);
933 return (vram->io_start + cursor.start) >> PAGE_SHIFT;
934}
935
936static void __xe_bo_vunmap(struct xe_bo *bo);
937
938/*
939 * TODO: Move this function to TTM so we don't rely on how TTM does its
940 * locking, thereby abusing TTM internals.
941 */
942static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo)
943{
944 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
945 bool locked;
946
947 xe_assert(xe, !kref_read(&ttm_bo->kref));
948
949 /*
950 * We can typically only race with TTM trylocking under the
951 * lru_lock, which will immediately be unlocked again since
952 * the ttm_bo refcount is zero at this point. So trylocking *should*
953 * always succeed here, as long as we hold the lru lock.
954 */
955 spin_lock(&ttm_bo->bdev->lru_lock);
956 locked = dma_resv_trylock(ttm_bo->base.resv);
957 spin_unlock(&ttm_bo->bdev->lru_lock);
958 xe_assert(xe, locked);
959
960 return locked;
961}
962
963static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo)
964{
965 struct dma_resv_iter cursor;
966 struct dma_fence *fence;
967 struct dma_fence *replacement = NULL;
968 struct xe_bo *bo;
969
970 if (!xe_bo_is_xe_bo(ttm_bo))
971 return;
972
973 bo = ttm_to_xe_bo(ttm_bo);
974 xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount)));
975
976 /*
977 * Corner case where TTM fails to allocate memory and this BOs resv
978 * still points the VMs resv
979 */
980 if (ttm_bo->base.resv != &ttm_bo->base._resv)
981 return;
982
983 if (!xe_ttm_bo_lock_in_destructor(ttm_bo))
984 return;
985
986 /*
987 * Scrub the preempt fences if any. The unbind fence is already
988 * attached to the resv.
989 * TODO: Don't do this for external bos once we scrub them after
990 * unbind.
991 */
992 dma_resv_for_each_fence(&cursor, ttm_bo->base.resv,
993 DMA_RESV_USAGE_BOOKKEEP, fence) {
994 if (xe_fence_is_xe_preempt(fence) &&
995 !dma_fence_is_signaled(fence)) {
996 if (!replacement)
997 replacement = dma_fence_get_stub();
998
999 dma_resv_replace_fences(ttm_bo->base.resv,
1000 fence->context,
1001 replacement,
1002 DMA_RESV_USAGE_BOOKKEEP);
1003 }
1004 }
1005 dma_fence_put(replacement);
1006
1007 dma_resv_unlock(ttm_bo->base.resv);
1008}
1009
1010static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo)
1011{
1012 if (!xe_bo_is_xe_bo(ttm_bo))
1013 return;
1014
1015 /*
1016 * Object is idle and about to be destroyed. Release the
1017 * dma-buf attachment.
1018 */
1019 if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) {
1020 struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm,
1021 struct xe_ttm_tt, ttm);
1022
1023 dma_buf_unmap_attachment(ttm_bo->base.import_attach, ttm_bo->sg,
1024 DMA_BIDIRECTIONAL);
1025 ttm_bo->sg = NULL;
1026 xe_tt->sg = NULL;
1027 }
1028}
1029
1030const struct ttm_device_funcs xe_ttm_funcs = {
1031 .ttm_tt_create = xe_ttm_tt_create,
1032 .ttm_tt_populate = xe_ttm_tt_populate,
1033 .ttm_tt_unpopulate = xe_ttm_tt_unpopulate,
1034 .ttm_tt_destroy = xe_ttm_tt_destroy,
1035 .evict_flags = xe_evict_flags,
1036 .move = xe_bo_move,
1037 .io_mem_reserve = xe_ttm_io_mem_reserve,
1038 .io_mem_pfn = xe_ttm_io_mem_pfn,
1039 .release_notify = xe_ttm_bo_release_notify,
1040 .eviction_valuable = ttm_bo_eviction_valuable,
1041 .delete_mem_notify = xe_ttm_bo_delete_mem_notify,
1042};
1043
1044static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
1045{
1046 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
1047 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
1048
1049 if (bo->ttm.base.import_attach)
1050 drm_prime_gem_destroy(&bo->ttm.base, NULL);
1051 drm_gem_object_release(&bo->ttm.base);
1052
1053 xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list));
1054
1055 if (bo->ggtt_node.size)
1056 xe_ggtt_remove_bo(bo->tile->mem.ggtt, bo);
1057
1058#ifdef CONFIG_PROC_FS
1059 if (bo->client)
1060 xe_drm_client_remove_bo(bo);
1061#endif
1062
1063 if (bo->vm && xe_bo_is_user(bo))
1064 xe_vm_put(bo->vm);
1065
1066 mutex_lock(&xe->mem_access.vram_userfault.lock);
1067 if (!list_empty(&bo->vram_userfault_link))
1068 list_del(&bo->vram_userfault_link);
1069 mutex_unlock(&xe->mem_access.vram_userfault.lock);
1070
1071 kfree(bo);
1072}
1073
1074static void xe_gem_object_free(struct drm_gem_object *obj)
1075{
1076 /* Our BO reference counting scheme works as follows:
1077 *
1078 * The gem object kref is typically used throughout the driver,
1079 * and the gem object holds a ttm_buffer_object refcount, so
1080 * that when the last gem object reference is put, which is when
1081 * we end up in this function, we put also that ttm_buffer_object
1082 * refcount. Anything using gem interfaces is then no longer
1083 * allowed to access the object in a way that requires a gem
1084 * refcount, including locking the object.
1085 *
1086 * driver ttm callbacks is allowed to use the ttm_buffer_object
1087 * refcount directly if needed.
1088 */
1089 __xe_bo_vunmap(gem_to_xe_bo(obj));
1090 ttm_bo_put(container_of(obj, struct ttm_buffer_object, base));
1091}
1092
1093static void xe_gem_object_close(struct drm_gem_object *obj,
1094 struct drm_file *file_priv)
1095{
1096 struct xe_bo *bo = gem_to_xe_bo(obj);
1097
1098 if (bo->vm && !xe_vm_in_fault_mode(bo->vm)) {
1099 xe_assert(xe_bo_device(bo), xe_bo_is_user(bo));
1100
1101 xe_bo_lock(bo, false);
1102 ttm_bo_set_bulk_move(&bo->ttm, NULL);
1103 xe_bo_unlock(bo);
1104 }
1105}
1106
1107static vm_fault_t xe_gem_fault(struct vm_fault *vmf)
1108{
1109 struct ttm_buffer_object *tbo = vmf->vma->vm_private_data;
1110 struct drm_device *ddev = tbo->base.dev;
1111 struct xe_device *xe = to_xe_device(ddev);
1112 struct xe_bo *bo = ttm_to_xe_bo(tbo);
1113 bool needs_rpm = bo->flags & XE_BO_CREATE_VRAM_MASK;
1114 vm_fault_t ret;
1115 int idx;
1116
1117 if (needs_rpm)
1118 xe_device_mem_access_get(xe);
1119
1120 ret = ttm_bo_vm_reserve(tbo, vmf);
1121 if (ret)
1122 goto out;
1123
1124 if (drm_dev_enter(ddev, &idx)) {
1125 trace_xe_bo_cpu_fault(bo);
1126
1127 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
1128 TTM_BO_VM_NUM_PREFAULT);
1129 drm_dev_exit(idx);
1130 } else {
1131 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
1132 }
1133
1134 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
1135 goto out;
1136 /*
1137 * ttm_bo_vm_reserve() already has dma_resv_lock.
1138 */
1139 if (ret == VM_FAULT_NOPAGE && mem_type_is_vram(tbo->resource->mem_type)) {
1140 mutex_lock(&xe->mem_access.vram_userfault.lock);
1141 if (list_empty(&bo->vram_userfault_link))
1142 list_add(&bo->vram_userfault_link, &xe->mem_access.vram_userfault.list);
1143 mutex_unlock(&xe->mem_access.vram_userfault.lock);
1144 }
1145
1146 dma_resv_unlock(tbo->base.resv);
1147out:
1148 if (needs_rpm)
1149 xe_device_mem_access_put(xe);
1150
1151 return ret;
1152}
1153
1154static const struct vm_operations_struct xe_gem_vm_ops = {
1155 .fault = xe_gem_fault,
1156 .open = ttm_bo_vm_open,
1157 .close = ttm_bo_vm_close,
1158 .access = ttm_bo_vm_access
1159};
1160
1161static const struct drm_gem_object_funcs xe_gem_object_funcs = {
1162 .free = xe_gem_object_free,
1163 .close = xe_gem_object_close,
1164 .mmap = drm_gem_ttm_mmap,
1165 .export = xe_gem_prime_export,
1166 .vm_ops = &xe_gem_vm_ops,
1167};
1168
1169/**
1170 * xe_bo_alloc - Allocate storage for a struct xe_bo
1171 *
1172 * This funcition is intended to allocate storage to be used for input
1173 * to __xe_bo_create_locked(), in the case a pointer to the bo to be
1174 * created is needed before the call to __xe_bo_create_locked().
1175 * If __xe_bo_create_locked ends up never to be called, then the
1176 * storage allocated with this function needs to be freed using
1177 * xe_bo_free().
1178 *
1179 * Return: A pointer to an uninitialized struct xe_bo on success,
1180 * ERR_PTR(-ENOMEM) on error.
1181 */
1182struct xe_bo *xe_bo_alloc(void)
1183{
1184 struct xe_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1185
1186 if (!bo)
1187 return ERR_PTR(-ENOMEM);
1188
1189 return bo;
1190}
1191
1192/**
1193 * xe_bo_free - Free storage allocated using xe_bo_alloc()
1194 * @bo: The buffer object storage.
1195 *
1196 * Refer to xe_bo_alloc() documentation for valid use-cases.
1197 */
1198void xe_bo_free(struct xe_bo *bo)
1199{
1200 kfree(bo);
1201}
1202
1203struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
1204 struct xe_tile *tile, struct dma_resv *resv,
1205 struct ttm_lru_bulk_move *bulk, size_t size,
1206 u16 cpu_caching, enum ttm_bo_type type,
1207 u32 flags)
1208{
1209 struct ttm_operation_ctx ctx = {
1210 .interruptible = true,
1211 .no_wait_gpu = false,
1212 };
1213 struct ttm_placement *placement;
1214 uint32_t alignment;
1215 size_t aligned_size;
1216 int err;
1217
1218 /* Only kernel objects should set GT */
1219 xe_assert(xe, !tile || type == ttm_bo_type_kernel);
1220
1221 if (XE_WARN_ON(!size)) {
1222 xe_bo_free(bo);
1223 return ERR_PTR(-EINVAL);
1224 }
1225
1226 if (flags & (XE_BO_CREATE_VRAM_MASK | XE_BO_CREATE_STOLEN_BIT) &&
1227 !(flags & XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT) &&
1228 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) {
1229 aligned_size = ALIGN(size, SZ_64K);
1230 if (type != ttm_bo_type_device)
1231 size = ALIGN(size, SZ_64K);
1232 flags |= XE_BO_INTERNAL_64K;
1233 alignment = SZ_64K >> PAGE_SHIFT;
1234
1235 } else {
1236 aligned_size = ALIGN(size, SZ_4K);
1237 flags &= ~XE_BO_INTERNAL_64K;
1238 alignment = SZ_4K >> PAGE_SHIFT;
1239 }
1240
1241 if (type == ttm_bo_type_device && aligned_size != size)
1242 return ERR_PTR(-EINVAL);
1243
1244 if (!bo) {
1245 bo = xe_bo_alloc();
1246 if (IS_ERR(bo))
1247 return bo;
1248 }
1249
1250 bo->ccs_cleared = false;
1251 bo->tile = tile;
1252 bo->size = size;
1253 bo->flags = flags;
1254 bo->cpu_caching = cpu_caching;
1255 bo->ttm.base.funcs = &xe_gem_object_funcs;
1256 bo->ttm.priority = XE_BO_PRIORITY_NORMAL;
1257 INIT_LIST_HEAD(&bo->pinned_link);
1258#ifdef CONFIG_PROC_FS
1259 INIT_LIST_HEAD(&bo->client_link);
1260#endif
1261 INIT_LIST_HEAD(&bo->vram_userfault_link);
1262
1263 drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size);
1264
1265 if (resv) {
1266 ctx.allow_res_evict = !(flags & XE_BO_CREATE_NO_RESV_EVICT);
1267 ctx.resv = resv;
1268 }
1269
1270 if (!(flags & XE_BO_FIXED_PLACEMENT_BIT)) {
1271 err = __xe_bo_placement_for_flags(xe, bo, bo->flags);
1272 if (WARN_ON(err)) {
1273 xe_ttm_bo_destroy(&bo->ttm);
1274 return ERR_PTR(err);
1275 }
1276 }
1277
1278 /* Defer populating type_sg bos */
1279 placement = (type == ttm_bo_type_sg ||
1280 bo->flags & XE_BO_DEFER_BACKING) ? &sys_placement :
1281 &bo->placement;
1282 err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type,
1283 placement, alignment,
1284 &ctx, NULL, resv, xe_ttm_bo_destroy);
1285 if (err)
1286 return ERR_PTR(err);
1287
1288 /*
1289 * The VRAM pages underneath are potentially still being accessed by the
1290 * GPU, as per async GPU clearing and async evictions. However TTM makes
1291 * sure to add any corresponding move/clear fences into the objects
1292 * dma-resv using the DMA_RESV_USAGE_KERNEL slot.
1293 *
1294 * For KMD internal buffers we don't care about GPU clearing, however we
1295 * still need to handle async evictions, where the VRAM is still being
1296 * accessed by the GPU. Most internal callers are not expecting this,
1297 * since they are missing the required synchronisation before accessing
1298 * the memory. To keep things simple just sync wait any kernel fences
1299 * here, if the buffer is designated KMD internal.
1300 *
1301 * For normal userspace objects we should already have the required
1302 * pipelining or sync waiting elsewhere, since we already have to deal
1303 * with things like async GPU clearing.
1304 */
1305 if (type == ttm_bo_type_kernel) {
1306 long timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
1307 DMA_RESV_USAGE_KERNEL,
1308 ctx.interruptible,
1309 MAX_SCHEDULE_TIMEOUT);
1310
1311 if (timeout < 0) {
1312 if (!resv)
1313 dma_resv_unlock(bo->ttm.base.resv);
1314 xe_bo_put(bo);
1315 return ERR_PTR(timeout);
1316 }
1317 }
1318
1319 bo->created = true;
1320 if (bulk)
1321 ttm_bo_set_bulk_move(&bo->ttm, bulk);
1322 else
1323 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
1324
1325 return bo;
1326}
1327
1328static int __xe_bo_fixed_placement(struct xe_device *xe,
1329 struct xe_bo *bo,
1330 u32 flags,
1331 u64 start, u64 end, u64 size)
1332{
1333 struct ttm_place *place = bo->placements;
1334
1335 if (flags & (XE_BO_CREATE_USER_BIT|XE_BO_CREATE_SYSTEM_BIT))
1336 return -EINVAL;
1337
1338 place->flags = TTM_PL_FLAG_CONTIGUOUS;
1339 place->fpfn = start >> PAGE_SHIFT;
1340 place->lpfn = end >> PAGE_SHIFT;
1341
1342 switch (flags & (XE_BO_CREATE_STOLEN_BIT | XE_BO_CREATE_VRAM_MASK)) {
1343 case XE_BO_CREATE_VRAM0_BIT:
1344 place->mem_type = XE_PL_VRAM0;
1345 break;
1346 case XE_BO_CREATE_VRAM1_BIT:
1347 place->mem_type = XE_PL_VRAM1;
1348 break;
1349 case XE_BO_CREATE_STOLEN_BIT:
1350 place->mem_type = XE_PL_STOLEN;
1351 break;
1352
1353 default:
1354 /* 0 or multiple of the above set */
1355 return -EINVAL;
1356 }
1357
1358 bo->placement = (struct ttm_placement) {
1359 .num_placement = 1,
1360 .placement = place,
1361 };
1362
1363 return 0;
1364}
1365
1366static struct xe_bo *
1367__xe_bo_create_locked(struct xe_device *xe,
1368 struct xe_tile *tile, struct xe_vm *vm,
1369 size_t size, u64 start, u64 end,
1370 u16 cpu_caching, enum ttm_bo_type type, u32 flags)
1371{
1372 struct xe_bo *bo = NULL;
1373 int err;
1374
1375 if (vm)
1376 xe_vm_assert_held(vm);
1377
1378 if (start || end != ~0ULL) {
1379 bo = xe_bo_alloc();
1380 if (IS_ERR(bo))
1381 return bo;
1382
1383 flags |= XE_BO_FIXED_PLACEMENT_BIT;
1384 err = __xe_bo_fixed_placement(xe, bo, flags, start, end, size);
1385 if (err) {
1386 xe_bo_free(bo);
1387 return ERR_PTR(err);
1388 }
1389 }
1390
1391 bo = ___xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL,
1392 vm && !xe_vm_in_fault_mode(vm) &&
1393 flags & XE_BO_CREATE_USER_BIT ?
1394 &vm->lru_bulk_move : NULL, size,
1395 cpu_caching, type, flags);
1396 if (IS_ERR(bo))
1397 return bo;
1398
1399 /*
1400 * Note that instead of taking a reference no the drm_gpuvm_resv_bo(),
1401 * to ensure the shared resv doesn't disappear under the bo, the bo
1402 * will keep a reference to the vm, and avoid circular references
1403 * by having all the vm's bo refereferences released at vm close
1404 * time.
1405 */
1406 if (vm && xe_bo_is_user(bo))
1407 xe_vm_get(vm);
1408 bo->vm = vm;
1409
1410 if (bo->flags & XE_BO_CREATE_GGTT_BIT) {
1411 if (!tile && flags & XE_BO_CREATE_STOLEN_BIT)
1412 tile = xe_device_get_root_tile(xe);
1413
1414 xe_assert(xe, tile);
1415
1416 if (flags & XE_BO_FIXED_PLACEMENT_BIT) {
1417 err = xe_ggtt_insert_bo_at(tile->mem.ggtt, bo,
1418 start + bo->size, U64_MAX);
1419 } else {
1420 err = xe_ggtt_insert_bo(tile->mem.ggtt, bo);
1421 }
1422 if (err)
1423 goto err_unlock_put_bo;
1424 }
1425
1426 return bo;
1427
1428err_unlock_put_bo:
1429 __xe_bo_unset_bulk_move(bo);
1430 xe_bo_unlock_vm_held(bo);
1431 xe_bo_put(bo);
1432 return ERR_PTR(err);
1433}
1434
1435struct xe_bo *
1436xe_bo_create_locked_range(struct xe_device *xe,
1437 struct xe_tile *tile, struct xe_vm *vm,
1438 size_t size, u64 start, u64 end,
1439 enum ttm_bo_type type, u32 flags)
1440{
1441 return __xe_bo_create_locked(xe, tile, vm, size, start, end, 0, type, flags);
1442}
1443
1444struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile,
1445 struct xe_vm *vm, size_t size,
1446 enum ttm_bo_type type, u32 flags)
1447{
1448 return __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 0, type, flags);
1449}
1450
1451struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile,
1452 struct xe_vm *vm, size_t size,
1453 u16 cpu_caching,
1454 enum ttm_bo_type type,
1455 u32 flags)
1456{
1457 struct xe_bo *bo = __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL,
1458 cpu_caching, type,
1459 flags | XE_BO_CREATE_USER_BIT);
1460 if (!IS_ERR(bo))
1461 xe_bo_unlock_vm_held(bo);
1462
1463 return bo;
1464}
1465
1466struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile,
1467 struct xe_vm *vm, size_t size,
1468 enum ttm_bo_type type, u32 flags)
1469{
1470 struct xe_bo *bo = xe_bo_create_locked(xe, tile, vm, size, type, flags);
1471
1472 if (!IS_ERR(bo))
1473 xe_bo_unlock_vm_held(bo);
1474
1475 return bo;
1476}
1477
1478struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile,
1479 struct xe_vm *vm,
1480 size_t size, u64 offset,
1481 enum ttm_bo_type type, u32 flags)
1482{
1483 struct xe_bo *bo;
1484 int err;
1485 u64 start = offset == ~0ull ? 0 : offset;
1486 u64 end = offset == ~0ull ? offset : start + size;
1487
1488 if (flags & XE_BO_CREATE_STOLEN_BIT &&
1489 xe_ttm_stolen_cpu_access_needs_ggtt(xe))
1490 flags |= XE_BO_CREATE_GGTT_BIT;
1491
1492 bo = xe_bo_create_locked_range(xe, tile, vm, size, start, end, type,
1493 flags | XE_BO_NEEDS_CPU_ACCESS);
1494 if (IS_ERR(bo))
1495 return bo;
1496
1497 err = xe_bo_pin(bo);
1498 if (err)
1499 goto err_put;
1500
1501 err = xe_bo_vmap(bo);
1502 if (err)
1503 goto err_unpin;
1504
1505 xe_bo_unlock_vm_held(bo);
1506
1507 return bo;
1508
1509err_unpin:
1510 xe_bo_unpin(bo);
1511err_put:
1512 xe_bo_unlock_vm_held(bo);
1513 xe_bo_put(bo);
1514 return ERR_PTR(err);
1515}
1516
1517struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,
1518 struct xe_vm *vm, size_t size,
1519 enum ttm_bo_type type, u32 flags)
1520{
1521 return xe_bo_create_pin_map_at(xe, tile, vm, size, ~0ull, type, flags);
1522}
1523
1524struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,
1525 const void *data, size_t size,
1526 enum ttm_bo_type type, u32 flags)
1527{
1528 struct xe_bo *bo = xe_bo_create_pin_map(xe, tile, NULL,
1529 ALIGN(size, PAGE_SIZE),
1530 type, flags);
1531 if (IS_ERR(bo))
1532 return bo;
1533
1534 xe_map_memcpy_to(xe, &bo->vmap, 0, data, size);
1535
1536 return bo;
1537}
1538
1539static void __xe_bo_unpin_map_no_vm(struct drm_device *drm, void *arg)
1540{
1541 xe_bo_unpin_map_no_vm(arg);
1542}
1543
1544struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,
1545 size_t size, u32 flags)
1546{
1547 struct xe_bo *bo;
1548 int ret;
1549
1550 bo = xe_bo_create_pin_map(xe, tile, NULL, size, ttm_bo_type_kernel, flags);
1551 if (IS_ERR(bo))
1552 return bo;
1553
1554 ret = drmm_add_action_or_reset(&xe->drm, __xe_bo_unpin_map_no_vm, bo);
1555 if (ret)
1556 return ERR_PTR(ret);
1557
1558 return bo;
1559}
1560
1561struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,
1562 const void *data, size_t size, u32 flags)
1563{
1564 struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, ALIGN(size, PAGE_SIZE), flags);
1565
1566 if (IS_ERR(bo))
1567 return bo;
1568
1569 xe_map_memcpy_to(xe, &bo->vmap, 0, data, size);
1570
1571 return bo;
1572}
1573
1574/**
1575 * xe_managed_bo_reinit_in_vram
1576 * @xe: xe device
1577 * @tile: Tile where the new buffer will be created
1578 * @src: Managed buffer object allocated in system memory
1579 *
1580 * Replace a managed src buffer object allocated in system memory with a new
1581 * one allocated in vram, copying the data between them.
1582 * Buffer object in VRAM is not going to have the same GGTT address, the caller
1583 * is responsible for making sure that any old references to it are updated.
1584 *
1585 * Returns 0 for success, negative error code otherwise.
1586 */
1587int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src)
1588{
1589 struct xe_bo *bo;
1590
1591 xe_assert(xe, IS_DGFX(xe));
1592 xe_assert(xe, !(*src)->vmap.is_iomem);
1593
1594 bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr, (*src)->size,
1595 XE_BO_CREATE_VRAM_IF_DGFX(tile) |
1596 XE_BO_CREATE_GGTT_BIT);
1597 if (IS_ERR(bo))
1598 return PTR_ERR(bo);
1599
1600 drmm_release_action(&xe->drm, __xe_bo_unpin_map_no_vm, *src);
1601 *src = bo;
1602
1603 return 0;
1604}
1605
1606/*
1607 * XXX: This is in the VM bind data path, likely should calculate this once and
1608 * store, with a recalculation if the BO is moved.
1609 */
1610uint64_t vram_region_gpu_offset(struct ttm_resource *res)
1611{
1612 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
1613
1614 if (res->mem_type == XE_PL_STOLEN)
1615 return xe_ttm_stolen_gpu_offset(xe);
1616
1617 return res_to_mem_region(res)->dpa_base;
1618}
1619
1620/**
1621 * xe_bo_pin_external - pin an external BO
1622 * @bo: buffer object to be pinned
1623 *
1624 * Pin an external (not tied to a VM, can be exported via dma-buf / prime FD)
1625 * BO. Unique call compared to xe_bo_pin as this function has it own set of
1626 * asserts and code to ensure evict / restore on suspend / resume.
1627 *
1628 * Returns 0 for success, negative error code otherwise.
1629 */
1630int xe_bo_pin_external(struct xe_bo *bo)
1631{
1632 struct xe_device *xe = xe_bo_device(bo);
1633 int err;
1634
1635 xe_assert(xe, !bo->vm);
1636 xe_assert(xe, xe_bo_is_user(bo));
1637
1638 if (!xe_bo_is_pinned(bo)) {
1639 err = xe_bo_validate(bo, NULL, false);
1640 if (err)
1641 return err;
1642
1643 if (xe_bo_is_vram(bo)) {
1644 spin_lock(&xe->pinned.lock);
1645 list_add_tail(&bo->pinned_link,
1646 &xe->pinned.external_vram);
1647 spin_unlock(&xe->pinned.lock);
1648 }
1649 }
1650
1651 ttm_bo_pin(&bo->ttm);
1652
1653 /*
1654 * FIXME: If we always use the reserve / unreserve functions for locking
1655 * we do not need this.
1656 */
1657 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
1658
1659 return 0;
1660}
1661
1662int xe_bo_pin(struct xe_bo *bo)
1663{
1664 struct xe_device *xe = xe_bo_device(bo);
1665 int err;
1666
1667 /* We currently don't expect user BO to be pinned */
1668 xe_assert(xe, !xe_bo_is_user(bo));
1669
1670 /* Pinned object must be in GGTT or have pinned flag */
1671 xe_assert(xe, bo->flags & (XE_BO_CREATE_PINNED_BIT |
1672 XE_BO_CREATE_GGTT_BIT));
1673
1674 /*
1675 * No reason we can't support pinning imported dma-bufs we just don't
1676 * expect to pin an imported dma-buf.
1677 */
1678 xe_assert(xe, !bo->ttm.base.import_attach);
1679
1680 /* We only expect at most 1 pin */
1681 xe_assert(xe, !xe_bo_is_pinned(bo));
1682
1683 err = xe_bo_validate(bo, NULL, false);
1684 if (err)
1685 return err;
1686
1687 /*
1688 * For pinned objects in on DGFX, which are also in vram, we expect
1689 * these to be in contiguous VRAM memory. Required eviction / restore
1690 * during suspend / resume (force restore to same physical address).
1691 */
1692 if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) &&
1693 bo->flags & XE_BO_INTERNAL_TEST)) {
1694 struct ttm_place *place = &(bo->placements[0]);
1695
1696 if (mem_type_is_vram(place->mem_type)) {
1697 xe_assert(xe, place->flags & TTM_PL_FLAG_CONTIGUOUS);
1698
1699 place->fpfn = (xe_bo_addr(bo, 0, PAGE_SIZE) -
1700 vram_region_gpu_offset(bo->ttm.resource)) >> PAGE_SHIFT;
1701 place->lpfn = place->fpfn + (bo->size >> PAGE_SHIFT);
1702
1703 spin_lock(&xe->pinned.lock);
1704 list_add_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present);
1705 spin_unlock(&xe->pinned.lock);
1706 }
1707 }
1708
1709 ttm_bo_pin(&bo->ttm);
1710
1711 /*
1712 * FIXME: If we always use the reserve / unreserve functions for locking
1713 * we do not need this.
1714 */
1715 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
1716
1717 return 0;
1718}
1719
1720/**
1721 * xe_bo_unpin_external - unpin an external BO
1722 * @bo: buffer object to be unpinned
1723 *
1724 * Unpin an external (not tied to a VM, can be exported via dma-buf / prime FD)
1725 * BO. Unique call compared to xe_bo_unpin as this function has it own set of
1726 * asserts and code to ensure evict / restore on suspend / resume.
1727 *
1728 * Returns 0 for success, negative error code otherwise.
1729 */
1730void xe_bo_unpin_external(struct xe_bo *bo)
1731{
1732 struct xe_device *xe = xe_bo_device(bo);
1733
1734 xe_assert(xe, !bo->vm);
1735 xe_assert(xe, xe_bo_is_pinned(bo));
1736 xe_assert(xe, xe_bo_is_user(bo));
1737
1738 if (bo->ttm.pin_count == 1 && !list_empty(&bo->pinned_link)) {
1739 spin_lock(&xe->pinned.lock);
1740 list_del_init(&bo->pinned_link);
1741 spin_unlock(&xe->pinned.lock);
1742 }
1743
1744 ttm_bo_unpin(&bo->ttm);
1745
1746 /*
1747 * FIXME: If we always use the reserve / unreserve functions for locking
1748 * we do not need this.
1749 */
1750 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
1751}
1752
1753void xe_bo_unpin(struct xe_bo *bo)
1754{
1755 struct xe_device *xe = xe_bo_device(bo);
1756
1757 xe_assert(xe, !bo->ttm.base.import_attach);
1758 xe_assert(xe, xe_bo_is_pinned(bo));
1759
1760 if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) &&
1761 bo->flags & XE_BO_INTERNAL_TEST)) {
1762 struct ttm_place *place = &(bo->placements[0]);
1763
1764 if (mem_type_is_vram(place->mem_type)) {
1765 xe_assert(xe, !list_empty(&bo->pinned_link));
1766
1767 spin_lock(&xe->pinned.lock);
1768 list_del_init(&bo->pinned_link);
1769 spin_unlock(&xe->pinned.lock);
1770 }
1771 }
1772
1773 ttm_bo_unpin(&bo->ttm);
1774}
1775
1776/**
1777 * xe_bo_validate() - Make sure the bo is in an allowed placement
1778 * @bo: The bo,
1779 * @vm: Pointer to a the vm the bo shares a locked dma_resv object with, or
1780 * NULL. Used together with @allow_res_evict.
1781 * @allow_res_evict: Whether it's allowed to evict bos sharing @vm's
1782 * reservation object.
1783 *
1784 * Make sure the bo is in allowed placement, migrating it if necessary. If
1785 * needed, other bos will be evicted. If bos selected for eviction shares
1786 * the @vm's reservation object, they can be evicted iff @allow_res_evict is
1787 * set to true, otherwise they will be bypassed.
1788 *
1789 * Return: 0 on success, negative error code on failure. May return
1790 * -EINTR or -ERESTARTSYS if internal waits are interrupted by a signal.
1791 */
1792int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict)
1793{
1794 struct ttm_operation_ctx ctx = {
1795 .interruptible = true,
1796 .no_wait_gpu = false,
1797 };
1798
1799 if (vm) {
1800 lockdep_assert_held(&vm->lock);
1801 xe_vm_assert_held(vm);
1802
1803 ctx.allow_res_evict = allow_res_evict;
1804 ctx.resv = xe_vm_resv(vm);
1805 }
1806
1807 return ttm_bo_validate(&bo->ttm, &bo->placement, &ctx);
1808}
1809
1810bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo)
1811{
1812 if (bo->destroy == &xe_ttm_bo_destroy)
1813 return true;
1814
1815 return false;
1816}
1817
1818/*
1819 * Resolve a BO address. There is no assert to check if the proper lock is held
1820 * so it should only be used in cases where it is not fatal to get the wrong
1821 * address, such as printing debug information, but not in cases where memory is
1822 * written based on this result.
1823 */
1824dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size)
1825{
1826 struct xe_device *xe = xe_bo_device(bo);
1827 struct xe_res_cursor cur;
1828 u64 page;
1829
1830 xe_assert(xe, page_size <= PAGE_SIZE);
1831 page = offset >> PAGE_SHIFT;
1832 offset &= (PAGE_SIZE - 1);
1833
1834 if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) {
1835 xe_assert(xe, bo->ttm.ttm);
1836
1837 xe_res_first_sg(xe_bo_sg(bo), page << PAGE_SHIFT,
1838 page_size, &cur);
1839 return xe_res_dma(&cur) + offset;
1840 } else {
1841 struct xe_res_cursor cur;
1842
1843 xe_res_first(bo->ttm.resource, page << PAGE_SHIFT,
1844 page_size, &cur);
1845 return cur.start + offset + vram_region_gpu_offset(bo->ttm.resource);
1846 }
1847}
1848
1849dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size)
1850{
1851 if (!READ_ONCE(bo->ttm.pin_count))
1852 xe_bo_assert_held(bo);
1853 return __xe_bo_addr(bo, offset, page_size);
1854}
1855
1856int xe_bo_vmap(struct xe_bo *bo)
1857{
1858 void *virtual;
1859 bool is_iomem;
1860 int ret;
1861
1862 xe_bo_assert_held(bo);
1863
1864 if (!(bo->flags & XE_BO_NEEDS_CPU_ACCESS))
1865 return -EINVAL;
1866
1867 if (!iosys_map_is_null(&bo->vmap))
1868 return 0;
1869
1870 /*
1871 * We use this more or less deprecated interface for now since
1872 * ttm_bo_vmap() doesn't offer the optimization of kmapping
1873 * single page bos, which is done here.
1874 * TODO: Fix up ttm_bo_vmap to do that, or fix up ttm_bo_kmap
1875 * to use struct iosys_map.
1876 */
1877 ret = ttm_bo_kmap(&bo->ttm, 0, bo->size >> PAGE_SHIFT, &bo->kmap);
1878 if (ret)
1879 return ret;
1880
1881 virtual = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
1882 if (is_iomem)
1883 iosys_map_set_vaddr_iomem(&bo->vmap, (void __iomem *)virtual);
1884 else
1885 iosys_map_set_vaddr(&bo->vmap, virtual);
1886
1887 return 0;
1888}
1889
1890static void __xe_bo_vunmap(struct xe_bo *bo)
1891{
1892 if (!iosys_map_is_null(&bo->vmap)) {
1893 iosys_map_clear(&bo->vmap);
1894 ttm_bo_kunmap(&bo->kmap);
1895 }
1896}
1897
1898void xe_bo_vunmap(struct xe_bo *bo)
1899{
1900 xe_bo_assert_held(bo);
1901 __xe_bo_vunmap(bo);
1902}
1903
1904int xe_gem_create_ioctl(struct drm_device *dev, void *data,
1905 struct drm_file *file)
1906{
1907 struct xe_device *xe = to_xe_device(dev);
1908 struct xe_file *xef = to_xe_file(file);
1909 struct drm_xe_gem_create *args = data;
1910 struct xe_vm *vm = NULL;
1911 struct xe_bo *bo;
1912 unsigned int bo_flags;
1913 u32 handle;
1914 int err;
1915
1916 if (XE_IOCTL_DBG(xe, args->extensions) ||
1917 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||
1918 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1919 return -EINVAL;
1920
1921 /* at least one valid memory placement must be specified */
1922 if (XE_IOCTL_DBG(xe, (args->placement & ~xe->info.mem_region_mask) ||
1923 !args->placement))
1924 return -EINVAL;
1925
1926 if (XE_IOCTL_DBG(xe, args->flags &
1927 ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING |
1928 DRM_XE_GEM_CREATE_FLAG_SCANOUT |
1929 DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM)))
1930 return -EINVAL;
1931
1932 if (XE_IOCTL_DBG(xe, args->handle))
1933 return -EINVAL;
1934
1935 if (XE_IOCTL_DBG(xe, !args->size))
1936 return -EINVAL;
1937
1938 if (XE_IOCTL_DBG(xe, args->size > SIZE_MAX))
1939 return -EINVAL;
1940
1941 if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK))
1942 return -EINVAL;
1943
1944 bo_flags = 0;
1945 if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING)
1946 bo_flags |= XE_BO_DEFER_BACKING;
1947
1948 if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)
1949 bo_flags |= XE_BO_SCANOUT_BIT;
1950
1951 bo_flags |= args->placement << (ffs(XE_BO_CREATE_SYSTEM_BIT) - 1);
1952
1953 if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) {
1954 if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_CREATE_VRAM_MASK)))
1955 return -EINVAL;
1956
1957 bo_flags |= XE_BO_NEEDS_CPU_ACCESS;
1958 }
1959
1960 if (XE_IOCTL_DBG(xe, !args->cpu_caching ||
1961 args->cpu_caching > DRM_XE_GEM_CPU_CACHING_WC))
1962 return -EINVAL;
1963
1964 if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_CREATE_VRAM_MASK &&
1965 args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC))
1966 return -EINVAL;
1967
1968 if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_SCANOUT_BIT &&
1969 args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB))
1970 return -EINVAL;
1971
1972 if (args->vm_id) {
1973 vm = xe_vm_lookup(xef, args->vm_id);
1974 if (XE_IOCTL_DBG(xe, !vm))
1975 return -ENOENT;
1976 err = xe_vm_lock(vm, true);
1977 if (err)
1978 goto out_vm;
1979 }
1980
1981 bo = xe_bo_create_user(xe, NULL, vm, args->size, args->cpu_caching,
1982 ttm_bo_type_device, bo_flags);
1983
1984 if (vm)
1985 xe_vm_unlock(vm);
1986
1987 if (IS_ERR(bo)) {
1988 err = PTR_ERR(bo);
1989 goto out_vm;
1990 }
1991
1992 err = drm_gem_handle_create(file, &bo->ttm.base, &handle);
1993 if (err)
1994 goto out_bulk;
1995
1996 args->handle = handle;
1997 goto out_put;
1998
1999out_bulk:
2000 if (vm && !xe_vm_in_fault_mode(vm)) {
2001 xe_vm_lock(vm, false);
2002 __xe_bo_unset_bulk_move(bo);
2003 xe_vm_unlock(vm);
2004 }
2005out_put:
2006 xe_bo_put(bo);
2007out_vm:
2008 if (vm)
2009 xe_vm_put(vm);
2010
2011 return err;
2012}
2013
2014int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
2015 struct drm_file *file)
2016{
2017 struct xe_device *xe = to_xe_device(dev);
2018 struct drm_xe_gem_mmap_offset *args = data;
2019 struct drm_gem_object *gem_obj;
2020
2021 if (XE_IOCTL_DBG(xe, args->extensions) ||
2022 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
2023 return -EINVAL;
2024
2025 if (XE_IOCTL_DBG(xe, args->flags))
2026 return -EINVAL;
2027
2028 gem_obj = drm_gem_object_lookup(file, args->handle);
2029 if (XE_IOCTL_DBG(xe, !gem_obj))
2030 return -ENOENT;
2031
2032 /* The mmap offset was set up at BO allocation time. */
2033 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
2034
2035 xe_bo_put(gem_to_xe_bo(gem_obj));
2036 return 0;
2037}
2038
2039/**
2040 * xe_bo_lock() - Lock the buffer object's dma_resv object
2041 * @bo: The struct xe_bo whose lock is to be taken
2042 * @intr: Whether to perform any wait interruptible
2043 *
2044 * Locks the buffer object's dma_resv object. If the buffer object is
2045 * pointing to a shared dma_resv object, that shared lock is locked.
2046 *
2047 * Return: 0 on success, -EINTR if @intr is true and the wait for a
2048 * contended lock was interrupted. If @intr is set to false, the
2049 * function always returns 0.
2050 */
2051int xe_bo_lock(struct xe_bo *bo, bool intr)
2052{
2053 if (intr)
2054 return dma_resv_lock_interruptible(bo->ttm.base.resv, NULL);
2055
2056 dma_resv_lock(bo->ttm.base.resv, NULL);
2057
2058 return 0;
2059}
2060
2061/**
2062 * xe_bo_unlock() - Unlock the buffer object's dma_resv object
2063 * @bo: The struct xe_bo whose lock is to be released.
2064 *
2065 * Unlock a buffer object lock that was locked by xe_bo_lock().
2066 */
2067void xe_bo_unlock(struct xe_bo *bo)
2068{
2069 dma_resv_unlock(bo->ttm.base.resv);
2070}
2071
2072/**
2073 * xe_bo_can_migrate - Whether a buffer object likely can be migrated
2074 * @bo: The buffer object to migrate
2075 * @mem_type: The TTM memory type intended to migrate to
2076 *
2077 * Check whether the buffer object supports migration to the
2078 * given memory type. Note that pinning may affect the ability to migrate as
2079 * returned by this function.
2080 *
2081 * This function is primarily intended as a helper for checking the
2082 * possibility to migrate buffer objects and can be called without
2083 * the object lock held.
2084 *
2085 * Return: true if migration is possible, false otherwise.
2086 */
2087bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type)
2088{
2089 unsigned int cur_place;
2090
2091 if (bo->ttm.type == ttm_bo_type_kernel)
2092 return true;
2093
2094 if (bo->ttm.type == ttm_bo_type_sg)
2095 return false;
2096
2097 for (cur_place = 0; cur_place < bo->placement.num_placement;
2098 cur_place++) {
2099 if (bo->placements[cur_place].mem_type == mem_type)
2100 return true;
2101 }
2102
2103 return false;
2104}
2105
2106static void xe_place_from_ttm_type(u32 mem_type, struct ttm_place *place)
2107{
2108 memset(place, 0, sizeof(*place));
2109 place->mem_type = mem_type;
2110}
2111
2112/**
2113 * xe_bo_migrate - Migrate an object to the desired region id
2114 * @bo: The buffer object to migrate.
2115 * @mem_type: The TTM region type to migrate to.
2116 *
2117 * Attempt to migrate the buffer object to the desired memory region. The
2118 * buffer object may not be pinned, and must be locked.
2119 * On successful completion, the object memory type will be updated,
2120 * but an async migration task may not have completed yet, and to
2121 * accomplish that, the object's kernel fences must be signaled with
2122 * the object lock held.
2123 *
2124 * Return: 0 on success. Negative error code on failure. In particular may
2125 * return -EINTR or -ERESTARTSYS if signal pending.
2126 */
2127int xe_bo_migrate(struct xe_bo *bo, u32 mem_type)
2128{
2129 struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev);
2130 struct ttm_operation_ctx ctx = {
2131 .interruptible = true,
2132 .no_wait_gpu = false,
2133 };
2134 struct ttm_placement placement;
2135 struct ttm_place requested;
2136
2137 xe_bo_assert_held(bo);
2138
2139 if (bo->ttm.resource->mem_type == mem_type)
2140 return 0;
2141
2142 if (xe_bo_is_pinned(bo))
2143 return -EBUSY;
2144
2145 if (!xe_bo_can_migrate(bo, mem_type))
2146 return -EINVAL;
2147
2148 xe_place_from_ttm_type(mem_type, &requested);
2149 placement.num_placement = 1;
2150 placement.placement = &requested;
2151
2152 /*
2153 * Stolen needs to be handled like below VRAM handling if we ever need
2154 * to support it.
2155 */
2156 drm_WARN_ON(&xe->drm, mem_type == XE_PL_STOLEN);
2157
2158 if (mem_type_is_vram(mem_type)) {
2159 u32 c = 0;
2160
2161 add_vram(xe, bo, &requested, bo->flags, mem_type, &c);
2162 }
2163
2164 return ttm_bo_validate(&bo->ttm, &placement, &ctx);
2165}
2166
2167/**
2168 * xe_bo_evict - Evict an object to evict placement
2169 * @bo: The buffer object to migrate.
2170 * @force_alloc: Set force_alloc in ttm_operation_ctx
2171 *
2172 * On successful completion, the object memory will be moved to evict
2173 * placement. Ths function blocks until the object has been fully moved.
2174 *
2175 * Return: 0 on success. Negative error code on failure.
2176 */
2177int xe_bo_evict(struct xe_bo *bo, bool force_alloc)
2178{
2179 struct ttm_operation_ctx ctx = {
2180 .interruptible = false,
2181 .no_wait_gpu = false,
2182 .force_alloc = force_alloc,
2183 };
2184 struct ttm_placement placement;
2185 int ret;
2186
2187 xe_evict_flags(&bo->ttm, &placement);
2188 ret = ttm_bo_validate(&bo->ttm, &placement, &ctx);
2189 if (ret)
2190 return ret;
2191
2192 dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
2193 false, MAX_SCHEDULE_TIMEOUT);
2194
2195 return 0;
2196}
2197
2198/**
2199 * xe_bo_needs_ccs_pages - Whether a bo needs to back up CCS pages when
2200 * placed in system memory.
2201 * @bo: The xe_bo
2202 *
2203 * Return: true if extra pages need to be allocated, false otherwise.
2204 */
2205bool xe_bo_needs_ccs_pages(struct xe_bo *bo)
2206{
2207 struct xe_device *xe = xe_bo_device(bo);
2208
2209 if (!xe_device_has_flat_ccs(xe) || bo->ttm.type != ttm_bo_type_device)
2210 return false;
2211
2212 /* On discrete GPUs, if the GPU can access this buffer from
2213 * system memory (i.e., it allows XE_PL_TT placement), FlatCCS
2214 * can't be used since there's no CCS storage associated with
2215 * non-VRAM addresses.
2216 */
2217 if (IS_DGFX(xe) && (bo->flags & XE_BO_CREATE_SYSTEM_BIT))
2218 return false;
2219
2220 return true;
2221}
2222
2223/**
2224 * __xe_bo_release_dummy() - Dummy kref release function
2225 * @kref: The embedded struct kref.
2226 *
2227 * Dummy release function for xe_bo_put_deferred(). Keep off.
2228 */
2229void __xe_bo_release_dummy(struct kref *kref)
2230{
2231}
2232
2233/**
2234 * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred().
2235 * @deferred: The lockless list used for the call to xe_bo_put_deferred().
2236 *
2237 * Puts all bos whose put was deferred by xe_bo_put_deferred().
2238 * The @deferred list can be either an onstack local list or a global
2239 * shared list used by a workqueue.
2240 */
2241void xe_bo_put_commit(struct llist_head *deferred)
2242{
2243 struct llist_node *freed;
2244 struct xe_bo *bo, *next;
2245
2246 if (!deferred)
2247 return;
2248
2249 freed = llist_del_all(deferred);
2250 if (!freed)
2251 return;
2252
2253 llist_for_each_entry_safe(bo, next, freed, freed)
2254 drm_gem_object_free(&bo->ttm.base.refcount);
2255}
2256
2257/**
2258 * xe_bo_dumb_create - Create a dumb bo as backing for a fb
2259 * @file_priv: ...
2260 * @dev: ...
2261 * @args: ...
2262 *
2263 * See dumb_create() hook in include/drm/drm_drv.h
2264 *
2265 * Return: ...
2266 */
2267int xe_bo_dumb_create(struct drm_file *file_priv,
2268 struct drm_device *dev,
2269 struct drm_mode_create_dumb *args)
2270{
2271 struct xe_device *xe = to_xe_device(dev);
2272 struct xe_bo *bo;
2273 uint32_t handle;
2274 int cpp = DIV_ROUND_UP(args->bpp, 8);
2275 int err;
2276 u32 page_size = max_t(u32, PAGE_SIZE,
2277 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K);
2278
2279 args->pitch = ALIGN(args->width * cpp, 64);
2280 args->size = ALIGN(mul_u32_u32(args->pitch, args->height),
2281 page_size);
2282
2283 bo = xe_bo_create_user(xe, NULL, NULL, args->size,
2284 DRM_XE_GEM_CPU_CACHING_WC,
2285 ttm_bo_type_device,
2286 XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
2287 XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT |
2288 XE_BO_NEEDS_CPU_ACCESS);
2289 if (IS_ERR(bo))
2290 return PTR_ERR(bo);
2291
2292 err = drm_gem_handle_create(file_priv, &bo->ttm.base, &handle);
2293 /* drop reference from allocate - handle holds it now */
2294 drm_gem_object_put(&bo->ttm.base);
2295 if (!err)
2296 args->handle = handle;
2297 return err;
2298}
2299
2300void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo)
2301{
2302 struct ttm_buffer_object *tbo = &bo->ttm;
2303 struct ttm_device *bdev = tbo->bdev;
2304
2305 drm_vma_node_unmap(&tbo->base.vma_node, bdev->dev_mapping);
2306
2307 list_del_init(&bo->vram_userfault_link);
2308}
2309
2310#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
2311#include "tests/xe_bo.c"
2312#endif