Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include "pvr_vm.h"
5
6#include "pvr_device.h"
7#include "pvr_drv.h"
8#include "pvr_gem.h"
9#include "pvr_mmu.h"
10#include "pvr_rogue_fwif.h"
11#include "pvr_rogue_heap_config.h"
12
13#include <drm/drm_exec.h>
14#include <drm/drm_gem.h>
15#include <drm/drm_gpuvm.h>
16
17#include <linux/container_of.h>
18#include <linux/err.h>
19#include <linux/errno.h>
20#include <linux/gfp_types.h>
21#include <linux/kref.h>
22#include <linux/mutex.h>
23#include <linux/stddef.h>
24
25/**
26 * DOC: Memory context
27 *
28 * This is the "top level" datatype in the VM code. It's exposed in the public
29 * API as an opaque handle.
30 */
31
32/**
33 * struct pvr_vm_context - Context type used to represent a single VM.
34 */
35struct pvr_vm_context {
36 /**
37 * @pvr_dev: The PowerVR device to which this context is bound.
38 * This binding is immutable for the life of the context.
39 */
40 struct pvr_device *pvr_dev;
41
42 /** @mmu_ctx: The context for binding to physical memory. */
43 struct pvr_mmu_context *mmu_ctx;
44
45 /** @gpuvm_mgr: GPUVM object associated with this context. */
46 struct drm_gpuvm gpuvm_mgr;
47
48 /** @lock: Global lock on this VM. */
49 struct mutex lock;
50
51 /**
52 * @fw_mem_ctx_obj: Firmware object representing firmware memory
53 * context.
54 */
55 struct pvr_fw_object *fw_mem_ctx_obj;
56
57 /** @ref_count: Reference count of object. */
58 struct kref ref_count;
59
60 /**
61 * @dummy_gem: GEM object to enable VM reservation. All private BOs
62 * should use the @dummy_gem.resv and not their own _resv field.
63 */
64 struct drm_gem_object dummy_gem;
65};
66
67static inline
68struct pvr_vm_context *to_pvr_vm_context(struct drm_gpuvm *gpuvm)
69{
70 return container_of(gpuvm, struct pvr_vm_context, gpuvm_mgr);
71}
72
73struct pvr_vm_context *pvr_vm_context_get(struct pvr_vm_context *vm_ctx)
74{
75 if (vm_ctx)
76 kref_get(&vm_ctx->ref_count);
77
78 return vm_ctx;
79}
80
81/**
82 * pvr_vm_get_page_table_root_addr() - Get the DMA address of the root of the
83 * page table structure behind a VM context.
84 * @vm_ctx: Target VM context.
85 */
86dma_addr_t pvr_vm_get_page_table_root_addr(struct pvr_vm_context *vm_ctx)
87{
88 return pvr_mmu_get_root_table_dma_addr(vm_ctx->mmu_ctx);
89}
90
91/**
92 * pvr_vm_get_dma_resv() - Expose the dma_resv owned by the VM context.
93 * @vm_ctx: Target VM context.
94 *
95 * This is used to allow private BOs to share a dma_resv for faster fence
96 * updates.
97 *
98 * Returns: The dma_resv pointer.
99 */
100struct dma_resv *pvr_vm_get_dma_resv(struct pvr_vm_context *vm_ctx)
101{
102 return vm_ctx->dummy_gem.resv;
103}
104
105/**
106 * DOC: Memory mappings
107 */
108
109/**
110 * struct pvr_vm_gpuva - Wrapper type representing a single VM mapping.
111 */
112struct pvr_vm_gpuva {
113 /** @base: The wrapped drm_gpuva object. */
114 struct drm_gpuva base;
115};
116
117enum pvr_vm_bind_type {
118 PVR_VM_BIND_TYPE_MAP,
119 PVR_VM_BIND_TYPE_UNMAP,
120};
121
122/**
123 * struct pvr_vm_bind_op - Context of a map/unmap operation.
124 */
125struct pvr_vm_bind_op {
126 /** @type: Map or unmap. */
127 enum pvr_vm_bind_type type;
128
129 /** @pvr_obj: Object associated with mapping (map only). */
130 struct pvr_gem_object *pvr_obj;
131
132 /**
133 * @vm_ctx: VM context where the mapping will be created or destroyed.
134 */
135 struct pvr_vm_context *vm_ctx;
136
137 /** @mmu_op_ctx: MMU op context. */
138 struct pvr_mmu_op_context *mmu_op_ctx;
139
140 /** @gpuvm_bo: Prealloced wrapped BO for attaching to the gpuvm. */
141 struct drm_gpuvm_bo *gpuvm_bo;
142
143 /**
144 * @new_va: Prealloced VA mapping object (init in callback).
145 * Used when creating a mapping.
146 */
147 struct pvr_vm_gpuva *new_va;
148
149 /**
150 * @prev_va: Prealloced VA mapping object (init in callback).
151 * Used when a mapping or unmapping operation overlaps an existing
152 * mapping and splits away the beginning into a new mapping.
153 */
154 struct pvr_vm_gpuva *prev_va;
155
156 /**
157 * @next_va: Prealloced VA mapping object (init in callback).
158 * Used when a mapping or unmapping operation overlaps an existing
159 * mapping and splits away the end into a new mapping.
160 */
161 struct pvr_vm_gpuva *next_va;
162
163 /** @offset: Offset into @pvr_obj to begin mapping from. */
164 u64 offset;
165
166 /** @device_addr: Device-virtual address at the start of the mapping. */
167 u64 device_addr;
168
169 /** @size: Size of the desired mapping. */
170 u64 size;
171};
172
173/**
174 * pvr_vm_bind_op_exec() - Execute a single bind op.
175 * @bind_op: Bind op context.
176 *
177 * Returns:
178 * * 0 on success,
179 * * Any error code returned by drm_gpuva_sm_map(), drm_gpuva_sm_unmap(), or
180 * a callback function.
181 */
182static int pvr_vm_bind_op_exec(struct pvr_vm_bind_op *bind_op)
183{
184 switch (bind_op->type) {
185 case PVR_VM_BIND_TYPE_MAP:
186 return drm_gpuvm_sm_map(&bind_op->vm_ctx->gpuvm_mgr,
187 bind_op, bind_op->device_addr,
188 bind_op->size,
189 gem_from_pvr_gem(bind_op->pvr_obj),
190 bind_op->offset);
191
192 case PVR_VM_BIND_TYPE_UNMAP:
193 return drm_gpuvm_sm_unmap(&bind_op->vm_ctx->gpuvm_mgr,
194 bind_op, bind_op->device_addr,
195 bind_op->size);
196 }
197
198 /*
199 * This shouldn't happen unless something went wrong
200 * in drm_sched.
201 */
202 WARN_ON(1);
203 return -EINVAL;
204}
205
206static void pvr_vm_bind_op_fini(struct pvr_vm_bind_op *bind_op)
207{
208 drm_gpuvm_bo_put(bind_op->gpuvm_bo);
209
210 kfree(bind_op->new_va);
211 kfree(bind_op->prev_va);
212 kfree(bind_op->next_va);
213
214 if (bind_op->pvr_obj)
215 pvr_gem_object_put(bind_op->pvr_obj);
216
217 if (bind_op->mmu_op_ctx)
218 pvr_mmu_op_context_destroy(bind_op->mmu_op_ctx);
219}
220
221static int
222pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op,
223 struct pvr_vm_context *vm_ctx,
224 struct pvr_gem_object *pvr_obj, u64 offset,
225 u64 device_addr, u64 size)
226{
227 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);
228 const bool is_user = vm_ctx != vm_ctx->pvr_dev->kernel_vm_ctx;
229 const u64 pvr_obj_size = pvr_gem_object_size(pvr_obj);
230 struct sg_table *sgt;
231 u64 offset_plus_size;
232 int err;
233
234 if (check_add_overflow(offset, size, &offset_plus_size))
235 return -EINVAL;
236
237 if (is_user &&
238 !pvr_find_heap_containing(vm_ctx->pvr_dev, device_addr, size)) {
239 return -EINVAL;
240 }
241
242 if (!pvr_device_addr_and_size_are_valid(vm_ctx, device_addr, size) ||
243 offset & ~PAGE_MASK || size & ~PAGE_MASK ||
244 offset >= pvr_obj_size || offset_plus_size > pvr_obj_size)
245 return -EINVAL;
246
247 bind_op->type = PVR_VM_BIND_TYPE_MAP;
248
249 dma_resv_lock(obj->resv, NULL);
250 bind_op->gpuvm_bo = drm_gpuvm_bo_obtain(&vm_ctx->gpuvm_mgr, obj);
251 dma_resv_unlock(obj->resv);
252 if (IS_ERR(bind_op->gpuvm_bo))
253 return PTR_ERR(bind_op->gpuvm_bo);
254
255 bind_op->new_va = kzalloc(sizeof(*bind_op->new_va), GFP_KERNEL);
256 bind_op->prev_va = kzalloc(sizeof(*bind_op->prev_va), GFP_KERNEL);
257 bind_op->next_va = kzalloc(sizeof(*bind_op->next_va), GFP_KERNEL);
258 if (!bind_op->new_va || !bind_op->prev_va || !bind_op->next_va) {
259 err = -ENOMEM;
260 goto err_bind_op_fini;
261 }
262
263 /* Pin pages so they're ready for use. */
264 sgt = pvr_gem_object_get_pages_sgt(pvr_obj);
265 err = PTR_ERR_OR_ZERO(sgt);
266 if (err)
267 goto err_bind_op_fini;
268
269 bind_op->mmu_op_ctx =
270 pvr_mmu_op_context_create(vm_ctx->mmu_ctx, sgt, offset, size);
271 err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx);
272 if (err) {
273 bind_op->mmu_op_ctx = NULL;
274 goto err_bind_op_fini;
275 }
276
277 bind_op->pvr_obj = pvr_obj;
278 bind_op->vm_ctx = vm_ctx;
279 bind_op->device_addr = device_addr;
280 bind_op->size = size;
281 bind_op->offset = offset;
282
283 return 0;
284
285err_bind_op_fini:
286 pvr_vm_bind_op_fini(bind_op);
287
288 return err;
289}
290
291static int
292pvr_vm_bind_op_unmap_init(struct pvr_vm_bind_op *bind_op,
293 struct pvr_vm_context *vm_ctx, u64 device_addr,
294 u64 size)
295{
296 int err;
297
298 if (!pvr_device_addr_and_size_are_valid(vm_ctx, device_addr, size))
299 return -EINVAL;
300
301 bind_op->type = PVR_VM_BIND_TYPE_UNMAP;
302
303 bind_op->prev_va = kzalloc(sizeof(*bind_op->prev_va), GFP_KERNEL);
304 bind_op->next_va = kzalloc(sizeof(*bind_op->next_va), GFP_KERNEL);
305 if (!bind_op->prev_va || !bind_op->next_va) {
306 err = -ENOMEM;
307 goto err_bind_op_fini;
308 }
309
310 bind_op->mmu_op_ctx =
311 pvr_mmu_op_context_create(vm_ctx->mmu_ctx, NULL, 0, 0);
312 err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx);
313 if (err) {
314 bind_op->mmu_op_ctx = NULL;
315 goto err_bind_op_fini;
316 }
317
318 bind_op->vm_ctx = vm_ctx;
319 bind_op->device_addr = device_addr;
320 bind_op->size = size;
321
322 return 0;
323
324err_bind_op_fini:
325 pvr_vm_bind_op_fini(bind_op);
326
327 return err;
328}
329
330/**
331 * pvr_vm_gpuva_map() - Insert a mapping into a memory context.
332 * @op: gpuva op containing the remap details.
333 * @op_ctx: Operation context.
334 *
335 * Context: Called by drm_gpuvm_sm_map following a successful mapping while
336 * @op_ctx.vm_ctx mutex is held.
337 *
338 * Return:
339 * * 0 on success, or
340 * * Any error returned by pvr_mmu_map().
341 */
342static int
343pvr_vm_gpuva_map(struct drm_gpuva_op *op, void *op_ctx)
344{
345 struct pvr_gem_object *pvr_gem = gem_to_pvr_gem(op->map.gem.obj);
346 struct pvr_vm_bind_op *ctx = op_ctx;
347 int err;
348
349 if ((op->map.gem.offset | op->map.va.range) & ~PVR_DEVICE_PAGE_MASK)
350 return -EINVAL;
351
352 err = pvr_mmu_map(ctx->mmu_op_ctx, op->map.va.range, pvr_gem->flags,
353 op->map.va.addr);
354 if (err)
355 return err;
356
357 drm_gpuva_map(&ctx->vm_ctx->gpuvm_mgr, &ctx->new_va->base, &op->map);
358 drm_gpuva_link(&ctx->new_va->base, ctx->gpuvm_bo);
359 ctx->new_va = NULL;
360
361 return 0;
362}
363
364/**
365 * pvr_vm_gpuva_unmap() - Remove a mapping from a memory context.
366 * @op: gpuva op containing the unmap details.
367 * @op_ctx: Operation context.
368 *
369 * Context: Called by drm_gpuvm_sm_unmap following a successful unmapping while
370 * @op_ctx.vm_ctx mutex is held.
371 *
372 * Return:
373 * * 0 on success, or
374 * * Any error returned by pvr_mmu_unmap().
375 */
376static int
377pvr_vm_gpuva_unmap(struct drm_gpuva_op *op, void *op_ctx)
378{
379 struct pvr_vm_bind_op *ctx = op_ctx;
380
381 int err = pvr_mmu_unmap(ctx->mmu_op_ctx, op->unmap.va->va.addr,
382 op->unmap.va->va.range);
383
384 if (err)
385 return err;
386
387 drm_gpuva_unmap(&op->unmap);
388 drm_gpuva_unlink(op->unmap.va);
389
390 return 0;
391}
392
393/**
394 * pvr_vm_gpuva_remap() - Remap a mapping within a memory context.
395 * @op: gpuva op containing the remap details.
396 * @op_ctx: Operation context.
397 *
398 * Context: Called by either drm_gpuvm_sm_map or drm_gpuvm_sm_unmap when a
399 * mapping or unmapping operation causes a region to be split. The
400 * @op_ctx.vm_ctx mutex is held.
401 *
402 * Return:
403 * * 0 on success, or
404 * * Any error returned by pvr_vm_gpuva_unmap() or pvr_vm_gpuva_unmap().
405 */
406static int
407pvr_vm_gpuva_remap(struct drm_gpuva_op *op, void *op_ctx)
408{
409 struct pvr_vm_bind_op *ctx = op_ctx;
410 u64 va_start = 0, va_range = 0;
411 int err;
412
413 drm_gpuva_op_remap_to_unmap_range(&op->remap, &va_start, &va_range);
414 err = pvr_mmu_unmap(ctx->mmu_op_ctx, va_start, va_range);
415 if (err)
416 return err;
417
418 /* No actual remap required: the page table tree depth is fixed to 3,
419 * and we use 4k page table entries only for now.
420 */
421 drm_gpuva_remap(&ctx->prev_va->base, &ctx->next_va->base, &op->remap);
422
423 if (op->remap.prev) {
424 pvr_gem_object_get(gem_to_pvr_gem(ctx->prev_va->base.gem.obj));
425 drm_gpuva_link(&ctx->prev_va->base, ctx->gpuvm_bo);
426 ctx->prev_va = NULL;
427 }
428
429 if (op->remap.next) {
430 pvr_gem_object_get(gem_to_pvr_gem(ctx->next_va->base.gem.obj));
431 drm_gpuva_link(&ctx->next_va->base, ctx->gpuvm_bo);
432 ctx->next_va = NULL;
433 }
434
435 drm_gpuva_unlink(op->remap.unmap->va);
436
437 return 0;
438}
439
440/*
441 * Public API
442 *
443 * For an overview of these functions, see *DOC: Public API* in "pvr_vm.h".
444 */
445
446/**
447 * pvr_device_addr_is_valid() - Tests whether a device-virtual address
448 * is valid.
449 * @device_addr: Virtual device address to test.
450 *
451 * Return:
452 * * %true if @device_addr is within the valid range for a device page
453 * table and is aligned to the device page size, or
454 * * %false otherwise.
455 */
456bool
457pvr_device_addr_is_valid(u64 device_addr)
458{
459 return (device_addr & ~PVR_PAGE_TABLE_ADDR_MASK) == 0 &&
460 (device_addr & ~PVR_DEVICE_PAGE_MASK) == 0;
461}
462
463/**
464 * pvr_device_addr_and_size_are_valid() - Tests whether a device-virtual
465 * address and associated size are both valid.
466 * @vm_ctx: Target VM context.
467 * @device_addr: Virtual device address to test.
468 * @size: Size of the range based at @device_addr to test.
469 *
470 * Calling pvr_device_addr_is_valid() twice (once on @size, and again on
471 * @device_addr + @size) to verify a device-virtual address range initially
472 * seems intuitive, but it produces a false-negative when the address range
473 * is right at the end of device-virtual address space.
474 *
475 * This function catches that corner case, as well as checking that
476 * @size is non-zero.
477 *
478 * Return:
479 * * %true if @device_addr is device page aligned; @size is device page
480 * aligned; the range specified by @device_addr and @size is within the
481 * bounds of the device-virtual address space, and @size is non-zero, or
482 * * %false otherwise.
483 */
484bool
485pvr_device_addr_and_size_are_valid(struct pvr_vm_context *vm_ctx,
486 u64 device_addr, u64 size)
487{
488 return pvr_device_addr_is_valid(device_addr) &&
489 drm_gpuvm_range_valid(&vm_ctx->gpuvm_mgr, device_addr, size) &&
490 size != 0 && (size & ~PVR_DEVICE_PAGE_MASK) == 0 &&
491 (device_addr + size <= PVR_PAGE_TABLE_ADDR_SPACE_SIZE);
492}
493
494static void pvr_gpuvm_free(struct drm_gpuvm *gpuvm)
495{
496 kfree(to_pvr_vm_context(gpuvm));
497}
498
499static const struct drm_gpuvm_ops pvr_vm_gpuva_ops = {
500 .vm_free = pvr_gpuvm_free,
501 .sm_step_map = pvr_vm_gpuva_map,
502 .sm_step_remap = pvr_vm_gpuva_remap,
503 .sm_step_unmap = pvr_vm_gpuva_unmap,
504};
505
506static void
507fw_mem_context_init(void *cpu_ptr, void *priv)
508{
509 struct rogue_fwif_fwmemcontext *fw_mem_ctx = cpu_ptr;
510 struct pvr_vm_context *vm_ctx = priv;
511
512 fw_mem_ctx->pc_dev_paddr = pvr_vm_get_page_table_root_addr(vm_ctx);
513 fw_mem_ctx->page_cat_base_reg_set = ROGUE_FW_BIF_INVALID_PCSET;
514}
515
516/**
517 * pvr_vm_create_context() - Create a new VM context.
518 * @pvr_dev: Target PowerVR device.
519 * @is_userspace_context: %true if this context is for userspace. This will
520 * create a firmware memory context for the VM context
521 * and disable warnings when tearing down mappings.
522 *
523 * Return:
524 * * A handle to the newly-minted VM context on success,
525 * * -%EINVAL if the feature "virtual address space bits" on @pvr_dev is
526 * missing or has an unsupported value,
527 * * -%ENOMEM if allocation of the structure behind the opaque handle fails,
528 * or
529 * * Any error encountered while setting up internal structures.
530 */
531struct pvr_vm_context *
532pvr_vm_create_context(struct pvr_device *pvr_dev, bool is_userspace_context)
533{
534 struct drm_device *drm_dev = from_pvr_device(pvr_dev);
535
536 struct pvr_vm_context *vm_ctx;
537 u16 device_addr_bits;
538
539 int err;
540
541 err = PVR_FEATURE_VALUE(pvr_dev, virtual_address_space_bits,
542 &device_addr_bits);
543 if (err) {
544 drm_err(drm_dev,
545 "Failed to get device virtual address space bits\n");
546 return ERR_PTR(err);
547 }
548
549 if (device_addr_bits != PVR_PAGE_TABLE_ADDR_BITS) {
550 drm_err(drm_dev,
551 "Device has unsupported virtual address space size\n");
552 return ERR_PTR(-EINVAL);
553 }
554
555 vm_ctx = kzalloc(sizeof(*vm_ctx), GFP_KERNEL);
556 if (!vm_ctx)
557 return ERR_PTR(-ENOMEM);
558
559 vm_ctx->pvr_dev = pvr_dev;
560
561 vm_ctx->mmu_ctx = pvr_mmu_context_create(pvr_dev);
562 err = PTR_ERR_OR_ZERO(vm_ctx->mmu_ctx);
563 if (err)
564 goto err_free;
565
566 if (is_userspace_context) {
567 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_fwmemcontext),
568 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
569 fw_mem_context_init, vm_ctx, &vm_ctx->fw_mem_ctx_obj);
570
571 if (err)
572 goto err_page_table_destroy;
573 }
574
575 drm_gem_private_object_init(&pvr_dev->base, &vm_ctx->dummy_gem, 0);
576 drm_gpuvm_init(&vm_ctx->gpuvm_mgr,
577 is_userspace_context ? "PowerVR-user-VM" : "PowerVR-FW-VM",
578 0, &pvr_dev->base, &vm_ctx->dummy_gem,
579 0, 1ULL << device_addr_bits, 0, 0, &pvr_vm_gpuva_ops);
580
581 mutex_init(&vm_ctx->lock);
582 kref_init(&vm_ctx->ref_count);
583
584 return vm_ctx;
585
586err_page_table_destroy:
587 pvr_mmu_context_destroy(vm_ctx->mmu_ctx);
588
589err_free:
590 kfree(vm_ctx);
591
592 return ERR_PTR(err);
593}
594
595/**
596 * pvr_vm_context_release() - Teardown a VM context.
597 * @ref_count: Pointer to reference counter of the VM context.
598 *
599 * This function ensures that no mappings are left dangling by unmapping them
600 * all in order of ascending device-virtual address.
601 */
602static void
603pvr_vm_context_release(struct kref *ref_count)
604{
605 struct pvr_vm_context *vm_ctx =
606 container_of(ref_count, struct pvr_vm_context, ref_count);
607
608 if (vm_ctx->fw_mem_ctx_obj)
609 pvr_fw_object_destroy(vm_ctx->fw_mem_ctx_obj);
610
611 WARN_ON(pvr_vm_unmap(vm_ctx, vm_ctx->gpuvm_mgr.mm_start,
612 vm_ctx->gpuvm_mgr.mm_range));
613
614 pvr_mmu_context_destroy(vm_ctx->mmu_ctx);
615 drm_gem_private_object_fini(&vm_ctx->dummy_gem);
616 mutex_destroy(&vm_ctx->lock);
617
618 drm_gpuvm_put(&vm_ctx->gpuvm_mgr);
619}
620
621/**
622 * pvr_vm_context_lookup() - Look up VM context from handle
623 * @pvr_file: Pointer to pvr_file structure.
624 * @handle: Object handle.
625 *
626 * Takes reference on VM context object. Call pvr_vm_context_put() to release.
627 *
628 * Returns:
629 * * The requested object on success, or
630 * * %NULL on failure (object does not exist in list, or is not a VM context)
631 */
632struct pvr_vm_context *
633pvr_vm_context_lookup(struct pvr_file *pvr_file, u32 handle)
634{
635 struct pvr_vm_context *vm_ctx;
636
637 xa_lock(&pvr_file->vm_ctx_handles);
638 vm_ctx = xa_load(&pvr_file->vm_ctx_handles, handle);
639 if (vm_ctx)
640 kref_get(&vm_ctx->ref_count);
641
642 xa_unlock(&pvr_file->vm_ctx_handles);
643
644 return vm_ctx;
645}
646
647/**
648 * pvr_vm_context_put() - Release a reference on a VM context
649 * @vm_ctx: Target VM context.
650 *
651 * Returns:
652 * * %true if the VM context was destroyed, or
653 * * %false if there are any references still remaining.
654 */
655bool
656pvr_vm_context_put(struct pvr_vm_context *vm_ctx)
657{
658 if (vm_ctx)
659 return kref_put(&vm_ctx->ref_count, pvr_vm_context_release);
660
661 return true;
662}
663
664/**
665 * pvr_destroy_vm_contexts_for_file: Destroy any VM contexts associated with the
666 * given file.
667 * @pvr_file: Pointer to pvr_file structure.
668 *
669 * Removes all vm_contexts associated with @pvr_file from the device VM context
670 * list and drops initial references. vm_contexts will then be destroyed once
671 * all outstanding references are dropped.
672 */
673void pvr_destroy_vm_contexts_for_file(struct pvr_file *pvr_file)
674{
675 struct pvr_vm_context *vm_ctx;
676 unsigned long handle;
677
678 xa_for_each(&pvr_file->vm_ctx_handles, handle, vm_ctx) {
679 /* vm_ctx is not used here because that would create a race with xa_erase */
680 pvr_vm_context_put(xa_erase(&pvr_file->vm_ctx_handles, handle));
681 }
682}
683
684static int
685pvr_vm_lock_extra(struct drm_gpuvm_exec *vm_exec)
686{
687 struct pvr_vm_bind_op *bind_op = vm_exec->extra.priv;
688 struct pvr_gem_object *pvr_obj = bind_op->pvr_obj;
689
690 /* Unmap operations don't have an object to lock. */
691 if (!pvr_obj)
692 return 0;
693
694 /* Acquire lock on the GEM being mapped. */
695 return drm_exec_lock_obj(&vm_exec->exec, gem_from_pvr_gem(pvr_obj));
696}
697
698/**
699 * pvr_vm_map() - Map a section of physical memory into a section of
700 * device-virtual memory.
701 * @vm_ctx: Target VM context.
702 * @pvr_obj: Target PowerVR memory object.
703 * @pvr_obj_offset: Offset into @pvr_obj to map from.
704 * @device_addr: Virtual device address at the start of the requested mapping.
705 * @size: Size of the requested mapping.
706 *
707 * No handle is returned to represent the mapping. Instead, callers should
708 * remember @device_addr and use that as a handle.
709 *
710 * Return:
711 * * 0 on success,
712 * * -%EINVAL if @device_addr is not a valid page-aligned device-virtual
713 * address; the region specified by @pvr_obj_offset and @size does not fall
714 * entirely within @pvr_obj, or any part of the specified region of @pvr_obj
715 * is not device-virtual page-aligned,
716 * * Any error encountered while performing internal operations required to
717 * destroy the mapping (returned from pvr_vm_gpuva_map or
718 * pvr_vm_gpuva_remap).
719 */
720int
721pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj,
722 u64 pvr_obj_offset, u64 device_addr, u64 size)
723{
724 struct pvr_vm_bind_op bind_op = {0};
725 struct drm_gpuvm_exec vm_exec = {
726 .vm = &vm_ctx->gpuvm_mgr,
727 .flags = DRM_EXEC_INTERRUPTIBLE_WAIT |
728 DRM_EXEC_IGNORE_DUPLICATES,
729 .extra = {
730 .fn = pvr_vm_lock_extra,
731 .priv = &bind_op,
732 },
733 };
734
735 int err = pvr_vm_bind_op_map_init(&bind_op, vm_ctx, pvr_obj,
736 pvr_obj_offset, device_addr,
737 size);
738
739 if (err)
740 return err;
741
742 pvr_gem_object_get(pvr_obj);
743
744 err = drm_gpuvm_exec_lock(&vm_exec);
745 if (err)
746 goto err_cleanup;
747
748 err = pvr_vm_bind_op_exec(&bind_op);
749
750 drm_gpuvm_exec_unlock(&vm_exec);
751
752err_cleanup:
753 pvr_vm_bind_op_fini(&bind_op);
754
755 return err;
756}
757
758/**
759 * pvr_vm_unmap() - Unmap an already mapped section of device-virtual memory.
760 * @vm_ctx: Target VM context.
761 * @device_addr: Virtual device address at the start of the target mapping.
762 * @size: Size of the target mapping.
763 *
764 * Return:
765 * * 0 on success,
766 * * -%EINVAL if @device_addr is not a valid page-aligned device-virtual
767 * address,
768 * * Any error encountered while performing internal operations required to
769 * destroy the mapping (returned from pvr_vm_gpuva_unmap or
770 * pvr_vm_gpuva_remap).
771 */
772int
773pvr_vm_unmap(struct pvr_vm_context *vm_ctx, u64 device_addr, u64 size)
774{
775 struct pvr_vm_bind_op bind_op = {0};
776 struct drm_gpuvm_exec vm_exec = {
777 .vm = &vm_ctx->gpuvm_mgr,
778 .flags = DRM_EXEC_INTERRUPTIBLE_WAIT |
779 DRM_EXEC_IGNORE_DUPLICATES,
780 .extra = {
781 .fn = pvr_vm_lock_extra,
782 .priv = &bind_op,
783 },
784 };
785
786 int err = pvr_vm_bind_op_unmap_init(&bind_op, vm_ctx, device_addr,
787 size);
788 if (err)
789 return err;
790
791 err = drm_gpuvm_exec_lock(&vm_exec);
792 if (err)
793 goto err_cleanup;
794
795 err = pvr_vm_bind_op_exec(&bind_op);
796
797 drm_gpuvm_exec_unlock(&vm_exec);
798
799err_cleanup:
800 pvr_vm_bind_op_fini(&bind_op);
801
802 return err;
803}
804
805/* Static data areas are determined by firmware. */
806static const struct drm_pvr_static_data_area static_data_areas[] = {
807 {
808 .area_usage = DRM_PVR_STATIC_DATA_AREA_FENCE,
809 .location_heap_id = DRM_PVR_HEAP_GENERAL,
810 .offset = 0,
811 .size = 128,
812 },
813 {
814 .area_usage = DRM_PVR_STATIC_DATA_AREA_YUV_CSC,
815 .location_heap_id = DRM_PVR_HEAP_GENERAL,
816 .offset = 128,
817 .size = 1024,
818 },
819 {
820 .area_usage = DRM_PVR_STATIC_DATA_AREA_VDM_SYNC,
821 .location_heap_id = DRM_PVR_HEAP_PDS_CODE_DATA,
822 .offset = 0,
823 .size = 128,
824 },
825 {
826 .area_usage = DRM_PVR_STATIC_DATA_AREA_EOT,
827 .location_heap_id = DRM_PVR_HEAP_PDS_CODE_DATA,
828 .offset = 128,
829 .size = 128,
830 },
831 {
832 .area_usage = DRM_PVR_STATIC_DATA_AREA_VDM_SYNC,
833 .location_heap_id = DRM_PVR_HEAP_USC_CODE,
834 .offset = 0,
835 .size = 128,
836 },
837};
838
839#define GET_RESERVED_SIZE(last_offset, last_size) round_up((last_offset) + (last_size), PAGE_SIZE)
840
841/*
842 * The values given to GET_RESERVED_SIZE() are taken from the last entry in the corresponding
843 * static data area for each heap.
844 */
845static const struct drm_pvr_heap pvr_heaps[] = {
846 [DRM_PVR_HEAP_GENERAL] = {
847 .base = ROGUE_GENERAL_HEAP_BASE,
848 .size = ROGUE_GENERAL_HEAP_SIZE,
849 .flags = 0,
850 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
851 },
852 [DRM_PVR_HEAP_PDS_CODE_DATA] = {
853 .base = ROGUE_PDSCODEDATA_HEAP_BASE,
854 .size = ROGUE_PDSCODEDATA_HEAP_SIZE,
855 .flags = 0,
856 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
857 },
858 [DRM_PVR_HEAP_USC_CODE] = {
859 .base = ROGUE_USCCODE_HEAP_BASE,
860 .size = ROGUE_USCCODE_HEAP_SIZE,
861 .flags = 0,
862 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
863 },
864 [DRM_PVR_HEAP_RGNHDR] = {
865 .base = ROGUE_RGNHDR_HEAP_BASE,
866 .size = ROGUE_RGNHDR_HEAP_SIZE,
867 .flags = 0,
868 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
869 },
870 [DRM_PVR_HEAP_VIS_TEST] = {
871 .base = ROGUE_VISTEST_HEAP_BASE,
872 .size = ROGUE_VISTEST_HEAP_SIZE,
873 .flags = 0,
874 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
875 },
876 [DRM_PVR_HEAP_TRANSFER_FRAG] = {
877 .base = ROGUE_TRANSFER_FRAG_HEAP_BASE,
878 .size = ROGUE_TRANSFER_FRAG_HEAP_SIZE,
879 .flags = 0,
880 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
881 },
882};
883
884int
885pvr_static_data_areas_get(const struct pvr_device *pvr_dev,
886 struct drm_pvr_ioctl_dev_query_args *args)
887{
888 struct drm_pvr_dev_query_static_data_areas query = {0};
889 int err;
890
891 if (!args->pointer) {
892 args->size = sizeof(struct drm_pvr_dev_query_static_data_areas);
893 return 0;
894 }
895
896 err = PVR_UOBJ_GET(query, args->size, args->pointer);
897 if (err < 0)
898 return err;
899
900 if (!query.static_data_areas.array) {
901 query.static_data_areas.count = ARRAY_SIZE(static_data_areas);
902 query.static_data_areas.stride = sizeof(struct drm_pvr_static_data_area);
903 goto copy_out;
904 }
905
906 if (query.static_data_areas.count > ARRAY_SIZE(static_data_areas))
907 query.static_data_areas.count = ARRAY_SIZE(static_data_areas);
908
909 err = PVR_UOBJ_SET_ARRAY(&query.static_data_areas, static_data_areas);
910 if (err < 0)
911 return err;
912
913copy_out:
914 err = PVR_UOBJ_SET(args->pointer, args->size, query);
915 if (err < 0)
916 return err;
917
918 args->size = sizeof(query);
919 return 0;
920}
921
922int
923pvr_heap_info_get(const struct pvr_device *pvr_dev,
924 struct drm_pvr_ioctl_dev_query_args *args)
925{
926 struct drm_pvr_dev_query_heap_info query = {0};
927 u64 dest;
928 int err;
929
930 if (!args->pointer) {
931 args->size = sizeof(struct drm_pvr_dev_query_heap_info);
932 return 0;
933 }
934
935 err = PVR_UOBJ_GET(query, args->size, args->pointer);
936 if (err < 0)
937 return err;
938
939 if (!query.heaps.array) {
940 query.heaps.count = ARRAY_SIZE(pvr_heaps);
941 query.heaps.stride = sizeof(struct drm_pvr_heap);
942 goto copy_out;
943 }
944
945 if (query.heaps.count > ARRAY_SIZE(pvr_heaps))
946 query.heaps.count = ARRAY_SIZE(pvr_heaps);
947
948 /* Region header heap is only present if BRN63142 is present. */
949 dest = query.heaps.array;
950 for (size_t i = 0; i < query.heaps.count; i++) {
951 struct drm_pvr_heap heap = pvr_heaps[i];
952
953 if (i == DRM_PVR_HEAP_RGNHDR && !PVR_HAS_QUIRK(pvr_dev, 63142))
954 heap.size = 0;
955
956 err = PVR_UOBJ_SET(dest, query.heaps.stride, heap);
957 if (err < 0)
958 return err;
959
960 dest += query.heaps.stride;
961 }
962
963copy_out:
964 err = PVR_UOBJ_SET(args->pointer, args->size, query);
965 if (err < 0)
966 return err;
967
968 args->size = sizeof(query);
969 return 0;
970}
971
972/**
973 * pvr_heap_contains_range() - Determine if a given heap contains the specified
974 * device-virtual address range.
975 * @pvr_heap: Target heap.
976 * @start: Inclusive start of the target range.
977 * @end: Inclusive end of the target range.
978 *
979 * It is an error to call this function with values of @start and @end that do
980 * not satisfy the condition @start <= @end.
981 */
982static __always_inline bool
983pvr_heap_contains_range(const struct drm_pvr_heap *pvr_heap, u64 start, u64 end)
984{
985 return pvr_heap->base <= start && end < pvr_heap->base + pvr_heap->size;
986}
987
988/**
989 * pvr_find_heap_containing() - Find a heap which contains the specified
990 * device-virtual address range.
991 * @pvr_dev: Target PowerVR device.
992 * @start: Start of the target range.
993 * @size: Size of the target range.
994 *
995 * Return:
996 * * A pointer to a constant instance of struct drm_pvr_heap representing the
997 * heap containing the entire range specified by @start and @size on
998 * success, or
999 * * %NULL if no such heap exists.
1000 */
1001const struct drm_pvr_heap *
1002pvr_find_heap_containing(struct pvr_device *pvr_dev, u64 start, u64 size)
1003{
1004 u64 end;
1005
1006 if (check_add_overflow(start, size - 1, &end))
1007 return NULL;
1008
1009 /*
1010 * There are no guarantees about the order of address ranges in
1011 * &pvr_heaps, so iterate over the entire array for a heap whose
1012 * range completely encompasses the given range.
1013 */
1014 for (u32 heap_id = 0; heap_id < ARRAY_SIZE(pvr_heaps); heap_id++) {
1015 /* Filter heaps that present only with an associated quirk */
1016 if (heap_id == DRM_PVR_HEAP_RGNHDR &&
1017 !PVR_HAS_QUIRK(pvr_dev, 63142)) {
1018 continue;
1019 }
1020
1021 if (pvr_heap_contains_range(&pvr_heaps[heap_id], start, end))
1022 return &pvr_heaps[heap_id];
1023 }
1024
1025 return NULL;
1026}
1027
1028/**
1029 * pvr_vm_find_gem_object() - Look up a buffer object from a given
1030 * device-virtual address.
1031 * @vm_ctx: [IN] Target VM context.
1032 * @device_addr: [IN] Virtual device address at the start of the required
1033 * object.
1034 * @mapped_offset_out: [OUT] Pointer to location to write offset of the start
1035 * of the mapped region within the buffer object. May be
1036 * %NULL if this information is not required.
1037 * @mapped_size_out: [OUT] Pointer to location to write size of the mapped
1038 * region. May be %NULL if this information is not required.
1039 *
1040 * If successful, a reference will be taken on the buffer object. The caller
1041 * must drop the reference with pvr_gem_object_put().
1042 *
1043 * Return:
1044 * * The PowerVR buffer object mapped at @device_addr if one exists, or
1045 * * %NULL otherwise.
1046 */
1047struct pvr_gem_object *
1048pvr_vm_find_gem_object(struct pvr_vm_context *vm_ctx, u64 device_addr,
1049 u64 *mapped_offset_out, u64 *mapped_size_out)
1050{
1051 struct pvr_gem_object *pvr_obj;
1052 struct drm_gpuva *va;
1053
1054 mutex_lock(&vm_ctx->lock);
1055
1056 va = drm_gpuva_find_first(&vm_ctx->gpuvm_mgr, device_addr, 1);
1057 if (!va)
1058 goto err_unlock;
1059
1060 pvr_obj = gem_to_pvr_gem(va->gem.obj);
1061 pvr_gem_object_get(pvr_obj);
1062
1063 if (mapped_offset_out)
1064 *mapped_offset_out = va->gem.offset;
1065 if (mapped_size_out)
1066 *mapped_size_out = va->va.range;
1067
1068 mutex_unlock(&vm_ctx->lock);
1069
1070 return pvr_obj;
1071
1072err_unlock:
1073 mutex_unlock(&vm_ctx->lock);
1074
1075 return NULL;
1076}
1077
1078/**
1079 * pvr_vm_get_fw_mem_context: Get object representing firmware memory context
1080 * @vm_ctx: Target VM context.
1081 *
1082 * Returns:
1083 * * FW object representing firmware memory context, or
1084 * * %NULL if this VM context does not have a firmware memory context.
1085 */
1086struct pvr_fw_object *
1087pvr_vm_get_fw_mem_context(struct pvr_vm_context *vm_ctx)
1088{
1089 return vm_ctx->fw_mem_ctx_obj;
1090}