Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/dma-buf.h>
7#include <linux/highmem.h>
8#include <linux/module.h>
9#include <linux/set_memory.h>
10#include <linux/xarray.h>
11
12#include <drm/drm_cache.h>
13#include <drm/drm_debugfs.h>
14#include <drm/drm_file.h>
15#include <drm/drm_utils.h>
16
17#include "ivpu_drv.h"
18#include "ivpu_gem.h"
19#include "ivpu_hw.h"
20#include "ivpu_mmu.h"
21#include "ivpu_mmu_context.h"
22
23static const struct drm_gem_object_funcs ivpu_gem_funcs;
24
25static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
26{
27 ivpu_dbg(vdev, BO,
28 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n",
29 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx ? bo->ctx->id : 0,
30 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
31 (bool)bo->base.base.import_attach);
32}
33
34/*
35 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
36 *
37 * This function pins physical memory pages, then maps the physical pages
38 * to IOMMU address space and finally updates the VPU MMU page tables
39 * to allow the VPU to translate VPU address to IOMMU address.
40 */
41int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
42{
43 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
44 int ret = 0;
45
46 mutex_lock(&bo->lock);
47
48 ivpu_dbg_bo(vdev, bo, "pin");
49 drm_WARN_ON(&vdev->drm, !bo->ctx);
50
51 if (!bo->mmu_mapped) {
52 struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
53
54 if (IS_ERR(sgt)) {
55 ret = PTR_ERR(sgt);
56 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
57 goto unlock;
58 }
59
60 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
61 ivpu_bo_is_snooped(bo));
62 if (ret) {
63 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
64 goto unlock;
65 }
66 bo->mmu_mapped = true;
67 }
68
69unlock:
70 mutex_unlock(&bo->lock);
71
72 return ret;
73}
74
75static int
76ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
77 const struct ivpu_addr_range *range)
78{
79 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
80 int idx, ret;
81
82 if (!drm_dev_enter(&vdev->drm, &idx))
83 return -ENODEV;
84
85 mutex_lock(&bo->lock);
86
87 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
88 if (!ret) {
89 bo->ctx = ctx;
90 bo->vpu_addr = bo->mm_node.start;
91 } else {
92 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
93 }
94
95 ivpu_dbg_bo(vdev, bo, "alloc");
96
97 mutex_unlock(&bo->lock);
98
99 drm_dev_exit(idx);
100
101 return ret;
102}
103
104static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
105{
106 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
107
108 lockdep_assert(lockdep_is_held(&bo->lock) || !kref_read(&bo->base.base.refcount));
109
110 if (bo->mmu_mapped) {
111 drm_WARN_ON(&vdev->drm, !bo->ctx);
112 drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
113 drm_WARN_ON(&vdev->drm, !bo->base.sgt);
114 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
115 bo->mmu_mapped = false;
116 }
117
118 if (bo->ctx) {
119 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
120 bo->ctx = NULL;
121 }
122
123 if (bo->base.base.import_attach)
124 return;
125
126 dma_resv_lock(bo->base.base.resv, NULL);
127 if (bo->base.sgt) {
128 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
129 sg_free_table(bo->base.sgt);
130 kfree(bo->base.sgt);
131 bo->base.sgt = NULL;
132 }
133 dma_resv_unlock(bo->base.base.resv);
134}
135
136void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
137{
138 struct ivpu_bo *bo;
139
140 if (drm_WARN_ON(&vdev->drm, !ctx))
141 return;
142
143 mutex_lock(&vdev->bo_list_lock);
144 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
145 mutex_lock(&bo->lock);
146 if (bo->ctx == ctx) {
147 ivpu_dbg_bo(vdev, bo, "unbind");
148 ivpu_bo_unbind_locked(bo);
149 }
150 mutex_unlock(&bo->lock);
151 }
152 mutex_unlock(&vdev->bo_list_lock);
153}
154
155struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
156{
157 struct ivpu_bo *bo;
158
159 if (size == 0 || !PAGE_ALIGNED(size))
160 return ERR_PTR(-EINVAL);
161
162 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
163 if (!bo)
164 return ERR_PTR(-ENOMEM);
165
166 bo->base.base.funcs = &ivpu_gem_funcs;
167 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
168
169 INIT_LIST_HEAD(&bo->bo_list_node);
170 mutex_init(&bo->lock);
171
172 return &bo->base.base;
173}
174
175static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags)
176{
177 struct drm_gem_shmem_object *shmem;
178 struct ivpu_bo *bo;
179
180 switch (flags & DRM_IVPU_BO_CACHE_MASK) {
181 case DRM_IVPU_BO_CACHED:
182 case DRM_IVPU_BO_WC:
183 break;
184 default:
185 return ERR_PTR(-EINVAL);
186 }
187
188 shmem = drm_gem_shmem_create(&vdev->drm, size);
189 if (IS_ERR(shmem))
190 return ERR_CAST(shmem);
191
192 bo = to_ivpu_bo(&shmem->base);
193 bo->base.map_wc = flags & DRM_IVPU_BO_WC;
194 bo->flags = flags;
195
196 mutex_lock(&vdev->bo_list_lock);
197 list_add_tail(&bo->bo_list_node, &vdev->bo_list);
198 mutex_unlock(&vdev->bo_list_lock);
199
200 return bo;
201}
202
203static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
204{
205 struct ivpu_file_priv *file_priv = file->driver_priv;
206 struct ivpu_device *vdev = file_priv->vdev;
207 struct ivpu_bo *bo = to_ivpu_bo(obj);
208 struct ivpu_addr_range *range;
209
210 if (bo->ctx) {
211 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
212 file_priv->ctx.id, bo->ctx->id);
213 return -EALREADY;
214 }
215
216 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
217 range = &vdev->hw->ranges.shave;
218 else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
219 range = &vdev->hw->ranges.dma;
220 else
221 range = &vdev->hw->ranges.user;
222
223 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
224}
225
226static void ivpu_gem_bo_free(struct drm_gem_object *obj)
227{
228 struct ivpu_device *vdev = to_ivpu_device(obj->dev);
229 struct ivpu_bo *bo = to_ivpu_bo(obj);
230
231 ivpu_dbg_bo(vdev, bo, "free");
232
233 mutex_lock(&vdev->bo_list_lock);
234 list_del(&bo->bo_list_node);
235 mutex_unlock(&vdev->bo_list_lock);
236
237 drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
238
239 ivpu_bo_unbind_locked(bo);
240 mutex_destroy(&bo->lock);
241
242 drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1);
243 drm_gem_shmem_free(&bo->base);
244}
245
246static const struct drm_gem_object_funcs ivpu_gem_funcs = {
247 .free = ivpu_gem_bo_free,
248 .open = ivpu_gem_bo_open,
249 .print_info = drm_gem_shmem_object_print_info,
250 .pin = drm_gem_shmem_object_pin,
251 .unpin = drm_gem_shmem_object_unpin,
252 .get_sg_table = drm_gem_shmem_object_get_sg_table,
253 .vmap = drm_gem_shmem_object_vmap,
254 .vunmap = drm_gem_shmem_object_vunmap,
255 .mmap = drm_gem_shmem_object_mmap,
256 .vm_ops = &drm_gem_shmem_vm_ops,
257};
258
259int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
260{
261 struct ivpu_file_priv *file_priv = file->driver_priv;
262 struct ivpu_device *vdev = file_priv->vdev;
263 struct drm_ivpu_bo_create *args = data;
264 u64 size = PAGE_ALIGN(args->size);
265 struct ivpu_bo *bo;
266 int ret;
267
268 if (args->flags & ~DRM_IVPU_BO_FLAGS)
269 return -EINVAL;
270
271 if (size == 0)
272 return -EINVAL;
273
274 bo = ivpu_bo_alloc(vdev, size, args->flags);
275 if (IS_ERR(bo)) {
276 ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
277 bo, file_priv->ctx.id, args->size, args->flags);
278 return PTR_ERR(bo);
279 }
280
281 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
282 if (!ret)
283 args->vpu_addr = bo->vpu_addr;
284
285 drm_gem_object_put(&bo->base.base);
286
287 return ret;
288}
289
290struct ivpu_bo *
291ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
292 struct ivpu_addr_range *range, u64 size, u32 flags)
293{
294 struct iosys_map map;
295 struct ivpu_bo *bo;
296 int ret;
297
298 if (drm_WARN_ON(&vdev->drm, !range))
299 return NULL;
300
301 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
302 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
303 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
304
305 bo = ivpu_bo_alloc(vdev, size, flags);
306 if (IS_ERR(bo)) {
307 ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
308 bo, range->start, size, flags);
309 return NULL;
310 }
311
312 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
313 if (ret)
314 goto err_put;
315
316 ret = ivpu_bo_pin(bo);
317 if (ret)
318 goto err_put;
319
320 if (flags & DRM_IVPU_BO_MAPPABLE) {
321 dma_resv_lock(bo->base.base.resv, NULL);
322 ret = drm_gem_shmem_vmap(&bo->base, &map);
323 dma_resv_unlock(bo->base.base.resv);
324
325 if (ret)
326 goto err_put;
327 }
328
329 return bo;
330
331err_put:
332 drm_gem_object_put(&bo->base.base);
333 return NULL;
334}
335
336struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
337{
338 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
339}
340
341void ivpu_bo_free(struct ivpu_bo *bo)
342{
343 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
344
345 if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
346 dma_resv_lock(bo->base.base.resv, NULL);
347 drm_gem_shmem_vunmap(&bo->base, &map);
348 dma_resv_unlock(bo->base.base.resv);
349 }
350
351 drm_gem_object_put(&bo->base.base);
352}
353
354int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
355{
356 struct drm_ivpu_bo_info *args = data;
357 struct drm_gem_object *obj;
358 struct ivpu_bo *bo;
359 int ret = 0;
360
361 obj = drm_gem_object_lookup(file, args->handle);
362 if (!obj)
363 return -ENOENT;
364
365 bo = to_ivpu_bo(obj);
366
367 mutex_lock(&bo->lock);
368 args->flags = bo->flags;
369 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
370 args->vpu_addr = bo->vpu_addr;
371 args->size = obj->size;
372 mutex_unlock(&bo->lock);
373
374 drm_gem_object_put(obj);
375 return ret;
376}
377
378int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
379{
380 struct drm_ivpu_bo_wait *args = data;
381 struct drm_gem_object *obj;
382 unsigned long timeout;
383 long ret;
384
385 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
386
387 /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
388 timeout += 1;
389
390 obj = drm_gem_object_lookup(file, args->handle);
391 if (!obj)
392 return -EINVAL;
393
394 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
395 if (ret == 0) {
396 ret = -ETIMEDOUT;
397 } else if (ret > 0) {
398 ret = 0;
399 args->job_status = to_ivpu_bo(obj)->job_status;
400 }
401
402 drm_gem_object_put(obj);
403
404 return ret;
405}
406
407static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
408{
409 mutex_lock(&bo->lock);
410
411 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
412 bo, bo->ctx ? bo->ctx->id : 0, bo->vpu_addr, bo->base.base.size,
413 bo->flags, kref_read(&bo->base.base.refcount));
414
415 if (bo->base.pages)
416 drm_printf(p, " has_pages");
417
418 if (bo->mmu_mapped)
419 drm_printf(p, " mmu_mapped");
420
421 if (bo->base.base.import_attach)
422 drm_printf(p, " imported");
423
424 drm_printf(p, "\n");
425
426 mutex_unlock(&bo->lock);
427}
428
429void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
430{
431 struct ivpu_device *vdev = to_ivpu_device(dev);
432 struct ivpu_bo *bo;
433
434 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
435 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
436
437 mutex_lock(&vdev->bo_list_lock);
438 list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
439 ivpu_bo_print_info(bo, p);
440 mutex_unlock(&vdev->bo_list_lock);
441}
442
443void ivpu_bo_list_print(struct drm_device *dev)
444{
445 struct drm_printer p = drm_info_printer(dev->dev);
446
447 ivpu_bo_list(dev, &p);
448}