1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
  3
  4#include <linux/mm.h>
  5#include <linux/sync_file.h>
  6#include <linux/pfn_t.h>
  7
  8#include <drm/drm_file.h>
  9#include <drm/drm_syncobj.h>
 10#include <drm/drm_utils.h>
 11
 12#include <drm/lima_drm.h>
 13
 14#include "lima_drv.h"
 15#include "lima_gem.h"
 16#include "lima_gem_prime.h"
 17#include "lima_vm.h"
 18#include "lima_object.h"
 19
 20int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file,
 21			   u32 size, u32 flags, u32 *handle)
 22{
 23	int err;
 24	struct lima_bo *bo;
 25	struct lima_device *ldev = to_lima_dev(dev);
 26
 27	bo = lima_bo_create(ldev, size, flags, NULL);
 28	if (IS_ERR(bo))
 29		return PTR_ERR(bo);
 30
 31	err = drm_gem_handle_create(file, &bo->gem, handle);
 32
 33	/* drop reference from allocate - handle holds it now */
 34	drm_gem_object_put_unlocked(&bo->gem);
 35
 36	return err;
 37}
 38
 39void lima_gem_free_object(struct drm_gem_object *obj)
 40{
 41	struct lima_bo *bo = to_lima_bo(obj);
 42
 43	if (!list_empty(&bo->va))
 44		dev_err(obj->dev->dev, "lima gem free bo still has va\n");
 45
 46	lima_bo_destroy(bo);
 47}
 48
 49int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file)
 50{
 51	struct lima_bo *bo = to_lima_bo(obj);
 52	struct lima_drm_priv *priv = to_lima_drm_priv(file);
 53	struct lima_vm *vm = priv->vm;
 54
 55	return lima_vm_bo_add(vm, bo, true);
 56}
 57
 58void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file)
 59{
 60	struct lima_bo *bo = to_lima_bo(obj);
 61	struct lima_drm_priv *priv = to_lima_drm_priv(file);
 62	struct lima_vm *vm = priv->vm;
 63
 64	lima_vm_bo_del(vm, bo);
 65}
 66
 67int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset)
 68{
 69	struct drm_gem_object *obj;
 70	struct lima_bo *bo;
 71	struct lima_drm_priv *priv = to_lima_drm_priv(file);
 72	struct lima_vm *vm = priv->vm;
 73	int err;
 74
 75	obj = drm_gem_object_lookup(file, handle);
 76	if (!obj)
 77		return -ENOENT;
 78
 79	bo = to_lima_bo(obj);
 80
 81	*va = lima_vm_get_va(vm, bo);
 82
 83	err = drm_gem_create_mmap_offset(obj);
 84	if (!err)
 85		*offset = drm_vma_node_offset_addr(&obj->vma_node);
 86
 87	drm_gem_object_put_unlocked(obj);
 88	return err;
 89}
 90
 91static vm_fault_t lima_gem_fault(struct vm_fault *vmf)
 92{
 93	struct vm_area_struct *vma = vmf->vma;
 94	struct drm_gem_object *obj = vma->vm_private_data;
 95	struct lima_bo *bo = to_lima_bo(obj);
 96	pfn_t pfn;
 97	pgoff_t pgoff;
 98
 99	/* We don't use vmf->pgoff since that has the fake offset: */
100	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
101	pfn = __pfn_to_pfn_t(page_to_pfn(bo->pages[pgoff]), PFN_DEV);
102
103	return vmf_insert_mixed(vma, vmf->address, pfn);
104}
105
106const struct vm_operations_struct lima_gem_vm_ops = {
107	.fault = lima_gem_fault,
108	.open = drm_gem_vm_open,
109	.close = drm_gem_vm_close,
110};
111
112void lima_set_vma_flags(struct vm_area_struct *vma)
113{
114	pgprot_t prot = vm_get_page_prot(vma->vm_flags);
115
116	vma->vm_flags |= VM_MIXEDMAP;
117	vma->vm_flags &= ~VM_PFNMAP;
118	vma->vm_page_prot = pgprot_writecombine(prot);
119}
120
121int lima_gem_mmap(struct file *filp, struct vm_area_struct *vma)
122{
123	int ret;
124
125	ret = drm_gem_mmap(filp, vma);
126	if (ret)
127		return ret;
128
129	lima_set_vma_flags(vma);
130	return 0;
131}
132
133static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo,
134			    bool write, bool explicit)
135{
136	int err = 0;
137
138	if (!write) {
139		err = dma_resv_reserve_shared(bo->gem.resv, 1);
140		if (err)
141			return err;
142	}
143
144	/* explicit sync use user passed dep fence */
145	if (explicit)
146		return 0;
147
148	return drm_gem_fence_array_add_implicit(&task->deps, &bo->gem, write);
149}
150
151static int lima_gem_lock_bos(struct lima_bo **bos, u32 nr_bos,
152			     struct ww_acquire_ctx *ctx)
153{
154	int i, ret = 0, contended, slow_locked = -1;
155
156	ww_acquire_init(ctx, &reservation_ww_class);
157
158retry:
159	for (i = 0; i < nr_bos; i++) {
160		if (i == slow_locked) {
161			slow_locked = -1;
162			continue;
163		}
164
165		ret = ww_mutex_lock_interruptible(&bos[i]->gem.resv->lock, ctx);
166		if (ret < 0) {
167			contended = i;
168			goto err;
169		}
170	}
171
172	ww_acquire_done(ctx);
173	return 0;
174
175err:
176	for (i--; i >= 0; i--)
177		ww_mutex_unlock(&bos[i]->gem.resv->lock);
178
179	if (slow_locked >= 0)
180		ww_mutex_unlock(&bos[slow_locked]->gem.resv->lock);
181
182	if (ret == -EDEADLK) {
183		/* we lost out in a seqno race, lock and retry.. */
184		ret = ww_mutex_lock_slow_interruptible(
185			&bos[contended]->gem.resv->lock, ctx);
186		if (!ret) {
187			slow_locked = contended;
188			goto retry;
189		}
190	}
191	ww_acquire_fini(ctx);
192
193	return ret;
194}
195
196static void lima_gem_unlock_bos(struct lima_bo **bos, u32 nr_bos,
197				struct ww_acquire_ctx *ctx)
198{
199	int i;
200
201	for (i = 0; i < nr_bos; i++)
202		ww_mutex_unlock(&bos[i]->gem.resv->lock);
203	ww_acquire_fini(ctx);
204}
205
206static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit)
207{
208	int i, err;
209
210	for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) {
211		struct dma_fence *fence = NULL;
212
213		if (!submit->in_sync[i])
214			continue;
215
216		err = drm_syncobj_find_fence(file, submit->in_sync[i],
217					     0, 0, &fence);
218		if (err)
219			return err;
220
221		err = drm_gem_fence_array_add(&submit->task->deps, fence);
222		if (err) {
223			dma_fence_put(fence);
224			return err;
225		}
226	}
227
228	return 0;
229}
230
231int lima_gem_submit(struct drm_file *file, struct lima_submit *submit)
232{
233	int i, err = 0;
234	struct ww_acquire_ctx ctx;
235	struct lima_drm_priv *priv = to_lima_drm_priv(file);
236	struct lima_vm *vm = priv->vm;
237	struct drm_syncobj *out_sync = NULL;
238	struct dma_fence *fence;
239	struct lima_bo **bos = submit->lbos;
240
241	if (submit->out_sync) {
242		out_sync = drm_syncobj_find(file, submit->out_sync);
243		if (!out_sync)
244			return -ENOENT;
245	}
246
247	for (i = 0; i < submit->nr_bos; i++) {
248		struct drm_gem_object *obj;
249		struct lima_bo *bo;
250
251		obj = drm_gem_object_lookup(file, submit->bos[i].handle);
252		if (!obj) {
253			err = -ENOENT;
254			goto err_out0;
255		}
256
257		bo = to_lima_bo(obj);
258
259		/* increase refcnt of gpu va map to prevent unmapped when executing,
260		 * will be decreased when task done
261		 */
262		err = lima_vm_bo_add(vm, bo, false);
263		if (err) {
264			drm_gem_object_put_unlocked(obj);
265			goto err_out0;
266		}
267
268		bos[i] = bo;
269	}
270
271	err = lima_gem_lock_bos(bos, submit->nr_bos, &ctx);
272	if (err)
273		goto err_out0;
274
275	err = lima_sched_task_init(
276		submit->task, submit->ctx->context + submit->pipe,
277		bos, submit->nr_bos, vm);
278	if (err)
279		goto err_out1;
280
281	err = lima_gem_add_deps(file, submit);
282	if (err)
283		goto err_out2;
284
285	for (i = 0; i < submit->nr_bos; i++) {
286		err = lima_gem_sync_bo(
287			submit->task, bos[i],
288			submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE,
289			submit->flags & LIMA_SUBMIT_FLAG_EXPLICIT_FENCE);
290		if (err)
291			goto err_out2;
292	}
293
294	fence = lima_sched_context_queue_task(
295		submit->ctx->context + submit->pipe, submit->task);
296
297	for (i = 0; i < submit->nr_bos; i++) {
298		if (submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE)
299			dma_resv_add_excl_fence(bos[i]->gem.resv, fence);
300		else
301			dma_resv_add_shared_fence(bos[i]->gem.resv, fence);
302	}
303
304	lima_gem_unlock_bos(bos, submit->nr_bos, &ctx);
305
306	for (i = 0; i < submit->nr_bos; i++)
307		drm_gem_object_put_unlocked(&bos[i]->gem);
308
309	if (out_sync) {
310		drm_syncobj_replace_fence(out_sync, fence);
311		drm_syncobj_put(out_sync);
312	}
313
314	dma_fence_put(fence);
315
316	return 0;
317
318err_out2:
319	lima_sched_task_fini(submit->task);
320err_out1:
321	lima_gem_unlock_bos(bos, submit->nr_bos, &ctx);
322err_out0:
323	for (i = 0; i < submit->nr_bos; i++) {
324		if (!bos[i])
325			break;
326		lima_vm_bo_del(vm, bos[i]);
327		drm_gem_object_put_unlocked(&bos[i]->gem);
328	}
329	if (out_sync)
330		drm_syncobj_put(out_sync);
331	return err;
332}
333
334int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns)
335{
336	bool write = op & LIMA_GEM_WAIT_WRITE;
337	long ret, timeout;
338
339	if (!op)
340		return 0;
341
342	timeout = drm_timeout_abs_to_jiffies(timeout_ns);
343
344	ret = drm_gem_dma_resv_wait(file, handle, write, timeout);
345	if (ret == -ETIME)
346		ret = timeout ? -ETIMEDOUT : -EBUSY;
347
348	return ret;
349}