1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
  3
  4#include <linux/mm.h>
  5#include <linux/sync_file.h>
  6#include <linux/pagemap.h>
  7#include <linux/shmem_fs.h>
  8#include <linux/dma-mapping.h>
  9
 10#include <drm/drm_file.h>
 11#include <drm/drm_syncobj.h>
 12#include <drm/drm_utils.h>
 13
 14#include <drm/lima_drm.h>
 15
 16#include "lima_drv.h"
 17#include "lima_gem.h"
 18#include "lima_vm.h"
 19
 20int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm)
 21{
 22	struct page **pages;
 23	struct address_space *mapping = bo->base.base.filp->f_mapping;
 24	struct device *dev = bo->base.base.dev->dev;
 25	size_t old_size = bo->heap_size;
 26	size_t new_size = bo->heap_size ? bo->heap_size * 2 :
 27		(lima_heap_init_nr_pages << PAGE_SHIFT);
 28	struct sg_table sgt;
 29	int i, ret;
 30
 31	if (bo->heap_size >= bo->base.base.size)
 32		return -ENOSPC;
 33
 34	new_size = min(new_size, bo->base.base.size);
 35
 36	mutex_lock(&bo->base.pages_lock);
 37
 38	if (bo->base.pages) {
 39		pages = bo->base.pages;
 40	} else {
 41		pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT,
 42				       sizeof(*pages), GFP_KERNEL | __GFP_ZERO);
 43		if (!pages) {
 44			mutex_unlock(&bo->base.pages_lock);
 45			return -ENOMEM;
 46		}
 47
 48		bo->base.pages = pages;
 49		bo->base.pages_use_count = 1;
 50
 51		mapping_set_unevictable(mapping);
 52	}
 53
 54	for (i = old_size >> PAGE_SHIFT; i < new_size >> PAGE_SHIFT; i++) {
 55		struct page *page = shmem_read_mapping_page(mapping, i);
 56
 57		if (IS_ERR(page)) {
 58			mutex_unlock(&bo->base.pages_lock);
 59			return PTR_ERR(page);
 60		}
 61		pages[i] = page;
 62	}
 63
 64	mutex_unlock(&bo->base.pages_lock);
 65
 66	ret = sg_alloc_table_from_pages(&sgt, pages, i, 0,
 67					new_size, GFP_KERNEL);
 68	if (ret)
 69		return ret;
 70
 71	if (bo->base.sgt) {
 72		dma_unmap_sg(dev, bo->base.sgt->sgl,
 73			     bo->base.sgt->nents, DMA_BIDIRECTIONAL);
 74		sg_free_table(bo->base.sgt);
 75	} else {
 76		bo->base.sgt = kmalloc(sizeof(*bo->base.sgt), GFP_KERNEL);
 77		if (!bo->base.sgt) {
 78			sg_free_table(&sgt);
 79			return -ENOMEM;
 80		}
 81	}
 82
 83	dma_map_sg(dev, sgt.sgl, sgt.nents, DMA_BIDIRECTIONAL);
 84
 85	*bo->base.sgt = sgt;
 86
 87	if (vm) {
 88		ret = lima_vm_map_bo(vm, bo, old_size >> PAGE_SHIFT);
 89		if (ret)
 90			return ret;
 91	}
 92
 93	bo->heap_size = new_size;
 94	return 0;
 95}
 96
 97int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file,
 98			   u32 size, u32 flags, u32 *handle)
 99{
100	int err;
101	gfp_t mask;
102	struct drm_gem_shmem_object *shmem;
103	struct drm_gem_object *obj;
104	struct lima_bo *bo;
105	bool is_heap = flags & LIMA_BO_FLAG_HEAP;
106
107	shmem = drm_gem_shmem_create(dev, size);
108	if (IS_ERR(shmem))
109		return PTR_ERR(shmem);
110
111	obj = &shmem->base;
112
113	/* Mali Utgard GPU can only support 32bit address space */
114	mask = mapping_gfp_mask(obj->filp->f_mapping);
115	mask &= ~__GFP_HIGHMEM;
116	mask |= __GFP_DMA32;
117	mapping_set_gfp_mask(obj->filp->f_mapping, mask);
118
119	if (is_heap) {
120		bo = to_lima_bo(obj);
121		err = lima_heap_alloc(bo, NULL);
122		if (err)
123			goto out;
124	} else {
125		struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(obj);
126
127		if (IS_ERR(sgt)) {
128			err = PTR_ERR(sgt);
129			goto out;
130		}
131	}
132
133	err = drm_gem_handle_create(file, obj, handle);
134
135out:
136	/* drop reference from allocate - handle holds it now */
137	drm_gem_object_put(obj);
138
139	return err;
140}
141
142static void lima_gem_free_object(struct drm_gem_object *obj)
143{
144	struct lima_bo *bo = to_lima_bo(obj);
145
146	if (!list_empty(&bo->va))
147		dev_err(obj->dev->dev, "lima gem free bo still has va\n");
148
149	drm_gem_shmem_free_object(obj);
150}
151
152static int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file)
153{
154	struct lima_bo *bo = to_lima_bo(obj);
155	struct lima_drm_priv *priv = to_lima_drm_priv(file);
156	struct lima_vm *vm = priv->vm;
157
158	return lima_vm_bo_add(vm, bo, true);
159}
160
161static void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file)
162{
163	struct lima_bo *bo = to_lima_bo(obj);
164	struct lima_drm_priv *priv = to_lima_drm_priv(file);
165	struct lima_vm *vm = priv->vm;
166
167	lima_vm_bo_del(vm, bo);
168}
169
170static int lima_gem_pin(struct drm_gem_object *obj)
171{
172	struct lima_bo *bo = to_lima_bo(obj);
173
174	if (bo->heap_size)
175		return -EINVAL;
176
177	return drm_gem_shmem_pin(obj);
178}
179
180static void *lima_gem_vmap(struct drm_gem_object *obj)
181{
182	struct lima_bo *bo = to_lima_bo(obj);
183
184	if (bo->heap_size)
185		return ERR_PTR(-EINVAL);
186
187	return drm_gem_shmem_vmap(obj);
188}
189
190static int lima_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
191{
192	struct lima_bo *bo = to_lima_bo(obj);
193
194	if (bo->heap_size)
195		return -EINVAL;
196
197	return drm_gem_shmem_mmap(obj, vma);
198}
199
200static const struct drm_gem_object_funcs lima_gem_funcs = {
201	.free = lima_gem_free_object,
202	.open = lima_gem_object_open,
203	.close = lima_gem_object_close,
204	.print_info = drm_gem_shmem_print_info,
205	.pin = lima_gem_pin,
206	.unpin = drm_gem_shmem_unpin,
207	.get_sg_table = drm_gem_shmem_get_sg_table,
208	.vmap = lima_gem_vmap,
209	.vunmap = drm_gem_shmem_vunmap,
210	.mmap = lima_gem_mmap,
211};
212
213struct drm_gem_object *lima_gem_create_object(struct drm_device *dev, size_t size)
214{
215	struct lima_bo *bo;
216
217	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
218	if (!bo)
219		return NULL;
220
221	mutex_init(&bo->lock);
222	INIT_LIST_HEAD(&bo->va);
223
224	bo->base.base.funcs = &lima_gem_funcs;
225
226	return &bo->base.base;
227}
228
229int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset)
230{
231	struct drm_gem_object *obj;
232	struct lima_bo *bo;
233	struct lima_drm_priv *priv = to_lima_drm_priv(file);
234	struct lima_vm *vm = priv->vm;
235
236	obj = drm_gem_object_lookup(file, handle);
237	if (!obj)
238		return -ENOENT;
239
240	bo = to_lima_bo(obj);
241
242	*va = lima_vm_get_va(vm, bo);
243
244	*offset = drm_vma_node_offset_addr(&obj->vma_node);
245
246	drm_gem_object_put(obj);
247	return 0;
248}
249
250static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo,
251			    bool write, bool explicit)
252{
253	int err = 0;
254
255	if (!write) {
256		err = dma_resv_reserve_shared(lima_bo_resv(bo), 1);
257		if (err)
258			return err;
259	}
260
261	/* explicit sync use user passed dep fence */
262	if (explicit)
263		return 0;
264
265	return drm_gem_fence_array_add_implicit(&task->deps, &bo->base.base, write);
266}
267
268static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit)
269{
270	int i, err;
271
272	for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) {
273		struct dma_fence *fence = NULL;
274
275		if (!submit->in_sync[i])
276			continue;
277
278		err = drm_syncobj_find_fence(file, submit->in_sync[i],
279					     0, 0, &fence);
280		if (err)
281			return err;
282
283		err = drm_gem_fence_array_add(&submit->task->deps, fence);
284		if (err) {
285			dma_fence_put(fence);
286			return err;
287		}
288	}
289
290	return 0;
291}
292
293int lima_gem_submit(struct drm_file *file, struct lima_submit *submit)
294{
295	int i, err = 0;
296	struct ww_acquire_ctx ctx;
297	struct lima_drm_priv *priv = to_lima_drm_priv(file);
298	struct lima_vm *vm = priv->vm;
299	struct drm_syncobj *out_sync = NULL;
300	struct dma_fence *fence;
301	struct lima_bo **bos = submit->lbos;
302
303	if (submit->out_sync) {
304		out_sync = drm_syncobj_find(file, submit->out_sync);
305		if (!out_sync)
306			return -ENOENT;
307	}
308
309	for (i = 0; i < submit->nr_bos; i++) {
310		struct drm_gem_object *obj;
311		struct lima_bo *bo;
312
313		obj = drm_gem_object_lookup(file, submit->bos[i].handle);
314		if (!obj) {
315			err = -ENOENT;
316			goto err_out0;
317		}
318
319		bo = to_lima_bo(obj);
320
321		/* increase refcnt of gpu va map to prevent unmapped when executing,
322		 * will be decreased when task done
323		 */
324		err = lima_vm_bo_add(vm, bo, false);
325		if (err) {
326			drm_gem_object_put(obj);
327			goto err_out0;
328		}
329
330		bos[i] = bo;
331	}
332
333	err = drm_gem_lock_reservations((struct drm_gem_object **)bos,
334					submit->nr_bos, &ctx);
335	if (err)
336		goto err_out0;
337
338	err = lima_sched_task_init(
339		submit->task, submit->ctx->context + submit->pipe,
340		bos, submit->nr_bos, vm);
341	if (err)
342		goto err_out1;
343
344	err = lima_gem_add_deps(file, submit);
345	if (err)
346		goto err_out2;
347
348	for (i = 0; i < submit->nr_bos; i++) {
349		err = lima_gem_sync_bo(
350			submit->task, bos[i],
351			submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE,
352			submit->flags & LIMA_SUBMIT_FLAG_EXPLICIT_FENCE);
353		if (err)
354			goto err_out2;
355	}
356
357	fence = lima_sched_context_queue_task(
358		submit->ctx->context + submit->pipe, submit->task);
359
360	for (i = 0; i < submit->nr_bos; i++) {
361		if (submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE)
362			dma_resv_add_excl_fence(lima_bo_resv(bos[i]), fence);
363		else
364			dma_resv_add_shared_fence(lima_bo_resv(bos[i]), fence);
365	}
366
367	drm_gem_unlock_reservations((struct drm_gem_object **)bos,
368				    submit->nr_bos, &ctx);
369
370	for (i = 0; i < submit->nr_bos; i++)
371		drm_gem_object_put(&bos[i]->base.base);
372
373	if (out_sync) {
374		drm_syncobj_replace_fence(out_sync, fence);
375		drm_syncobj_put(out_sync);
376	}
377
378	dma_fence_put(fence);
379
380	return 0;
381
382err_out2:
383	lima_sched_task_fini(submit->task);
384err_out1:
385	drm_gem_unlock_reservations((struct drm_gem_object **)bos,
386				    submit->nr_bos, &ctx);
387err_out0:
388	for (i = 0; i < submit->nr_bos; i++) {
389		if (!bos[i])
390			break;
391		lima_vm_bo_del(vm, bos[i]);
392		drm_gem_object_put(&bos[i]->base.base);
393	}
394	if (out_sync)
395		drm_syncobj_put(out_sync);
396	return err;
397}
398
399int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns)
400{
401	bool write = op & LIMA_GEM_WAIT_WRITE;
402	long ret, timeout;
403
404	if (!op)
405		return 0;
406
407	timeout = drm_timeout_abs_to_jiffies(timeout_ns);
408
409	ret = drm_gem_dma_resv_wait(file, handle, write, timeout);
410	if (ret == -ETIME)
411		ret = timeout ? -ETIMEDOUT : -EBUSY;
412
413	return ret;
414}

  1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
  3
  4#include <linux/mm.h>
  5#include <linux/sync_file.h>
  6#include <linux/pagemap.h>
  7#include <linux/shmem_fs.h>
  8#include <linux/dma-mapping.h>
  9
 10#include <drm/drm_file.h>
 11#include <drm/drm_syncobj.h>
 12#include <drm/drm_utils.h>
 13
 14#include <drm/lima_drm.h>
 15
 16#include "lima_drv.h"
 17#include "lima_gem.h"
 18#include "lima_vm.h"
 19
 20int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm)
 21{
 22	struct page **pages;
 23	struct address_space *mapping = bo->base.base.filp->f_mapping;
 24	struct device *dev = bo->base.base.dev->dev;
 25	size_t old_size = bo->heap_size;
 26	size_t new_size = bo->heap_size ? bo->heap_size * 2 :
 27		(lima_heap_init_nr_pages << PAGE_SHIFT);
 28	struct sg_table sgt;
 29	int i, ret;
 30
 31	if (bo->heap_size >= bo->base.base.size)
 32		return -ENOSPC;
 33
 34	new_size = min(new_size, bo->base.base.size);
 35
 36	mutex_lock(&bo->base.pages_lock);
 37
 38	if (bo->base.pages) {
 39		pages = bo->base.pages;
 40	} else {
 41		pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT,
 42				       sizeof(*pages), GFP_KERNEL | __GFP_ZERO);
 43		if (!pages) {
 44			mutex_unlock(&bo->base.pages_lock);
 45			return -ENOMEM;
 46		}
 47
 48		bo->base.pages = pages;
 49		bo->base.pages_use_count = 1;
 50
 51		mapping_set_unevictable(mapping);
 52	}
 53
 54	for (i = old_size >> PAGE_SHIFT; i < new_size >> PAGE_SHIFT; i++) {
 55		struct page *page = shmem_read_mapping_page(mapping, i);
 56
 57		if (IS_ERR(page)) {
 58			mutex_unlock(&bo->base.pages_lock);
 59			return PTR_ERR(page);
 60		}
 61		pages[i] = page;
 62	}
 63
 64	mutex_unlock(&bo->base.pages_lock);
 65
 66	ret = sg_alloc_table_from_pages(&sgt, pages, i, 0,
 67					new_size, GFP_KERNEL);
 68	if (ret)
 69		return ret;
 70
 71	if (bo->base.sgt) {
 72		dma_unmap_sg(dev, bo->base.sgt->sgl,
 73			     bo->base.sgt->nents, DMA_BIDIRECTIONAL);
 74		sg_free_table(bo->base.sgt);
 75	} else {
 76		bo->base.sgt = kmalloc(sizeof(*bo->base.sgt), GFP_KERNEL);
 77		if (!bo->base.sgt) {
 78			sg_free_table(&sgt);
 79			return -ENOMEM;
 80		}
 81	}
 82
 83	dma_map_sg(dev, sgt.sgl, sgt.nents, DMA_BIDIRECTIONAL);
 84
 85	*bo->base.sgt = sgt;
 86
 87	if (vm) {
 88		ret = lima_vm_map_bo(vm, bo, old_size >> PAGE_SHIFT);
 89		if (ret)
 90			return ret;
 91	}
 92
 93	bo->heap_size = new_size;
 94	return 0;
 95}
 96
 97int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file,
 98			   u32 size, u32 flags, u32 *handle)
 99{
100	int err;
101	gfp_t mask;
102	struct drm_gem_shmem_object *shmem;
103	struct drm_gem_object *obj;
104	struct lima_bo *bo;
105	bool is_heap = flags & LIMA_BO_FLAG_HEAP;
106
107	shmem = drm_gem_shmem_create(dev, size);
108	if (IS_ERR(shmem))
109		return PTR_ERR(shmem);
110
111	obj = &shmem->base;
112
113	/* Mali Utgard GPU can only support 32bit address space */
114	mask = mapping_gfp_mask(obj->filp->f_mapping);
115	mask &= ~__GFP_HIGHMEM;
116	mask |= __GFP_DMA32;
117	mapping_set_gfp_mask(obj->filp->f_mapping, mask);
118
119	if (is_heap) {
120		bo = to_lima_bo(obj);
121		err = lima_heap_alloc(bo, NULL);
122		if (err)
123			goto out;
124	} else {
125		struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(obj);
126
127		if (IS_ERR(sgt)) {
128			err = PTR_ERR(sgt);
129			goto out;
130		}
131	}
132
133	err = drm_gem_handle_create(file, obj, handle);
134
135out:
136	/* drop reference from allocate - handle holds it now */
137	drm_gem_object_put(obj);
138
139	return err;
140}
141
142static void lima_gem_free_object(struct drm_gem_object *obj)
143{
144	struct lima_bo *bo = to_lima_bo(obj);
145
146	if (!list_empty(&bo->va))
147		dev_err(obj->dev->dev, "lima gem free bo still has va\n");
148
149	drm_gem_shmem_free_object(obj);
150}
151
152static int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file)
153{
154	struct lima_bo *bo = to_lima_bo(obj);
155	struct lima_drm_priv *priv = to_lima_drm_priv(file);
156	struct lima_vm *vm = priv->vm;
157
158	return lima_vm_bo_add(vm, bo, true);
159}
160
161static void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file)
162{
163	struct lima_bo *bo = to_lima_bo(obj);
164	struct lima_drm_priv *priv = to_lima_drm_priv(file);
165	struct lima_vm *vm = priv->vm;
166
167	lima_vm_bo_del(vm, bo);
168}
169
170static int lima_gem_pin(struct drm_gem_object *obj)
171{
172	struct lima_bo *bo = to_lima_bo(obj);
173
174	if (bo->heap_size)
175		return -EINVAL;
176
177	return drm_gem_shmem_pin(obj);
178}
179
180static void *lima_gem_vmap(struct drm_gem_object *obj)
181{
182	struct lima_bo *bo = to_lima_bo(obj);
183
184	if (bo->heap_size)
185		return ERR_PTR(-EINVAL);
186
187	return drm_gem_shmem_vmap(obj);
188}
189
190static int lima_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
191{
192	struct lima_bo *bo = to_lima_bo(obj);
193
194	if (bo->heap_size)
195		return -EINVAL;
196
197	return drm_gem_shmem_mmap(obj, vma);
198}
199
200static const struct drm_gem_object_funcs lima_gem_funcs = {
201	.free = lima_gem_free_object,
202	.open = lima_gem_object_open,
203	.close = lima_gem_object_close,
204	.print_info = drm_gem_shmem_print_info,
205	.pin = lima_gem_pin,
206	.unpin = drm_gem_shmem_unpin,
207	.get_sg_table = drm_gem_shmem_get_sg_table,
208	.vmap = lima_gem_vmap,
209	.vunmap = drm_gem_shmem_vunmap,
210	.mmap = lima_gem_mmap,
211};
212
213struct drm_gem_object *lima_gem_create_object(struct drm_device *dev, size_t size)
214{
215	struct lima_bo *bo;
216
217	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
218	if (!bo)
219		return NULL;
220
221	mutex_init(&bo->lock);
222	INIT_LIST_HEAD(&bo->va);
223
224	bo->base.base.funcs = &lima_gem_funcs;
225
226	return &bo->base.base;
227}
228
229int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset)
230{
231	struct drm_gem_object *obj;
232	struct lima_bo *bo;
233	struct lima_drm_priv *priv = to_lima_drm_priv(file);
234	struct lima_vm *vm = priv->vm;
235
236	obj = drm_gem_object_lookup(file, handle);
237	if (!obj)
238		return -ENOENT;
239
240	bo = to_lima_bo(obj);
241
242	*va = lima_vm_get_va(vm, bo);
243
244	*offset = drm_vma_node_offset_addr(&obj->vma_node);
245
246	drm_gem_object_put(obj);
247	return 0;
248}
249
250static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo,
251			    bool write, bool explicit)
252{
253	int err = 0;
254
255	if (!write) {
256		err = dma_resv_reserve_shared(lima_bo_resv(bo), 1);
257		if (err)
258			return err;
259	}
260
261	/* explicit sync use user passed dep fence */
262	if (explicit)
263		return 0;
264
265	return drm_gem_fence_array_add_implicit(&task->deps, &bo->base.base, write);
266}
267
268static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit)
269{
270	int i, err;
271
272	for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) {
273		struct dma_fence *fence = NULL;
274
275		if (!submit->in_sync[i])
276			continue;
277
278		err = drm_syncobj_find_fence(file, submit->in_sync[i],
279					     0, 0, &fence);
280		if (err)
281			return err;
282
283		err = drm_gem_fence_array_add(&submit->task->deps, fence);
284		if (err) {
285			dma_fence_put(fence);
286			return err;
287		}
288	}
289
290	return 0;
291}
292
293int lima_gem_submit(struct drm_file *file, struct lima_submit *submit)
294{
295	int i, err = 0;
296	struct ww_acquire_ctx ctx;
297	struct lima_drm_priv *priv = to_lima_drm_priv(file);
298	struct lima_vm *vm = priv->vm;
299	struct drm_syncobj *out_sync = NULL;
300	struct dma_fence *fence;
301	struct lima_bo **bos = submit->lbos;
302
303	if (submit->out_sync) {
304		out_sync = drm_syncobj_find(file, submit->out_sync);
305		if (!out_sync)
306			return -ENOENT;
307	}
308
309	for (i = 0; i < submit->nr_bos; i++) {
310		struct drm_gem_object *obj;
311		struct lima_bo *bo;
312
313		obj = drm_gem_object_lookup(file, submit->bos[i].handle);
314		if (!obj) {
315			err = -ENOENT;
316			goto err_out0;
317		}
318
319		bo = to_lima_bo(obj);
320
321		/* increase refcnt of gpu va map to prevent unmapped when executing,
322		 * will be decreased when task done
323		 */
324		err = lima_vm_bo_add(vm, bo, false);
325		if (err) {
326			drm_gem_object_put(obj);
327			goto err_out0;
328		}
329
330		bos[i] = bo;
331	}
332
333	err = drm_gem_lock_reservations((struct drm_gem_object **)bos,
334					submit->nr_bos, &ctx);
335	if (err)
336		goto err_out0;
337
338	err = lima_sched_task_init(
339		submit->task, submit->ctx->context + submit->pipe,
340		bos, submit->nr_bos, vm);
341	if (err)
342		goto err_out1;
343
344	err = lima_gem_add_deps(file, submit);
345	if (err)
346		goto err_out2;
347
348	for (i = 0; i < submit->nr_bos; i++) {
349		err = lima_gem_sync_bo(
350			submit->task, bos[i],
351			submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE,
352			submit->flags & LIMA_SUBMIT_FLAG_EXPLICIT_FENCE);
353		if (err)
354			goto err_out2;
355	}
356
357	fence = lima_sched_context_queue_task(
358		submit->ctx->context + submit->pipe, submit->task);
359
360	for (i = 0; i < submit->nr_bos; i++) {
361		if (submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE)
362			dma_resv_add_excl_fence(lima_bo_resv(bos[i]), fence);
363		else
364			dma_resv_add_shared_fence(lima_bo_resv(bos[i]), fence);
365	}
366
367	drm_gem_unlock_reservations((struct drm_gem_object **)bos,
368				    submit->nr_bos, &ctx);
369
370	for (i = 0; i < submit->nr_bos; i++)
371		drm_gem_object_put(&bos[i]->base.base);
372
373	if (out_sync) {
374		drm_syncobj_replace_fence(out_sync, fence);
375		drm_syncobj_put(out_sync);
376	}
377
378	dma_fence_put(fence);
379
380	return 0;
381
382err_out2:
383	lima_sched_task_fini(submit->task);
384err_out1:
385	drm_gem_unlock_reservations((struct drm_gem_object **)bos,
386				    submit->nr_bos, &ctx);
387err_out0:
388	for (i = 0; i < submit->nr_bos; i++) {
389		if (!bos[i])
390			break;
391		lima_vm_bo_del(vm, bos[i]);
392		drm_gem_object_put(&bos[i]->base.base);
393	}
394	if (out_sync)
395		drm_syncobj_put(out_sync);
396	return err;
397}
398
399int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns)
400{
401	bool write = op & LIMA_GEM_WAIT_WRITE;
402	long ret, timeout;
403
404	if (!op)
405		return 0;
406
407	timeout = drm_timeout_abs_to_jiffies(timeout_ns);
408
409	ret = drm_gem_dma_resv_wait(file, handle, write, timeout);
410	if (ret == -ETIME)
411		ret = timeout ? -ETIMEDOUT : -EBUSY;
412
413	return ret;
414}