Loading...
1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/hardirq.h>
5#include <linux/iosys-map.h>
6#include <linux/kthread.h>
7#include <linux/slab.h>
8#include <linux/vmalloc.h>
9#include <linux/pm_runtime.h>
10
11#include "lima_devfreq.h"
12#include "lima_drv.h"
13#include "lima_sched.h"
14#include "lima_vm.h"
15#include "lima_mmu.h"
16#include "lima_l2_cache.h"
17#include "lima_gem.h"
18#include "lima_trace.h"
19
20struct lima_fence {
21 struct dma_fence base;
22 struct lima_sched_pipe *pipe;
23};
24
25static struct kmem_cache *lima_fence_slab;
26static int lima_fence_slab_refcnt;
27
28int lima_sched_slab_init(void)
29{
30 if (!lima_fence_slab) {
31 lima_fence_slab = kmem_cache_create(
32 "lima_fence", sizeof(struct lima_fence), 0,
33 SLAB_HWCACHE_ALIGN, NULL);
34 if (!lima_fence_slab)
35 return -ENOMEM;
36 }
37
38 lima_fence_slab_refcnt++;
39 return 0;
40}
41
42void lima_sched_slab_fini(void)
43{
44 if (!--lima_fence_slab_refcnt) {
45 kmem_cache_destroy(lima_fence_slab);
46 lima_fence_slab = NULL;
47 }
48}
49
50static inline struct lima_fence *to_lima_fence(struct dma_fence *fence)
51{
52 return container_of(fence, struct lima_fence, base);
53}
54
55static const char *lima_fence_get_driver_name(struct dma_fence *fence)
56{
57 return "lima";
58}
59
60static const char *lima_fence_get_timeline_name(struct dma_fence *fence)
61{
62 struct lima_fence *f = to_lima_fence(fence);
63
64 return f->pipe->base.name;
65}
66
67static void lima_fence_release_rcu(struct rcu_head *rcu)
68{
69 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
70 struct lima_fence *fence = to_lima_fence(f);
71
72 kmem_cache_free(lima_fence_slab, fence);
73}
74
75static void lima_fence_release(struct dma_fence *fence)
76{
77 struct lima_fence *f = to_lima_fence(fence);
78
79 call_rcu(&f->base.rcu, lima_fence_release_rcu);
80}
81
82static const struct dma_fence_ops lima_fence_ops = {
83 .get_driver_name = lima_fence_get_driver_name,
84 .get_timeline_name = lima_fence_get_timeline_name,
85 .release = lima_fence_release,
86};
87
88static struct lima_fence *lima_fence_create(struct lima_sched_pipe *pipe)
89{
90 struct lima_fence *fence;
91
92 fence = kmem_cache_zalloc(lima_fence_slab, GFP_KERNEL);
93 if (!fence)
94 return NULL;
95
96 fence->pipe = pipe;
97 dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,
98 pipe->fence_context, ++pipe->fence_seqno);
99
100 return fence;
101}
102
103static inline struct lima_sched_task *to_lima_task(struct drm_sched_job *job)
104{
105 return container_of(job, struct lima_sched_task, base);
106}
107
108static inline struct lima_sched_pipe *to_lima_pipe(struct drm_gpu_scheduler *sched)
109{
110 return container_of(sched, struct lima_sched_pipe, base);
111}
112
113int lima_sched_task_init(struct lima_sched_task *task,
114 struct lima_sched_context *context,
115 struct lima_bo **bos, int num_bos,
116 struct lima_vm *vm)
117{
118 int err, i;
119
120 task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);
121 if (!task->bos)
122 return -ENOMEM;
123
124 for (i = 0; i < num_bos; i++)
125 drm_gem_object_get(&bos[i]->base.base);
126
127 err = drm_sched_job_init(&task->base, &context->base, 1, vm);
128 if (err) {
129 kfree(task->bos);
130 return err;
131 }
132
133 drm_sched_job_arm(&task->base);
134
135 task->num_bos = num_bos;
136 task->vm = lima_vm_get(vm);
137
138 return 0;
139}
140
141void lima_sched_task_fini(struct lima_sched_task *task)
142{
143 int i;
144
145 drm_sched_job_cleanup(&task->base);
146
147 if (task->bos) {
148 for (i = 0; i < task->num_bos; i++)
149 drm_gem_object_put(&task->bos[i]->base.base);
150 kfree(task->bos);
151 }
152
153 lima_vm_put(task->vm);
154}
155
156int lima_sched_context_init(struct lima_sched_pipe *pipe,
157 struct lima_sched_context *context)
158{
159 struct drm_gpu_scheduler *sched = &pipe->base;
160
161 return drm_sched_entity_init(&context->base, DRM_SCHED_PRIORITY_NORMAL,
162 &sched, 1, NULL);
163}
164
165void lima_sched_context_fini(struct lima_sched_pipe *pipe,
166 struct lima_sched_context *context)
167{
168 drm_sched_entity_destroy(&context->base);
169}
170
171struct dma_fence *lima_sched_context_queue_task(struct lima_sched_task *task)
172{
173 struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
174
175 trace_lima_task_submit(task);
176 drm_sched_entity_push_job(&task->base);
177 return fence;
178}
179
180static int lima_pm_busy(struct lima_device *ldev)
181{
182 int ret;
183
184 /* resume GPU if it has been suspended by runtime PM */
185 ret = pm_runtime_resume_and_get(ldev->dev);
186 if (ret < 0)
187 return ret;
188
189 lima_devfreq_record_busy(&ldev->devfreq);
190 return 0;
191}
192
193static void lima_pm_idle(struct lima_device *ldev)
194{
195 lima_devfreq_record_idle(&ldev->devfreq);
196
197 /* GPU can do auto runtime suspend */
198 pm_runtime_mark_last_busy(ldev->dev);
199 pm_runtime_put_autosuspend(ldev->dev);
200}
201
202static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
203{
204 struct lima_sched_task *task = to_lima_task(job);
205 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
206 struct lima_device *ldev = pipe->ldev;
207 struct lima_fence *fence;
208 int i, err;
209
210 /* after GPU reset */
211 if (job->s_fence->finished.error < 0)
212 return NULL;
213
214 fence = lima_fence_create(pipe);
215 if (!fence)
216 return NULL;
217
218 err = lima_pm_busy(ldev);
219 if (err < 0) {
220 dma_fence_put(&fence->base);
221 return NULL;
222 }
223
224 task->fence = &fence->base;
225
226 /* for caller usage of the fence, otherwise irq handler
227 * may consume the fence before caller use it
228 */
229 dma_fence_get(task->fence);
230
231 pipe->current_task = task;
232
233 /* this is needed for MMU to work correctly, otherwise GP/PP
234 * will hang or page fault for unknown reason after running for
235 * a while.
236 *
237 * Need to investigate:
238 * 1. is it related to TLB
239 * 2. how much performance will be affected by L2 cache flush
240 * 3. can we reduce the calling of this function because all
241 * GP/PP use the same L2 cache on mali400
242 *
243 * TODO:
244 * 1. move this to task fini to save some wait time?
245 * 2. when GP/PP use different l2 cache, need PP wait GP l2
246 * cache flush?
247 */
248 for (i = 0; i < pipe->num_l2_cache; i++)
249 lima_l2_cache_flush(pipe->l2_cache[i]);
250
251 lima_vm_put(pipe->current_vm);
252 pipe->current_vm = lima_vm_get(task->vm);
253
254 if (pipe->bcast_mmu)
255 lima_mmu_switch_vm(pipe->bcast_mmu, pipe->current_vm);
256 else {
257 for (i = 0; i < pipe->num_mmu; i++)
258 lima_mmu_switch_vm(pipe->mmu[i], pipe->current_vm);
259 }
260
261 trace_lima_task_run(task);
262
263 pipe->error = false;
264 pipe->task_run(pipe, task);
265
266 return task->fence;
267}
268
269static void lima_sched_build_error_task_list(struct lima_sched_task *task)
270{
271 struct lima_sched_error_task *et;
272 struct lima_sched_pipe *pipe = to_lima_pipe(task->base.sched);
273 struct lima_ip *ip = pipe->processor[0];
274 int pipe_id = ip->id == lima_ip_gp ? lima_pipe_gp : lima_pipe_pp;
275 struct lima_device *dev = ip->dev;
276 struct lima_sched_context *sched_ctx =
277 container_of(task->base.entity,
278 struct lima_sched_context, base);
279 struct lima_ctx *ctx =
280 container_of(sched_ctx, struct lima_ctx, context[pipe_id]);
281 struct lima_dump_task *dt;
282 struct lima_dump_chunk *chunk;
283 struct lima_dump_chunk_pid *pid_chunk;
284 struct lima_dump_chunk_buffer *buffer_chunk;
285 u32 size, task_size, mem_size;
286 int i;
287 struct iosys_map map;
288 int ret;
289
290 mutex_lock(&dev->error_task_list_lock);
291
292 if (dev->dump.num_tasks >= lima_max_error_tasks) {
293 dev_info(dev->dev, "fail to save task state from %s pid %d: "
294 "error task list is full\n", ctx->pname, ctx->pid);
295 goto out;
296 }
297
298 /* frame chunk */
299 size = sizeof(struct lima_dump_chunk) + pipe->frame_size;
300 /* process name chunk */
301 size += sizeof(struct lima_dump_chunk) + sizeof(ctx->pname);
302 /* pid chunk */
303 size += sizeof(struct lima_dump_chunk);
304 /* buffer chunks */
305 for (i = 0; i < task->num_bos; i++) {
306 struct lima_bo *bo = task->bos[i];
307
308 size += sizeof(struct lima_dump_chunk);
309 size += bo->heap_size ? bo->heap_size : lima_bo_size(bo);
310 }
311
312 task_size = size + sizeof(struct lima_dump_task);
313 mem_size = task_size + sizeof(*et);
314 et = kvmalloc(mem_size, GFP_KERNEL);
315 if (!et) {
316 dev_err(dev->dev, "fail to alloc task dump buffer of size %x\n",
317 mem_size);
318 goto out;
319 }
320
321 et->data = et + 1;
322 et->size = task_size;
323
324 dt = et->data;
325 memset(dt, 0, sizeof(*dt));
326 dt->id = pipe_id;
327 dt->size = size;
328
329 chunk = (struct lima_dump_chunk *)(dt + 1);
330 memset(chunk, 0, sizeof(*chunk));
331 chunk->id = LIMA_DUMP_CHUNK_FRAME;
332 chunk->size = pipe->frame_size;
333 memcpy(chunk + 1, task->frame, pipe->frame_size);
334 dt->num_chunks++;
335
336 chunk = (void *)(chunk + 1) + chunk->size;
337 memset(chunk, 0, sizeof(*chunk));
338 chunk->id = LIMA_DUMP_CHUNK_PROCESS_NAME;
339 chunk->size = sizeof(ctx->pname);
340 memcpy(chunk + 1, ctx->pname, sizeof(ctx->pname));
341 dt->num_chunks++;
342
343 pid_chunk = (void *)(chunk + 1) + chunk->size;
344 memset(pid_chunk, 0, sizeof(*pid_chunk));
345 pid_chunk->id = LIMA_DUMP_CHUNK_PROCESS_ID;
346 pid_chunk->pid = ctx->pid;
347 dt->num_chunks++;
348
349 buffer_chunk = (void *)(pid_chunk + 1) + pid_chunk->size;
350 for (i = 0; i < task->num_bos; i++) {
351 struct lima_bo *bo = task->bos[i];
352 void *data;
353
354 memset(buffer_chunk, 0, sizeof(*buffer_chunk));
355 buffer_chunk->id = LIMA_DUMP_CHUNK_BUFFER;
356 buffer_chunk->va = lima_vm_get_va(task->vm, bo);
357
358 if (bo->heap_size) {
359 buffer_chunk->size = bo->heap_size;
360
361 data = vmap(bo->base.pages, bo->heap_size >> PAGE_SHIFT,
362 VM_MAP, pgprot_writecombine(PAGE_KERNEL));
363 if (!data) {
364 kvfree(et);
365 goto out;
366 }
367
368 memcpy(buffer_chunk + 1, data, buffer_chunk->size);
369
370 vunmap(data);
371 } else {
372 buffer_chunk->size = lima_bo_size(bo);
373
374 ret = drm_gem_vmap_unlocked(&bo->base.base, &map);
375 if (ret) {
376 kvfree(et);
377 goto out;
378 }
379
380 memcpy(buffer_chunk + 1, map.vaddr, buffer_chunk->size);
381
382 drm_gem_vunmap_unlocked(&bo->base.base, &map);
383 }
384
385 buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size;
386 dt->num_chunks++;
387 }
388
389 list_add(&et->list, &dev->error_task_list);
390 dev->dump.size += et->size;
391 dev->dump.num_tasks++;
392
393 dev_info(dev->dev, "save error task state success\n");
394
395out:
396 mutex_unlock(&dev->error_task_list_lock);
397}
398
399static enum drm_gpu_sched_stat lima_sched_timedout_job(struct drm_sched_job *job)
400{
401 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
402 struct lima_sched_task *task = to_lima_task(job);
403 struct lima_device *ldev = pipe->ldev;
404 struct lima_ip *ip = pipe->processor[0];
405 int i;
406
407 /*
408 * If the GPU managed to complete this jobs fence, the timeout is
409 * spurious. Bail out.
410 */
411 if (dma_fence_is_signaled(task->fence)) {
412 DRM_WARN("%s spurious timeout\n", lima_ip_name(ip));
413 return DRM_GPU_SCHED_STAT_NOMINAL;
414 }
415
416 /*
417 * Lima IRQ handler may take a long time to process an interrupt
418 * if there is another IRQ handler hogging the processing.
419 * In order to catch such cases and not report spurious Lima job
420 * timeouts, synchronize the IRQ handler and re-check the fence
421 * status.
422 */
423 for (i = 0; i < pipe->num_processor; i++)
424 synchronize_irq(pipe->processor[i]->irq);
425 if (pipe->bcast_processor)
426 synchronize_irq(pipe->bcast_processor->irq);
427
428 if (dma_fence_is_signaled(task->fence)) {
429 DRM_WARN("%s unexpectedly high interrupt latency\n", lima_ip_name(ip));
430 return DRM_GPU_SCHED_STAT_NOMINAL;
431 }
432
433 /*
434 * The task might still finish while this timeout handler runs.
435 * To prevent a race condition on its completion, mask all irqs
436 * on the running core until the next hard reset completes.
437 */
438 pipe->task_mask_irq(pipe);
439
440 if (!pipe->error)
441 DRM_ERROR("%s job timeout\n", lima_ip_name(ip));
442
443 drm_sched_stop(&pipe->base, &task->base);
444
445 drm_sched_increase_karma(&task->base);
446
447 if (lima_max_error_tasks)
448 lima_sched_build_error_task_list(task);
449
450 pipe->task_error(pipe);
451
452 if (pipe->bcast_mmu)
453 lima_mmu_page_fault_resume(pipe->bcast_mmu);
454 else {
455 for (i = 0; i < pipe->num_mmu; i++)
456 lima_mmu_page_fault_resume(pipe->mmu[i]);
457 }
458
459 lima_vm_put(pipe->current_vm);
460 pipe->current_vm = NULL;
461 pipe->current_task = NULL;
462
463 lima_pm_idle(ldev);
464
465 drm_sched_resubmit_jobs(&pipe->base);
466 drm_sched_start(&pipe->base, 0);
467
468 return DRM_GPU_SCHED_STAT_NOMINAL;
469}
470
471static void lima_sched_free_job(struct drm_sched_job *job)
472{
473 struct lima_sched_task *task = to_lima_task(job);
474 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
475 struct lima_vm *vm = task->vm;
476 struct lima_bo **bos = task->bos;
477 int i;
478
479 dma_fence_put(task->fence);
480
481 for (i = 0; i < task->num_bos; i++)
482 lima_vm_bo_del(vm, bos[i]);
483
484 lima_sched_task_fini(task);
485 kmem_cache_free(pipe->task_slab, task);
486}
487
488static const struct drm_sched_backend_ops lima_sched_ops = {
489 .run_job = lima_sched_run_job,
490 .timedout_job = lima_sched_timedout_job,
491 .free_job = lima_sched_free_job,
492};
493
494static void lima_sched_recover_work(struct work_struct *work)
495{
496 struct lima_sched_pipe *pipe =
497 container_of(work, struct lima_sched_pipe, recover_work);
498 int i;
499
500 for (i = 0; i < pipe->num_l2_cache; i++)
501 lima_l2_cache_flush(pipe->l2_cache[i]);
502
503 if (pipe->bcast_mmu) {
504 lima_mmu_flush_tlb(pipe->bcast_mmu);
505 } else {
506 for (i = 0; i < pipe->num_mmu; i++)
507 lima_mmu_flush_tlb(pipe->mmu[i]);
508 }
509
510 if (pipe->task_recover(pipe))
511 drm_sched_fault(&pipe->base);
512}
513
514int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
515{
516 unsigned int timeout = lima_sched_timeout_ms > 0 ?
517 lima_sched_timeout_ms : 10000;
518
519 pipe->fence_context = dma_fence_context_alloc(1);
520 spin_lock_init(&pipe->fence_lock);
521
522 INIT_WORK(&pipe->recover_work, lima_sched_recover_work);
523
524 return drm_sched_init(&pipe->base, &lima_sched_ops, NULL,
525 DRM_SCHED_PRIORITY_COUNT,
526 1,
527 lima_job_hang_limit,
528 msecs_to_jiffies(timeout), NULL,
529 NULL, name, pipe->ldev->dev);
530}
531
532void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
533{
534 drm_sched_fini(&pipe->base);
535}
536
537void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
538{
539 struct lima_sched_task *task = pipe->current_task;
540 struct lima_device *ldev = pipe->ldev;
541
542 if (pipe->error) {
543 if (task && task->recoverable)
544 schedule_work(&pipe->recover_work);
545 else
546 drm_sched_fault(&pipe->base);
547 } else {
548 pipe->task_fini(pipe);
549 dma_fence_signal(task->fence);
550
551 lima_pm_idle(ldev);
552 }
553}
1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/kthread.h>
5#include <linux/slab.h>
6#include <linux/xarray.h>
7
8#include "lima_drv.h"
9#include "lima_sched.h"
10#include "lima_vm.h"
11#include "lima_mmu.h"
12#include "lima_l2_cache.h"
13#include "lima_object.h"
14
15struct lima_fence {
16 struct dma_fence base;
17 struct lima_sched_pipe *pipe;
18};
19
20static struct kmem_cache *lima_fence_slab;
21static int lima_fence_slab_refcnt;
22
23int lima_sched_slab_init(void)
24{
25 if (!lima_fence_slab) {
26 lima_fence_slab = kmem_cache_create(
27 "lima_fence", sizeof(struct lima_fence), 0,
28 SLAB_HWCACHE_ALIGN, NULL);
29 if (!lima_fence_slab)
30 return -ENOMEM;
31 }
32
33 lima_fence_slab_refcnt++;
34 return 0;
35}
36
37void lima_sched_slab_fini(void)
38{
39 if (!--lima_fence_slab_refcnt) {
40 kmem_cache_destroy(lima_fence_slab);
41 lima_fence_slab = NULL;
42 }
43}
44
45static inline struct lima_fence *to_lima_fence(struct dma_fence *fence)
46{
47 return container_of(fence, struct lima_fence, base);
48}
49
50static const char *lima_fence_get_driver_name(struct dma_fence *fence)
51{
52 return "lima";
53}
54
55static const char *lima_fence_get_timeline_name(struct dma_fence *fence)
56{
57 struct lima_fence *f = to_lima_fence(fence);
58
59 return f->pipe->base.name;
60}
61
62static void lima_fence_release_rcu(struct rcu_head *rcu)
63{
64 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
65 struct lima_fence *fence = to_lima_fence(f);
66
67 kmem_cache_free(lima_fence_slab, fence);
68}
69
70static void lima_fence_release(struct dma_fence *fence)
71{
72 struct lima_fence *f = to_lima_fence(fence);
73
74 call_rcu(&f->base.rcu, lima_fence_release_rcu);
75}
76
77static const struct dma_fence_ops lima_fence_ops = {
78 .get_driver_name = lima_fence_get_driver_name,
79 .get_timeline_name = lima_fence_get_timeline_name,
80 .release = lima_fence_release,
81};
82
83static struct lima_fence *lima_fence_create(struct lima_sched_pipe *pipe)
84{
85 struct lima_fence *fence;
86
87 fence = kmem_cache_zalloc(lima_fence_slab, GFP_KERNEL);
88 if (!fence)
89 return NULL;
90
91 fence->pipe = pipe;
92 dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,
93 pipe->fence_context, ++pipe->fence_seqno);
94
95 return fence;
96}
97
98static inline struct lima_sched_task *to_lima_task(struct drm_sched_job *job)
99{
100 return container_of(job, struct lima_sched_task, base);
101}
102
103static inline struct lima_sched_pipe *to_lima_pipe(struct drm_gpu_scheduler *sched)
104{
105 return container_of(sched, struct lima_sched_pipe, base);
106}
107
108int lima_sched_task_init(struct lima_sched_task *task,
109 struct lima_sched_context *context,
110 struct lima_bo **bos, int num_bos,
111 struct lima_vm *vm)
112{
113 int err, i;
114
115 task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);
116 if (!task->bos)
117 return -ENOMEM;
118
119 for (i = 0; i < num_bos; i++)
120 drm_gem_object_get(&bos[i]->gem);
121
122 err = drm_sched_job_init(&task->base, &context->base, vm);
123 if (err) {
124 kfree(task->bos);
125 return err;
126 }
127
128 task->num_bos = num_bos;
129 task->vm = lima_vm_get(vm);
130
131 xa_init_flags(&task->deps, XA_FLAGS_ALLOC);
132
133 return 0;
134}
135
136void lima_sched_task_fini(struct lima_sched_task *task)
137{
138 struct dma_fence *fence;
139 unsigned long index;
140 int i;
141
142 drm_sched_job_cleanup(&task->base);
143
144 xa_for_each(&task->deps, index, fence) {
145 dma_fence_put(fence);
146 }
147 xa_destroy(&task->deps);
148
149 if (task->bos) {
150 for (i = 0; i < task->num_bos; i++)
151 drm_gem_object_put_unlocked(&task->bos[i]->gem);
152 kfree(task->bos);
153 }
154
155 lima_vm_put(task->vm);
156}
157
158int lima_sched_context_init(struct lima_sched_pipe *pipe,
159 struct lima_sched_context *context,
160 atomic_t *guilty)
161{
162 struct drm_sched_rq *rq = pipe->base.sched_rq + DRM_SCHED_PRIORITY_NORMAL;
163
164 return drm_sched_entity_init(&context->base, &rq, 1, guilty);
165}
166
167void lima_sched_context_fini(struct lima_sched_pipe *pipe,
168 struct lima_sched_context *context)
169{
170 drm_sched_entity_fini(&context->base);
171}
172
173struct dma_fence *lima_sched_context_queue_task(struct lima_sched_context *context,
174 struct lima_sched_task *task)
175{
176 struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
177
178 drm_sched_entity_push_job(&task->base, &context->base);
179 return fence;
180}
181
182static struct dma_fence *lima_sched_dependency(struct drm_sched_job *job,
183 struct drm_sched_entity *entity)
184{
185 struct lima_sched_task *task = to_lima_task(job);
186
187 if (!xa_empty(&task->deps))
188 return xa_erase(&task->deps, task->last_dep++);
189
190 return NULL;
191}
192
193static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
194{
195 struct lima_sched_task *task = to_lima_task(job);
196 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
197 struct lima_fence *fence;
198 struct dma_fence *ret;
199 struct lima_vm *vm = NULL, *last_vm = NULL;
200 int i;
201
202 /* after GPU reset */
203 if (job->s_fence->finished.error < 0)
204 return NULL;
205
206 fence = lima_fence_create(pipe);
207 if (!fence)
208 return NULL;
209 task->fence = &fence->base;
210
211 /* for caller usage of the fence, otherwise irq handler
212 * may consume the fence before caller use it
213 */
214 ret = dma_fence_get(task->fence);
215
216 pipe->current_task = task;
217
218 /* this is needed for MMU to work correctly, otherwise GP/PP
219 * will hang or page fault for unknown reason after running for
220 * a while.
221 *
222 * Need to investigate:
223 * 1. is it related to TLB
224 * 2. how much performance will be affected by L2 cache flush
225 * 3. can we reduce the calling of this function because all
226 * GP/PP use the same L2 cache on mali400
227 *
228 * TODO:
229 * 1. move this to task fini to save some wait time?
230 * 2. when GP/PP use different l2 cache, need PP wait GP l2
231 * cache flush?
232 */
233 for (i = 0; i < pipe->num_l2_cache; i++)
234 lima_l2_cache_flush(pipe->l2_cache[i]);
235
236 if (task->vm != pipe->current_vm) {
237 vm = lima_vm_get(task->vm);
238 last_vm = pipe->current_vm;
239 pipe->current_vm = task->vm;
240 }
241
242 if (pipe->bcast_mmu)
243 lima_mmu_switch_vm(pipe->bcast_mmu, vm);
244 else {
245 for (i = 0; i < pipe->num_mmu; i++)
246 lima_mmu_switch_vm(pipe->mmu[i], vm);
247 }
248
249 if (last_vm)
250 lima_vm_put(last_vm);
251
252 pipe->error = false;
253 pipe->task_run(pipe, task);
254
255 return task->fence;
256}
257
258static void lima_sched_handle_error_task(struct lima_sched_pipe *pipe,
259 struct lima_sched_task *task)
260{
261 drm_sched_stop(&pipe->base, &task->base);
262
263 if (task)
264 drm_sched_increase_karma(&task->base);
265
266 pipe->task_error(pipe);
267
268 if (pipe->bcast_mmu)
269 lima_mmu_page_fault_resume(pipe->bcast_mmu);
270 else {
271 int i;
272
273 for (i = 0; i < pipe->num_mmu; i++)
274 lima_mmu_page_fault_resume(pipe->mmu[i]);
275 }
276
277 if (pipe->current_vm)
278 lima_vm_put(pipe->current_vm);
279
280 pipe->current_vm = NULL;
281 pipe->current_task = NULL;
282
283 drm_sched_resubmit_jobs(&pipe->base);
284 drm_sched_start(&pipe->base, true);
285}
286
287static void lima_sched_timedout_job(struct drm_sched_job *job)
288{
289 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
290 struct lima_sched_task *task = to_lima_task(job);
291
292 DRM_ERROR("lima job timeout\n");
293
294 lima_sched_handle_error_task(pipe, task);
295}
296
297static void lima_sched_free_job(struct drm_sched_job *job)
298{
299 struct lima_sched_task *task = to_lima_task(job);
300 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
301 struct lima_vm *vm = task->vm;
302 struct lima_bo **bos = task->bos;
303 int i;
304
305 dma_fence_put(task->fence);
306
307 for (i = 0; i < task->num_bos; i++)
308 lima_vm_bo_del(vm, bos[i]);
309
310 lima_sched_task_fini(task);
311 kmem_cache_free(pipe->task_slab, task);
312}
313
314static const struct drm_sched_backend_ops lima_sched_ops = {
315 .dependency = lima_sched_dependency,
316 .run_job = lima_sched_run_job,
317 .timedout_job = lima_sched_timedout_job,
318 .free_job = lima_sched_free_job,
319};
320
321static void lima_sched_error_work(struct work_struct *work)
322{
323 struct lima_sched_pipe *pipe =
324 container_of(work, struct lima_sched_pipe, error_work);
325 struct lima_sched_task *task = pipe->current_task;
326
327 lima_sched_handle_error_task(pipe, task);
328}
329
330int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
331{
332 unsigned int timeout = lima_sched_timeout_ms > 0 ?
333 lima_sched_timeout_ms : 500;
334
335 pipe->fence_context = dma_fence_context_alloc(1);
336 spin_lock_init(&pipe->fence_lock);
337
338 INIT_WORK(&pipe->error_work, lima_sched_error_work);
339
340 return drm_sched_init(&pipe->base, &lima_sched_ops, 1, 0,
341 msecs_to_jiffies(timeout), name);
342}
343
344void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
345{
346 drm_sched_fini(&pipe->base);
347}
348
349void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
350{
351 if (pipe->error)
352 schedule_work(&pipe->error_work);
353 else {
354 struct lima_sched_task *task = pipe->current_task;
355
356 pipe->task_fini(pipe);
357 dma_fence_signal(task->fence);
358 }
359}