Loading...
1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/kthread.h>
5#include <linux/slab.h>
6#include <linux/vmalloc.h>
7#include <linux/pm_runtime.h>
8
9#include "lima_devfreq.h"
10#include "lima_drv.h"
11#include "lima_sched.h"
12#include "lima_vm.h"
13#include "lima_mmu.h"
14#include "lima_l2_cache.h"
15#include "lima_gem.h"
16#include "lima_trace.h"
17
18struct lima_fence {
19 struct dma_fence base;
20 struct lima_sched_pipe *pipe;
21};
22
23static struct kmem_cache *lima_fence_slab;
24static int lima_fence_slab_refcnt;
25
26int lima_sched_slab_init(void)
27{
28 if (!lima_fence_slab) {
29 lima_fence_slab = kmem_cache_create(
30 "lima_fence", sizeof(struct lima_fence), 0,
31 SLAB_HWCACHE_ALIGN, NULL);
32 if (!lima_fence_slab)
33 return -ENOMEM;
34 }
35
36 lima_fence_slab_refcnt++;
37 return 0;
38}
39
40void lima_sched_slab_fini(void)
41{
42 if (!--lima_fence_slab_refcnt) {
43 kmem_cache_destroy(lima_fence_slab);
44 lima_fence_slab = NULL;
45 }
46}
47
48static inline struct lima_fence *to_lima_fence(struct dma_fence *fence)
49{
50 return container_of(fence, struct lima_fence, base);
51}
52
53static const char *lima_fence_get_driver_name(struct dma_fence *fence)
54{
55 return "lima";
56}
57
58static const char *lima_fence_get_timeline_name(struct dma_fence *fence)
59{
60 struct lima_fence *f = to_lima_fence(fence);
61
62 return f->pipe->base.name;
63}
64
65static void lima_fence_release_rcu(struct rcu_head *rcu)
66{
67 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
68 struct lima_fence *fence = to_lima_fence(f);
69
70 kmem_cache_free(lima_fence_slab, fence);
71}
72
73static void lima_fence_release(struct dma_fence *fence)
74{
75 struct lima_fence *f = to_lima_fence(fence);
76
77 call_rcu(&f->base.rcu, lima_fence_release_rcu);
78}
79
80static const struct dma_fence_ops lima_fence_ops = {
81 .get_driver_name = lima_fence_get_driver_name,
82 .get_timeline_name = lima_fence_get_timeline_name,
83 .release = lima_fence_release,
84};
85
86static struct lima_fence *lima_fence_create(struct lima_sched_pipe *pipe)
87{
88 struct lima_fence *fence;
89
90 fence = kmem_cache_zalloc(lima_fence_slab, GFP_KERNEL);
91 if (!fence)
92 return NULL;
93
94 fence->pipe = pipe;
95 dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,
96 pipe->fence_context, ++pipe->fence_seqno);
97
98 return fence;
99}
100
101static inline struct lima_sched_task *to_lima_task(struct drm_sched_job *job)
102{
103 return container_of(job, struct lima_sched_task, base);
104}
105
106static inline struct lima_sched_pipe *to_lima_pipe(struct drm_gpu_scheduler *sched)
107{
108 return container_of(sched, struct lima_sched_pipe, base);
109}
110
111int lima_sched_task_init(struct lima_sched_task *task,
112 struct lima_sched_context *context,
113 struct lima_bo **bos, int num_bos,
114 struct lima_vm *vm)
115{
116 int err, i;
117
118 task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);
119 if (!task->bos)
120 return -ENOMEM;
121
122 for (i = 0; i < num_bos; i++)
123 drm_gem_object_get(&bos[i]->base.base);
124
125 err = drm_sched_job_init(&task->base, &context->base, vm);
126 if (err) {
127 kfree(task->bos);
128 return err;
129 }
130
131 task->num_bos = num_bos;
132 task->vm = lima_vm_get(vm);
133
134 xa_init_flags(&task->deps, XA_FLAGS_ALLOC);
135
136 return 0;
137}
138
139void lima_sched_task_fini(struct lima_sched_task *task)
140{
141 struct dma_fence *fence;
142 unsigned long index;
143 int i;
144
145 drm_sched_job_cleanup(&task->base);
146
147 xa_for_each(&task->deps, index, fence) {
148 dma_fence_put(fence);
149 }
150 xa_destroy(&task->deps);
151
152 if (task->bos) {
153 for (i = 0; i < task->num_bos; i++)
154 drm_gem_object_put(&task->bos[i]->base.base);
155 kfree(task->bos);
156 }
157
158 lima_vm_put(task->vm);
159}
160
161int lima_sched_context_init(struct lima_sched_pipe *pipe,
162 struct lima_sched_context *context,
163 atomic_t *guilty)
164{
165 struct drm_gpu_scheduler *sched = &pipe->base;
166
167 return drm_sched_entity_init(&context->base, DRM_SCHED_PRIORITY_NORMAL,
168 &sched, 1, guilty);
169}
170
171void lima_sched_context_fini(struct lima_sched_pipe *pipe,
172 struct lima_sched_context *context)
173{
174 drm_sched_entity_fini(&context->base);
175}
176
177struct dma_fence *lima_sched_context_queue_task(struct lima_sched_context *context,
178 struct lima_sched_task *task)
179{
180 struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
181
182 trace_lima_task_submit(task);
183 drm_sched_entity_push_job(&task->base, &context->base);
184 return fence;
185}
186
187static struct dma_fence *lima_sched_dependency(struct drm_sched_job *job,
188 struct drm_sched_entity *entity)
189{
190 struct lima_sched_task *task = to_lima_task(job);
191
192 if (!xa_empty(&task->deps))
193 return xa_erase(&task->deps, task->last_dep++);
194
195 return NULL;
196}
197
198static int lima_pm_busy(struct lima_device *ldev)
199{
200 int ret;
201
202 /* resume GPU if it has been suspended by runtime PM */
203 ret = pm_runtime_get_sync(ldev->dev);
204 if (ret < 0)
205 return ret;
206
207 lima_devfreq_record_busy(&ldev->devfreq);
208 return 0;
209}
210
211static void lima_pm_idle(struct lima_device *ldev)
212{
213 lima_devfreq_record_idle(&ldev->devfreq);
214
215 /* GPU can do auto runtime suspend */
216 pm_runtime_mark_last_busy(ldev->dev);
217 pm_runtime_put_autosuspend(ldev->dev);
218}
219
220static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
221{
222 struct lima_sched_task *task = to_lima_task(job);
223 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
224 struct lima_device *ldev = pipe->ldev;
225 struct lima_fence *fence;
226 struct dma_fence *ret;
227 int i, err;
228
229 /* after GPU reset */
230 if (job->s_fence->finished.error < 0)
231 return NULL;
232
233 fence = lima_fence_create(pipe);
234 if (!fence)
235 return NULL;
236
237 err = lima_pm_busy(ldev);
238 if (err < 0) {
239 dma_fence_put(&fence->base);
240 return NULL;
241 }
242
243 task->fence = &fence->base;
244
245 /* for caller usage of the fence, otherwise irq handler
246 * may consume the fence before caller use it
247 */
248 ret = dma_fence_get(task->fence);
249
250 pipe->current_task = task;
251
252 /* this is needed for MMU to work correctly, otherwise GP/PP
253 * will hang or page fault for unknown reason after running for
254 * a while.
255 *
256 * Need to investigate:
257 * 1. is it related to TLB
258 * 2. how much performance will be affected by L2 cache flush
259 * 3. can we reduce the calling of this function because all
260 * GP/PP use the same L2 cache on mali400
261 *
262 * TODO:
263 * 1. move this to task fini to save some wait time?
264 * 2. when GP/PP use different l2 cache, need PP wait GP l2
265 * cache flush?
266 */
267 for (i = 0; i < pipe->num_l2_cache; i++)
268 lima_l2_cache_flush(pipe->l2_cache[i]);
269
270 lima_vm_put(pipe->current_vm);
271 pipe->current_vm = lima_vm_get(task->vm);
272
273 if (pipe->bcast_mmu)
274 lima_mmu_switch_vm(pipe->bcast_mmu, pipe->current_vm);
275 else {
276 for (i = 0; i < pipe->num_mmu; i++)
277 lima_mmu_switch_vm(pipe->mmu[i], pipe->current_vm);
278 }
279
280 trace_lima_task_run(task);
281
282 pipe->error = false;
283 pipe->task_run(pipe, task);
284
285 return task->fence;
286}
287
288static void lima_sched_build_error_task_list(struct lima_sched_task *task)
289{
290 struct lima_sched_error_task *et;
291 struct lima_sched_pipe *pipe = to_lima_pipe(task->base.sched);
292 struct lima_ip *ip = pipe->processor[0];
293 int pipe_id = ip->id == lima_ip_gp ? lima_pipe_gp : lima_pipe_pp;
294 struct lima_device *dev = ip->dev;
295 struct lima_sched_context *sched_ctx =
296 container_of(task->base.entity,
297 struct lima_sched_context, base);
298 struct lima_ctx *ctx =
299 container_of(sched_ctx, struct lima_ctx, context[pipe_id]);
300 struct lima_dump_task *dt;
301 struct lima_dump_chunk *chunk;
302 struct lima_dump_chunk_pid *pid_chunk;
303 struct lima_dump_chunk_buffer *buffer_chunk;
304 u32 size, task_size, mem_size;
305 int i;
306
307 mutex_lock(&dev->error_task_list_lock);
308
309 if (dev->dump.num_tasks >= lima_max_error_tasks) {
310 dev_info(dev->dev, "fail to save task state from %s pid %d: "
311 "error task list is full\n", ctx->pname, ctx->pid);
312 goto out;
313 }
314
315 /* frame chunk */
316 size = sizeof(struct lima_dump_chunk) + pipe->frame_size;
317 /* process name chunk */
318 size += sizeof(struct lima_dump_chunk) + sizeof(ctx->pname);
319 /* pid chunk */
320 size += sizeof(struct lima_dump_chunk);
321 /* buffer chunks */
322 for (i = 0; i < task->num_bos; i++) {
323 struct lima_bo *bo = task->bos[i];
324
325 size += sizeof(struct lima_dump_chunk);
326 size += bo->heap_size ? bo->heap_size : lima_bo_size(bo);
327 }
328
329 task_size = size + sizeof(struct lima_dump_task);
330 mem_size = task_size + sizeof(*et);
331 et = kvmalloc(mem_size, GFP_KERNEL);
332 if (!et) {
333 dev_err(dev->dev, "fail to alloc task dump buffer of size %x\n",
334 mem_size);
335 goto out;
336 }
337
338 et->data = et + 1;
339 et->size = task_size;
340
341 dt = et->data;
342 memset(dt, 0, sizeof(*dt));
343 dt->id = pipe_id;
344 dt->size = size;
345
346 chunk = (struct lima_dump_chunk *)(dt + 1);
347 memset(chunk, 0, sizeof(*chunk));
348 chunk->id = LIMA_DUMP_CHUNK_FRAME;
349 chunk->size = pipe->frame_size;
350 memcpy(chunk + 1, task->frame, pipe->frame_size);
351 dt->num_chunks++;
352
353 chunk = (void *)(chunk + 1) + chunk->size;
354 memset(chunk, 0, sizeof(*chunk));
355 chunk->id = LIMA_DUMP_CHUNK_PROCESS_NAME;
356 chunk->size = sizeof(ctx->pname);
357 memcpy(chunk + 1, ctx->pname, sizeof(ctx->pname));
358 dt->num_chunks++;
359
360 pid_chunk = (void *)(chunk + 1) + chunk->size;
361 memset(pid_chunk, 0, sizeof(*pid_chunk));
362 pid_chunk->id = LIMA_DUMP_CHUNK_PROCESS_ID;
363 pid_chunk->pid = ctx->pid;
364 dt->num_chunks++;
365
366 buffer_chunk = (void *)(pid_chunk + 1) + pid_chunk->size;
367 for (i = 0; i < task->num_bos; i++) {
368 struct lima_bo *bo = task->bos[i];
369 void *data;
370
371 memset(buffer_chunk, 0, sizeof(*buffer_chunk));
372 buffer_chunk->id = LIMA_DUMP_CHUNK_BUFFER;
373 buffer_chunk->va = lima_vm_get_va(task->vm, bo);
374
375 if (bo->heap_size) {
376 buffer_chunk->size = bo->heap_size;
377
378 data = vmap(bo->base.pages, bo->heap_size >> PAGE_SHIFT,
379 VM_MAP, pgprot_writecombine(PAGE_KERNEL));
380 if (!data) {
381 kvfree(et);
382 goto out;
383 }
384
385 memcpy(buffer_chunk + 1, data, buffer_chunk->size);
386
387 vunmap(data);
388 } else {
389 buffer_chunk->size = lima_bo_size(bo);
390
391 data = drm_gem_shmem_vmap(&bo->base.base);
392 if (IS_ERR_OR_NULL(data)) {
393 kvfree(et);
394 goto out;
395 }
396
397 memcpy(buffer_chunk + 1, data, buffer_chunk->size);
398
399 drm_gem_shmem_vunmap(&bo->base.base, data);
400 }
401
402 buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size;
403 dt->num_chunks++;
404 }
405
406 list_add(&et->list, &dev->error_task_list);
407 dev->dump.size += et->size;
408 dev->dump.num_tasks++;
409
410 dev_info(dev->dev, "save error task state success\n");
411
412out:
413 mutex_unlock(&dev->error_task_list_lock);
414}
415
416static void lima_sched_timedout_job(struct drm_sched_job *job)
417{
418 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
419 struct lima_sched_task *task = to_lima_task(job);
420 struct lima_device *ldev = pipe->ldev;
421
422 if (!pipe->error)
423 DRM_ERROR("lima job timeout\n");
424
425 drm_sched_stop(&pipe->base, &task->base);
426
427 drm_sched_increase_karma(&task->base);
428
429 lima_sched_build_error_task_list(task);
430
431 pipe->task_error(pipe);
432
433 if (pipe->bcast_mmu)
434 lima_mmu_page_fault_resume(pipe->bcast_mmu);
435 else {
436 int i;
437
438 for (i = 0; i < pipe->num_mmu; i++)
439 lima_mmu_page_fault_resume(pipe->mmu[i]);
440 }
441
442 lima_vm_put(pipe->current_vm);
443 pipe->current_vm = NULL;
444 pipe->current_task = NULL;
445
446 lima_pm_idle(ldev);
447
448 drm_sched_resubmit_jobs(&pipe->base);
449 drm_sched_start(&pipe->base, true);
450}
451
452static void lima_sched_free_job(struct drm_sched_job *job)
453{
454 struct lima_sched_task *task = to_lima_task(job);
455 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
456 struct lima_vm *vm = task->vm;
457 struct lima_bo **bos = task->bos;
458 int i;
459
460 dma_fence_put(task->fence);
461
462 for (i = 0; i < task->num_bos; i++)
463 lima_vm_bo_del(vm, bos[i]);
464
465 lima_sched_task_fini(task);
466 kmem_cache_free(pipe->task_slab, task);
467}
468
469static const struct drm_sched_backend_ops lima_sched_ops = {
470 .dependency = lima_sched_dependency,
471 .run_job = lima_sched_run_job,
472 .timedout_job = lima_sched_timedout_job,
473 .free_job = lima_sched_free_job,
474};
475
476static void lima_sched_recover_work(struct work_struct *work)
477{
478 struct lima_sched_pipe *pipe =
479 container_of(work, struct lima_sched_pipe, recover_work);
480 int i;
481
482 for (i = 0; i < pipe->num_l2_cache; i++)
483 lima_l2_cache_flush(pipe->l2_cache[i]);
484
485 if (pipe->bcast_mmu) {
486 lima_mmu_flush_tlb(pipe->bcast_mmu);
487 } else {
488 for (i = 0; i < pipe->num_mmu; i++)
489 lima_mmu_flush_tlb(pipe->mmu[i]);
490 }
491
492 if (pipe->task_recover(pipe))
493 drm_sched_fault(&pipe->base);
494}
495
496int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
497{
498 unsigned int timeout = lima_sched_timeout_ms > 0 ?
499 lima_sched_timeout_ms : 500;
500
501 pipe->fence_context = dma_fence_context_alloc(1);
502 spin_lock_init(&pipe->fence_lock);
503
504 INIT_WORK(&pipe->recover_work, lima_sched_recover_work);
505
506 return drm_sched_init(&pipe->base, &lima_sched_ops, 1,
507 lima_job_hang_limit, msecs_to_jiffies(timeout),
508 name);
509}
510
511void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
512{
513 drm_sched_fini(&pipe->base);
514}
515
516void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
517{
518 struct lima_sched_task *task = pipe->current_task;
519 struct lima_device *ldev = pipe->ldev;
520
521 if (pipe->error) {
522 if (task && task->recoverable)
523 schedule_work(&pipe->recover_work);
524 else
525 drm_sched_fault(&pipe->base);
526 } else {
527 pipe->task_fini(pipe);
528 dma_fence_signal(task->fence);
529
530 lima_pm_idle(ldev);
531 }
532}
1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/dma-buf-map.h>
5#include <linux/kthread.h>
6#include <linux/slab.h>
7#include <linux/vmalloc.h>
8#include <linux/pm_runtime.h>
9
10#include "lima_devfreq.h"
11#include "lima_drv.h"
12#include "lima_sched.h"
13#include "lima_vm.h"
14#include "lima_mmu.h"
15#include "lima_l2_cache.h"
16#include "lima_gem.h"
17#include "lima_trace.h"
18
19struct lima_fence {
20 struct dma_fence base;
21 struct lima_sched_pipe *pipe;
22};
23
24static struct kmem_cache *lima_fence_slab;
25static int lima_fence_slab_refcnt;
26
27int lima_sched_slab_init(void)
28{
29 if (!lima_fence_slab) {
30 lima_fence_slab = kmem_cache_create(
31 "lima_fence", sizeof(struct lima_fence), 0,
32 SLAB_HWCACHE_ALIGN, NULL);
33 if (!lima_fence_slab)
34 return -ENOMEM;
35 }
36
37 lima_fence_slab_refcnt++;
38 return 0;
39}
40
41void lima_sched_slab_fini(void)
42{
43 if (!--lima_fence_slab_refcnt) {
44 kmem_cache_destroy(lima_fence_slab);
45 lima_fence_slab = NULL;
46 }
47}
48
49static inline struct lima_fence *to_lima_fence(struct dma_fence *fence)
50{
51 return container_of(fence, struct lima_fence, base);
52}
53
54static const char *lima_fence_get_driver_name(struct dma_fence *fence)
55{
56 return "lima";
57}
58
59static const char *lima_fence_get_timeline_name(struct dma_fence *fence)
60{
61 struct lima_fence *f = to_lima_fence(fence);
62
63 return f->pipe->base.name;
64}
65
66static void lima_fence_release_rcu(struct rcu_head *rcu)
67{
68 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
69 struct lima_fence *fence = to_lima_fence(f);
70
71 kmem_cache_free(lima_fence_slab, fence);
72}
73
74static void lima_fence_release(struct dma_fence *fence)
75{
76 struct lima_fence *f = to_lima_fence(fence);
77
78 call_rcu(&f->base.rcu, lima_fence_release_rcu);
79}
80
81static const struct dma_fence_ops lima_fence_ops = {
82 .get_driver_name = lima_fence_get_driver_name,
83 .get_timeline_name = lima_fence_get_timeline_name,
84 .release = lima_fence_release,
85};
86
87static struct lima_fence *lima_fence_create(struct lima_sched_pipe *pipe)
88{
89 struct lima_fence *fence;
90
91 fence = kmem_cache_zalloc(lima_fence_slab, GFP_KERNEL);
92 if (!fence)
93 return NULL;
94
95 fence->pipe = pipe;
96 dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,
97 pipe->fence_context, ++pipe->fence_seqno);
98
99 return fence;
100}
101
102static inline struct lima_sched_task *to_lima_task(struct drm_sched_job *job)
103{
104 return container_of(job, struct lima_sched_task, base);
105}
106
107static inline struct lima_sched_pipe *to_lima_pipe(struct drm_gpu_scheduler *sched)
108{
109 return container_of(sched, struct lima_sched_pipe, base);
110}
111
112int lima_sched_task_init(struct lima_sched_task *task,
113 struct lima_sched_context *context,
114 struct lima_bo **bos, int num_bos,
115 struct lima_vm *vm)
116{
117 int err, i;
118
119 task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);
120 if (!task->bos)
121 return -ENOMEM;
122
123 for (i = 0; i < num_bos; i++)
124 drm_gem_object_get(&bos[i]->base.base);
125
126 err = drm_sched_job_init(&task->base, &context->base, vm);
127 if (err) {
128 kfree(task->bos);
129 return err;
130 }
131
132 task->num_bos = num_bos;
133 task->vm = lima_vm_get(vm);
134
135 xa_init_flags(&task->deps, XA_FLAGS_ALLOC);
136
137 return 0;
138}
139
140void lima_sched_task_fini(struct lima_sched_task *task)
141{
142 struct dma_fence *fence;
143 unsigned long index;
144 int i;
145
146 drm_sched_job_cleanup(&task->base);
147
148 xa_for_each(&task->deps, index, fence) {
149 dma_fence_put(fence);
150 }
151 xa_destroy(&task->deps);
152
153 if (task->bos) {
154 for (i = 0; i < task->num_bos; i++)
155 drm_gem_object_put(&task->bos[i]->base.base);
156 kfree(task->bos);
157 }
158
159 lima_vm_put(task->vm);
160}
161
162int lima_sched_context_init(struct lima_sched_pipe *pipe,
163 struct lima_sched_context *context,
164 atomic_t *guilty)
165{
166 struct drm_gpu_scheduler *sched = &pipe->base;
167
168 return drm_sched_entity_init(&context->base, DRM_SCHED_PRIORITY_NORMAL,
169 &sched, 1, guilty);
170}
171
172void lima_sched_context_fini(struct lima_sched_pipe *pipe,
173 struct lima_sched_context *context)
174{
175 drm_sched_entity_fini(&context->base);
176}
177
178struct dma_fence *lima_sched_context_queue_task(struct lima_sched_context *context,
179 struct lima_sched_task *task)
180{
181 struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
182
183 trace_lima_task_submit(task);
184 drm_sched_entity_push_job(&task->base, &context->base);
185 return fence;
186}
187
188static struct dma_fence *lima_sched_dependency(struct drm_sched_job *job,
189 struct drm_sched_entity *entity)
190{
191 struct lima_sched_task *task = to_lima_task(job);
192
193 if (!xa_empty(&task->deps))
194 return xa_erase(&task->deps, task->last_dep++);
195
196 return NULL;
197}
198
199static int lima_pm_busy(struct lima_device *ldev)
200{
201 int ret;
202
203 /* resume GPU if it has been suspended by runtime PM */
204 ret = pm_runtime_resume_and_get(ldev->dev);
205 if (ret < 0)
206 return ret;
207
208 lima_devfreq_record_busy(&ldev->devfreq);
209 return 0;
210}
211
212static void lima_pm_idle(struct lima_device *ldev)
213{
214 lima_devfreq_record_idle(&ldev->devfreq);
215
216 /* GPU can do auto runtime suspend */
217 pm_runtime_mark_last_busy(ldev->dev);
218 pm_runtime_put_autosuspend(ldev->dev);
219}
220
221static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
222{
223 struct lima_sched_task *task = to_lima_task(job);
224 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
225 struct lima_device *ldev = pipe->ldev;
226 struct lima_fence *fence;
227 int i, err;
228
229 /* after GPU reset */
230 if (job->s_fence->finished.error < 0)
231 return NULL;
232
233 fence = lima_fence_create(pipe);
234 if (!fence)
235 return NULL;
236
237 err = lima_pm_busy(ldev);
238 if (err < 0) {
239 dma_fence_put(&fence->base);
240 return NULL;
241 }
242
243 task->fence = &fence->base;
244
245 /* for caller usage of the fence, otherwise irq handler
246 * may consume the fence before caller use it
247 */
248 dma_fence_get(task->fence);
249
250 pipe->current_task = task;
251
252 /* this is needed for MMU to work correctly, otherwise GP/PP
253 * will hang or page fault for unknown reason after running for
254 * a while.
255 *
256 * Need to investigate:
257 * 1. is it related to TLB
258 * 2. how much performance will be affected by L2 cache flush
259 * 3. can we reduce the calling of this function because all
260 * GP/PP use the same L2 cache on mali400
261 *
262 * TODO:
263 * 1. move this to task fini to save some wait time?
264 * 2. when GP/PP use different l2 cache, need PP wait GP l2
265 * cache flush?
266 */
267 for (i = 0; i < pipe->num_l2_cache; i++)
268 lima_l2_cache_flush(pipe->l2_cache[i]);
269
270 lima_vm_put(pipe->current_vm);
271 pipe->current_vm = lima_vm_get(task->vm);
272
273 if (pipe->bcast_mmu)
274 lima_mmu_switch_vm(pipe->bcast_mmu, pipe->current_vm);
275 else {
276 for (i = 0; i < pipe->num_mmu; i++)
277 lima_mmu_switch_vm(pipe->mmu[i], pipe->current_vm);
278 }
279
280 trace_lima_task_run(task);
281
282 pipe->error = false;
283 pipe->task_run(pipe, task);
284
285 return task->fence;
286}
287
288static void lima_sched_build_error_task_list(struct lima_sched_task *task)
289{
290 struct lima_sched_error_task *et;
291 struct lima_sched_pipe *pipe = to_lima_pipe(task->base.sched);
292 struct lima_ip *ip = pipe->processor[0];
293 int pipe_id = ip->id == lima_ip_gp ? lima_pipe_gp : lima_pipe_pp;
294 struct lima_device *dev = ip->dev;
295 struct lima_sched_context *sched_ctx =
296 container_of(task->base.entity,
297 struct lima_sched_context, base);
298 struct lima_ctx *ctx =
299 container_of(sched_ctx, struct lima_ctx, context[pipe_id]);
300 struct lima_dump_task *dt;
301 struct lima_dump_chunk *chunk;
302 struct lima_dump_chunk_pid *pid_chunk;
303 struct lima_dump_chunk_buffer *buffer_chunk;
304 u32 size, task_size, mem_size;
305 int i;
306 struct dma_buf_map map;
307 int ret;
308
309 mutex_lock(&dev->error_task_list_lock);
310
311 if (dev->dump.num_tasks >= lima_max_error_tasks) {
312 dev_info(dev->dev, "fail to save task state from %s pid %d: "
313 "error task list is full\n", ctx->pname, ctx->pid);
314 goto out;
315 }
316
317 /* frame chunk */
318 size = sizeof(struct lima_dump_chunk) + pipe->frame_size;
319 /* process name chunk */
320 size += sizeof(struct lima_dump_chunk) + sizeof(ctx->pname);
321 /* pid chunk */
322 size += sizeof(struct lima_dump_chunk);
323 /* buffer chunks */
324 for (i = 0; i < task->num_bos; i++) {
325 struct lima_bo *bo = task->bos[i];
326
327 size += sizeof(struct lima_dump_chunk);
328 size += bo->heap_size ? bo->heap_size : lima_bo_size(bo);
329 }
330
331 task_size = size + sizeof(struct lima_dump_task);
332 mem_size = task_size + sizeof(*et);
333 et = kvmalloc(mem_size, GFP_KERNEL);
334 if (!et) {
335 dev_err(dev->dev, "fail to alloc task dump buffer of size %x\n",
336 mem_size);
337 goto out;
338 }
339
340 et->data = et + 1;
341 et->size = task_size;
342
343 dt = et->data;
344 memset(dt, 0, sizeof(*dt));
345 dt->id = pipe_id;
346 dt->size = size;
347
348 chunk = (struct lima_dump_chunk *)(dt + 1);
349 memset(chunk, 0, sizeof(*chunk));
350 chunk->id = LIMA_DUMP_CHUNK_FRAME;
351 chunk->size = pipe->frame_size;
352 memcpy(chunk + 1, task->frame, pipe->frame_size);
353 dt->num_chunks++;
354
355 chunk = (void *)(chunk + 1) + chunk->size;
356 memset(chunk, 0, sizeof(*chunk));
357 chunk->id = LIMA_DUMP_CHUNK_PROCESS_NAME;
358 chunk->size = sizeof(ctx->pname);
359 memcpy(chunk + 1, ctx->pname, sizeof(ctx->pname));
360 dt->num_chunks++;
361
362 pid_chunk = (void *)(chunk + 1) + chunk->size;
363 memset(pid_chunk, 0, sizeof(*pid_chunk));
364 pid_chunk->id = LIMA_DUMP_CHUNK_PROCESS_ID;
365 pid_chunk->pid = ctx->pid;
366 dt->num_chunks++;
367
368 buffer_chunk = (void *)(pid_chunk + 1) + pid_chunk->size;
369 for (i = 0; i < task->num_bos; i++) {
370 struct lima_bo *bo = task->bos[i];
371 void *data;
372
373 memset(buffer_chunk, 0, sizeof(*buffer_chunk));
374 buffer_chunk->id = LIMA_DUMP_CHUNK_BUFFER;
375 buffer_chunk->va = lima_vm_get_va(task->vm, bo);
376
377 if (bo->heap_size) {
378 buffer_chunk->size = bo->heap_size;
379
380 data = vmap(bo->base.pages, bo->heap_size >> PAGE_SHIFT,
381 VM_MAP, pgprot_writecombine(PAGE_KERNEL));
382 if (!data) {
383 kvfree(et);
384 goto out;
385 }
386
387 memcpy(buffer_chunk + 1, data, buffer_chunk->size);
388
389 vunmap(data);
390 } else {
391 buffer_chunk->size = lima_bo_size(bo);
392
393 ret = drm_gem_shmem_vmap(&bo->base.base, &map);
394 if (ret) {
395 kvfree(et);
396 goto out;
397 }
398
399 memcpy(buffer_chunk + 1, map.vaddr, buffer_chunk->size);
400
401 drm_gem_shmem_vunmap(&bo->base.base, &map);
402 }
403
404 buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size;
405 dt->num_chunks++;
406 }
407
408 list_add(&et->list, &dev->error_task_list);
409 dev->dump.size += et->size;
410 dev->dump.num_tasks++;
411
412 dev_info(dev->dev, "save error task state success\n");
413
414out:
415 mutex_unlock(&dev->error_task_list_lock);
416}
417
418static enum drm_gpu_sched_stat lima_sched_timedout_job(struct drm_sched_job *job)
419{
420 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
421 struct lima_sched_task *task = to_lima_task(job);
422 struct lima_device *ldev = pipe->ldev;
423
424 if (!pipe->error)
425 DRM_ERROR("lima job timeout\n");
426
427 drm_sched_stop(&pipe->base, &task->base);
428
429 drm_sched_increase_karma(&task->base);
430
431 lima_sched_build_error_task_list(task);
432
433 pipe->task_error(pipe);
434
435 if (pipe->bcast_mmu)
436 lima_mmu_page_fault_resume(pipe->bcast_mmu);
437 else {
438 int i;
439
440 for (i = 0; i < pipe->num_mmu; i++)
441 lima_mmu_page_fault_resume(pipe->mmu[i]);
442 }
443
444 lima_vm_put(pipe->current_vm);
445 pipe->current_vm = NULL;
446 pipe->current_task = NULL;
447
448 lima_pm_idle(ldev);
449
450 drm_sched_resubmit_jobs(&pipe->base);
451 drm_sched_start(&pipe->base, true);
452
453 return DRM_GPU_SCHED_STAT_NOMINAL;
454}
455
456static void lima_sched_free_job(struct drm_sched_job *job)
457{
458 struct lima_sched_task *task = to_lima_task(job);
459 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
460 struct lima_vm *vm = task->vm;
461 struct lima_bo **bos = task->bos;
462 int i;
463
464 dma_fence_put(task->fence);
465
466 for (i = 0; i < task->num_bos; i++)
467 lima_vm_bo_del(vm, bos[i]);
468
469 lima_sched_task_fini(task);
470 kmem_cache_free(pipe->task_slab, task);
471}
472
473static const struct drm_sched_backend_ops lima_sched_ops = {
474 .dependency = lima_sched_dependency,
475 .run_job = lima_sched_run_job,
476 .timedout_job = lima_sched_timedout_job,
477 .free_job = lima_sched_free_job,
478};
479
480static void lima_sched_recover_work(struct work_struct *work)
481{
482 struct lima_sched_pipe *pipe =
483 container_of(work, struct lima_sched_pipe, recover_work);
484 int i;
485
486 for (i = 0; i < pipe->num_l2_cache; i++)
487 lima_l2_cache_flush(pipe->l2_cache[i]);
488
489 if (pipe->bcast_mmu) {
490 lima_mmu_flush_tlb(pipe->bcast_mmu);
491 } else {
492 for (i = 0; i < pipe->num_mmu; i++)
493 lima_mmu_flush_tlb(pipe->mmu[i]);
494 }
495
496 if (pipe->task_recover(pipe))
497 drm_sched_fault(&pipe->base);
498}
499
500int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
501{
502 unsigned int timeout = lima_sched_timeout_ms > 0 ?
503 lima_sched_timeout_ms : 500;
504
505 pipe->fence_context = dma_fence_context_alloc(1);
506 spin_lock_init(&pipe->fence_lock);
507
508 INIT_WORK(&pipe->recover_work, lima_sched_recover_work);
509
510 return drm_sched_init(&pipe->base, &lima_sched_ops, 1,
511 lima_job_hang_limit, msecs_to_jiffies(timeout),
512 NULL, name);
513}
514
515void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
516{
517 drm_sched_fini(&pipe->base);
518}
519
520void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
521{
522 struct lima_sched_task *task = pipe->current_task;
523 struct lima_device *ldev = pipe->ldev;
524
525 if (pipe->error) {
526 if (task && task->recoverable)
527 schedule_work(&pipe->recover_work);
528 else
529 drm_sched_fault(&pipe->base);
530 } else {
531 pipe->task_fini(pipe);
532 dma_fence_signal(task->fence);
533
534 lima_pm_idle(ldev);
535 }
536}