Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Tegra host1x Command DMA
4 *
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 */
7
8
9#include <asm/cacheflush.h>
10#include <linux/device.h>
11#include <linux/dma-mapping.h>
12#include <linux/host1x.h>
13#include <linux/interrupt.h>
14#include <linux/kernel.h>
15#include <linux/kfifo.h>
16#include <linux/slab.h>
17#include <trace/events/host1x.h>
18
19#include "cdma.h"
20#include "channel.h"
21#include "dev.h"
22#include "debug.h"
23#include "job.h"
24
25/*
26 * push_buffer
27 *
28 * The push buffer is a circular array of words to be fetched by command DMA.
29 * Note that it works slightly differently to the sync queue; fence == pos
30 * means that the push buffer is full, not empty.
31 */
32
33/*
34 * Typically the commands written into the push buffer are a pair of words. We
35 * use slots to represent each of these pairs and to simplify things. Note the
36 * strange number of slots allocated here. 512 slots will fit exactly within a
37 * single memory page. We also need one additional word at the end of the push
38 * buffer for the RESTART opcode that will instruct the CDMA to jump back to
39 * the beginning of the push buffer. With 512 slots, this means that we'll use
40 * 2 memory pages and waste 4092 bytes of the second page that will never be
41 * used.
42 */
43#define HOST1X_PUSHBUFFER_SLOTS 511
44
45/*
46 * Clean up push buffer resources
47 */
48static void host1x_pushbuffer_destroy(struct push_buffer *pb)
49{
50 struct host1x_cdma *cdma = pb_to_cdma(pb);
51 struct host1x *host1x = cdma_to_host1x(cdma);
52
53 if (!pb->mapped)
54 return;
55
56 if (host1x->domain) {
57 iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
58 free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
59 }
60
61 dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
62
63 pb->mapped = NULL;
64 pb->phys = 0;
65}
66
67/*
68 * Init push buffer resources
69 */
70static int host1x_pushbuffer_init(struct push_buffer *pb)
71{
72 struct host1x_cdma *cdma = pb_to_cdma(pb);
73 struct host1x *host1x = cdma_to_host1x(cdma);
74 struct iova *alloc;
75 u32 size;
76 int err;
77
78 pb->mapped = NULL;
79 pb->phys = 0;
80 pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;
81
82 size = pb->size + 4;
83
84 /* initialize buffer pointers */
85 pb->fence = pb->size - 8;
86 pb->pos = 0;
87
88 if (host1x->domain) {
89 unsigned long shift;
90
91 size = iova_align(&host1x->iova, size);
92
93 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
94 GFP_KERNEL);
95 if (!pb->mapped)
96 return -ENOMEM;
97
98 shift = iova_shift(&host1x->iova);
99 alloc = alloc_iova(&host1x->iova, size >> shift,
100 host1x->iova_end >> shift, true);
101 if (!alloc) {
102 err = -ENOMEM;
103 goto iommu_free_mem;
104 }
105
106 pb->dma = iova_dma_addr(&host1x->iova, alloc);
107 err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
108 IOMMU_READ);
109 if (err)
110 goto iommu_free_iova;
111 } else {
112 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
113 GFP_KERNEL);
114 if (!pb->mapped)
115 return -ENOMEM;
116
117 pb->dma = pb->phys;
118 }
119
120 pb->alloc_size = size;
121
122 host1x_hw_pushbuffer_init(host1x, pb);
123
124 return 0;
125
126iommu_free_iova:
127 __free_iova(&host1x->iova, alloc);
128iommu_free_mem:
129 dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
130
131 return err;
132}
133
134/*
135 * Push two words to the push buffer
136 * Caller must ensure push buffer is not full
137 */
138static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
139{
140 u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
141
142 WARN_ON(pb->pos == pb->fence);
143 *(p++) = op1;
144 *(p++) = op2;
145 pb->pos += 8;
146
147 if (pb->pos >= pb->size)
148 pb->pos -= pb->size;
149}
150
151/*
152 * Pop a number of two word slots from the push buffer
153 * Caller must ensure push buffer is not empty
154 */
155static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
156{
157 /* Advance the next write position */
158 pb->fence += slots * 8;
159
160 if (pb->fence >= pb->size)
161 pb->fence -= pb->size;
162}
163
164/*
165 * Return the number of two word slots free in the push buffer
166 */
167static u32 host1x_pushbuffer_space(struct push_buffer *pb)
168{
169 unsigned int fence = pb->fence;
170
171 if (pb->fence < pb->pos)
172 fence += pb->size;
173
174 return (fence - pb->pos) / 8;
175}
176
177/*
178 * Sleep (if necessary) until the requested event happens
179 * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
180 * - Returns 1
181 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
182 * - Return the amount of space (> 0)
183 * Must be called with the cdma lock held.
184 */
185unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
186 enum cdma_event event)
187{
188 for (;;) {
189 struct push_buffer *pb = &cdma->push_buffer;
190 unsigned int space;
191
192 switch (event) {
193 case CDMA_EVENT_SYNC_QUEUE_EMPTY:
194 space = list_empty(&cdma->sync_queue) ? 1 : 0;
195 break;
196
197 case CDMA_EVENT_PUSH_BUFFER_SPACE:
198 space = host1x_pushbuffer_space(pb);
199 break;
200
201 default:
202 WARN_ON(1);
203 return -EINVAL;
204 }
205
206 if (space)
207 return space;
208
209 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
210 event);
211
212 /* If somebody has managed to already start waiting, yield */
213 if (cdma->event != CDMA_EVENT_NONE) {
214 mutex_unlock(&cdma->lock);
215 schedule();
216 mutex_lock(&cdma->lock);
217 continue;
218 }
219
220 cdma->event = event;
221
222 mutex_unlock(&cdma->lock);
223 wait_for_completion(&cdma->complete);
224 mutex_lock(&cdma->lock);
225 }
226
227 return 0;
228}
229
230/*
231 * Sleep (if necessary) until the push buffer has enough free space.
232 *
233 * Must be called with the cdma lock held.
234 */
235int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
236 struct host1x_cdma *cdma,
237 unsigned int needed)
238{
239 while (true) {
240 struct push_buffer *pb = &cdma->push_buffer;
241 unsigned int space;
242
243 space = host1x_pushbuffer_space(pb);
244 if (space >= needed)
245 break;
246
247 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
248 CDMA_EVENT_PUSH_BUFFER_SPACE);
249
250 host1x_hw_cdma_flush(host1x, cdma);
251
252 /* If somebody has managed to already start waiting, yield */
253 if (cdma->event != CDMA_EVENT_NONE) {
254 mutex_unlock(&cdma->lock);
255 schedule();
256 mutex_lock(&cdma->lock);
257 continue;
258 }
259
260 cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;
261
262 mutex_unlock(&cdma->lock);
263 wait_for_completion(&cdma->complete);
264 mutex_lock(&cdma->lock);
265 }
266
267 return 0;
268}
269/*
270 * Start timer that tracks the time spent by the job.
271 * Must be called with the cdma lock held.
272 */
273static void cdma_start_timer_locked(struct host1x_cdma *cdma,
274 struct host1x_job *job)
275{
276 struct host1x *host = cdma_to_host1x(cdma);
277
278 if (cdma->timeout.client) {
279 /* timer already started */
280 return;
281 }
282
283 cdma->timeout.client = job->client;
284 cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
285 cdma->timeout.syncpt_val = job->syncpt_end;
286 cdma->timeout.start_ktime = ktime_get();
287
288 schedule_delayed_work(&cdma->timeout.wq,
289 msecs_to_jiffies(job->timeout));
290}
291
292/*
293 * Stop timer when a buffer submission completes.
294 * Must be called with the cdma lock held.
295 */
296static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
297{
298 cancel_delayed_work(&cdma->timeout.wq);
299 cdma->timeout.client = NULL;
300}
301
302/*
303 * For all sync queue entries that have already finished according to the
304 * current sync point registers:
305 * - unpin & unref their mems
306 * - pop their push buffer slots
307 * - remove them from the sync queue
308 * This is normally called from the host code's worker thread, but can be
309 * called manually if necessary.
310 * Must be called with the cdma lock held.
311 */
312static void update_cdma_locked(struct host1x_cdma *cdma)
313{
314 bool signal = false;
315 struct host1x *host1x = cdma_to_host1x(cdma);
316 struct host1x_job *job, *n;
317
318 /* If CDMA is stopped, queue is cleared and we can return */
319 if (!cdma->running)
320 return;
321
322 /*
323 * Walk the sync queue, reading the sync point registers as necessary,
324 * to consume as many sync queue entries as possible without blocking
325 */
326 list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
327 struct host1x_syncpt *sp =
328 host1x_syncpt_get(host1x, job->syncpt_id);
329
330 /* Check whether this syncpt has completed, and bail if not */
331 if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
332 /* Start timer on next pending syncpt */
333 if (job->timeout)
334 cdma_start_timer_locked(cdma, job);
335
336 break;
337 }
338
339 /* Cancel timeout, when a buffer completes */
340 if (cdma->timeout.client)
341 stop_cdma_timer_locked(cdma);
342
343 /* Unpin the memory */
344 host1x_job_unpin(job);
345
346 /* Pop push buffer slots */
347 if (job->num_slots) {
348 struct push_buffer *pb = &cdma->push_buffer;
349
350 host1x_pushbuffer_pop(pb, job->num_slots);
351
352 if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
353 signal = true;
354 }
355
356 list_del(&job->list);
357 host1x_job_put(job);
358 }
359
360 if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
361 list_empty(&cdma->sync_queue))
362 signal = true;
363
364 if (signal) {
365 cdma->event = CDMA_EVENT_NONE;
366 complete(&cdma->complete);
367 }
368}
369
370void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
371 struct device *dev)
372{
373 struct host1x *host1x = cdma_to_host1x(cdma);
374 u32 restart_addr, syncpt_incrs, syncpt_val;
375 struct host1x_job *job, *next_job = NULL;
376
377 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
378
379 dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
380 __func__, syncpt_val);
381
382 /*
383 * Move the sync_queue read pointer to the first entry that hasn't
384 * completed based on the current HW syncpt value. It's likely there
385 * won't be any (i.e. we're still at the head), but covers the case
386 * where a syncpt incr happens just prior/during the teardown.
387 */
388
389 dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
390 __func__);
391
392 list_for_each_entry(job, &cdma->sync_queue, list) {
393 if (syncpt_val < job->syncpt_end) {
394
395 if (!list_is_last(&job->list, &cdma->sync_queue))
396 next_job = list_next_entry(job, list);
397
398 goto syncpt_incr;
399 }
400
401 host1x_job_dump(dev, job);
402 }
403
404 /* all jobs have been completed */
405 job = NULL;
406
407syncpt_incr:
408
409 /*
410 * Increment with CPU the remaining syncpts of a partially executed job.
411 *
412 * CDMA will continue execution starting with the next job or will get
413 * into idle state.
414 */
415 if (next_job)
416 restart_addr = next_job->first_get;
417 else
418 restart_addr = cdma->last_pos;
419
420 /* do CPU increments for the remaining syncpts */
421 if (job) {
422 dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
423 __func__);
424
425 /* won't need a timeout when replayed */
426 job->timeout = 0;
427
428 syncpt_incrs = job->syncpt_end - syncpt_val;
429 dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
430
431 host1x_job_dump(dev, job);
432
433 /* safe to use CPU to incr syncpts */
434 host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
435 syncpt_incrs, job->syncpt_end,
436 job->num_slots);
437
438 dev_dbg(dev, "%s: finished sync_queue modification\n",
439 __func__);
440 }
441
442 /* roll back DMAGET and start up channel again */
443 host1x_hw_cdma_resume(host1x, cdma, restart_addr);
444}
445
446/*
447 * Create a cdma
448 */
449int host1x_cdma_init(struct host1x_cdma *cdma)
450{
451 int err;
452
453 mutex_init(&cdma->lock);
454 init_completion(&cdma->complete);
455
456 INIT_LIST_HEAD(&cdma->sync_queue);
457
458 cdma->event = CDMA_EVENT_NONE;
459 cdma->running = false;
460 cdma->torndown = false;
461
462 err = host1x_pushbuffer_init(&cdma->push_buffer);
463 if (err)
464 return err;
465
466 return 0;
467}
468
469/*
470 * Destroy a cdma
471 */
472int host1x_cdma_deinit(struct host1x_cdma *cdma)
473{
474 struct push_buffer *pb = &cdma->push_buffer;
475 struct host1x *host1x = cdma_to_host1x(cdma);
476
477 if (cdma->running) {
478 pr_warn("%s: CDMA still running\n", __func__);
479 return -EBUSY;
480 }
481
482 host1x_pushbuffer_destroy(pb);
483 host1x_hw_cdma_timeout_destroy(host1x, cdma);
484
485 return 0;
486}
487
488/*
489 * Begin a cdma submit
490 */
491int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
492{
493 struct host1x *host1x = cdma_to_host1x(cdma);
494
495 mutex_lock(&cdma->lock);
496
497 if (job->timeout) {
498 /* init state on first submit with timeout value */
499 if (!cdma->timeout.initialized) {
500 int err;
501
502 err = host1x_hw_cdma_timeout_init(host1x, cdma,
503 job->syncpt_id);
504 if (err) {
505 mutex_unlock(&cdma->lock);
506 return err;
507 }
508 }
509 }
510
511 if (!cdma->running)
512 host1x_hw_cdma_start(host1x, cdma);
513
514 cdma->slots_free = 0;
515 cdma->slots_used = 0;
516 cdma->first_get = cdma->push_buffer.pos;
517
518 trace_host1x_cdma_begin(dev_name(job->channel->dev));
519 return 0;
520}
521
522/*
523 * Push two words into a push buffer slot
524 * Blocks as necessary if the push buffer is full.
525 */
526void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
527{
528 struct host1x *host1x = cdma_to_host1x(cdma);
529 struct push_buffer *pb = &cdma->push_buffer;
530 u32 slots_free = cdma->slots_free;
531
532 if (host1x_debug_trace_cmdbuf)
533 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
534 op1, op2);
535
536 if (slots_free == 0) {
537 host1x_hw_cdma_flush(host1x, cdma);
538 slots_free = host1x_cdma_wait_locked(cdma,
539 CDMA_EVENT_PUSH_BUFFER_SPACE);
540 }
541
542 cdma->slots_free = slots_free - 1;
543 cdma->slots_used++;
544 host1x_pushbuffer_push(pb, op1, op2);
545}
546
547/*
548 * Push four words into two consecutive push buffer slots. Note that extra
549 * care needs to be taken not to split the two slots across the end of the
550 * push buffer. Otherwise the RESTART opcode at the end of the push buffer
551 * that ensures processing will restart at the beginning will break up the
552 * four words.
553 *
554 * Blocks as necessary if the push buffer is full.
555 */
556void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
557 u32 op3, u32 op4)
558{
559 struct host1x_channel *channel = cdma_to_channel(cdma);
560 struct host1x *host1x = cdma_to_host1x(cdma);
561 struct push_buffer *pb = &cdma->push_buffer;
562 unsigned int needed = 2, extra = 0, i;
563 unsigned int space = cdma->slots_free;
564
565 if (host1x_debug_trace_cmdbuf)
566 trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
567 op3, op4);
568
569 /* compute number of extra slots needed for padding */
570 if (pb->pos + 16 > pb->size) {
571 extra = (pb->size - pb->pos) / 8;
572 needed += extra;
573 }
574
575 host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
576 space = host1x_pushbuffer_space(pb);
577
578 cdma->slots_free = space - needed;
579 cdma->slots_used += needed;
580
581 /*
582 * Note that we rely on the fact that this is only used to submit wide
583 * gather opcodes, which consist of 3 words, and they are padded with
584 * a NOP to avoid having to deal with fractional slots (a slot always
585 * represents 2 words). The fourth opcode passed to this function will
586 * therefore always be a NOP.
587 *
588 * This works around a slight ambiguity when it comes to opcodes. For
589 * all current host1x incarnations the NOP opcode uses the exact same
590 * encoding (0x20000000), so we could hard-code the value here, but a
591 * new incarnation may change it and break that assumption.
592 */
593 for (i = 0; i < extra; i++)
594 host1x_pushbuffer_push(pb, op4, op4);
595
596 host1x_pushbuffer_push(pb, op1, op2);
597 host1x_pushbuffer_push(pb, op3, op4);
598}
599
600/*
601 * End a cdma submit
602 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
603 * from the pushbuffer. The handles for a submit must all be pinned at the same
604 * time, but they can be unpinned in smaller chunks.
605 */
606void host1x_cdma_end(struct host1x_cdma *cdma,
607 struct host1x_job *job)
608{
609 struct host1x *host1x = cdma_to_host1x(cdma);
610 bool idle = list_empty(&cdma->sync_queue);
611
612 host1x_hw_cdma_flush(host1x, cdma);
613
614 job->first_get = cdma->first_get;
615 job->num_slots = cdma->slots_used;
616 host1x_job_get(job);
617 list_add_tail(&job->list, &cdma->sync_queue);
618
619 /* start timer on idle -> active transitions */
620 if (job->timeout && idle)
621 cdma_start_timer_locked(cdma, job);
622
623 trace_host1x_cdma_end(dev_name(job->channel->dev));
624 mutex_unlock(&cdma->lock);
625}
626
627/*
628 * Update cdma state according to current sync point values
629 */
630void host1x_cdma_update(struct host1x_cdma *cdma)
631{
632 mutex_lock(&cdma->lock);
633 update_cdma_locked(cdma);
634 mutex_unlock(&cdma->lock);
635}
1/*
2 * Tegra host1x Command DMA
3 *
4 * Copyright (c) 2010-2013, NVIDIA Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19
20#include <asm/cacheflush.h>
21#include <linux/device.h>
22#include <linux/dma-mapping.h>
23#include <linux/host1x.h>
24#include <linux/interrupt.h>
25#include <linux/kernel.h>
26#include <linux/kfifo.h>
27#include <linux/slab.h>
28#include <trace/events/host1x.h>
29
30#include "cdma.h"
31#include "channel.h"
32#include "dev.h"
33#include "debug.h"
34#include "job.h"
35
36/*
37 * push_buffer
38 *
39 * The push buffer is a circular array of words to be fetched by command DMA.
40 * Note that it works slightly differently to the sync queue; fence == pos
41 * means that the push buffer is full, not empty.
42 */
43
44#define HOST1X_PUSHBUFFER_SLOTS 512
45
46/*
47 * Clean up push buffer resources
48 */
49static void host1x_pushbuffer_destroy(struct push_buffer *pb)
50{
51 struct host1x_cdma *cdma = pb_to_cdma(pb);
52 struct host1x *host1x = cdma_to_host1x(cdma);
53
54 if (pb->phys != 0)
55 dma_free_wc(host1x->dev, pb->size_bytes + 4, pb->mapped,
56 pb->phys);
57
58 pb->mapped = NULL;
59 pb->phys = 0;
60}
61
62/*
63 * Init push buffer resources
64 */
65static int host1x_pushbuffer_init(struct push_buffer *pb)
66{
67 struct host1x_cdma *cdma = pb_to_cdma(pb);
68 struct host1x *host1x = cdma_to_host1x(cdma);
69
70 pb->mapped = NULL;
71 pb->phys = 0;
72 pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8;
73
74 /* initialize buffer pointers */
75 pb->fence = pb->size_bytes - 8;
76 pb->pos = 0;
77
78 /* allocate and map pushbuffer memory */
79 pb->mapped = dma_alloc_wc(host1x->dev, pb->size_bytes + 4, &pb->phys,
80 GFP_KERNEL);
81 if (!pb->mapped)
82 goto fail;
83
84 host1x_hw_pushbuffer_init(host1x, pb);
85
86 return 0;
87
88fail:
89 host1x_pushbuffer_destroy(pb);
90 return -ENOMEM;
91}
92
93/*
94 * Push two words to the push buffer
95 * Caller must ensure push buffer is not full
96 */
97static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
98{
99 u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
100
101 WARN_ON(pb->pos == pb->fence);
102 *(p++) = op1;
103 *(p++) = op2;
104 pb->pos = (pb->pos + 8) & (pb->size_bytes - 1);
105}
106
107/*
108 * Pop a number of two word slots from the push buffer
109 * Caller must ensure push buffer is not empty
110 */
111static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
112{
113 /* Advance the next write position */
114 pb->fence = (pb->fence + slots * 8) & (pb->size_bytes - 1);
115}
116
117/*
118 * Return the number of two word slots free in the push buffer
119 */
120static u32 host1x_pushbuffer_space(struct push_buffer *pb)
121{
122 return ((pb->fence - pb->pos) & (pb->size_bytes - 1)) / 8;
123}
124
125/*
126 * Sleep (if necessary) until the requested event happens
127 * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
128 * - Returns 1
129 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
130 * - Return the amount of space (> 0)
131 * Must be called with the cdma lock held.
132 */
133unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
134 enum cdma_event event)
135{
136 for (;;) {
137 struct push_buffer *pb = &cdma->push_buffer;
138 unsigned int space;
139
140 switch (event) {
141 case CDMA_EVENT_SYNC_QUEUE_EMPTY:
142 space = list_empty(&cdma->sync_queue) ? 1 : 0;
143 break;
144
145 case CDMA_EVENT_PUSH_BUFFER_SPACE:
146 space = host1x_pushbuffer_space(pb);
147 break;
148
149 default:
150 WARN_ON(1);
151 return -EINVAL;
152 }
153
154 if (space)
155 return space;
156
157 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
158 event);
159
160 /* If somebody has managed to already start waiting, yield */
161 if (cdma->event != CDMA_EVENT_NONE) {
162 mutex_unlock(&cdma->lock);
163 schedule();
164 mutex_lock(&cdma->lock);
165 continue;
166 }
167
168 cdma->event = event;
169
170 mutex_unlock(&cdma->lock);
171 down(&cdma->sem);
172 mutex_lock(&cdma->lock);
173 }
174
175 return 0;
176}
177
178/*
179 * Start timer that tracks the time spent by the job.
180 * Must be called with the cdma lock held.
181 */
182static void cdma_start_timer_locked(struct host1x_cdma *cdma,
183 struct host1x_job *job)
184{
185 struct host1x *host = cdma_to_host1x(cdma);
186
187 if (cdma->timeout.client) {
188 /* timer already started */
189 return;
190 }
191
192 cdma->timeout.client = job->client;
193 cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
194 cdma->timeout.syncpt_val = job->syncpt_end;
195 cdma->timeout.start_ktime = ktime_get();
196
197 schedule_delayed_work(&cdma->timeout.wq,
198 msecs_to_jiffies(job->timeout));
199}
200
201/*
202 * Stop timer when a buffer submission completes.
203 * Must be called with the cdma lock held.
204 */
205static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
206{
207 cancel_delayed_work(&cdma->timeout.wq);
208 cdma->timeout.client = 0;
209}
210
211/*
212 * For all sync queue entries that have already finished according to the
213 * current sync point registers:
214 * - unpin & unref their mems
215 * - pop their push buffer slots
216 * - remove them from the sync queue
217 * This is normally called from the host code's worker thread, but can be
218 * called manually if necessary.
219 * Must be called with the cdma lock held.
220 */
221static void update_cdma_locked(struct host1x_cdma *cdma)
222{
223 bool signal = false;
224 struct host1x *host1x = cdma_to_host1x(cdma);
225 struct host1x_job *job, *n;
226
227 /* If CDMA is stopped, queue is cleared and we can return */
228 if (!cdma->running)
229 return;
230
231 /*
232 * Walk the sync queue, reading the sync point registers as necessary,
233 * to consume as many sync queue entries as possible without blocking
234 */
235 list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
236 struct host1x_syncpt *sp =
237 host1x_syncpt_get(host1x, job->syncpt_id);
238
239 /* Check whether this syncpt has completed, and bail if not */
240 if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
241 /* Start timer on next pending syncpt */
242 if (job->timeout)
243 cdma_start_timer_locked(cdma, job);
244
245 break;
246 }
247
248 /* Cancel timeout, when a buffer completes */
249 if (cdma->timeout.client)
250 stop_cdma_timer_locked(cdma);
251
252 /* Unpin the memory */
253 host1x_job_unpin(job);
254
255 /* Pop push buffer slots */
256 if (job->num_slots) {
257 struct push_buffer *pb = &cdma->push_buffer;
258
259 host1x_pushbuffer_pop(pb, job->num_slots);
260
261 if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
262 signal = true;
263 }
264
265 list_del(&job->list);
266 host1x_job_put(job);
267 }
268
269 if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
270 list_empty(&cdma->sync_queue))
271 signal = true;
272
273 if (signal) {
274 cdma->event = CDMA_EVENT_NONE;
275 up(&cdma->sem);
276 }
277}
278
279void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
280 struct device *dev)
281{
282 struct host1x *host1x = cdma_to_host1x(cdma);
283 u32 restart_addr, syncpt_incrs, syncpt_val;
284 struct host1x_job *job = NULL;
285
286 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
287
288 dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
289 __func__, syncpt_val);
290
291 /*
292 * Move the sync_queue read pointer to the first entry that hasn't
293 * completed based on the current HW syncpt value. It's likely there
294 * won't be any (i.e. we're still at the head), but covers the case
295 * where a syncpt incr happens just prior/during the teardown.
296 */
297
298 dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
299 __func__);
300
301 list_for_each_entry(job, &cdma->sync_queue, list) {
302 if (syncpt_val < job->syncpt_end)
303 break;
304
305 host1x_job_dump(dev, job);
306 }
307
308 /*
309 * Walk the sync_queue, first incrementing with the CPU syncpts that
310 * are partially executed (the first buffer) or fully skipped while
311 * still in the current context (slots are also NOP-ed).
312 *
313 * At the point contexts are interleaved, syncpt increments must be
314 * done inline with the pushbuffer from a GATHER buffer to maintain
315 * the order (slots are modified to be a GATHER of syncpt incrs).
316 *
317 * Note: save in restart_addr the location where the timed out buffer
318 * started in the PB, so we can start the refetch from there (with the
319 * modified NOP-ed PB slots). This lets things appear to have completed
320 * properly for this buffer and resources are freed.
321 */
322
323 dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n",
324 __func__);
325
326 if (!list_empty(&cdma->sync_queue))
327 restart_addr = job->first_get;
328 else
329 restart_addr = cdma->last_pos;
330
331 /* do CPU increments as long as this context continues */
332 list_for_each_entry_from(job, &cdma->sync_queue, list) {
333 /* different context, gets us out of this loop */
334 if (job->client != cdma->timeout.client)
335 break;
336
337 /* won't need a timeout when replayed */
338 job->timeout = 0;
339
340 syncpt_incrs = job->syncpt_end - syncpt_val;
341 dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
342
343 host1x_job_dump(dev, job);
344
345 /* safe to use CPU to incr syncpts */
346 host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
347 syncpt_incrs, job->syncpt_end,
348 job->num_slots);
349
350 syncpt_val += syncpt_incrs;
351 }
352
353 /*
354 * The following sumbits from the same client may be dependent on the
355 * failed submit and therefore they may fail. Force a small timeout
356 * to make the queue cleanup faster.
357 */
358
359 list_for_each_entry_from(job, &cdma->sync_queue, list)
360 if (job->client == cdma->timeout.client)
361 job->timeout = min_t(unsigned int, job->timeout, 500);
362
363 dev_dbg(dev, "%s: finished sync_queue modification\n", __func__);
364
365 /* roll back DMAGET and start up channel again */
366 host1x_hw_cdma_resume(host1x, cdma, restart_addr);
367}
368
369/*
370 * Create a cdma
371 */
372int host1x_cdma_init(struct host1x_cdma *cdma)
373{
374 int err;
375
376 mutex_init(&cdma->lock);
377 sema_init(&cdma->sem, 0);
378
379 INIT_LIST_HEAD(&cdma->sync_queue);
380
381 cdma->event = CDMA_EVENT_NONE;
382 cdma->running = false;
383 cdma->torndown = false;
384
385 err = host1x_pushbuffer_init(&cdma->push_buffer);
386 if (err)
387 return err;
388
389 return 0;
390}
391
392/*
393 * Destroy a cdma
394 */
395int host1x_cdma_deinit(struct host1x_cdma *cdma)
396{
397 struct push_buffer *pb = &cdma->push_buffer;
398 struct host1x *host1x = cdma_to_host1x(cdma);
399
400 if (cdma->running) {
401 pr_warn("%s: CDMA still running\n", __func__);
402 return -EBUSY;
403 }
404
405 host1x_pushbuffer_destroy(pb);
406 host1x_hw_cdma_timeout_destroy(host1x, cdma);
407
408 return 0;
409}
410
411/*
412 * Begin a cdma submit
413 */
414int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
415{
416 struct host1x *host1x = cdma_to_host1x(cdma);
417
418 mutex_lock(&cdma->lock);
419
420 if (job->timeout) {
421 /* init state on first submit with timeout value */
422 if (!cdma->timeout.initialized) {
423 int err;
424
425 err = host1x_hw_cdma_timeout_init(host1x, cdma,
426 job->syncpt_id);
427 if (err) {
428 mutex_unlock(&cdma->lock);
429 return err;
430 }
431 }
432 }
433
434 if (!cdma->running)
435 host1x_hw_cdma_start(host1x, cdma);
436
437 cdma->slots_free = 0;
438 cdma->slots_used = 0;
439 cdma->first_get = cdma->push_buffer.pos;
440
441 trace_host1x_cdma_begin(dev_name(job->channel->dev));
442 return 0;
443}
444
445/*
446 * Push two words into a push buffer slot
447 * Blocks as necessary if the push buffer is full.
448 */
449void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
450{
451 struct host1x *host1x = cdma_to_host1x(cdma);
452 struct push_buffer *pb = &cdma->push_buffer;
453 u32 slots_free = cdma->slots_free;
454
455 if (host1x_debug_trace_cmdbuf)
456 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
457 op1, op2);
458
459 if (slots_free == 0) {
460 host1x_hw_cdma_flush(host1x, cdma);
461 slots_free = host1x_cdma_wait_locked(cdma,
462 CDMA_EVENT_PUSH_BUFFER_SPACE);
463 }
464
465 cdma->slots_free = slots_free - 1;
466 cdma->slots_used++;
467 host1x_pushbuffer_push(pb, op1, op2);
468}
469
470/*
471 * End a cdma submit
472 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
473 * from the pushbuffer. The handles for a submit must all be pinned at the same
474 * time, but they can be unpinned in smaller chunks.
475 */
476void host1x_cdma_end(struct host1x_cdma *cdma,
477 struct host1x_job *job)
478{
479 struct host1x *host1x = cdma_to_host1x(cdma);
480 bool idle = list_empty(&cdma->sync_queue);
481
482 host1x_hw_cdma_flush(host1x, cdma);
483
484 job->first_get = cdma->first_get;
485 job->num_slots = cdma->slots_used;
486 host1x_job_get(job);
487 list_add_tail(&job->list, &cdma->sync_queue);
488
489 /* start timer on idle -> active transitions */
490 if (job->timeout && idle)
491 cdma_start_timer_locked(cdma, job);
492
493 trace_host1x_cdma_end(dev_name(job->channel->dev));
494 mutex_unlock(&cdma->lock);
495}
496
497/*
498 * Update cdma state according to current sync point values
499 */
500void host1x_cdma_update(struct host1x_cdma *cdma)
501{
502 mutex_lock(&cdma->lock);
503 update_cdma_locked(cdma);
504 mutex_unlock(&cdma->lock);
505}