Loading...
1/*
2 * Copyright (C) 2007 Ben Skeggs.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 *
25 */
26
27#include "drmP.h"
28#include "drm.h"
29
30#include <linux/ktime.h>
31#include <linux/hrtimer.h>
32
33#include "nouveau_drv.h"
34#include "nouveau_ramht.h"
35#include "nouveau_dma.h"
36
37#define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
38#define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
39
40struct nouveau_fence {
41 struct nouveau_channel *channel;
42 struct kref refcount;
43 struct list_head entry;
44
45 uint32_t sequence;
46 bool signalled;
47
48 void (*work)(void *priv, bool signalled);
49 void *priv;
50};
51
52struct nouveau_semaphore {
53 struct kref ref;
54 struct drm_device *dev;
55 struct drm_mm_node *mem;
56};
57
58static inline struct nouveau_fence *
59nouveau_fence(void *sync_obj)
60{
61 return (struct nouveau_fence *)sync_obj;
62}
63
64static void
65nouveau_fence_del(struct kref *ref)
66{
67 struct nouveau_fence *fence =
68 container_of(ref, struct nouveau_fence, refcount);
69
70 nouveau_channel_ref(NULL, &fence->channel);
71 kfree(fence);
72}
73
74void
75nouveau_fence_update(struct nouveau_channel *chan)
76{
77 struct drm_device *dev = chan->dev;
78 struct nouveau_fence *tmp, *fence;
79 uint32_t sequence;
80
81 spin_lock(&chan->fence.lock);
82
83 /* Fetch the last sequence if the channel is still up and running */
84 if (likely(!list_empty(&chan->fence.pending))) {
85 if (USE_REFCNT(dev))
86 sequence = nvchan_rd32(chan, 0x48);
87 else
88 sequence = atomic_read(&chan->fence.last_sequence_irq);
89
90 if (chan->fence.sequence_ack == sequence)
91 goto out;
92 chan->fence.sequence_ack = sequence;
93 }
94
95 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
96 sequence = fence->sequence;
97 fence->signalled = true;
98 list_del(&fence->entry);
99
100 if (unlikely(fence->work))
101 fence->work(fence->priv, true);
102
103 kref_put(&fence->refcount, nouveau_fence_del);
104
105 if (sequence == chan->fence.sequence_ack)
106 break;
107 }
108out:
109 spin_unlock(&chan->fence.lock);
110}
111
112int
113nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
114 bool emit)
115{
116 struct nouveau_fence *fence;
117 int ret = 0;
118
119 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
120 if (!fence)
121 return -ENOMEM;
122 kref_init(&fence->refcount);
123 nouveau_channel_ref(chan, &fence->channel);
124
125 if (emit)
126 ret = nouveau_fence_emit(fence);
127
128 if (ret)
129 nouveau_fence_unref(&fence);
130 *pfence = fence;
131 return ret;
132}
133
134struct nouveau_channel *
135nouveau_fence_channel(struct nouveau_fence *fence)
136{
137 return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
138}
139
140int
141nouveau_fence_emit(struct nouveau_fence *fence)
142{
143 struct nouveau_channel *chan = fence->channel;
144 struct drm_device *dev = chan->dev;
145 struct drm_nouveau_private *dev_priv = dev->dev_private;
146 int ret;
147
148 ret = RING_SPACE(chan, 2);
149 if (ret)
150 return ret;
151
152 if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
153 nouveau_fence_update(chan);
154
155 BUG_ON(chan->fence.sequence ==
156 chan->fence.sequence_ack - 1);
157 }
158
159 fence->sequence = ++chan->fence.sequence;
160
161 kref_get(&fence->refcount);
162 spin_lock(&chan->fence.lock);
163 list_add_tail(&fence->entry, &chan->fence.pending);
164 spin_unlock(&chan->fence.lock);
165
166 if (USE_REFCNT(dev)) {
167 if (dev_priv->card_type < NV_C0)
168 BEGIN_RING(chan, NvSubSw, 0x0050, 1);
169 else
170 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1);
171 } else {
172 BEGIN_RING(chan, NvSubSw, 0x0150, 1);
173 }
174 OUT_RING (chan, fence->sequence);
175 FIRE_RING(chan);
176
177 return 0;
178}
179
180void
181nouveau_fence_work(struct nouveau_fence *fence,
182 void (*work)(void *priv, bool signalled),
183 void *priv)
184{
185 BUG_ON(fence->work);
186
187 spin_lock(&fence->channel->fence.lock);
188
189 if (fence->signalled) {
190 work(priv, true);
191 } else {
192 fence->work = work;
193 fence->priv = priv;
194 }
195
196 spin_unlock(&fence->channel->fence.lock);
197}
198
199void
200__nouveau_fence_unref(void **sync_obj)
201{
202 struct nouveau_fence *fence = nouveau_fence(*sync_obj);
203
204 if (fence)
205 kref_put(&fence->refcount, nouveau_fence_del);
206 *sync_obj = NULL;
207}
208
209void *
210__nouveau_fence_ref(void *sync_obj)
211{
212 struct nouveau_fence *fence = nouveau_fence(sync_obj);
213
214 kref_get(&fence->refcount);
215 return sync_obj;
216}
217
218bool
219__nouveau_fence_signalled(void *sync_obj, void *sync_arg)
220{
221 struct nouveau_fence *fence = nouveau_fence(sync_obj);
222 struct nouveau_channel *chan = fence->channel;
223
224 if (fence->signalled)
225 return true;
226
227 nouveau_fence_update(chan);
228 return fence->signalled;
229}
230
231int
232__nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
233{
234 unsigned long timeout = jiffies + (3 * DRM_HZ);
235 unsigned long sleep_time = NSEC_PER_MSEC / 1000;
236 ktime_t t;
237 int ret = 0;
238
239 while (1) {
240 if (__nouveau_fence_signalled(sync_obj, sync_arg))
241 break;
242
243 if (time_after_eq(jiffies, timeout)) {
244 ret = -EBUSY;
245 break;
246 }
247
248 __set_current_state(intr ? TASK_INTERRUPTIBLE
249 : TASK_UNINTERRUPTIBLE);
250 if (lazy) {
251 t = ktime_set(0, sleep_time);
252 schedule_hrtimeout(&t, HRTIMER_MODE_REL);
253 sleep_time *= 2;
254 if (sleep_time > NSEC_PER_MSEC)
255 sleep_time = NSEC_PER_MSEC;
256 }
257
258 if (intr && signal_pending(current)) {
259 ret = -ERESTARTSYS;
260 break;
261 }
262 }
263
264 __set_current_state(TASK_RUNNING);
265
266 return ret;
267}
268
269static struct nouveau_semaphore *
270semaphore_alloc(struct drm_device *dev)
271{
272 struct drm_nouveau_private *dev_priv = dev->dev_private;
273 struct nouveau_semaphore *sema;
274 int size = (dev_priv->chipset < 0x84) ? 4 : 16;
275 int ret, i;
276
277 if (!USE_SEMA(dev))
278 return NULL;
279
280 sema = kmalloc(sizeof(*sema), GFP_KERNEL);
281 if (!sema)
282 goto fail;
283
284 ret = drm_mm_pre_get(&dev_priv->fence.heap);
285 if (ret)
286 goto fail;
287
288 spin_lock(&dev_priv->fence.lock);
289 sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
290 if (sema->mem)
291 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
292 spin_unlock(&dev_priv->fence.lock);
293
294 if (!sema->mem)
295 goto fail;
296
297 kref_init(&sema->ref);
298 sema->dev = dev;
299 for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
300 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
301
302 return sema;
303fail:
304 kfree(sema);
305 return NULL;
306}
307
308static void
309semaphore_free(struct kref *ref)
310{
311 struct nouveau_semaphore *sema =
312 container_of(ref, struct nouveau_semaphore, ref);
313 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
314
315 spin_lock(&dev_priv->fence.lock);
316 drm_mm_put_block(sema->mem);
317 spin_unlock(&dev_priv->fence.lock);
318
319 kfree(sema);
320}
321
322static void
323semaphore_work(void *priv, bool signalled)
324{
325 struct nouveau_semaphore *sema = priv;
326 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
327
328 if (unlikely(!signalled))
329 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
330
331 kref_put(&sema->ref, semaphore_free);
332}
333
334static int
335semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
336{
337 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
338 struct nouveau_fence *fence = NULL;
339 u64 offset = chan->fence.vma.offset + sema->mem->start;
340 int ret;
341
342 if (dev_priv->chipset < 0x84) {
343 ret = RING_SPACE(chan, 4);
344 if (ret)
345 return ret;
346
347 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3);
348 OUT_RING (chan, NvSema);
349 OUT_RING (chan, offset);
350 OUT_RING (chan, 1);
351 } else
352 if (dev_priv->chipset < 0xc0) {
353 ret = RING_SPACE(chan, 7);
354 if (ret)
355 return ret;
356
357 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
358 OUT_RING (chan, chan->vram_handle);
359 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
360 OUT_RING (chan, upper_32_bits(offset));
361 OUT_RING (chan, lower_32_bits(offset));
362 OUT_RING (chan, 1);
363 OUT_RING (chan, 1); /* ACQUIRE_EQ */
364 } else {
365 ret = RING_SPACE(chan, 5);
366 if (ret)
367 return ret;
368
369 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
370 OUT_RING (chan, upper_32_bits(offset));
371 OUT_RING (chan, lower_32_bits(offset));
372 OUT_RING (chan, 1);
373 OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */
374 }
375
376 /* Delay semaphore destruction until its work is done */
377 ret = nouveau_fence_new(chan, &fence, true);
378 if (ret)
379 return ret;
380
381 kref_get(&sema->ref);
382 nouveau_fence_work(fence, semaphore_work, sema);
383 nouveau_fence_unref(&fence);
384 return 0;
385}
386
387static int
388semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
389{
390 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
391 struct nouveau_fence *fence = NULL;
392 u64 offset = chan->fence.vma.offset + sema->mem->start;
393 int ret;
394
395 if (dev_priv->chipset < 0x84) {
396 ret = RING_SPACE(chan, 5);
397 if (ret)
398 return ret;
399
400 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2);
401 OUT_RING (chan, NvSema);
402 OUT_RING (chan, offset);
403 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
404 OUT_RING (chan, 1);
405 } else
406 if (dev_priv->chipset < 0xc0) {
407 ret = RING_SPACE(chan, 7);
408 if (ret)
409 return ret;
410
411 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
412 OUT_RING (chan, chan->vram_handle);
413 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
414 OUT_RING (chan, upper_32_bits(offset));
415 OUT_RING (chan, lower_32_bits(offset));
416 OUT_RING (chan, 1);
417 OUT_RING (chan, 2); /* RELEASE */
418 } else {
419 ret = RING_SPACE(chan, 5);
420 if (ret)
421 return ret;
422
423 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
424 OUT_RING (chan, upper_32_bits(offset));
425 OUT_RING (chan, lower_32_bits(offset));
426 OUT_RING (chan, 1);
427 OUT_RING (chan, 0x1002); /* RELEASE */
428 }
429
430 /* Delay semaphore destruction until its work is done */
431 ret = nouveau_fence_new(chan, &fence, true);
432 if (ret)
433 return ret;
434
435 kref_get(&sema->ref);
436 nouveau_fence_work(fence, semaphore_work, sema);
437 nouveau_fence_unref(&fence);
438 return 0;
439}
440
441int
442nouveau_fence_sync(struct nouveau_fence *fence,
443 struct nouveau_channel *wchan)
444{
445 struct nouveau_channel *chan = nouveau_fence_channel(fence);
446 struct drm_device *dev = wchan->dev;
447 struct nouveau_semaphore *sema;
448 int ret = 0;
449
450 if (likely(!chan || chan == wchan ||
451 nouveau_fence_signalled(fence)))
452 goto out;
453
454 sema = semaphore_alloc(dev);
455 if (!sema) {
456 /* Early card or broken userspace, fall back to
457 * software sync. */
458 ret = nouveau_fence_wait(fence, true, false);
459 goto out;
460 }
461
462 /* try to take chan's mutex, if we can't take it right away
463 * we have to fallback to software sync to prevent locking
464 * order issues
465 */
466 if (!mutex_trylock(&chan->mutex)) {
467 ret = nouveau_fence_wait(fence, true, false);
468 goto out_unref;
469 }
470
471 /* Make wchan wait until it gets signalled */
472 ret = semaphore_acquire(wchan, sema);
473 if (ret)
474 goto out_unlock;
475
476 /* Signal the semaphore from chan */
477 ret = semaphore_release(chan, sema);
478
479out_unlock:
480 mutex_unlock(&chan->mutex);
481out_unref:
482 kref_put(&sema->ref, semaphore_free);
483out:
484 if (chan)
485 nouveau_channel_put_unlocked(&chan);
486 return ret;
487}
488
489int
490__nouveau_fence_flush(void *sync_obj, void *sync_arg)
491{
492 return 0;
493}
494
495int
496nouveau_fence_channel_init(struct nouveau_channel *chan)
497{
498 struct drm_device *dev = chan->dev;
499 struct drm_nouveau_private *dev_priv = dev->dev_private;
500 struct nouveau_gpuobj *obj = NULL;
501 int ret;
502
503 if (dev_priv->card_type < NV_C0) {
504 /* Create an NV_SW object for various sync purposes */
505 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
506 if (ret)
507 return ret;
508
509 ret = RING_SPACE(chan, 2);
510 if (ret)
511 return ret;
512
513 BEGIN_RING(chan, NvSubSw, 0, 1);
514 OUT_RING (chan, NvSw);
515 FIRE_RING (chan);
516 }
517
518 /* Setup area of memory shared between all channels for x-chan sync */
519 if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
520 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
521
522 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
523 mem->start << PAGE_SHIFT,
524 mem->size, NV_MEM_ACCESS_RW,
525 NV_MEM_TARGET_VRAM, &obj);
526 if (ret)
527 return ret;
528
529 ret = nouveau_ramht_insert(chan, NvSema, obj);
530 nouveau_gpuobj_ref(NULL, &obj);
531 if (ret)
532 return ret;
533 } else
534 if (USE_SEMA(dev)) {
535 /* map fence bo into channel's vm */
536 ret = nouveau_bo_vma_add(dev_priv->fence.bo, chan->vm,
537 &chan->fence.vma);
538 if (ret)
539 return ret;
540 }
541
542 INIT_LIST_HEAD(&chan->fence.pending);
543 spin_lock_init(&chan->fence.lock);
544 atomic_set(&chan->fence.last_sequence_irq, 0);
545 return 0;
546}
547
548void
549nouveau_fence_channel_fini(struct nouveau_channel *chan)
550{
551 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
552 struct nouveau_fence *tmp, *fence;
553
554 spin_lock(&chan->fence.lock);
555 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
556 fence->signalled = true;
557 list_del(&fence->entry);
558
559 if (unlikely(fence->work))
560 fence->work(fence->priv, false);
561
562 kref_put(&fence->refcount, nouveau_fence_del);
563 }
564 spin_unlock(&chan->fence.lock);
565
566 nouveau_bo_vma_del(dev_priv->fence.bo, &chan->fence.vma);
567}
568
569int
570nouveau_fence_init(struct drm_device *dev)
571{
572 struct drm_nouveau_private *dev_priv = dev->dev_private;
573 int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
574 int ret;
575
576 /* Create a shared VRAM heap for cross-channel sync. */
577 if (USE_SEMA(dev)) {
578 ret = nouveau_bo_new(dev, size, 0, TTM_PL_FLAG_VRAM,
579 0, 0, &dev_priv->fence.bo);
580 if (ret)
581 return ret;
582
583 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
584 if (ret)
585 goto fail;
586
587 ret = nouveau_bo_map(dev_priv->fence.bo);
588 if (ret)
589 goto fail;
590
591 ret = drm_mm_init(&dev_priv->fence.heap, 0,
592 dev_priv->fence.bo->bo.mem.size);
593 if (ret)
594 goto fail;
595
596 spin_lock_init(&dev_priv->fence.lock);
597 }
598
599 return 0;
600fail:
601 nouveau_bo_unmap(dev_priv->fence.bo);
602 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
603 return ret;
604}
605
606void
607nouveau_fence_fini(struct drm_device *dev)
608{
609 struct drm_nouveau_private *dev_priv = dev->dev_private;
610
611 if (USE_SEMA(dev)) {
612 drm_mm_takedown(&dev_priv->fence.heap);
613 nouveau_bo_unmap(dev_priv->fence.bo);
614 nouveau_bo_unpin(dev_priv->fence.bo);
615 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
616 }
617}
1/*
2 * Copyright (C) 2007 Ben Skeggs.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 *
25 */
26
27#include <drm/drmP.h>
28
29#include <linux/ktime.h>
30#include <linux/hrtimer.h>
31#include <trace/events/dma_fence.h>
32
33#include <nvif/cl826e.h>
34#include <nvif/notify.h>
35#include <nvif/event.h>
36
37#include "nouveau_drv.h"
38#include "nouveau_dma.h"
39#include "nouveau_fence.h"
40
41static const struct dma_fence_ops nouveau_fence_ops_uevent;
42static const struct dma_fence_ops nouveau_fence_ops_legacy;
43
44static inline struct nouveau_fence *
45from_fence(struct dma_fence *fence)
46{
47 return container_of(fence, struct nouveau_fence, base);
48}
49
50static inline struct nouveau_fence_chan *
51nouveau_fctx(struct nouveau_fence *fence)
52{
53 return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
54}
55
56static int
57nouveau_fence_signal(struct nouveau_fence *fence)
58{
59 int drop = 0;
60
61 dma_fence_signal_locked(&fence->base);
62 list_del(&fence->head);
63 rcu_assign_pointer(fence->channel, NULL);
64
65 if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
66 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
67
68 if (!--fctx->notify_ref)
69 drop = 1;
70 }
71
72 dma_fence_put(&fence->base);
73 return drop;
74}
75
76static struct nouveau_fence *
77nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm) {
78 struct nouveau_fence_priv *priv = (void*)drm->fence;
79
80 if (fence->ops != &nouveau_fence_ops_legacy &&
81 fence->ops != &nouveau_fence_ops_uevent)
82 return NULL;
83
84 if (fence->context < priv->context_base ||
85 fence->context >= priv->context_base + priv->contexts)
86 return NULL;
87
88 return from_fence(fence);
89}
90
91void
92nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
93{
94 struct nouveau_fence *fence;
95
96 spin_lock_irq(&fctx->lock);
97 while (!list_empty(&fctx->pending)) {
98 fence = list_entry(fctx->pending.next, typeof(*fence), head);
99
100 if (nouveau_fence_signal(fence))
101 nvif_notify_put(&fctx->notify);
102 }
103 spin_unlock_irq(&fctx->lock);
104
105 nvif_notify_fini(&fctx->notify);
106 fctx->dead = 1;
107
108 /*
109 * Ensure that all accesses to fence->channel complete before freeing
110 * the channel.
111 */
112 synchronize_rcu();
113}
114
115static void
116nouveau_fence_context_put(struct kref *fence_ref)
117{
118 kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
119}
120
121void
122nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
123{
124 kref_put(&fctx->fence_ref, nouveau_fence_context_put);
125}
126
127static int
128nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
129{
130 struct nouveau_fence *fence;
131 int drop = 0;
132 u32 seq = fctx->read(chan);
133
134 while (!list_empty(&fctx->pending)) {
135 fence = list_entry(fctx->pending.next, typeof(*fence), head);
136
137 if ((int)(seq - fence->base.seqno) < 0)
138 break;
139
140 drop |= nouveau_fence_signal(fence);
141 }
142
143 return drop;
144}
145
146static int
147nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
148{
149 struct nouveau_fence_chan *fctx =
150 container_of(notify, typeof(*fctx), notify);
151 unsigned long flags;
152 int ret = NVIF_NOTIFY_KEEP;
153
154 spin_lock_irqsave(&fctx->lock, flags);
155 if (!list_empty(&fctx->pending)) {
156 struct nouveau_fence *fence;
157 struct nouveau_channel *chan;
158
159 fence = list_entry(fctx->pending.next, typeof(*fence), head);
160 chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
161 if (nouveau_fence_update(fence->channel, fctx))
162 ret = NVIF_NOTIFY_DROP;
163 }
164 spin_unlock_irqrestore(&fctx->lock, flags);
165
166 return ret;
167}
168
169void
170nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
171{
172 struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
173 struct nouveau_cli *cli = (void *)chan->user.client;
174 int ret;
175
176 INIT_LIST_HEAD(&fctx->flip);
177 INIT_LIST_HEAD(&fctx->pending);
178 spin_lock_init(&fctx->lock);
179 fctx->context = priv->context_base + chan->chid;
180
181 if (chan == chan->drm->cechan)
182 strcpy(fctx->name, "copy engine channel");
183 else if (chan == chan->drm->channel)
184 strcpy(fctx->name, "generic kernel channel");
185 else
186 strcpy(fctx->name, nvxx_client(&cli->base)->name);
187
188 kref_init(&fctx->fence_ref);
189 if (!priv->uevent)
190 return;
191
192 ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler,
193 false, G82_CHANNEL_DMA_V0_NTFY_UEVENT,
194 &(struct nvif_notify_uevent_req) { },
195 sizeof(struct nvif_notify_uevent_req),
196 sizeof(struct nvif_notify_uevent_rep),
197 &fctx->notify);
198
199 WARN_ON(ret);
200}
201
202struct nouveau_fence_work {
203 struct work_struct work;
204 struct dma_fence_cb cb;
205 void (*func)(void *);
206 void *data;
207};
208
209static void
210nouveau_fence_work_handler(struct work_struct *kwork)
211{
212 struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work);
213 work->func(work->data);
214 kfree(work);
215}
216
217static void nouveau_fence_work_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
218{
219 struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);
220
221 schedule_work(&work->work);
222}
223
224void
225nouveau_fence_work(struct dma_fence *fence,
226 void (*func)(void *), void *data)
227{
228 struct nouveau_fence_work *work;
229
230 if (dma_fence_is_signaled(fence))
231 goto err;
232
233 work = kmalloc(sizeof(*work), GFP_KERNEL);
234 if (!work) {
235 /*
236 * this might not be a nouveau fence any more,
237 * so force a lazy wait here
238 */
239 WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
240 true, false));
241 goto err;
242 }
243
244 INIT_WORK(&work->work, nouveau_fence_work_handler);
245 work->func = func;
246 work->data = data;
247
248 if (dma_fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
249 goto err_free;
250 return;
251
252err_free:
253 kfree(work);
254err:
255 func(data);
256}
257
258int
259nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
260{
261 struct nouveau_fence_chan *fctx = chan->fence;
262 struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
263 int ret;
264
265 fence->channel = chan;
266 fence->timeout = jiffies + (15 * HZ);
267
268 if (priv->uevent)
269 dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
270 &fctx->lock, fctx->context, ++fctx->sequence);
271 else
272 dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
273 &fctx->lock, fctx->context, ++fctx->sequence);
274 kref_get(&fctx->fence_ref);
275
276 trace_dma_fence_emit(&fence->base);
277 ret = fctx->emit(fence);
278 if (!ret) {
279 dma_fence_get(&fence->base);
280 spin_lock_irq(&fctx->lock);
281
282 if (nouveau_fence_update(chan, fctx))
283 nvif_notify_put(&fctx->notify);
284
285 list_add_tail(&fence->head, &fctx->pending);
286 spin_unlock_irq(&fctx->lock);
287 }
288
289 return ret;
290}
291
292bool
293nouveau_fence_done(struct nouveau_fence *fence)
294{
295 if (fence->base.ops == &nouveau_fence_ops_legacy ||
296 fence->base.ops == &nouveau_fence_ops_uevent) {
297 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
298 struct nouveau_channel *chan;
299 unsigned long flags;
300
301 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
302 return true;
303
304 spin_lock_irqsave(&fctx->lock, flags);
305 chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
306 if (chan && nouveau_fence_update(chan, fctx))
307 nvif_notify_put(&fctx->notify);
308 spin_unlock_irqrestore(&fctx->lock, flags);
309 }
310 return dma_fence_is_signaled(&fence->base);
311}
312
313static long
314nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
315{
316 struct nouveau_fence *fence = from_fence(f);
317 unsigned long sleep_time = NSEC_PER_MSEC / 1000;
318 unsigned long t = jiffies, timeout = t + wait;
319
320 while (!nouveau_fence_done(fence)) {
321 ktime_t kt;
322
323 t = jiffies;
324
325 if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
326 __set_current_state(TASK_RUNNING);
327 return 0;
328 }
329
330 __set_current_state(intr ? TASK_INTERRUPTIBLE :
331 TASK_UNINTERRUPTIBLE);
332
333 kt = sleep_time;
334 schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
335 sleep_time *= 2;
336 if (sleep_time > NSEC_PER_MSEC)
337 sleep_time = NSEC_PER_MSEC;
338
339 if (intr && signal_pending(current))
340 return -ERESTARTSYS;
341 }
342
343 __set_current_state(TASK_RUNNING);
344
345 return timeout - t;
346}
347
348static int
349nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
350{
351 int ret = 0;
352
353 while (!nouveau_fence_done(fence)) {
354 if (time_after_eq(jiffies, fence->timeout)) {
355 ret = -EBUSY;
356 break;
357 }
358
359 __set_current_state(intr ?
360 TASK_INTERRUPTIBLE :
361 TASK_UNINTERRUPTIBLE);
362
363 if (intr && signal_pending(current)) {
364 ret = -ERESTARTSYS;
365 break;
366 }
367 }
368
369 __set_current_state(TASK_RUNNING);
370 return ret;
371}
372
373int
374nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
375{
376 long ret;
377
378 if (!lazy)
379 return nouveau_fence_wait_busy(fence, intr);
380
381 ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
382 if (ret < 0)
383 return ret;
384 else if (!ret)
385 return -EBUSY;
386 else
387 return 0;
388}
389
390int
391nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
392{
393 struct nouveau_fence_chan *fctx = chan->fence;
394 struct dma_fence *fence;
395 struct reservation_object *resv = nvbo->bo.resv;
396 struct reservation_object_list *fobj;
397 struct nouveau_fence *f;
398 int ret = 0, i;
399
400 if (!exclusive) {
401 ret = reservation_object_reserve_shared(resv);
402
403 if (ret)
404 return ret;
405 }
406
407 fobj = reservation_object_get_list(resv);
408 fence = reservation_object_get_excl(resv);
409
410 if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
411 struct nouveau_channel *prev = NULL;
412 bool must_wait = true;
413
414 f = nouveau_local_fence(fence, chan->drm);
415 if (f) {
416 rcu_read_lock();
417 prev = rcu_dereference(f->channel);
418 if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
419 must_wait = false;
420 rcu_read_unlock();
421 }
422
423 if (must_wait)
424 ret = dma_fence_wait(fence, intr);
425
426 return ret;
427 }
428
429 if (!exclusive || !fobj)
430 return ret;
431
432 for (i = 0; i < fobj->shared_count && !ret; ++i) {
433 struct nouveau_channel *prev = NULL;
434 bool must_wait = true;
435
436 fence = rcu_dereference_protected(fobj->shared[i],
437 reservation_object_held(resv));
438
439 f = nouveau_local_fence(fence, chan->drm);
440 if (f) {
441 rcu_read_lock();
442 prev = rcu_dereference(f->channel);
443 if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
444 must_wait = false;
445 rcu_read_unlock();
446 }
447
448 if (must_wait)
449 ret = dma_fence_wait(fence, intr);
450 }
451
452 return ret;
453}
454
455void
456nouveau_fence_unref(struct nouveau_fence **pfence)
457{
458 if (*pfence)
459 dma_fence_put(&(*pfence)->base);
460 *pfence = NULL;
461}
462
463int
464nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
465 struct nouveau_fence **pfence)
466{
467 struct nouveau_fence *fence;
468 int ret = 0;
469
470 if (unlikely(!chan->fence))
471 return -ENODEV;
472
473 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
474 if (!fence)
475 return -ENOMEM;
476
477 fence->sysmem = sysmem;
478
479 ret = nouveau_fence_emit(fence, chan);
480 if (ret)
481 nouveau_fence_unref(&fence);
482
483 *pfence = fence;
484 return ret;
485}
486
487static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
488{
489 return "nouveau";
490}
491
492static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
493{
494 struct nouveau_fence *fence = from_fence(f);
495 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
496
497 return !fctx->dead ? fctx->name : "dead channel";
498}
499
500/*
501 * In an ideal world, read would not assume the channel context is still alive.
502 * This function may be called from another device, running into free memory as a
503 * result. The drm node should still be there, so we can derive the index from
504 * the fence context.
505 */
506static bool nouveau_fence_is_signaled(struct dma_fence *f)
507{
508 struct nouveau_fence *fence = from_fence(f);
509 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
510 struct nouveau_channel *chan;
511 bool ret = false;
512
513 rcu_read_lock();
514 chan = rcu_dereference(fence->channel);
515 if (chan)
516 ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
517 rcu_read_unlock();
518
519 return ret;
520}
521
522static bool nouveau_fence_no_signaling(struct dma_fence *f)
523{
524 struct nouveau_fence *fence = from_fence(f);
525
526 /*
527 * caller should have a reference on the fence,
528 * else fence could get freed here
529 */
530 WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
531
532 /*
533 * This needs uevents to work correctly, but dma_fence_add_callback relies on
534 * being able to enable signaling. It will still get signaled eventually,
535 * just not right away.
536 */
537 if (nouveau_fence_is_signaled(f)) {
538 list_del(&fence->head);
539
540 dma_fence_put(&fence->base);
541 return false;
542 }
543
544 return true;
545}
546
547static void nouveau_fence_release(struct dma_fence *f)
548{
549 struct nouveau_fence *fence = from_fence(f);
550 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
551
552 kref_put(&fctx->fence_ref, nouveau_fence_context_put);
553 dma_fence_free(&fence->base);
554}
555
556static const struct dma_fence_ops nouveau_fence_ops_legacy = {
557 .get_driver_name = nouveau_fence_get_get_driver_name,
558 .get_timeline_name = nouveau_fence_get_timeline_name,
559 .enable_signaling = nouveau_fence_no_signaling,
560 .signaled = nouveau_fence_is_signaled,
561 .wait = nouveau_fence_wait_legacy,
562 .release = nouveau_fence_release
563};
564
565static bool nouveau_fence_enable_signaling(struct dma_fence *f)
566{
567 struct nouveau_fence *fence = from_fence(f);
568 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
569 bool ret;
570
571 if (!fctx->notify_ref++)
572 nvif_notify_get(&fctx->notify);
573
574 ret = nouveau_fence_no_signaling(f);
575 if (ret)
576 set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
577 else if (!--fctx->notify_ref)
578 nvif_notify_put(&fctx->notify);
579
580 return ret;
581}
582
583static const struct dma_fence_ops nouveau_fence_ops_uevent = {
584 .get_driver_name = nouveau_fence_get_get_driver_name,
585 .get_timeline_name = nouveau_fence_get_timeline_name,
586 .enable_signaling = nouveau_fence_enable_signaling,
587 .signaled = nouveau_fence_is_signaled,
588 .wait = dma_fence_default_wait,
589 .release = nouveau_fence_release
590};