Linux Audio

Check our new training course

Loading...
v6.9.4
  1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * (C) Copyright 2016 Intel Corporation
 
 
 
 
 
  5 */
  6
  7#include <linux/slab.h>
  8#include <linux/dma-fence.h>
  9#include <linux/irq_work.h>
 10#include <linux/dma-resv.h>
 11
 12#include "i915_sw_fence.h"
 13#include "i915_selftest.h"
 14
 15#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
 16#define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr)
 17#else
 18#define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
 19#endif
 20
 21#ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
 22static DEFINE_SPINLOCK(i915_sw_fence_lock);
 23#endif
 24
 25#define WQ_FLAG_BITS \
 26	BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags))
 27
 28/* after WQ_FLAG_* for safety */
 29#define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1)
 30#define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2)
 31
 32enum {
 33	DEBUG_FENCE_IDLE = 0,
 34	DEBUG_FENCE_NOTIFY,
 35};
 36
 37static void *i915_sw_fence_debug_hint(void *addr)
 38{
 39	return (void *)(((struct i915_sw_fence *)addr)->fn);
 40}
 41
 42#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
 43
 44static const struct debug_obj_descr i915_sw_fence_debug_descr = {
 45	.name = "i915_sw_fence",
 46	.debug_hint = i915_sw_fence_debug_hint,
 47};
 48
 49static inline void debug_fence_init(struct i915_sw_fence *fence)
 50{
 51	debug_object_init(fence, &i915_sw_fence_debug_descr);
 52}
 53
 54static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
 55{
 56	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
 57}
 58
 59static inline void debug_fence_activate(struct i915_sw_fence *fence)
 60{
 61	debug_object_activate(fence, &i915_sw_fence_debug_descr);
 62}
 63
 64static inline void debug_fence_set_state(struct i915_sw_fence *fence,
 65					 int old, int new)
 66{
 67	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
 68}
 69
 70static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
 71{
 72	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
 73}
 74
 75static inline void debug_fence_destroy(struct i915_sw_fence *fence)
 76{
 77	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
 78}
 79
 80static inline void debug_fence_free(struct i915_sw_fence *fence)
 81{
 82	debug_object_free(fence, &i915_sw_fence_debug_descr);
 83	smp_wmb(); /* flush the change in state before reallocation */
 84}
 85
 86static inline void debug_fence_assert(struct i915_sw_fence *fence)
 87{
 88	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
 89}
 90
 91#else
 92
 93static inline void debug_fence_init(struct i915_sw_fence *fence)
 94{
 95}
 96
 97static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
 98{
 99}
100
101static inline void debug_fence_activate(struct i915_sw_fence *fence)
102{
103}
104
105static inline void debug_fence_set_state(struct i915_sw_fence *fence,
106					 int old, int new)
107{
108}
109
110static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
111{
112}
113
114static inline void debug_fence_destroy(struct i915_sw_fence *fence)
115{
116}
117
118static inline void debug_fence_free(struct i915_sw_fence *fence)
119{
120}
121
122static inline void debug_fence_assert(struct i915_sw_fence *fence)
123{
 
 
124}
125
126#endif
127
128static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
129				  enum i915_sw_fence_notify state)
130{
131	return fence->fn(fence, state);
132}
133
134#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
135void i915_sw_fence_fini(struct i915_sw_fence *fence)
136{
137	debug_fence_free(fence);
 
138}
139#endif
140
141static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
142					struct list_head *continuation)
143{
144	wait_queue_head_t *x = &fence->wait;
145	wait_queue_entry_t *pos, *next;
146	unsigned long flags;
147
148	debug_fence_deactivate(fence);
149	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
150
151	/*
152	 * To prevent unbounded recursion as we traverse the graph of
153	 * i915_sw_fences, we move the entry list from this, the next ready
154	 * fence, to the tail of the original fence's entry list
155	 * (and so added to the list to be woken).
156	 */
157
158	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
159	if (continuation) {
160		list_for_each_entry_safe(pos, next, &x->head, entry) {
161			if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
162				list_move_tail(&pos->entry, continuation);
163			else
164				pos->func(pos, TASK_NORMAL, 0, continuation);
 
 
165		}
166	} else {
167		LIST_HEAD(extra);
168
169		do {
170			list_for_each_entry_safe(pos, next, &x->head, entry) {
171				int wake_flags;
172
173				wake_flags = 0;
174				if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
175					wake_flags = fence->error;
176
177				pos->func(pos, TASK_NORMAL, wake_flags, &extra);
178			}
179
180			if (list_empty(&extra))
181				break;
182
183			list_splice_tail_init(&extra, &x->head);
184		} while (1);
185	}
186	spin_unlock_irqrestore(&x->lock, flags);
187
188	debug_fence_assert(fence);
189}
190
191static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
192				     struct list_head *continuation)
193{
194	debug_fence_assert(fence);
195
196	if (!atomic_dec_and_test(&fence->pending))
197		return;
198
199	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
200
201	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
202		return;
203
204	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
205
206	__i915_sw_fence_wake_up_all(fence, continuation);
207
208	debug_fence_destroy(fence);
209	__i915_sw_fence_notify(fence, FENCE_FREE);
210}
211
212void i915_sw_fence_complete(struct i915_sw_fence *fence)
213{
214	debug_fence_assert(fence);
215
216	if (WARN_ON(i915_sw_fence_done(fence)))
217		return;
218
219	__i915_sw_fence_complete(fence, NULL);
220}
221
222bool i915_sw_fence_await(struct i915_sw_fence *fence)
223{
224	int pending;
225
226	/*
227	 * It is only safe to add a new await to the fence while it has
228	 * not yet been signaled (i.e. there are still existing signalers).
229	 */
230	pending = atomic_read(&fence->pending);
231	do {
232		if (pending < 1)
233			return false;
234	} while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1));
235
236	return true;
237}
238
239void __i915_sw_fence_init(struct i915_sw_fence *fence,
240			  i915_sw_fence_notify_t fn,
241			  const char *name,
242			  struct lock_class_key *key)
243{
244	__init_waitqueue_head(&fence->wait, name, key);
245	fence->fn = fn;
246#ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
247	fence->flags = 0;
248#endif
249
250	i915_sw_fence_reinit(fence);
251}
252
253void i915_sw_fence_reinit(struct i915_sw_fence *fence)
254{
255	debug_fence_init(fence);
256
 
 
257	atomic_set(&fence->pending, 1);
258	fence->error = 0;
259
260	I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head));
261}
262
263void i915_sw_fence_commit(struct i915_sw_fence *fence)
264{
265	debug_fence_activate(fence);
266	i915_sw_fence_complete(fence);
 
267}
268
269static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
270{
271	i915_sw_fence_set_error_once(wq->private, flags);
272
273	list_del(&wq->entry);
274	__i915_sw_fence_complete(wq->private, key);
275
276	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
277		kfree(wq);
278	return 0;
279}
280
281#ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
282static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
283				    const struct i915_sw_fence * const signaler)
284{
285	wait_queue_entry_t *wq;
286
287	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
288		return false;
289
290	if (fence == signaler)
291		return true;
292
293	list_for_each_entry(wq, &fence->wait.head, entry) {
294		if (wq->func != i915_sw_fence_wake)
295			continue;
296
297		if (__i915_sw_fence_check_if_after(wq->private, signaler))
298			return true;
299	}
300
301	return false;
302}
303
304static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
305{
306	wait_queue_entry_t *wq;
307
308	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
309		return;
310
311	list_for_each_entry(wq, &fence->wait.head, entry) {
312		if (wq->func != i915_sw_fence_wake)
313			continue;
314
315		__i915_sw_fence_clear_checked_bit(wq->private);
316	}
317}
318
319static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
320				  const struct i915_sw_fence * const signaler)
321{
322	unsigned long flags;
323	bool err;
324
 
 
 
325	spin_lock_irqsave(&i915_sw_fence_lock, flags);
326	err = __i915_sw_fence_check_if_after(fence, signaler);
327	__i915_sw_fence_clear_checked_bit(fence);
328	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
329
330	return err;
331}
332#else
333static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
334					 const struct i915_sw_fence * const signaler)
335{
336	return false;
337}
338#endif
339
340static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
341					  struct i915_sw_fence *signaler,
342					  wait_queue_entry_t *wq, gfp_t gfp)
343{
344	unsigned int pending;
345	unsigned long flags;
 
346
347	debug_fence_assert(fence);
348	might_sleep_if(gfpflags_allow_blocking(gfp));
349
350	if (i915_sw_fence_done(signaler)) {
351		i915_sw_fence_set_error_once(fence, signaler->error);
352		return 0;
353	}
354
355	debug_fence_assert(signaler);
356
357	/* The dependency graph must be acyclic. */
358	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
359		return -EINVAL;
360
361	pending = I915_SW_FENCE_FLAG_FENCE;
362	if (!wq) {
363		wq = kmalloc(sizeof(*wq), gfp);
364		if (!wq) {
365			if (!gfpflags_allow_blocking(gfp))
366				return -ENOMEM;
367
368			i915_sw_fence_wait(signaler);
369			i915_sw_fence_set_error_once(fence, signaler->error);
370			return 0;
371		}
372
373		pending |= I915_SW_FENCE_FLAG_ALLOC;
374	}
375
376	INIT_LIST_HEAD(&wq->entry);
377	wq->flags = pending;
378	wq->func = i915_sw_fence_wake;
379	wq->private = fence;
380
381	i915_sw_fence_await(fence);
382
383	spin_lock_irqsave(&signaler->wait.lock, flags);
384	if (likely(!i915_sw_fence_done(signaler))) {
385		__add_wait_queue_entry_tail(&signaler->wait, wq);
386		pending = 1;
387	} else {
388		i915_sw_fence_wake(wq, 0, signaler->error, NULL);
389		pending = 0;
390	}
391	spin_unlock_irqrestore(&signaler->wait.lock, flags);
392
393	return pending;
394}
395
396int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
397				 struct i915_sw_fence *signaler,
398				 wait_queue_entry_t *wq)
399{
400	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
401}
402
403int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
404				     struct i915_sw_fence *signaler,
405				     gfp_t gfp)
406{
407	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
408}
409
410struct i915_sw_dma_fence_cb_timer {
411	struct i915_sw_dma_fence_cb base;
 
412	struct dma_fence *dma;
413	struct timer_list timer;
414	struct irq_work work;
415	struct rcu_head rcu;
416};
417
418static void dma_i915_sw_fence_wake(struct dma_fence *dma,
419				   struct dma_fence_cb *data)
420{
421	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
422
423	i915_sw_fence_set_error_once(cb->fence, dma->error);
424	i915_sw_fence_complete(cb->fence);
425	kfree(cb);
426}
427
428static void timer_i915_sw_fence_wake(struct timer_list *t)
429{
430	struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
431	struct i915_sw_fence *fence;
432
433	fence = xchg(&cb->base.fence, NULL);
434	if (!fence)
435		return;
436
437	pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n",
438		  cb->dma->ops->get_driver_name(cb->dma),
439		  cb->dma->ops->get_timeline_name(cb->dma),
440		  cb->dma->seqno,
441		  i915_sw_fence_debug_hint(fence));
442
443	i915_sw_fence_set_error_once(fence, -ETIMEDOUT);
444	i915_sw_fence_complete(fence);
445}
446
447static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
448					 struct dma_fence_cb *data)
449{
450	struct i915_sw_dma_fence_cb_timer *cb =
451		container_of(data, typeof(*cb), base.base);
452	struct i915_sw_fence *fence;
453
454	fence = xchg(&cb->base.fence, NULL);
455	if (fence) {
456		i915_sw_fence_set_error_once(fence, dma->error);
457		i915_sw_fence_complete(fence);
458	}
 
459
460	irq_work_queue(&cb->work);
 
461}
462
463static void irq_i915_sw_fence_work(struct irq_work *wrk)
 
464{
465	struct i915_sw_dma_fence_cb_timer *cb =
466		container_of(wrk, typeof(*cb), work);
467
468	timer_shutdown_sync(&cb->timer);
 
 
469	dma_fence_put(cb->dma);
470
471	kfree_rcu(cb, rcu);
472}
473
474int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
475				  struct dma_fence *dma,
476				  unsigned long timeout,
477				  gfp_t gfp)
478{
479	struct i915_sw_dma_fence_cb *cb;
480	dma_fence_func_t func;
481	int ret;
482
483	debug_fence_assert(fence);
484	might_sleep_if(gfpflags_allow_blocking(gfp));
485
486	if (dma_fence_is_signaled(dma)) {
487		i915_sw_fence_set_error_once(fence, dma->error);
488		return 0;
489	}
490
491	cb = kmalloc(timeout ?
492		     sizeof(struct i915_sw_dma_fence_cb_timer) :
493		     sizeof(struct i915_sw_dma_fence_cb),
494		     gfp);
495	if (!cb) {
496		if (!gfpflags_allow_blocking(gfp))
497			return -ENOMEM;
498
499		ret = dma_fence_wait(dma, false);
500		if (ret)
501			return ret;
502
503		i915_sw_fence_set_error_once(fence, dma->error);
504		return 0;
505	}
506
507	cb->fence = fence;
508	i915_sw_fence_await(fence);
509
510	func = dma_i915_sw_fence_wake;
 
 
 
511	if (timeout) {
512		struct i915_sw_dma_fence_cb_timer *timer =
513			container_of(cb, typeof(*timer), base);
514
515		timer->dma = dma_fence_get(dma);
516		init_irq_work(&timer->work, irq_i915_sw_fence_work);
517
518		timer_setup(&timer->timer,
519			    timer_i915_sw_fence_wake, TIMER_IRQSAFE);
520		mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));
521
522		func = dma_i915_sw_fence_wake_timer;
523	}
524
525	ret = dma_fence_add_callback(dma, &cb->base, func);
526	if (ret == 0) {
527		ret = 1;
528	} else {
529		func(dma, &cb->base);
530		if (ret == -ENOENT) /* fence already signaled */
531			ret = 0;
532	}
533
534	return ret;
535}
536
537static void __dma_i915_sw_fence_wake(struct dma_fence *dma,
538				     struct dma_fence_cb *data)
539{
540	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
541
542	i915_sw_fence_set_error_once(cb->fence, dma->error);
543	i915_sw_fence_complete(cb->fence);
544}
545
546int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
547				    struct dma_fence *dma,
548				    struct i915_sw_dma_fence_cb *cb)
549{
550	int ret;
551
552	debug_fence_assert(fence);
553
554	if (dma_fence_is_signaled(dma)) {
555		i915_sw_fence_set_error_once(fence, dma->error);
556		return 0;
557	}
558
559	cb->fence = fence;
560	i915_sw_fence_await(fence);
561
562	ret = 1;
563	if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) {
564		/* fence already signaled */
565		__dma_i915_sw_fence_wake(dma, &cb->base);
566		ret = 0;
567	}
568
569	return ret;
570}
571
572int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
573				    struct dma_resv *resv,
 
574				    bool write,
575				    unsigned long timeout,
576				    gfp_t gfp)
577{
578	struct dma_resv_iter cursor;
579	struct dma_fence *f;
580	int ret = 0, pending;
581
582	debug_fence_assert(fence);
583	might_sleep_if(gfpflags_allow_blocking(gfp));
 
584
585	dma_resv_iter_begin(&cursor, resv, dma_resv_usage_rw(write));
586	dma_resv_for_each_fence_unlocked(&cursor, f) {
587		pending = i915_sw_fence_await_dma_fence(fence, f, timeout,
588							gfp);
589		if (pending < 0) {
590			ret = pending;
591			break;
 
 
 
 
 
 
 
 
 
 
 
 
592		}
593
594		ret |= pending;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
595	}
596	dma_resv_iter_end(&cursor);
 
 
597	return ret;
598}
599
600#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
601#include "selftests/lib_sw_fence.c"
602#include "selftests/i915_sw_fence.c"
603#endif
v4.10.11
  1/*
 
 
  2 * (C) Copyright 2016 Intel Corporation
  3 *
  4 * This program is free software; you can redistribute it and/or
  5 * modify it under the terms of the GNU General Public License
  6 * as published by the Free Software Foundation; version 2
  7 * of the License.
  8 */
  9
 10#include <linux/slab.h>
 11#include <linux/dma-fence.h>
 12#include <linux/reservation.h>
 
 13
 14#include "i915_sw_fence.h"
 
 15
 16#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
 
 
 
 
 17
 
 18static DEFINE_SPINLOCK(i915_sw_fence_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 19
 20static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
 21				  enum i915_sw_fence_notify state)
 22{
 23	i915_sw_fence_notify_t fn;
 24
 25	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
 26	return fn(fence, state);
 27}
 28
 29static void i915_sw_fence_free(struct kref *kref)
 30{
 31	struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref);
 32
 33	WARN_ON(atomic_read(&fence->pending) > 0);
 
 
 34
 35	if (fence->flags & I915_SW_FENCE_MASK)
 36		__i915_sw_fence_notify(fence, FENCE_FREE);
 37	else
 38		kfree(fence);
 39}
 40
 41static void i915_sw_fence_put(struct i915_sw_fence *fence)
 
 
 
 42{
 43	kref_put(&fence->kref, i915_sw_fence_free);
 44}
 45
 46static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence)
 
 47{
 48	kref_get(&fence->kref);
 49	return fence;
 50}
 
 51
 52static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
 53					struct list_head *continuation)
 54{
 55	wait_queue_head_t *x = &fence->wait;
 56	wait_queue_t *pos, *next;
 57	unsigned long flags;
 58
 
 59	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
 60
 61	/*
 62	 * To prevent unbounded recursion as we traverse the graph of
 63	 * i915_sw_fences, we move the task_list from this, the next ready
 64	 * fence, to the tail of the original fence's task_list
 65	 * (and so added to the list to be woken).
 66	 */
 67
 68	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
 69	if (continuation) {
 70		list_for_each_entry_safe(pos, next, &x->task_list, task_list) {
 71			if (pos->func == autoremove_wake_function)
 
 
 72				pos->func(pos, TASK_NORMAL, 0, continuation);
 73			else
 74				list_move_tail(&pos->task_list, continuation);
 75		}
 76	} else {
 77		LIST_HEAD(extra);
 78
 79		do {
 80			list_for_each_entry_safe(pos, next,
 81						 &x->task_list, task_list)
 82				pos->func(pos, TASK_NORMAL, 0, &extra);
 
 
 
 
 
 
 83
 84			if (list_empty(&extra))
 85				break;
 86
 87			list_splice_tail_init(&extra, &x->task_list);
 88		} while (1);
 89	}
 90	spin_unlock_irqrestore(&x->lock, flags);
 
 
 91}
 92
 93static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 94				     struct list_head *continuation)
 95{
 
 
 96	if (!atomic_dec_and_test(&fence->pending))
 97		return;
 98
 99	if (fence->flags & I915_SW_FENCE_MASK &&
100	    __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
 
101		return;
102
 
 
103	__i915_sw_fence_wake_up_all(fence, continuation);
 
 
 
104}
105
106static void i915_sw_fence_complete(struct i915_sw_fence *fence)
107{
 
 
108	if (WARN_ON(i915_sw_fence_done(fence)))
109		return;
110
111	__i915_sw_fence_complete(fence, NULL);
112}
113
114static void i915_sw_fence_await(struct i915_sw_fence *fence)
115{
116	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
 
 
 
 
 
 
 
 
 
 
 
 
117}
118
119void __i915_sw_fence_init(struct i915_sw_fence *fence,
120			  i915_sw_fence_notify_t fn,
121			  const char *name,
122			  struct lock_class_key *key)
123{
124	BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);
 
 
 
 
 
 
 
 
 
 
 
125
126	__init_waitqueue_head(&fence->wait, name, key);
127	kref_init(&fence->kref);
128	atomic_set(&fence->pending, 1);
129	fence->flags = (unsigned long)fn;
 
 
130}
131
132void i915_sw_fence_commit(struct i915_sw_fence *fence)
133{
 
134	i915_sw_fence_complete(fence);
135	i915_sw_fence_put(fence);
136}
137
138static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key)
139{
140	list_del(&wq->task_list);
 
 
141	__i915_sw_fence_complete(wq->private, key);
142	i915_sw_fence_put(wq->private);
143	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
144		kfree(wq);
145	return 0;
146}
147
 
148static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
149				    const struct i915_sw_fence * const signaler)
150{
151	wait_queue_t *wq;
152
153	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
154		return false;
155
156	if (fence == signaler)
157		return true;
158
159	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
160		if (wq->func != i915_sw_fence_wake)
161			continue;
162
163		if (__i915_sw_fence_check_if_after(wq->private, signaler))
164			return true;
165	}
166
167	return false;
168}
169
170static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
171{
172	wait_queue_t *wq;
173
174	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
175		return;
176
177	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
178		if (wq->func != i915_sw_fence_wake)
179			continue;
180
181		__i915_sw_fence_clear_checked_bit(wq->private);
182	}
183}
184
185static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
186				  const struct i915_sw_fence * const signaler)
187{
188	unsigned long flags;
189	bool err;
190
191	if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG))
192		return false;
193
194	spin_lock_irqsave(&i915_sw_fence_lock, flags);
195	err = __i915_sw_fence_check_if_after(fence, signaler);
196	__i915_sw_fence_clear_checked_bit(fence);
197	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
198
199	return err;
200}
 
 
 
 
 
 
 
201
202static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
203					  struct i915_sw_fence *signaler,
204					  wait_queue_t *wq, gfp_t gfp)
205{
 
206	unsigned long flags;
207	int pending;
208
209	if (i915_sw_fence_done(signaler))
 
 
 
 
210		return 0;
 
 
 
211
212	/* The dependency graph must be acyclic. */
213	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
214		return -EINVAL;
215
216	pending = 0;
217	if (!wq) {
218		wq = kmalloc(sizeof(*wq), gfp);
219		if (!wq) {
220			if (!gfpflags_allow_blocking(gfp))
221				return -ENOMEM;
222
223			i915_sw_fence_wait(signaler);
 
224			return 0;
225		}
226
227		pending |= I915_SW_FENCE_FLAG_ALLOC;
228	}
229
230	INIT_LIST_HEAD(&wq->task_list);
231	wq->flags = pending;
232	wq->func = i915_sw_fence_wake;
233	wq->private = i915_sw_fence_get(fence);
234
235	i915_sw_fence_await(fence);
236
237	spin_lock_irqsave(&signaler->wait.lock, flags);
238	if (likely(!i915_sw_fence_done(signaler))) {
239		__add_wait_queue_tail(&signaler->wait, wq);
240		pending = 1;
241	} else {
242		i915_sw_fence_wake(wq, 0, 0, NULL);
243		pending = 0;
244	}
245	spin_unlock_irqrestore(&signaler->wait.lock, flags);
246
247	return pending;
248}
249
250int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
251				 struct i915_sw_fence *signaler,
252				 wait_queue_t *wq)
253{
254	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
255}
256
257int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
258				     struct i915_sw_fence *signaler,
259				     gfp_t gfp)
260{
261	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
262}
263
264struct i915_sw_dma_fence_cb {
265	struct dma_fence_cb base;
266	struct i915_sw_fence *fence;
267	struct dma_fence *dma;
268	struct timer_list timer;
 
 
269};
270
271static void timer_i915_sw_fence_wake(unsigned long data)
 
 
 
 
 
 
 
 
 
 
272{
273	struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
274
275	printk(KERN_WARNING "asynchronous wait on fence %s:%s:%x timed out\n",
276	       cb->dma->ops->get_driver_name(cb->dma),
277	       cb->dma->ops->get_timeline_name(cb->dma),
278	       cb->dma->seqno);
279	dma_fence_put(cb->dma);
280	cb->dma = NULL;
281
282	i915_sw_fence_commit(cb->fence);
283	cb->timer.function = NULL;
284}
285
286static void dma_i915_sw_fence_wake(struct dma_fence *dma,
287				   struct dma_fence_cb *data)
288{
289	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 
290
291	del_timer_sync(&cb->timer);
292	if (cb->timer.function)
293		i915_sw_fence_commit(cb->fence);
294	dma_fence_put(cb->dma);
295
296	kfree(cb);
297}
298
299int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
300				  struct dma_fence *dma,
301				  unsigned long timeout,
302				  gfp_t gfp)
303{
304	struct i915_sw_dma_fence_cb *cb;
 
305	int ret;
306
307	if (dma_fence_is_signaled(dma))
 
 
 
 
308		return 0;
 
309
310	cb = kmalloc(sizeof(*cb), gfp);
 
 
 
311	if (!cb) {
312		if (!gfpflags_allow_blocking(gfp))
313			return -ENOMEM;
314
315		return dma_fence_wait(dma, false);
 
 
 
 
 
316	}
317
318	cb->fence = i915_sw_fence_get(fence);
319	i915_sw_fence_await(fence);
320
321	cb->dma = NULL;
322	__setup_timer(&cb->timer,
323		      timer_i915_sw_fence_wake, (unsigned long)cb,
324		      TIMER_IRQSAFE);
325	if (timeout) {
326		cb->dma = dma_fence_get(dma);
327		mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
 
 
 
 
 
 
 
 
 
328	}
329
330	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
331	if (ret == 0) {
332		ret = 1;
333	} else {
334		dma_i915_sw_fence_wake(dma, &cb->base);
335		if (ret == -ENOENT) /* fence already signaled */
336			ret = 0;
337	}
338
339	return ret;
340}
341
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
342int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
343				    struct reservation_object *resv,
344				    const struct dma_fence_ops *exclude,
345				    bool write,
346				    unsigned long timeout,
347				    gfp_t gfp)
348{
349	struct dma_fence *excl;
 
350	int ret = 0, pending;
351
352	if (write) {
353		struct dma_fence **shared;
354		unsigned int count, i;
355
356		ret = reservation_object_get_fences_rcu(resv,
357							&excl, &count, &shared);
358		if (ret)
359			return ret;
360
361		for (i = 0; i < count; i++) {
362			if (shared[i]->ops == exclude)
363				continue;
364
365			pending = i915_sw_fence_await_dma_fence(fence,
366								shared[i],
367								timeout,
368								gfp);
369			if (pending < 0) {
370				ret = pending;
371				break;
372			}
373
374			ret |= pending;
375		}
376
377		for (i = 0; i < count; i++)
378			dma_fence_put(shared[i]);
379		kfree(shared);
380	} else {
381		excl = reservation_object_get_excl_rcu(resv);
382	}
383
384	if (ret >= 0 && excl && excl->ops != exclude) {
385		pending = i915_sw_fence_await_dma_fence(fence,
386							excl,
387							timeout,
388							gfp);
389		if (pending < 0)
390			ret = pending;
391		else
392			ret |= pending;
393	}
394
395	dma_fence_put(excl);
396
397	return ret;
398}