1/*
  2 * Copyright © 2015 Intel Corporation
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the next
 12 * paragraph) shall be included in all copies or substantial portions of the
 13 * Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 21 * IN THE SOFTWARE.
 22 *
 23 */
 24
 25#include <linux/kthread.h>
 26#include <trace/events/dma_fence.h>
 27#include <uapi/linux/sched/types.h>
 28
 29#include "i915_drv.h"
 30#include "i915_trace.h"
 31
 32static void irq_enable(struct intel_engine_cs *engine)
 33{
 34	if (!engine->irq_enable)
 35		return;
 36
 37	/* Caller disables interrupts */
 38	spin_lock(&engine->gt->irq_lock);
 39	engine->irq_enable(engine);
 40	spin_unlock(&engine->gt->irq_lock);
 41}
 42
 43static void irq_disable(struct intel_engine_cs *engine)
 44{
 45	if (!engine->irq_disable)
 46		return;
 47
 48	/* Caller disables interrupts */
 49	spin_lock(&engine->gt->irq_lock);
 50	engine->irq_disable(engine);
 51	spin_unlock(&engine->gt->irq_lock);
 52}
 53
 54static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
 55{
 56	lockdep_assert_held(&b->irq_lock);
 57
 58	GEM_BUG_ON(!b->irq_enabled);
 59	if (!--b->irq_enabled)
 60		irq_disable(container_of(b,
 61					 struct intel_engine_cs,
 62					 breadcrumbs));
 63
 64	b->irq_armed = false;
 65}
 66
 67void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 68{
 69	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 70	unsigned long flags;
 71
 72	if (!b->irq_armed)
 73		return;
 74
 75	spin_lock_irqsave(&b->irq_lock, flags);
 76	if (b->irq_armed)
 77		__intel_breadcrumbs_disarm_irq(b);
 78	spin_unlock_irqrestore(&b->irq_lock, flags);
 79}
 80
 81static inline bool __request_completed(const struct i915_request *rq)
 82{
 83	return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
 84}
 85
 86__maybe_unused static bool
 87check_signal_order(struct intel_context *ce, struct i915_request *rq)
 88{
 89	if (!list_is_last(&rq->signal_link, &ce->signals) &&
 90	    i915_seqno_passed(rq->fence.seqno,
 91			      list_next_entry(rq, signal_link)->fence.seqno))
 92		return false;
 93
 94	if (!list_is_first(&rq->signal_link, &ce->signals) &&
 95	    i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
 96			      rq->fence.seqno))
 97		return false;
 98
 99	return true;
100}
101
102static bool
103__dma_fence_signal(struct dma_fence *fence)
104{
105	return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
106}
107
108static void
109__dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
110{
111	fence->timestamp = timestamp;
112	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
113	trace_dma_fence_signaled(fence);
114}
115
116static void
117__dma_fence_signal__notify(struct dma_fence *fence,
118			   const struct list_head *list)
119{
120	struct dma_fence_cb *cur, *tmp;
121
122	lockdep_assert_held(fence->lock);
123	lockdep_assert_irqs_disabled();
124
125	list_for_each_entry_safe(cur, tmp, list, node) {
126		INIT_LIST_HEAD(&cur->node);
127		cur->func(fence, cur);
128	}
129}
130
131void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
132{
133	struct intel_breadcrumbs *b = &engine->breadcrumbs;
134	const ktime_t timestamp = ktime_get();
135	struct intel_context *ce, *cn;
136	struct list_head *pos, *next;
137	LIST_HEAD(signal);
138
139	spin_lock(&b->irq_lock);
140
141	if (b->irq_armed && list_empty(&b->signalers))
142		__intel_breadcrumbs_disarm_irq(b);
143
144	list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
145		GEM_BUG_ON(list_empty(&ce->signals));
146
147		list_for_each_safe(pos, next, &ce->signals) {
148			struct i915_request *rq =
149				list_entry(pos, typeof(*rq), signal_link);
150
151			GEM_BUG_ON(!check_signal_order(ce, rq));
152
153			if (!__request_completed(rq))
154				break;
155
156			GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
157					     &rq->fence.flags));
158			clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
159
160			if (!__dma_fence_signal(&rq->fence))
161				continue;
162
163			/*
164			 * Queue for execution after dropping the signaling
165			 * spinlock as the callback chain may end up adding
166			 * more signalers to the same context or engine.
167			 */
168			i915_request_get(rq);
169			list_add_tail(&rq->signal_link, &signal);
170		}
171
172		/*
173		 * We process the list deletion in bulk, only using a list_add
174		 * (not list_move) above but keeping the status of
175		 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
176		 */
177		if (!list_is_first(pos, &ce->signals)) {
178			/* Advance the list to the first incomplete request */
179			__list_del_many(&ce->signals, pos);
180			if (&ce->signals == pos) /* now empty */
181				list_del_init(&ce->signal_link);
182		}
183	}
184
185	spin_unlock(&b->irq_lock);
186
187	list_for_each_safe(pos, next, &signal) {
188		struct i915_request *rq =
189			list_entry(pos, typeof(*rq), signal_link);
190		struct list_head cb_list;
191
192		spin_lock(&rq->lock);
193		list_replace(&rq->fence.cb_list, &cb_list);
194		__dma_fence_signal__timestamp(&rq->fence, timestamp);
195		__dma_fence_signal__notify(&rq->fence, &cb_list);
196		spin_unlock(&rq->lock);
197
198		i915_request_put(rq);
199	}
200}
201
202void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine)
203{
204	local_irq_disable();
205	intel_engine_breadcrumbs_irq(engine);
206	local_irq_enable();
207}
208
209static void signal_irq_work(struct irq_work *work)
210{
211	struct intel_engine_cs *engine =
212		container_of(work, typeof(*engine), breadcrumbs.irq_work);
213
214	intel_engine_breadcrumbs_irq(engine);
215}
216
217static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
218{
219	struct intel_engine_cs *engine =
220		container_of(b, struct intel_engine_cs, breadcrumbs);
221
222	lockdep_assert_held(&b->irq_lock);
223	if (b->irq_armed)
224		return;
225
226	/*
227	 * The breadcrumb irq will be disarmed on the interrupt after the
228	 * waiters are signaled. This gives us a single interrupt window in
229	 * which we can add a new waiter and avoid the cost of re-enabling
230	 * the irq.
231	 */
232	b->irq_armed = true;
233
234	/*
235	 * Since we are waiting on a request, the GPU should be busy
236	 * and should have its own rpm reference. This is tracked
237	 * by i915->gt.awake, we can forgo holding our own wakref
238	 * for the interrupt as before i915->gt.awake is released (when
239	 * the driver is idle) we disarm the breadcrumbs.
240	 */
241
242	if (!b->irq_enabled++)
243		irq_enable(engine);
244}
245
246void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
247{
248	struct intel_breadcrumbs *b = &engine->breadcrumbs;
249
250	spin_lock_init(&b->irq_lock);
251	INIT_LIST_HEAD(&b->signalers);
252
253	init_irq_work(&b->irq_work, signal_irq_work);
254}
255
256void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
257{
258	struct intel_breadcrumbs *b = &engine->breadcrumbs;
259	unsigned long flags;
260
261	spin_lock_irqsave(&b->irq_lock, flags);
262
263	if (b->irq_enabled)
264		irq_enable(engine);
265	else
266		irq_disable(engine);
267
268	spin_unlock_irqrestore(&b->irq_lock, flags);
269}
270
271void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
272{
273}
274
275bool i915_request_enable_breadcrumb(struct i915_request *rq)
276{
277	lockdep_assert_held(&rq->lock);
278	lockdep_assert_irqs_disabled();
279
280	if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
281		struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
282		struct intel_context *ce = rq->hw_context;
283		struct list_head *pos;
284
285		spin_lock(&b->irq_lock);
286		GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
287
288		__intel_breadcrumbs_arm_irq(b);
289
290		/*
291		 * We keep the seqno in retirement order, so we can break
292		 * inside intel_engine_breadcrumbs_irq as soon as we've passed
293		 * the last completed request (or seen a request that hasn't
294		 * event started). We could iterate the timeline->requests list,
295		 * but keeping a separate signalers_list has the advantage of
296		 * hopefully being much smaller than the full list and so
297		 * provides faster iteration and detection when there are no
298		 * more interrupts required for this context.
299		 *
300		 * We typically expect to add new signalers in order, so we
301		 * start looking for our insertion point from the tail of
302		 * the list.
303		 */
304		list_for_each_prev(pos, &ce->signals) {
305			struct i915_request *it =
306				list_entry(pos, typeof(*it), signal_link);
307
308			if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
309				break;
310		}
311		list_add(&rq->signal_link, pos);
312		if (pos == &ce->signals) /* catch transitions from empty list */
313			list_move_tail(&ce->signal_link, &b->signalers);
314		GEM_BUG_ON(!check_signal_order(ce, rq));
315
316		set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
317		spin_unlock(&b->irq_lock);
318	}
319
320	return !__request_completed(rq);
321}
322
323void i915_request_cancel_breadcrumb(struct i915_request *rq)
324{
325	struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
326
327	lockdep_assert_held(&rq->lock);
328	lockdep_assert_irqs_disabled();
329
330	/*
331	 * We must wait for b->irq_lock so that we know the interrupt handler
332	 * has released its reference to the intel_context and has completed
333	 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
334	 * required).
335	 */
336	spin_lock(&b->irq_lock);
337	if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
338		struct intel_context *ce = rq->hw_context;
339
340		list_del(&rq->signal_link);
341		if (list_empty(&ce->signals))
342			list_del_init(&ce->signal_link);
343
344		clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
345	}
346	spin_unlock(&b->irq_lock);
347}
348
349void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
350				    struct drm_printer *p)
351{
352	struct intel_breadcrumbs *b = &engine->breadcrumbs;
353	struct intel_context *ce;
354	struct i915_request *rq;
355
356	if (list_empty(&b->signalers))
357		return;
358
359	drm_printf(p, "Signals:\n");
360
361	spin_lock_irq(&b->irq_lock);
362	list_for_each_entry(ce, &b->signalers, signal_link) {
363		list_for_each_entry(rq, &ce->signals, signal_link) {
364			drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
365				   rq->fence.context, rq->fence.seqno,
366				   i915_request_completed(rq) ? "!" :
367				   i915_request_started(rq) ? "*" :
368				   "",
369				   jiffies_to_msecs(jiffies - rq->emitted_jiffies));
370		}
371	}
372	spin_unlock_irq(&b->irq_lock);
373}