1/*
  2 * Copyright © 2015 Intel Corporation
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the next
 12 * paragraph) shall be included in all copies or substantial portions of the
 13 * Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 21 * IN THE SOFTWARE.
 22 *
 23 */
 24
 25#include <linux/kthread.h>
 26#include <trace/events/dma_fence.h>
 27#include <uapi/linux/sched/types.h>
 28
 29#include "i915_drv.h"
 30#include "i915_trace.h"
 31#include "intel_gt_pm.h"
 32#include "intel_gt_requests.h"
 33
 34static void irq_enable(struct intel_engine_cs *engine)
 35{
 36	if (!engine->irq_enable)
 37		return;
 38
 39	/* Caller disables interrupts */
 40	spin_lock(&engine->gt->irq_lock);
 41	engine->irq_enable(engine);
 42	spin_unlock(&engine->gt->irq_lock);
 43}
 44
 45static void irq_disable(struct intel_engine_cs *engine)
 46{
 47	if (!engine->irq_disable)
 48		return;
 49
 50	/* Caller disables interrupts */
 51	spin_lock(&engine->gt->irq_lock);
 52	engine->irq_disable(engine);
 53	spin_unlock(&engine->gt->irq_lock);
 54}
 55
 56static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
 57{
 58	struct intel_engine_cs *engine =
 59		container_of(b, struct intel_engine_cs, breadcrumbs);
 60
 61	lockdep_assert_held(&b->irq_lock);
 62
 63	GEM_BUG_ON(!b->irq_enabled);
 64	if (!--b->irq_enabled)
 65		irq_disable(engine);
 66
 67	WRITE_ONCE(b->irq_armed, false);
 68	intel_gt_pm_put_async(engine->gt);
 69}
 70
 71void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 72{
 73	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 74	unsigned long flags;
 75
 76	if (!READ_ONCE(b->irq_armed))
 77		return;
 78
 79	spin_lock_irqsave(&b->irq_lock, flags);
 80	if (b->irq_armed)
 81		__intel_breadcrumbs_disarm_irq(b);
 82	spin_unlock_irqrestore(&b->irq_lock, flags);
 83}
 84
 85static inline bool __request_completed(const struct i915_request *rq)
 86{
 87	return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
 88}
 89
 90__maybe_unused static bool
 91check_signal_order(struct intel_context *ce, struct i915_request *rq)
 92{
 93	if (!list_is_last(&rq->signal_link, &ce->signals) &&
 94	    i915_seqno_passed(rq->fence.seqno,
 95			      list_next_entry(rq, signal_link)->fence.seqno))
 96		return false;
 97
 98	if (!list_is_first(&rq->signal_link, &ce->signals) &&
 99	    i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
100			      rq->fence.seqno))
101		return false;
102
103	return true;
104}
105
106static bool
107__dma_fence_signal(struct dma_fence *fence)
108{
109	return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
110}
111
112static void
113__dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
114{
115	fence->timestamp = timestamp;
116	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
117	trace_dma_fence_signaled(fence);
118}
119
120static void
121__dma_fence_signal__notify(struct dma_fence *fence,
122			   const struct list_head *list)
123{
124	struct dma_fence_cb *cur, *tmp;
125
126	lockdep_assert_held(fence->lock);
127
128	list_for_each_entry_safe(cur, tmp, list, node) {
129		INIT_LIST_HEAD(&cur->node);
130		cur->func(fence, cur);
131	}
132}
133
134static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
135{
136	struct intel_engine_cs *engine =
137		container_of(b, struct intel_engine_cs, breadcrumbs);
138
139	if (unlikely(intel_engine_is_virtual(engine)))
140		engine = intel_virtual_engine_get_sibling(engine, 0);
141
142	intel_engine_add_retire(engine, tl);
143}
144
145static void __signal_request(struct i915_request *rq, struct list_head *signals)
146{
147	GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
148	clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
149
150	if (!__dma_fence_signal(&rq->fence))
151		return;
152
153	i915_request_get(rq);
154	list_add_tail(&rq->signal_link, signals);
155}
156
157static void signal_irq_work(struct irq_work *work)
158{
159	struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
160	const ktime_t timestamp = ktime_get();
161	struct intel_context *ce, *cn;
162	struct list_head *pos, *next;
163	LIST_HEAD(signal);
164
165	spin_lock(&b->irq_lock);
166
167	if (b->irq_armed && list_empty(&b->signalers))
168		__intel_breadcrumbs_disarm_irq(b);
169
170	list_splice_init(&b->signaled_requests, &signal);
171
172	list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
173		GEM_BUG_ON(list_empty(&ce->signals));
174
175		list_for_each_safe(pos, next, &ce->signals) {
176			struct i915_request *rq =
177				list_entry(pos, typeof(*rq), signal_link);
178
179			GEM_BUG_ON(!check_signal_order(ce, rq));
180			if (!__request_completed(rq))
181				break;
182
183			/*
184			 * Queue for execution after dropping the signaling
185			 * spinlock as the callback chain may end up adding
186			 * more signalers to the same context or engine.
187			 */
188			__signal_request(rq, &signal);
189		}
190
191		/*
192		 * We process the list deletion in bulk, only using a list_add
193		 * (not list_move) above but keeping the status of
194		 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
195		 */
196		if (!list_is_first(pos, &ce->signals)) {
197			/* Advance the list to the first incomplete request */
198			__list_del_many(&ce->signals, pos);
199			if (&ce->signals == pos) { /* now empty */
200				list_del_init(&ce->signal_link);
201				add_retire(b, ce->timeline);
202			}
203		}
204	}
205
206	spin_unlock(&b->irq_lock);
207
208	list_for_each_safe(pos, next, &signal) {
209		struct i915_request *rq =
210			list_entry(pos, typeof(*rq), signal_link);
211		struct list_head cb_list;
212
213		spin_lock(&rq->lock);
214		list_replace(&rq->fence.cb_list, &cb_list);
215		__dma_fence_signal__timestamp(&rq->fence, timestamp);
216		__dma_fence_signal__notify(&rq->fence, &cb_list);
217		spin_unlock(&rq->lock);
218
219		i915_request_put(rq);
220	}
221}
222
223static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
224{
225	struct intel_engine_cs *engine =
226		container_of(b, struct intel_engine_cs, breadcrumbs);
227
228	lockdep_assert_held(&b->irq_lock);
229	if (b->irq_armed)
230		return true;
231
232	if (!intel_gt_pm_get_if_awake(engine->gt))
233		return false;
234
235	/*
236	 * The breadcrumb irq will be disarmed on the interrupt after the
237	 * waiters are signaled. This gives us a single interrupt window in
238	 * which we can add a new waiter and avoid the cost of re-enabling
239	 * the irq.
240	 */
241	WRITE_ONCE(b->irq_armed, true);
242
243	/*
244	 * Since we are waiting on a request, the GPU should be busy
245	 * and should have its own rpm reference. This is tracked
246	 * by i915->gt.awake, we can forgo holding our own wakref
247	 * for the interrupt as before i915->gt.awake is released (when
248	 * the driver is idle) we disarm the breadcrumbs.
249	 */
250
251	if (!b->irq_enabled++)
252		irq_enable(engine);
253
254	return true;
255}
256
257void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
258{
259	struct intel_breadcrumbs *b = &engine->breadcrumbs;
260
261	spin_lock_init(&b->irq_lock);
262	INIT_LIST_HEAD(&b->signalers);
263	INIT_LIST_HEAD(&b->signaled_requests);
264
265	init_irq_work(&b->irq_work, signal_irq_work);
266}
267
268void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
269{
270	struct intel_breadcrumbs *b = &engine->breadcrumbs;
271	unsigned long flags;
272
273	spin_lock_irqsave(&b->irq_lock, flags);
274
275	if (b->irq_enabled)
276		irq_enable(engine);
277	else
278		irq_disable(engine);
279
280	spin_unlock_irqrestore(&b->irq_lock, flags);
281}
282
283void intel_engine_transfer_stale_breadcrumbs(struct intel_engine_cs *engine,
284					     struct intel_context *ce)
285{
286	struct intel_breadcrumbs *b = &engine->breadcrumbs;
287	unsigned long flags;
288
289	spin_lock_irqsave(&b->irq_lock, flags);
290	if (!list_empty(&ce->signals)) {
291		struct i915_request *rq, *next;
292
293		/* Queue for executing the signal callbacks in the irq_work */
294		list_for_each_entry_safe(rq, next, &ce->signals, signal_link) {
295			GEM_BUG_ON(rq->engine != engine);
296			GEM_BUG_ON(!__request_completed(rq));
297
298			__signal_request(rq, &b->signaled_requests);
299		}
300
301		INIT_LIST_HEAD(&ce->signals);
302		list_del_init(&ce->signal_link);
303
304		irq_work_queue(&b->irq_work);
305	}
306	spin_unlock_irqrestore(&b->irq_lock, flags);
307}
308
309void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
310{
311}
312
313bool i915_request_enable_breadcrumb(struct i915_request *rq)
314{
315	lockdep_assert_held(&rq->lock);
316
317	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
318		return true;
319
320	if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
321		struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
322		struct intel_context *ce = rq->context;
323		struct list_head *pos;
324
325		spin_lock(&b->irq_lock);
326
327		if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))
328			goto unlock;
329
330		if (!__intel_breadcrumbs_arm_irq(b))
331			goto unlock;
332
333		/*
334		 * We keep the seqno in retirement order, so we can break
335		 * inside intel_engine_signal_breadcrumbs as soon as we've
336		 * passed the last completed request (or seen a request that
337		 * hasn't event started). We could walk the timeline->requests,
338		 * but keeping a separate signalers_list has the advantage of
339		 * hopefully being much smaller than the full list and so
340		 * provides faster iteration and detection when there are no
341		 * more interrupts required for this context.
342		 *
343		 * We typically expect to add new signalers in order, so we
344		 * start looking for our insertion point from the tail of
345		 * the list.
346		 */
347		list_for_each_prev(pos, &ce->signals) {
348			struct i915_request *it =
349				list_entry(pos, typeof(*it), signal_link);
350
351			if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
352				break;
353		}
354		list_add(&rq->signal_link, pos);
355		if (pos == &ce->signals) /* catch transitions from empty list */
356			list_move_tail(&ce->signal_link, &b->signalers);
357		GEM_BUG_ON(!check_signal_order(ce, rq));
358
359		set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
360unlock:
361		spin_unlock(&b->irq_lock);
362	}
363
364	return !__request_completed(rq);
365}
366
367void i915_request_cancel_breadcrumb(struct i915_request *rq)
368{
369	struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
370
371	lockdep_assert_held(&rq->lock);
372
373	/*
374	 * We must wait for b->irq_lock so that we know the interrupt handler
375	 * has released its reference to the intel_context and has completed
376	 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
377	 * required).
378	 */
379	spin_lock(&b->irq_lock);
380	if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
381		struct intel_context *ce = rq->context;
382
383		list_del(&rq->signal_link);
384		if (list_empty(&ce->signals))
385			list_del_init(&ce->signal_link);
386
387		clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
388	}
389	spin_unlock(&b->irq_lock);
390}
391
392void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
393				    struct drm_printer *p)
394{
395	struct intel_breadcrumbs *b = &engine->breadcrumbs;
396	struct intel_context *ce;
397	struct i915_request *rq;
398
399	if (list_empty(&b->signalers))
400		return;
401
402	drm_printf(p, "Signals:\n");
403
404	spin_lock_irq(&b->irq_lock);
405	list_for_each_entry(ce, &b->signalers, signal_link) {
406		list_for_each_entry(rq, &ce->signals, signal_link) {
407			drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
408				   rq->fence.context, rq->fence.seqno,
409				   i915_request_completed(rq) ? "!" :
410				   i915_request_started(rq) ? "*" :
411				   "",
412				   jiffies_to_msecs(jiffies - rq->emitted_jiffies));
413		}
414	}
415	spin_unlock_irq(&b->irq_lock);
416}