Loading...
Note: File does not exist in v3.1.
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/dma-fence-array.h>
26#include <linux/dma-fence-chain.h>
27#include <linux/irq_work.h>
28#include <linux/prefetch.h>
29#include <linux/sched.h>
30#include <linux/sched/clock.h>
31#include <linux/sched/signal.h>
32
33#include "gem/i915_gem_context.h"
34#include "gt/intel_context.h"
35#include "gt/intel_ring.h"
36#include "gt/intel_rps.h"
37
38#include "i915_active.h"
39#include "i915_drv.h"
40#include "i915_globals.h"
41#include "i915_trace.h"
42#include "intel_pm.h"
43
44struct execute_cb {
45 struct irq_work work;
46 struct i915_sw_fence *fence;
47 void (*hook)(struct i915_request *rq, struct dma_fence *signal);
48 struct i915_request *signal;
49};
50
51static struct i915_global_request {
52 struct i915_global base;
53 struct kmem_cache *slab_requests;
54 struct kmem_cache *slab_execute_cbs;
55} global;
56
57static const char *i915_fence_get_driver_name(struct dma_fence *fence)
58{
59 return dev_name(to_request(fence)->engine->i915->drm.dev);
60}
61
62static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
63{
64 const struct i915_gem_context *ctx;
65
66 /*
67 * The timeline struct (as part of the ppgtt underneath a context)
68 * may be freed when the request is no longer in use by the GPU.
69 * We could extend the life of a context to beyond that of all
70 * fences, possibly keeping the hw resource around indefinitely,
71 * or we just give them a false name. Since
72 * dma_fence_ops.get_timeline_name is a debug feature, the occasional
73 * lie seems justifiable.
74 */
75 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
76 return "signaled";
77
78 ctx = i915_request_gem_context(to_request(fence));
79 if (!ctx)
80 return "[" DRIVER_NAME "]";
81
82 return ctx->name;
83}
84
85static bool i915_fence_signaled(struct dma_fence *fence)
86{
87 return i915_request_completed(to_request(fence));
88}
89
90static bool i915_fence_enable_signaling(struct dma_fence *fence)
91{
92 return i915_request_enable_breadcrumb(to_request(fence));
93}
94
95static signed long i915_fence_wait(struct dma_fence *fence,
96 bool interruptible,
97 signed long timeout)
98{
99 return i915_request_wait(to_request(fence),
100 interruptible | I915_WAIT_PRIORITY,
101 timeout);
102}
103
104struct kmem_cache *i915_request_slab_cache(void)
105{
106 return global.slab_requests;
107}
108
109static void i915_fence_release(struct dma_fence *fence)
110{
111 struct i915_request *rq = to_request(fence);
112
113 /*
114 * The request is put onto a RCU freelist (i.e. the address
115 * is immediately reused), mark the fences as being freed now.
116 * Otherwise the debugobjects for the fences are only marked as
117 * freed when the slab cache itself is freed, and so we would get
118 * caught trying to reuse dead objects.
119 */
120 i915_sw_fence_fini(&rq->submit);
121 i915_sw_fence_fini(&rq->semaphore);
122
123 /*
124 * Keep one request on each engine for reserved use under mempressure
125 *
126 * We do not hold a reference to the engine here and so have to be
127 * very careful in what rq->engine we poke. The virtual engine is
128 * referenced via the rq->context and we released that ref during
129 * i915_request_retire(), ergo we must not dereference a virtual
130 * engine here. Not that we would want to, as the only consumer of
131 * the reserved engine->request_pool is the power management parking,
132 * which must-not-fail, and that is only run on the physical engines.
133 *
134 * Since the request must have been executed to be have completed,
135 * we know that it will have been processed by the HW and will
136 * not be unsubmitted again, so rq->engine and rq->execution_mask
137 * at this point is stable. rq->execution_mask will be a single
138 * bit if the last and _only_ engine it could execution on was a
139 * physical engine, if it's multiple bits then it started on and
140 * could still be on a virtual engine. Thus if the mask is not a
141 * power-of-two we assume that rq->engine may still be a virtual
142 * engine and so a dangling invalid pointer that we cannot dereference
143 *
144 * For example, consider the flow of a bonded request through a virtual
145 * engine. The request is created with a wide engine mask (all engines
146 * that we might execute on). On processing the bond, the request mask
147 * is reduced to one or more engines. If the request is subsequently
148 * bound to a single engine, it will then be constrained to only
149 * execute on that engine and never returned to the virtual engine
150 * after timeslicing away, see __unwind_incomplete_requests(). Thus we
151 * know that if the rq->execution_mask is a single bit, rq->engine
152 * can be a physical engine with the exact corresponding mask.
153 */
154 if (is_power_of_2(rq->execution_mask) &&
155 !cmpxchg(&rq->engine->request_pool, NULL, rq))
156 return;
157
158 kmem_cache_free(global.slab_requests, rq);
159}
160
161const struct dma_fence_ops i915_fence_ops = {
162 .get_driver_name = i915_fence_get_driver_name,
163 .get_timeline_name = i915_fence_get_timeline_name,
164 .enable_signaling = i915_fence_enable_signaling,
165 .signaled = i915_fence_signaled,
166 .wait = i915_fence_wait,
167 .release = i915_fence_release,
168};
169
170static void irq_execute_cb(struct irq_work *wrk)
171{
172 struct execute_cb *cb = container_of(wrk, typeof(*cb), work);
173
174 i915_sw_fence_complete(cb->fence);
175 kmem_cache_free(global.slab_execute_cbs, cb);
176}
177
178static void irq_execute_cb_hook(struct irq_work *wrk)
179{
180 struct execute_cb *cb = container_of(wrk, typeof(*cb), work);
181
182 cb->hook(container_of(cb->fence, struct i915_request, submit),
183 &cb->signal->fence);
184 i915_request_put(cb->signal);
185
186 irq_execute_cb(wrk);
187}
188
189static void __notify_execute_cb(struct i915_request *rq)
190{
191 struct execute_cb *cb, *cn;
192
193 lockdep_assert_held(&rq->lock);
194
195 GEM_BUG_ON(!i915_request_is_active(rq));
196 if (llist_empty(&rq->execute_cb))
197 return;
198
199 llist_for_each_entry_safe(cb, cn, rq->execute_cb.first, work.llnode)
200 irq_work_queue(&cb->work);
201
202 /*
203 * XXX Rollback on __i915_request_unsubmit()
204 *
205 * In the future, perhaps when we have an active time-slicing scheduler,
206 * it will be interesting to unsubmit parallel execution and remove
207 * busywaits from the GPU until their master is restarted. This is
208 * quite hairy, we have to carefully rollback the fence and do a
209 * preempt-to-idle cycle on the target engine, all the while the
210 * master execute_cb may refire.
211 */
212 init_llist_head(&rq->execute_cb);
213}
214
215static inline void
216remove_from_client(struct i915_request *request)
217{
218 struct drm_i915_file_private *file_priv;
219
220 if (!READ_ONCE(request->file_priv))
221 return;
222
223 rcu_read_lock();
224 file_priv = xchg(&request->file_priv, NULL);
225 if (file_priv) {
226 spin_lock(&file_priv->mm.lock);
227 list_del(&request->client_link);
228 spin_unlock(&file_priv->mm.lock);
229 }
230 rcu_read_unlock();
231}
232
233static void free_capture_list(struct i915_request *request)
234{
235 struct i915_capture_list *capture;
236
237 capture = fetch_and_zero(&request->capture_list);
238 while (capture) {
239 struct i915_capture_list *next = capture->next;
240
241 kfree(capture);
242 capture = next;
243 }
244}
245
246static void __i915_request_fill(struct i915_request *rq, u8 val)
247{
248 void *vaddr = rq->ring->vaddr;
249 u32 head;
250
251 head = rq->infix;
252 if (rq->postfix < head) {
253 memset(vaddr + head, val, rq->ring->size - head);
254 head = 0;
255 }
256 memset(vaddr + head, val, rq->postfix - head);
257}
258
259static void remove_from_engine(struct i915_request *rq)
260{
261 struct intel_engine_cs *engine, *locked;
262
263 /*
264 * Virtual engines complicate acquiring the engine timeline lock,
265 * as their rq->engine pointer is not stable until under that
266 * engine lock. The simple ploy we use is to take the lock then
267 * check that the rq still belongs to the newly locked engine.
268 */
269 locked = READ_ONCE(rq->engine);
270 spin_lock_irq(&locked->active.lock);
271 while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
272 spin_unlock(&locked->active.lock);
273 spin_lock(&engine->active.lock);
274 locked = engine;
275 }
276 list_del_init(&rq->sched.link);
277 clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
278 clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);
279 spin_unlock_irq(&locked->active.lock);
280}
281
282bool i915_request_retire(struct i915_request *rq)
283{
284 if (!i915_request_completed(rq))
285 return false;
286
287 RQ_TRACE(rq, "\n");
288
289 GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
290 trace_i915_request_retire(rq);
291
292 /*
293 * We know the GPU must have read the request to have
294 * sent us the seqno + interrupt, so use the position
295 * of tail of the request to update the last known position
296 * of the GPU head.
297 *
298 * Note this requires that we are always called in request
299 * completion order.
300 */
301 GEM_BUG_ON(!list_is_first(&rq->link,
302 &i915_request_timeline(rq)->requests));
303 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
304 /* Poison before we release our space in the ring */
305 __i915_request_fill(rq, POISON_FREE);
306 rq->ring->head = rq->postfix;
307
308 /*
309 * We only loosely track inflight requests across preemption,
310 * and so we may find ourselves attempting to retire a _completed_
311 * request that we have removed from the HW and put back on a run
312 * queue.
313 */
314 remove_from_engine(rq);
315
316 spin_lock_irq(&rq->lock);
317 i915_request_mark_complete(rq);
318 if (!i915_request_signaled(rq))
319 dma_fence_signal_locked(&rq->fence);
320 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))
321 i915_request_cancel_breadcrumb(rq);
322 if (i915_request_has_waitboost(rq)) {
323 GEM_BUG_ON(!atomic_read(&rq->engine->gt->rps.num_waiters));
324 atomic_dec(&rq->engine->gt->rps.num_waiters);
325 }
326 if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
327 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
328 __notify_execute_cb(rq);
329 }
330 GEM_BUG_ON(!llist_empty(&rq->execute_cb));
331 spin_unlock_irq(&rq->lock);
332
333 remove_from_client(rq);
334 __list_del_entry(&rq->link); /* poison neither prev/next (RCU walks) */
335
336 intel_context_exit(rq->context);
337 intel_context_unpin(rq->context);
338
339 free_capture_list(rq);
340 i915_sched_node_fini(&rq->sched);
341 i915_request_put(rq);
342
343 return true;
344}
345
346void i915_request_retire_upto(struct i915_request *rq)
347{
348 struct intel_timeline * const tl = i915_request_timeline(rq);
349 struct i915_request *tmp;
350
351 RQ_TRACE(rq, "\n");
352
353 GEM_BUG_ON(!i915_request_completed(rq));
354
355 do {
356 tmp = list_first_entry(&tl->requests, typeof(*tmp), link);
357 } while (i915_request_retire(tmp) && tmp != rq);
358}
359
360static void __llist_add(struct llist_node *node, struct llist_head *head)
361{
362 node->next = head->first;
363 head->first = node;
364}
365
366static struct i915_request * const *
367__engine_active(struct intel_engine_cs *engine)
368{
369 return READ_ONCE(engine->execlists.active);
370}
371
372static bool __request_in_flight(const struct i915_request *signal)
373{
374 struct i915_request * const *port, *rq;
375 bool inflight = false;
376
377 if (!i915_request_is_ready(signal))
378 return false;
379
380 /*
381 * Even if we have unwound the request, it may still be on
382 * the GPU (preempt-to-busy). If that request is inside an
383 * unpreemptible critical section, it will not be removed. Some
384 * GPU functions may even be stuck waiting for the paired request
385 * (__await_execution) to be submitted and cannot be preempted
386 * until the bond is executing.
387 *
388 * As we know that there are always preemption points between
389 * requests, we know that only the currently executing request
390 * may be still active even though we have cleared the flag.
391 * However, we can't rely on our tracking of ELSP[0] to know
392 * which request is currently active and so maybe stuck, as
393 * the tracking maybe an event behind. Instead assume that
394 * if the context is still inflight, then it is still active
395 * even if the active flag has been cleared.
396 *
397 * To further complicate matters, if there a pending promotion, the HW
398 * may either perform a context switch to the second inflight execlists,
399 * or it may switch to the pending set of execlists. In the case of the
400 * latter, it may send the ACK and we process the event copying the
401 * pending[] over top of inflight[], _overwriting_ our *active. Since
402 * this implies the HW is arbitrating and not struck in *active, we do
403 * not worry about complete accuracy, but we do require no read/write
404 * tearing of the pointer [the read of the pointer must be valid, even
405 * as the array is being overwritten, for which we require the writes
406 * to avoid tearing.]
407 *
408 * Note that the read of *execlists->active may race with the promotion
409 * of execlists->pending[] to execlists->inflight[], overwritting
410 * the value at *execlists->active. This is fine. The promotion implies
411 * that we received an ACK from the HW, and so the context is not
412 * stuck -- if we do not see ourselves in *active, the inflight status
413 * is valid. If instead we see ourselves being copied into *active,
414 * we are inflight and may signal the callback.
415 */
416 if (!intel_context_inflight(signal->context))
417 return false;
418
419 rcu_read_lock();
420 for (port = __engine_active(signal->engine);
421 (rq = READ_ONCE(*port)); /* may race with promotion of pending[] */
422 port++) {
423 if (rq->context == signal->context) {
424 inflight = i915_seqno_passed(rq->fence.seqno,
425 signal->fence.seqno);
426 break;
427 }
428 }
429 rcu_read_unlock();
430
431 return inflight;
432}
433
434static int
435__await_execution(struct i915_request *rq,
436 struct i915_request *signal,
437 void (*hook)(struct i915_request *rq,
438 struct dma_fence *signal),
439 gfp_t gfp)
440{
441 struct execute_cb *cb;
442
443 if (i915_request_is_active(signal)) {
444 if (hook)
445 hook(rq, &signal->fence);
446 return 0;
447 }
448
449 cb = kmem_cache_alloc(global.slab_execute_cbs, gfp);
450 if (!cb)
451 return -ENOMEM;
452
453 cb->fence = &rq->submit;
454 i915_sw_fence_await(cb->fence);
455 init_irq_work(&cb->work, irq_execute_cb);
456
457 if (hook) {
458 cb->hook = hook;
459 cb->signal = i915_request_get(signal);
460 cb->work.func = irq_execute_cb_hook;
461 }
462
463 spin_lock_irq(&signal->lock);
464 if (i915_request_is_active(signal) || __request_in_flight(signal)) {
465 if (hook) {
466 hook(rq, &signal->fence);
467 i915_request_put(signal);
468 }
469 i915_sw_fence_complete(cb->fence);
470 kmem_cache_free(global.slab_execute_cbs, cb);
471 } else {
472 __llist_add(&cb->work.llnode, &signal->execute_cb);
473 }
474 spin_unlock_irq(&signal->lock);
475
476 return 0;
477}
478
479static bool fatal_error(int error)
480{
481 switch (error) {
482 case 0: /* not an error! */
483 case -EAGAIN: /* innocent victim of a GT reset (__i915_request_reset) */
484 case -ETIMEDOUT: /* waiting for Godot (timer_i915_sw_fence_wake) */
485 return false;
486 default:
487 return true;
488 }
489}
490
491void __i915_request_skip(struct i915_request *rq)
492{
493 GEM_BUG_ON(!fatal_error(rq->fence.error));
494
495 if (rq->infix == rq->postfix)
496 return;
497
498 /*
499 * As this request likely depends on state from the lost
500 * context, clear out all the user operations leaving the
501 * breadcrumb at the end (so we get the fence notifications).
502 */
503 __i915_request_fill(rq, 0);
504 rq->infix = rq->postfix;
505}
506
507void i915_request_set_error_once(struct i915_request *rq, int error)
508{
509 int old;
510
511 GEM_BUG_ON(!IS_ERR_VALUE((long)error));
512
513 if (i915_request_signaled(rq))
514 return;
515
516 old = READ_ONCE(rq->fence.error);
517 do {
518 if (fatal_error(old))
519 return;
520 } while (!try_cmpxchg(&rq->fence.error, &old, error));
521}
522
523bool __i915_request_submit(struct i915_request *request)
524{
525 struct intel_engine_cs *engine = request->engine;
526 bool result = false;
527
528 RQ_TRACE(request, "\n");
529
530 GEM_BUG_ON(!irqs_disabled());
531 lockdep_assert_held(&engine->active.lock);
532
533 /*
534 * With the advent of preempt-to-busy, we frequently encounter
535 * requests that we have unsubmitted from HW, but left running
536 * until the next ack and so have completed in the meantime. On
537 * resubmission of that completed request, we can skip
538 * updating the payload, and execlists can even skip submitting
539 * the request.
540 *
541 * We must remove the request from the caller's priority queue,
542 * and the caller must only call us when the request is in their
543 * priority queue, under the active.lock. This ensures that the
544 * request has *not* yet been retired and we can safely move
545 * the request into the engine->active.list where it will be
546 * dropped upon retiring. (Otherwise if resubmit a *retired*
547 * request, this would be a horrible use-after-free.)
548 */
549 if (i915_request_completed(request))
550 goto xfer;
551
552 if (unlikely(intel_context_is_banned(request->context)))
553 i915_request_set_error_once(request, -EIO);
554 if (unlikely(fatal_error(request->fence.error)))
555 __i915_request_skip(request);
556
557 /*
558 * Are we using semaphores when the gpu is already saturated?
559 *
560 * Using semaphores incurs a cost in having the GPU poll a
561 * memory location, busywaiting for it to change. The continual
562 * memory reads can have a noticeable impact on the rest of the
563 * system with the extra bus traffic, stalling the cpu as it too
564 * tries to access memory across the bus (perf stat -e bus-cycles).
565 *
566 * If we installed a semaphore on this request and we only submit
567 * the request after the signaler completed, that indicates the
568 * system is overloaded and using semaphores at this time only
569 * increases the amount of work we are doing. If so, we disable
570 * further use of semaphores until we are idle again, whence we
571 * optimistically try again.
572 */
573 if (request->sched.semaphores &&
574 i915_sw_fence_signaled(&request->semaphore))
575 engine->saturated |= request->sched.semaphores;
576
577 engine->emit_fini_breadcrumb(request,
578 request->ring->vaddr + request->postfix);
579
580 trace_i915_request_execute(request);
581 engine->serial++;
582 result = true;
583
584xfer:
585 if (!test_and_set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)) {
586 list_move_tail(&request->sched.link, &engine->active.requests);
587 clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags);
588 }
589
590 /* We may be recursing from the signal callback of another i915 fence */
591 if (!i915_request_signaled(request)) {
592 spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
593
594 __notify_execute_cb(request);
595 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
596 &request->fence.flags) &&
597 !i915_request_enable_breadcrumb(request))
598 intel_engine_signal_breadcrumbs(engine);
599
600 spin_unlock(&request->lock);
601 GEM_BUG_ON(!llist_empty(&request->execute_cb));
602 }
603
604 return result;
605}
606
607void i915_request_submit(struct i915_request *request)
608{
609 struct intel_engine_cs *engine = request->engine;
610 unsigned long flags;
611
612 /* Will be called from irq-context when using foreign fences. */
613 spin_lock_irqsave(&engine->active.lock, flags);
614
615 __i915_request_submit(request);
616
617 spin_unlock_irqrestore(&engine->active.lock, flags);
618}
619
620void __i915_request_unsubmit(struct i915_request *request)
621{
622 struct intel_engine_cs *engine = request->engine;
623
624 RQ_TRACE(request, "\n");
625
626 GEM_BUG_ON(!irqs_disabled());
627 lockdep_assert_held(&engine->active.lock);
628
629 /*
630 * Only unwind in reverse order, required so that the per-context list
631 * is kept in seqno/ring order.
632 */
633
634 /* We may be recursing from the signal callback of another i915 fence */
635 spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
636
637 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
638 i915_request_cancel_breadcrumb(request);
639
640 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));
641 clear_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);
642
643 spin_unlock(&request->lock);
644
645 /* We've already spun, don't charge on resubmitting. */
646 if (request->sched.semaphores && i915_request_started(request))
647 request->sched.semaphores = 0;
648
649 /*
650 * We don't need to wake_up any waiters on request->execute, they
651 * will get woken by any other event or us re-adding this request
652 * to the engine timeline (__i915_request_submit()). The waiters
653 * should be quite adapt at finding that the request now has a new
654 * global_seqno to the one they went to sleep on.
655 */
656}
657
658void i915_request_unsubmit(struct i915_request *request)
659{
660 struct intel_engine_cs *engine = request->engine;
661 unsigned long flags;
662
663 /* Will be called from irq-context when using foreign fences. */
664 spin_lock_irqsave(&engine->active.lock, flags);
665
666 __i915_request_unsubmit(request);
667
668 spin_unlock_irqrestore(&engine->active.lock, flags);
669}
670
671static int __i915_sw_fence_call
672submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
673{
674 struct i915_request *request =
675 container_of(fence, typeof(*request), submit);
676
677 switch (state) {
678 case FENCE_COMPLETE:
679 trace_i915_request_submit(request);
680
681 if (unlikely(fence->error))
682 i915_request_set_error_once(request, fence->error);
683
684 /*
685 * We need to serialize use of the submit_request() callback
686 * with its hotplugging performed during an emergency
687 * i915_gem_set_wedged(). We use the RCU mechanism to mark the
688 * critical section in order to force i915_gem_set_wedged() to
689 * wait until the submit_request() is completed before
690 * proceeding.
691 */
692 rcu_read_lock();
693 request->engine->submit_request(request);
694 rcu_read_unlock();
695 break;
696
697 case FENCE_FREE:
698 i915_request_put(request);
699 break;
700 }
701
702 return NOTIFY_DONE;
703}
704
705static int __i915_sw_fence_call
706semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
707{
708 struct i915_request *rq = container_of(fence, typeof(*rq), semaphore);
709
710 switch (state) {
711 case FENCE_COMPLETE:
712 break;
713
714 case FENCE_FREE:
715 i915_request_put(rq);
716 break;
717 }
718
719 return NOTIFY_DONE;
720}
721
722static void retire_requests(struct intel_timeline *tl)
723{
724 struct i915_request *rq, *rn;
725
726 list_for_each_entry_safe(rq, rn, &tl->requests, link)
727 if (!i915_request_retire(rq))
728 break;
729}
730
731static noinline struct i915_request *
732request_alloc_slow(struct intel_timeline *tl,
733 struct i915_request **rsvd,
734 gfp_t gfp)
735{
736 struct i915_request *rq;
737
738 /* If we cannot wait, dip into our reserves */
739 if (!gfpflags_allow_blocking(gfp)) {
740 rq = xchg(rsvd, NULL);
741 if (!rq) /* Use the normal failure path for one final WARN */
742 goto out;
743
744 return rq;
745 }
746
747 if (list_empty(&tl->requests))
748 goto out;
749
750 /* Move our oldest request to the slab-cache (if not in use!) */
751 rq = list_first_entry(&tl->requests, typeof(*rq), link);
752 i915_request_retire(rq);
753
754 rq = kmem_cache_alloc(global.slab_requests,
755 gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
756 if (rq)
757 return rq;
758
759 /* Ratelimit ourselves to prevent oom from malicious clients */
760 rq = list_last_entry(&tl->requests, typeof(*rq), link);
761 cond_synchronize_rcu(rq->rcustate);
762
763 /* Retire our old requests in the hope that we free some */
764 retire_requests(tl);
765
766out:
767 return kmem_cache_alloc(global.slab_requests, gfp);
768}
769
770static void __i915_request_ctor(void *arg)
771{
772 struct i915_request *rq = arg;
773
774 spin_lock_init(&rq->lock);
775 i915_sched_node_init(&rq->sched);
776 i915_sw_fence_init(&rq->submit, submit_notify);
777 i915_sw_fence_init(&rq->semaphore, semaphore_notify);
778
779 dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock, 0, 0);
780
781 rq->file_priv = NULL;
782 rq->capture_list = NULL;
783
784 init_llist_head(&rq->execute_cb);
785}
786
787struct i915_request *
788__i915_request_create(struct intel_context *ce, gfp_t gfp)
789{
790 struct intel_timeline *tl = ce->timeline;
791 struct i915_request *rq;
792 u32 seqno;
793 int ret;
794
795 might_sleep_if(gfpflags_allow_blocking(gfp));
796
797 /* Check that the caller provided an already pinned context */
798 __intel_context_pin(ce);
799
800 /*
801 * Beware: Dragons be flying overhead.
802 *
803 * We use RCU to look up requests in flight. The lookups may
804 * race with the request being allocated from the slab freelist.
805 * That is the request we are writing to here, may be in the process
806 * of being read by __i915_active_request_get_rcu(). As such,
807 * we have to be very careful when overwriting the contents. During
808 * the RCU lookup, we change chase the request->engine pointer,
809 * read the request->global_seqno and increment the reference count.
810 *
811 * The reference count is incremented atomically. If it is zero,
812 * the lookup knows the request is unallocated and complete. Otherwise,
813 * it is either still in use, or has been reallocated and reset
814 * with dma_fence_init(). This increment is safe for release as we
815 * check that the request we have a reference to and matches the active
816 * request.
817 *
818 * Before we increment the refcount, we chase the request->engine
819 * pointer. We must not call kmem_cache_zalloc() or else we set
820 * that pointer to NULL and cause a crash during the lookup. If
821 * we see the request is completed (based on the value of the
822 * old engine and seqno), the lookup is complete and reports NULL.
823 * If we decide the request is not completed (new engine or seqno),
824 * then we grab a reference and double check that it is still the
825 * active request - which it won't be and restart the lookup.
826 *
827 * Do not use kmem_cache_zalloc() here!
828 */
829 rq = kmem_cache_alloc(global.slab_requests,
830 gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
831 if (unlikely(!rq)) {
832 rq = request_alloc_slow(tl, &ce->engine->request_pool, gfp);
833 if (!rq) {
834 ret = -ENOMEM;
835 goto err_unreserve;
836 }
837 }
838
839 rq->context = ce;
840 rq->engine = ce->engine;
841 rq->ring = ce->ring;
842 rq->execution_mask = ce->engine->mask;
843
844 kref_init(&rq->fence.refcount);
845 rq->fence.flags = 0;
846 rq->fence.error = 0;
847 INIT_LIST_HEAD(&rq->fence.cb_list);
848
849 ret = intel_timeline_get_seqno(tl, rq, &seqno);
850 if (ret)
851 goto err_free;
852
853 rq->fence.context = tl->fence_context;
854 rq->fence.seqno = seqno;
855
856 RCU_INIT_POINTER(rq->timeline, tl);
857 RCU_INIT_POINTER(rq->hwsp_cacheline, tl->hwsp_cacheline);
858 rq->hwsp_seqno = tl->hwsp_seqno;
859 GEM_BUG_ON(i915_request_completed(rq));
860
861 rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */
862
863 /* We bump the ref for the fence chain */
864 i915_sw_fence_reinit(&i915_request_get(rq)->submit);
865 i915_sw_fence_reinit(&i915_request_get(rq)->semaphore);
866
867 i915_sched_node_reinit(&rq->sched);
868
869 /* No zalloc, everything must be cleared after use */
870 rq->batch = NULL;
871 GEM_BUG_ON(rq->file_priv);
872 GEM_BUG_ON(rq->capture_list);
873 GEM_BUG_ON(!llist_empty(&rq->execute_cb));
874
875 /*
876 * Reserve space in the ring buffer for all the commands required to
877 * eventually emit this request. This is to guarantee that the
878 * i915_request_add() call can't fail. Note that the reserve may need
879 * to be redone if the request is not actually submitted straight
880 * away, e.g. because a GPU scheduler has deferred it.
881 *
882 * Note that due to how we add reserved_space to intel_ring_begin()
883 * we need to double our request to ensure that if we need to wrap
884 * around inside i915_request_add() there is sufficient space at
885 * the beginning of the ring as well.
886 */
887 rq->reserved_space =
888 2 * rq->engine->emit_fini_breadcrumb_dw * sizeof(u32);
889
890 /*
891 * Record the position of the start of the request so that
892 * should we detect the updated seqno part-way through the
893 * GPU processing the request, we never over-estimate the
894 * position of the head.
895 */
896 rq->head = rq->ring->emit;
897
898 ret = rq->engine->request_alloc(rq);
899 if (ret)
900 goto err_unwind;
901
902 rq->infix = rq->ring->emit; /* end of header; start of user payload */
903
904 intel_context_mark_active(ce);
905 list_add_tail_rcu(&rq->link, &tl->requests);
906
907 return rq;
908
909err_unwind:
910 ce->ring->emit = rq->head;
911
912 /* Make sure we didn't add ourselves to external state before freeing */
913 GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
914 GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
915
916err_free:
917 kmem_cache_free(global.slab_requests, rq);
918err_unreserve:
919 intel_context_unpin(ce);
920 return ERR_PTR(ret);
921}
922
923struct i915_request *
924i915_request_create(struct intel_context *ce)
925{
926 struct i915_request *rq;
927 struct intel_timeline *tl;
928
929 tl = intel_context_timeline_lock(ce);
930 if (IS_ERR(tl))
931 return ERR_CAST(tl);
932
933 /* Move our oldest request to the slab-cache (if not in use!) */
934 rq = list_first_entry(&tl->requests, typeof(*rq), link);
935 if (!list_is_last(&rq->link, &tl->requests))
936 i915_request_retire(rq);
937
938 intel_context_enter(ce);
939 rq = __i915_request_create(ce, GFP_KERNEL);
940 intel_context_exit(ce); /* active reference transferred to request */
941 if (IS_ERR(rq))
942 goto err_unlock;
943
944 /* Check that we do not interrupt ourselves with a new request */
945 rq->cookie = lockdep_pin_lock(&tl->mutex);
946
947 return rq;
948
949err_unlock:
950 intel_context_timeline_unlock(tl);
951 return rq;
952}
953
954static int
955i915_request_await_start(struct i915_request *rq, struct i915_request *signal)
956{
957 struct dma_fence *fence;
958 int err;
959
960 if (i915_request_timeline(rq) == rcu_access_pointer(signal->timeline))
961 return 0;
962
963 if (i915_request_started(signal))
964 return 0;
965
966 fence = NULL;
967 rcu_read_lock();
968 spin_lock_irq(&signal->lock);
969 do {
970 struct list_head *pos = READ_ONCE(signal->link.prev);
971 struct i915_request *prev;
972
973 /* Confirm signal has not been retired, the link is valid */
974 if (unlikely(i915_request_started(signal)))
975 break;
976
977 /* Is signal the earliest request on its timeline? */
978 if (pos == &rcu_dereference(signal->timeline)->requests)
979 break;
980
981 /*
982 * Peek at the request before us in the timeline. That
983 * request will only be valid before it is retired, so
984 * after acquiring a reference to it, confirm that it is
985 * still part of the signaler's timeline.
986 */
987 prev = list_entry(pos, typeof(*prev), link);
988 if (!i915_request_get_rcu(prev))
989 break;
990
991 /* After the strong barrier, confirm prev is still attached */
992 if (unlikely(READ_ONCE(prev->link.next) != &signal->link)) {
993 i915_request_put(prev);
994 break;
995 }
996
997 fence = &prev->fence;
998 } while (0);
999 spin_unlock_irq(&signal->lock);
1000 rcu_read_unlock();
1001 if (!fence)
1002 return 0;
1003
1004 err = 0;
1005 if (!intel_timeline_sync_is_later(i915_request_timeline(rq), fence))
1006 err = i915_sw_fence_await_dma_fence(&rq->submit,
1007 fence, 0,
1008 I915_FENCE_GFP);
1009 dma_fence_put(fence);
1010
1011 return err;
1012}
1013
1014static intel_engine_mask_t
1015already_busywaiting(struct i915_request *rq)
1016{
1017 /*
1018 * Polling a semaphore causes bus traffic, delaying other users of
1019 * both the GPU and CPU. We want to limit the impact on others,
1020 * while taking advantage of early submission to reduce GPU
1021 * latency. Therefore we restrict ourselves to not using more
1022 * than one semaphore from each source, and not using a semaphore
1023 * if we have detected the engine is saturated (i.e. would not be
1024 * submitted early and cause bus traffic reading an already passed
1025 * semaphore).
1026 *
1027 * See the are-we-too-late? check in __i915_request_submit().
1028 */
1029 return rq->sched.semaphores | READ_ONCE(rq->engine->saturated);
1030}
1031
1032static int
1033__emit_semaphore_wait(struct i915_request *to,
1034 struct i915_request *from,
1035 u32 seqno)
1036{
1037 const int has_token = INTEL_GEN(to->engine->i915) >= 12;
1038 u32 hwsp_offset;
1039 int len, err;
1040 u32 *cs;
1041
1042 GEM_BUG_ON(INTEL_GEN(to->engine->i915) < 8);
1043 GEM_BUG_ON(i915_request_has_initial_breadcrumb(to));
1044
1045 /* We need to pin the signaler's HWSP until we are finished reading. */
1046 err = intel_timeline_read_hwsp(from, to, &hwsp_offset);
1047 if (err)
1048 return err;
1049
1050 len = 4;
1051 if (has_token)
1052 len += 2;
1053
1054 cs = intel_ring_begin(to, len);
1055 if (IS_ERR(cs))
1056 return PTR_ERR(cs);
1057
1058 /*
1059 * Using greater-than-or-equal here means we have to worry
1060 * about seqno wraparound. To side step that issue, we swap
1061 * the timeline HWSP upon wrapping, so that everyone listening
1062 * for the old (pre-wrap) values do not see the much smaller
1063 * (post-wrap) values than they were expecting (and so wait
1064 * forever).
1065 */
1066 *cs++ = (MI_SEMAPHORE_WAIT |
1067 MI_SEMAPHORE_GLOBAL_GTT |
1068 MI_SEMAPHORE_POLL |
1069 MI_SEMAPHORE_SAD_GTE_SDD) +
1070 has_token;
1071 *cs++ = seqno;
1072 *cs++ = hwsp_offset;
1073 *cs++ = 0;
1074 if (has_token) {
1075 *cs++ = 0;
1076 *cs++ = MI_NOOP;
1077 }
1078
1079 intel_ring_advance(to, cs);
1080 return 0;
1081}
1082
1083static int
1084emit_semaphore_wait(struct i915_request *to,
1085 struct i915_request *from,
1086 gfp_t gfp)
1087{
1088 const intel_engine_mask_t mask = READ_ONCE(from->engine)->mask;
1089 struct i915_sw_fence *wait = &to->submit;
1090
1091 if (!intel_context_use_semaphores(to->context))
1092 goto await_fence;
1093
1094 if (i915_request_has_initial_breadcrumb(to))
1095 goto await_fence;
1096
1097 if (!rcu_access_pointer(from->hwsp_cacheline))
1098 goto await_fence;
1099
1100 /*
1101 * If this or its dependents are waiting on an external fence
1102 * that may fail catastrophically, then we want to avoid using
1103 * sempahores as they bypass the fence signaling metadata, and we
1104 * lose the fence->error propagation.
1105 */
1106 if (from->sched.flags & I915_SCHED_HAS_EXTERNAL_CHAIN)
1107 goto await_fence;
1108
1109 /* Just emit the first semaphore we see as request space is limited. */
1110 if (already_busywaiting(to) & mask)
1111 goto await_fence;
1112
1113 if (i915_request_await_start(to, from) < 0)
1114 goto await_fence;
1115
1116 /* Only submit our spinner after the signaler is running! */
1117 if (__await_execution(to, from, NULL, gfp))
1118 goto await_fence;
1119
1120 if (__emit_semaphore_wait(to, from, from->fence.seqno))
1121 goto await_fence;
1122
1123 to->sched.semaphores |= mask;
1124 wait = &to->semaphore;
1125
1126await_fence:
1127 return i915_sw_fence_await_dma_fence(wait,
1128 &from->fence, 0,
1129 I915_FENCE_GFP);
1130}
1131
1132static bool intel_timeline_sync_has_start(struct intel_timeline *tl,
1133 struct dma_fence *fence)
1134{
1135 return __intel_timeline_sync_is_later(tl,
1136 fence->context,
1137 fence->seqno - 1);
1138}
1139
1140static int intel_timeline_sync_set_start(struct intel_timeline *tl,
1141 const struct dma_fence *fence)
1142{
1143 return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1);
1144}
1145
1146static int
1147__i915_request_await_execution(struct i915_request *to,
1148 struct i915_request *from,
1149 void (*hook)(struct i915_request *rq,
1150 struct dma_fence *signal))
1151{
1152 int err;
1153
1154 GEM_BUG_ON(intel_context_is_barrier(from->context));
1155
1156 /* Submit both requests at the same time */
1157 err = __await_execution(to, from, hook, I915_FENCE_GFP);
1158 if (err)
1159 return err;
1160
1161 /* Squash repeated depenendices to the same timelines */
1162 if (intel_timeline_sync_has_start(i915_request_timeline(to),
1163 &from->fence))
1164 return 0;
1165
1166 /*
1167 * Wait until the start of this request.
1168 *
1169 * The execution cb fires when we submit the request to HW. But in
1170 * many cases this may be long before the request itself is ready to
1171 * run (consider that we submit 2 requests for the same context, where
1172 * the request of interest is behind an indefinite spinner). So we hook
1173 * up to both to reduce our queues and keep the execution lag minimised
1174 * in the worst case, though we hope that the await_start is elided.
1175 */
1176 err = i915_request_await_start(to, from);
1177 if (err < 0)
1178 return err;
1179
1180 /*
1181 * Ensure both start together [after all semaphores in signal]
1182 *
1183 * Now that we are queued to the HW at roughly the same time (thanks
1184 * to the execute cb) and are ready to run at roughly the same time
1185 * (thanks to the await start), our signaler may still be indefinitely
1186 * delayed by waiting on a semaphore from a remote engine. If our
1187 * signaler depends on a semaphore, so indirectly do we, and we do not
1188 * want to start our payload until our signaler also starts theirs.
1189 * So we wait.
1190 *
1191 * However, there is also a second condition for which we need to wait
1192 * for the precise start of the signaler. Consider that the signaler
1193 * was submitted in a chain of requests following another context
1194 * (with just an ordinary intra-engine fence dependency between the
1195 * two). In this case the signaler is queued to HW, but not for
1196 * immediate execution, and so we must wait until it reaches the
1197 * active slot.
1198 */
1199 if (intel_engine_has_semaphores(to->engine) &&
1200 !i915_request_has_initial_breadcrumb(to)) {
1201 err = __emit_semaphore_wait(to, from, from->fence.seqno - 1);
1202 if (err < 0)
1203 return err;
1204 }
1205
1206 /* Couple the dependency tree for PI on this exposed to->fence */
1207 if (to->engine->schedule) {
1208 err = i915_sched_node_add_dependency(&to->sched,
1209 &from->sched,
1210 I915_DEPENDENCY_WEAK);
1211 if (err < 0)
1212 return err;
1213 }
1214
1215 return intel_timeline_sync_set_start(i915_request_timeline(to),
1216 &from->fence);
1217}
1218
1219static void mark_external(struct i915_request *rq)
1220{
1221 /*
1222 * The downside of using semaphores is that we lose metadata passing
1223 * along the signaling chain. This is particularly nasty when we
1224 * need to pass along a fatal error such as EFAULT or EDEADLK. For
1225 * fatal errors we want to scrub the request before it is executed,
1226 * which means that we cannot preload the request onto HW and have
1227 * it wait upon a semaphore.
1228 */
1229 rq->sched.flags |= I915_SCHED_HAS_EXTERNAL_CHAIN;
1230}
1231
1232static int
1233__i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
1234{
1235 mark_external(rq);
1236 return i915_sw_fence_await_dma_fence(&rq->submit, fence,
1237 i915_fence_context_timeout(rq->engine->i915,
1238 fence->context),
1239 I915_FENCE_GFP);
1240}
1241
1242static int
1243i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
1244{
1245 struct dma_fence *iter;
1246 int err = 0;
1247
1248 if (!to_dma_fence_chain(fence))
1249 return __i915_request_await_external(rq, fence);
1250
1251 dma_fence_chain_for_each(iter, fence) {
1252 struct dma_fence_chain *chain = to_dma_fence_chain(iter);
1253
1254 if (!dma_fence_is_i915(chain->fence)) {
1255 err = __i915_request_await_external(rq, iter);
1256 break;
1257 }
1258
1259 err = i915_request_await_dma_fence(rq, chain->fence);
1260 if (err < 0)
1261 break;
1262 }
1263
1264 dma_fence_put(iter);
1265 return err;
1266}
1267
1268int
1269i915_request_await_execution(struct i915_request *rq,
1270 struct dma_fence *fence,
1271 void (*hook)(struct i915_request *rq,
1272 struct dma_fence *signal))
1273{
1274 struct dma_fence **child = &fence;
1275 unsigned int nchild = 1;
1276 int ret;
1277
1278 if (dma_fence_is_array(fence)) {
1279 struct dma_fence_array *array = to_dma_fence_array(fence);
1280
1281 /* XXX Error for signal-on-any fence arrays */
1282
1283 child = array->fences;
1284 nchild = array->num_fences;
1285 GEM_BUG_ON(!nchild);
1286 }
1287
1288 do {
1289 fence = *child++;
1290 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
1291 i915_sw_fence_set_error_once(&rq->submit, fence->error);
1292 continue;
1293 }
1294
1295 if (fence->context == rq->fence.context)
1296 continue;
1297
1298 /*
1299 * We don't squash repeated fence dependencies here as we
1300 * want to run our callback in all cases.
1301 */
1302
1303 if (dma_fence_is_i915(fence))
1304 ret = __i915_request_await_execution(rq,
1305 to_request(fence),
1306 hook);
1307 else
1308 ret = i915_request_await_external(rq, fence);
1309 if (ret < 0)
1310 return ret;
1311 } while (--nchild);
1312
1313 return 0;
1314}
1315
1316static int
1317await_request_submit(struct i915_request *to, struct i915_request *from)
1318{
1319 /*
1320 * If we are waiting on a virtual engine, then it may be
1321 * constrained to execute on a single engine *prior* to submission.
1322 * When it is submitted, it will be first submitted to the virtual
1323 * engine and then passed to the physical engine. We cannot allow
1324 * the waiter to be submitted immediately to the physical engine
1325 * as it may then bypass the virtual request.
1326 */
1327 if (to->engine == READ_ONCE(from->engine))
1328 return i915_sw_fence_await_sw_fence_gfp(&to->submit,
1329 &from->submit,
1330 I915_FENCE_GFP);
1331 else
1332 return __i915_request_await_execution(to, from, NULL);
1333}
1334
1335static int
1336i915_request_await_request(struct i915_request *to, struct i915_request *from)
1337{
1338 int ret;
1339
1340 GEM_BUG_ON(to == from);
1341 GEM_BUG_ON(to->timeline == from->timeline);
1342
1343 if (i915_request_completed(from)) {
1344 i915_sw_fence_set_error_once(&to->submit, from->fence.error);
1345 return 0;
1346 }
1347
1348 if (to->engine->schedule) {
1349 ret = i915_sched_node_add_dependency(&to->sched,
1350 &from->sched,
1351 I915_DEPENDENCY_EXTERNAL);
1352 if (ret < 0)
1353 return ret;
1354 }
1355
1356 if (is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask)))
1357 ret = await_request_submit(to, from);
1358 else
1359 ret = emit_semaphore_wait(to, from, I915_FENCE_GFP);
1360 if (ret < 0)
1361 return ret;
1362
1363 return 0;
1364}
1365
1366int
1367i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
1368{
1369 struct dma_fence **child = &fence;
1370 unsigned int nchild = 1;
1371 int ret;
1372
1373 /*
1374 * Note that if the fence-array was created in signal-on-any mode,
1375 * we should *not* decompose it into its individual fences. However,
1376 * we don't currently store which mode the fence-array is operating
1377 * in. Fortunately, the only user of signal-on-any is private to
1378 * amdgpu and we should not see any incoming fence-array from
1379 * sync-file being in signal-on-any mode.
1380 */
1381 if (dma_fence_is_array(fence)) {
1382 struct dma_fence_array *array = to_dma_fence_array(fence);
1383
1384 child = array->fences;
1385 nchild = array->num_fences;
1386 GEM_BUG_ON(!nchild);
1387 }
1388
1389 do {
1390 fence = *child++;
1391 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
1392 i915_sw_fence_set_error_once(&rq->submit, fence->error);
1393 continue;
1394 }
1395
1396 /*
1397 * Requests on the same timeline are explicitly ordered, along
1398 * with their dependencies, by i915_request_add() which ensures
1399 * that requests are submitted in-order through each ring.
1400 */
1401 if (fence->context == rq->fence.context)
1402 continue;
1403
1404 /* Squash repeated waits to the same timelines */
1405 if (fence->context &&
1406 intel_timeline_sync_is_later(i915_request_timeline(rq),
1407 fence))
1408 continue;
1409
1410 if (dma_fence_is_i915(fence))
1411 ret = i915_request_await_request(rq, to_request(fence));
1412 else
1413 ret = i915_request_await_external(rq, fence);
1414 if (ret < 0)
1415 return ret;
1416
1417 /* Record the latest fence used against each timeline */
1418 if (fence->context)
1419 intel_timeline_sync_set(i915_request_timeline(rq),
1420 fence);
1421 } while (--nchild);
1422
1423 return 0;
1424}
1425
1426/**
1427 * i915_request_await_object - set this request to (async) wait upon a bo
1428 * @to: request we are wishing to use
1429 * @obj: object which may be in use on another ring.
1430 * @write: whether the wait is on behalf of a writer
1431 *
1432 * This code is meant to abstract object synchronization with the GPU.
1433 * Conceptually we serialise writes between engines inside the GPU.
1434 * We only allow one engine to write into a buffer at any time, but
1435 * multiple readers. To ensure each has a coherent view of memory, we must:
1436 *
1437 * - If there is an outstanding write request to the object, the new
1438 * request must wait for it to complete (either CPU or in hw, requests
1439 * on the same ring will be naturally ordered).
1440 *
1441 * - If we are a write request (pending_write_domain is set), the new
1442 * request must wait for outstanding read requests to complete.
1443 *
1444 * Returns 0 if successful, else propagates up the lower layer error.
1445 */
1446int
1447i915_request_await_object(struct i915_request *to,
1448 struct drm_i915_gem_object *obj,
1449 bool write)
1450{
1451 struct dma_fence *excl;
1452 int ret = 0;
1453
1454 if (write) {
1455 struct dma_fence **shared;
1456 unsigned int count, i;
1457
1458 ret = dma_resv_get_fences_rcu(obj->base.resv,
1459 &excl, &count, &shared);
1460 if (ret)
1461 return ret;
1462
1463 for (i = 0; i < count; i++) {
1464 ret = i915_request_await_dma_fence(to, shared[i]);
1465 if (ret)
1466 break;
1467
1468 dma_fence_put(shared[i]);
1469 }
1470
1471 for (; i < count; i++)
1472 dma_fence_put(shared[i]);
1473 kfree(shared);
1474 } else {
1475 excl = dma_resv_get_excl_rcu(obj->base.resv);
1476 }
1477
1478 if (excl) {
1479 if (ret == 0)
1480 ret = i915_request_await_dma_fence(to, excl);
1481
1482 dma_fence_put(excl);
1483 }
1484
1485 return ret;
1486}
1487
1488static struct i915_request *
1489__i915_request_add_to_timeline(struct i915_request *rq)
1490{
1491 struct intel_timeline *timeline = i915_request_timeline(rq);
1492 struct i915_request *prev;
1493
1494 /*
1495 * Dependency tracking and request ordering along the timeline
1496 * is special cased so that we can eliminate redundant ordering
1497 * operations while building the request (we know that the timeline
1498 * itself is ordered, and here we guarantee it).
1499 *
1500 * As we know we will need to emit tracking along the timeline,
1501 * we embed the hooks into our request struct -- at the cost of
1502 * having to have specialised no-allocation interfaces (which will
1503 * be beneficial elsewhere).
1504 *
1505 * A second benefit to open-coding i915_request_await_request is
1506 * that we can apply a slight variant of the rules specialised
1507 * for timelines that jump between engines (such as virtual engines).
1508 * If we consider the case of virtual engine, we must emit a dma-fence
1509 * to prevent scheduling of the second request until the first is
1510 * complete (to maximise our greedy late load balancing) and this
1511 * precludes optimising to use semaphores serialisation of a single
1512 * timeline across engines.
1513 */
1514 prev = to_request(__i915_active_fence_set(&timeline->last_request,
1515 &rq->fence));
1516 if (prev && !i915_request_completed(prev)) {
1517 /*
1518 * The requests are supposed to be kept in order. However,
1519 * we need to be wary in case the timeline->last_request
1520 * is used as a barrier for external modification to this
1521 * context.
1522 */
1523 GEM_BUG_ON(prev->context == rq->context &&
1524 i915_seqno_passed(prev->fence.seqno,
1525 rq->fence.seqno));
1526
1527 if (is_power_of_2(READ_ONCE(prev->engine)->mask | rq->engine->mask))
1528 i915_sw_fence_await_sw_fence(&rq->submit,
1529 &prev->submit,
1530 &rq->submitq);
1531 else
1532 __i915_sw_fence_await_dma_fence(&rq->submit,
1533 &prev->fence,
1534 &rq->dmaq);
1535 if (rq->engine->schedule)
1536 __i915_sched_node_add_dependency(&rq->sched,
1537 &prev->sched,
1538 &rq->dep,
1539 0);
1540 }
1541
1542 /*
1543 * Make sure that no request gazumped us - if it was allocated after
1544 * our i915_request_alloc() and called __i915_request_add() before
1545 * us, the timeline will hold its seqno which is later than ours.
1546 */
1547 GEM_BUG_ON(timeline->seqno != rq->fence.seqno);
1548
1549 return prev;
1550}
1551
1552/*
1553 * NB: This function is not allowed to fail. Doing so would mean the the
1554 * request is not being tracked for completion but the work itself is
1555 * going to happen on the hardware. This would be a Bad Thing(tm).
1556 */
1557struct i915_request *__i915_request_commit(struct i915_request *rq)
1558{
1559 struct intel_engine_cs *engine = rq->engine;
1560 struct intel_ring *ring = rq->ring;
1561 u32 *cs;
1562
1563 RQ_TRACE(rq, "\n");
1564
1565 /*
1566 * To ensure that this call will not fail, space for its emissions
1567 * should already have been reserved in the ring buffer. Let the ring
1568 * know that it is time to use that space up.
1569 */
1570 GEM_BUG_ON(rq->reserved_space > ring->space);
1571 rq->reserved_space = 0;
1572 rq->emitted_jiffies = jiffies;
1573
1574 /*
1575 * Record the position of the start of the breadcrumb so that
1576 * should we detect the updated seqno part-way through the
1577 * GPU processing the request, we never over-estimate the
1578 * position of the ring's HEAD.
1579 */
1580 cs = intel_ring_begin(rq, engine->emit_fini_breadcrumb_dw);
1581 GEM_BUG_ON(IS_ERR(cs));
1582 rq->postfix = intel_ring_offset(rq, cs);
1583
1584 return __i915_request_add_to_timeline(rq);
1585}
1586
1587void __i915_request_queue(struct i915_request *rq,
1588 const struct i915_sched_attr *attr)
1589{
1590 /*
1591 * Let the backend know a new request has arrived that may need
1592 * to adjust the existing execution schedule due to a high priority
1593 * request - i.e. we may want to preempt the current request in order
1594 * to run a high priority dependency chain *before* we can execute this
1595 * request.
1596 *
1597 * This is called before the request is ready to run so that we can
1598 * decide whether to preempt the entire chain so that it is ready to
1599 * run at the earliest possible convenience.
1600 */
1601 if (attr && rq->engine->schedule)
1602 rq->engine->schedule(rq, attr);
1603 i915_sw_fence_commit(&rq->semaphore);
1604 i915_sw_fence_commit(&rq->submit);
1605}
1606
1607void i915_request_add(struct i915_request *rq)
1608{
1609 struct intel_timeline * const tl = i915_request_timeline(rq);
1610 struct i915_sched_attr attr = {};
1611 struct i915_gem_context *ctx;
1612
1613 lockdep_assert_held(&tl->mutex);
1614 lockdep_unpin_lock(&tl->mutex, rq->cookie);
1615
1616 trace_i915_request_add(rq);
1617 __i915_request_commit(rq);
1618
1619 /* XXX placeholder for selftests */
1620 rcu_read_lock();
1621 ctx = rcu_dereference(rq->context->gem_context);
1622 if (ctx)
1623 attr = ctx->sched;
1624 rcu_read_unlock();
1625
1626 __i915_request_queue(rq, &attr);
1627
1628 mutex_unlock(&tl->mutex);
1629}
1630
1631static unsigned long local_clock_ns(unsigned int *cpu)
1632{
1633 unsigned long t;
1634
1635 /*
1636 * Cheaply and approximately convert from nanoseconds to microseconds.
1637 * The result and subsequent calculations are also defined in the same
1638 * approximate microseconds units. The principal source of timing
1639 * error here is from the simple truncation.
1640 *
1641 * Note that local_clock() is only defined wrt to the current CPU;
1642 * the comparisons are no longer valid if we switch CPUs. Instead of
1643 * blocking preemption for the entire busywait, we can detect the CPU
1644 * switch and use that as indicator of system load and a reason to
1645 * stop busywaiting, see busywait_stop().
1646 */
1647 *cpu = get_cpu();
1648 t = local_clock();
1649 put_cpu();
1650
1651 return t;
1652}
1653
1654static bool busywait_stop(unsigned long timeout, unsigned int cpu)
1655{
1656 unsigned int this_cpu;
1657
1658 if (time_after(local_clock_ns(&this_cpu), timeout))
1659 return true;
1660
1661 return this_cpu != cpu;
1662}
1663
1664static bool __i915_spin_request(const struct i915_request * const rq, int state)
1665{
1666 unsigned long timeout_ns;
1667 unsigned int cpu;
1668
1669 /*
1670 * Only wait for the request if we know it is likely to complete.
1671 *
1672 * We don't track the timestamps around requests, nor the average
1673 * request length, so we do not have a good indicator that this
1674 * request will complete within the timeout. What we do know is the
1675 * order in which requests are executed by the context and so we can
1676 * tell if the request has been started. If the request is not even
1677 * running yet, it is a fair assumption that it will not complete
1678 * within our relatively short timeout.
1679 */
1680 if (!i915_request_is_running(rq))
1681 return false;
1682
1683 /*
1684 * When waiting for high frequency requests, e.g. during synchronous
1685 * rendering split between the CPU and GPU, the finite amount of time
1686 * required to set up the irq and wait upon it limits the response
1687 * rate. By busywaiting on the request completion for a short while we
1688 * can service the high frequency waits as quick as possible. However,
1689 * if it is a slow request, we want to sleep as quickly as possible.
1690 * The tradeoff between waiting and sleeping is roughly the time it
1691 * takes to sleep on a request, on the order of a microsecond.
1692 */
1693
1694 timeout_ns = READ_ONCE(rq->engine->props.max_busywait_duration_ns);
1695 timeout_ns += local_clock_ns(&cpu);
1696 do {
1697 if (i915_request_completed(rq))
1698 return true;
1699
1700 if (signal_pending_state(state, current))
1701 break;
1702
1703 if (busywait_stop(timeout_ns, cpu))
1704 break;
1705
1706 cpu_relax();
1707 } while (!need_resched());
1708
1709 return false;
1710}
1711
1712struct request_wait {
1713 struct dma_fence_cb cb;
1714 struct task_struct *tsk;
1715};
1716
1717static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb)
1718{
1719 struct request_wait *wait = container_of(cb, typeof(*wait), cb);
1720
1721 wake_up_process(wait->tsk);
1722}
1723
1724/**
1725 * i915_request_wait - wait until execution of request has finished
1726 * @rq: the request to wait upon
1727 * @flags: how to wait
1728 * @timeout: how long to wait in jiffies
1729 *
1730 * i915_request_wait() waits for the request to be completed, for a
1731 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
1732 * unbounded wait).
1733 *
1734 * Returns the remaining time (in jiffies) if the request completed, which may
1735 * be zero or -ETIME if the request is unfinished after the timeout expires.
1736 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
1737 * pending before the request completes.
1738 */
1739long i915_request_wait(struct i915_request *rq,
1740 unsigned int flags,
1741 long timeout)
1742{
1743 const int state = flags & I915_WAIT_INTERRUPTIBLE ?
1744 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1745 struct request_wait wait;
1746
1747 might_sleep();
1748 GEM_BUG_ON(timeout < 0);
1749
1750 if (dma_fence_is_signaled(&rq->fence))
1751 return timeout;
1752
1753 if (!timeout)
1754 return -ETIME;
1755
1756 trace_i915_request_wait_begin(rq, flags);
1757
1758 /*
1759 * We must never wait on the GPU while holding a lock as we
1760 * may need to perform a GPU reset. So while we don't need to
1761 * serialise wait/reset with an explicit lock, we do want
1762 * lockdep to detect potential dependency cycles.
1763 */
1764 mutex_acquire(&rq->engine->gt->reset.mutex.dep_map, 0, 0, _THIS_IP_);
1765
1766 /*
1767 * Optimistic spin before touching IRQs.
1768 *
1769 * We may use a rather large value here to offset the penalty of
1770 * switching away from the active task. Frequently, the client will
1771 * wait upon an old swapbuffer to throttle itself to remain within a
1772 * frame of the gpu. If the client is running in lockstep with the gpu,
1773 * then it should not be waiting long at all, and a sleep now will incur
1774 * extra scheduler latency in producing the next frame. To try to
1775 * avoid adding the cost of enabling/disabling the interrupt to the
1776 * short wait, we first spin to see if the request would have completed
1777 * in the time taken to setup the interrupt.
1778 *
1779 * We need upto 5us to enable the irq, and upto 20us to hide the
1780 * scheduler latency of a context switch, ignoring the secondary
1781 * impacts from a context switch such as cache eviction.
1782 *
1783 * The scheme used for low-latency IO is called "hybrid interrupt
1784 * polling". The suggestion there is to sleep until just before you
1785 * expect to be woken by the device interrupt and then poll for its
1786 * completion. That requires having a good predictor for the request
1787 * duration, which we currently lack.
1788 */
1789 if (IS_ACTIVE(CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT) &&
1790 __i915_spin_request(rq, state)) {
1791 dma_fence_signal(&rq->fence);
1792 goto out;
1793 }
1794
1795 /*
1796 * This client is about to stall waiting for the GPU. In many cases
1797 * this is undesirable and limits the throughput of the system, as
1798 * many clients cannot continue processing user input/output whilst
1799 * blocked. RPS autotuning may take tens of milliseconds to respond
1800 * to the GPU load and thus incurs additional latency for the client.
1801 * We can circumvent that by promoting the GPU frequency to maximum
1802 * before we sleep. This makes the GPU throttle up much more quickly
1803 * (good for benchmarks and user experience, e.g. window animations),
1804 * but at a cost of spending more power processing the workload
1805 * (bad for battery).
1806 */
1807 if (flags & I915_WAIT_PRIORITY) {
1808 if (!i915_request_started(rq) &&
1809 INTEL_GEN(rq->engine->i915) >= 6)
1810 intel_rps_boost(rq);
1811 }
1812
1813 wait.tsk = current;
1814 if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake))
1815 goto out;
1816
1817 for (;;) {
1818 set_current_state(state);
1819
1820 if (i915_request_completed(rq)) {
1821 dma_fence_signal(&rq->fence);
1822 break;
1823 }
1824
1825 intel_engine_flush_submission(rq->engine);
1826
1827 if (signal_pending_state(state, current)) {
1828 timeout = -ERESTARTSYS;
1829 break;
1830 }
1831
1832 if (!timeout) {
1833 timeout = -ETIME;
1834 break;
1835 }
1836
1837 timeout = io_schedule_timeout(timeout);
1838 }
1839 __set_current_state(TASK_RUNNING);
1840
1841 dma_fence_remove_callback(&rq->fence, &wait.cb);
1842
1843out:
1844 mutex_release(&rq->engine->gt->reset.mutex.dep_map, _THIS_IP_);
1845 trace_i915_request_wait_end(rq);
1846 return timeout;
1847}
1848
1849#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1850#include "selftests/mock_request.c"
1851#include "selftests/i915_request.c"
1852#endif
1853
1854static void i915_global_request_shrink(void)
1855{
1856 kmem_cache_shrink(global.slab_execute_cbs);
1857 kmem_cache_shrink(global.slab_requests);
1858}
1859
1860static void i915_global_request_exit(void)
1861{
1862 kmem_cache_destroy(global.slab_execute_cbs);
1863 kmem_cache_destroy(global.slab_requests);
1864}
1865
1866static struct i915_global_request global = { {
1867 .shrink = i915_global_request_shrink,
1868 .exit = i915_global_request_exit,
1869} };
1870
1871int __init i915_global_request_init(void)
1872{
1873 global.slab_requests =
1874 kmem_cache_create("i915_request",
1875 sizeof(struct i915_request),
1876 __alignof__(struct i915_request),
1877 SLAB_HWCACHE_ALIGN |
1878 SLAB_RECLAIM_ACCOUNT |
1879 SLAB_TYPESAFE_BY_RCU,
1880 __i915_request_ctor);
1881 if (!global.slab_requests)
1882 return -ENOMEM;
1883
1884 global.slab_execute_cbs = KMEM_CACHE(execute_cb,
1885 SLAB_HWCACHE_ALIGN |
1886 SLAB_RECLAIM_ACCOUNT |
1887 SLAB_TYPESAFE_BY_RCU);
1888 if (!global.slab_execute_cbs)
1889 goto err_requests;
1890
1891 i915_global_register(&global.base);
1892 return 0;
1893
1894err_requests:
1895 kmem_cache_destroy(global.slab_requests);
1896 return -ENOMEM;
1897}