Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2016 Intel Corporation
4 */
5
6#include <linux/kthread.h>
7
8#include "gem/i915_gem_context.h"
9
10#include "intel_gt.h"
11#include "intel_engine_heartbeat.h"
12#include "intel_engine_pm.h"
13#include "selftest_engine_heartbeat.h"
14
15#include "i915_selftest.h"
16#include "selftests/i915_random.h"
17#include "selftests/igt_flush_test.h"
18#include "selftests/igt_reset.h"
19#include "selftests/igt_atomic.h"
20
21#include "selftests/mock_drm.h"
22
23#include "gem/selftests/mock_context.h"
24#include "gem/selftests/igt_gem_utils.h"
25
26#define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */
27
28struct hang {
29 struct intel_gt *gt;
30 struct drm_i915_gem_object *hws;
31 struct drm_i915_gem_object *obj;
32 struct i915_gem_context *ctx;
33 u32 *seqno;
34 u32 *batch;
35};
36
37static int hang_init(struct hang *h, struct intel_gt *gt)
38{
39 void *vaddr;
40 int err;
41
42 memset(h, 0, sizeof(*h));
43 h->gt = gt;
44
45 h->ctx = kernel_context(gt->i915);
46 if (IS_ERR(h->ctx))
47 return PTR_ERR(h->ctx);
48
49 GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx));
50
51 h->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
52 if (IS_ERR(h->hws)) {
53 err = PTR_ERR(h->hws);
54 goto err_ctx;
55 }
56
57 h->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
58 if (IS_ERR(h->obj)) {
59 err = PTR_ERR(h->obj);
60 goto err_hws;
61 }
62
63 i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);
64 vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB);
65 if (IS_ERR(vaddr)) {
66 err = PTR_ERR(vaddr);
67 goto err_obj;
68 }
69 h->seqno = memset(vaddr, 0xff, PAGE_SIZE);
70
71 vaddr = i915_gem_object_pin_map_unlocked(h->obj,
72 i915_coherent_map_type(gt->i915, h->obj, false));
73 if (IS_ERR(vaddr)) {
74 err = PTR_ERR(vaddr);
75 goto err_unpin_hws;
76 }
77 h->batch = vaddr;
78
79 return 0;
80
81err_unpin_hws:
82 i915_gem_object_unpin_map(h->hws);
83err_obj:
84 i915_gem_object_put(h->obj);
85err_hws:
86 i915_gem_object_put(h->hws);
87err_ctx:
88 kernel_context_close(h->ctx);
89 return err;
90}
91
92static u64 hws_address(const struct i915_vma *hws,
93 const struct i915_request *rq)
94{
95 return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context);
96}
97
98static int move_to_active(struct i915_vma *vma,
99 struct i915_request *rq,
100 unsigned int flags)
101{
102 int err;
103
104 i915_vma_lock(vma);
105 err = i915_request_await_object(rq, vma->obj,
106 flags & EXEC_OBJECT_WRITE);
107 if (err == 0)
108 err = i915_vma_move_to_active(vma, rq, flags);
109 i915_vma_unlock(vma);
110
111 return err;
112}
113
114static struct i915_request *
115hang_create_request(struct hang *h, struct intel_engine_cs *engine)
116{
117 struct intel_gt *gt = h->gt;
118 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(h->ctx);
119 struct drm_i915_gem_object *obj;
120 struct i915_request *rq = NULL;
121 struct i915_vma *hws, *vma;
122 unsigned int flags;
123 void *vaddr;
124 u32 *batch;
125 int err;
126
127 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
128 if (IS_ERR(obj)) {
129 i915_vm_put(vm);
130 return ERR_CAST(obj);
131 }
132
133 vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915, obj, false));
134 if (IS_ERR(vaddr)) {
135 i915_gem_object_put(obj);
136 i915_vm_put(vm);
137 return ERR_CAST(vaddr);
138 }
139
140 i915_gem_object_unpin_map(h->obj);
141 i915_gem_object_put(h->obj);
142
143 h->obj = obj;
144 h->batch = vaddr;
145
146 vma = i915_vma_instance(h->obj, vm, NULL);
147 if (IS_ERR(vma)) {
148 i915_vm_put(vm);
149 return ERR_CAST(vma);
150 }
151
152 hws = i915_vma_instance(h->hws, vm, NULL);
153 if (IS_ERR(hws)) {
154 i915_vm_put(vm);
155 return ERR_CAST(hws);
156 }
157
158 err = i915_vma_pin(vma, 0, 0, PIN_USER);
159 if (err) {
160 i915_vm_put(vm);
161 return ERR_PTR(err);
162 }
163
164 err = i915_vma_pin(hws, 0, 0, PIN_USER);
165 if (err)
166 goto unpin_vma;
167
168 rq = igt_request_alloc(h->ctx, engine);
169 if (IS_ERR(rq)) {
170 err = PTR_ERR(rq);
171 goto unpin_hws;
172 }
173
174 err = move_to_active(vma, rq, 0);
175 if (err)
176 goto cancel_rq;
177
178 err = move_to_active(hws, rq, 0);
179 if (err)
180 goto cancel_rq;
181
182 batch = h->batch;
183 if (GRAPHICS_VER(gt->i915) >= 8) {
184 *batch++ = MI_STORE_DWORD_IMM_GEN4;
185 *batch++ = lower_32_bits(hws_address(hws, rq));
186 *batch++ = upper_32_bits(hws_address(hws, rq));
187 *batch++ = rq->fence.seqno;
188 *batch++ = MI_NOOP;
189
190 memset(batch, 0, 1024);
191 batch += 1024 / sizeof(*batch);
192
193 *batch++ = MI_NOOP;
194 *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
195 *batch++ = lower_32_bits(vma->node.start);
196 *batch++ = upper_32_bits(vma->node.start);
197 } else if (GRAPHICS_VER(gt->i915) >= 6) {
198 *batch++ = MI_STORE_DWORD_IMM_GEN4;
199 *batch++ = 0;
200 *batch++ = lower_32_bits(hws_address(hws, rq));
201 *batch++ = rq->fence.seqno;
202 *batch++ = MI_NOOP;
203
204 memset(batch, 0, 1024);
205 batch += 1024 / sizeof(*batch);
206
207 *batch++ = MI_NOOP;
208 *batch++ = MI_BATCH_BUFFER_START | 1 << 8;
209 *batch++ = lower_32_bits(vma->node.start);
210 } else if (GRAPHICS_VER(gt->i915) >= 4) {
211 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
212 *batch++ = 0;
213 *batch++ = lower_32_bits(hws_address(hws, rq));
214 *batch++ = rq->fence.seqno;
215 *batch++ = MI_NOOP;
216
217 memset(batch, 0, 1024);
218 batch += 1024 / sizeof(*batch);
219
220 *batch++ = MI_NOOP;
221 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
222 *batch++ = lower_32_bits(vma->node.start);
223 } else {
224 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
225 *batch++ = lower_32_bits(hws_address(hws, rq));
226 *batch++ = rq->fence.seqno;
227 *batch++ = MI_NOOP;
228
229 memset(batch, 0, 1024);
230 batch += 1024 / sizeof(*batch);
231
232 *batch++ = MI_NOOP;
233 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
234 *batch++ = lower_32_bits(vma->node.start);
235 }
236 *batch++ = MI_BATCH_BUFFER_END; /* not reached */
237 intel_gt_chipset_flush(engine->gt);
238
239 if (rq->engine->emit_init_breadcrumb) {
240 err = rq->engine->emit_init_breadcrumb(rq);
241 if (err)
242 goto cancel_rq;
243 }
244
245 flags = 0;
246 if (GRAPHICS_VER(gt->i915) <= 5)
247 flags |= I915_DISPATCH_SECURE;
248
249 err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
250
251cancel_rq:
252 if (err) {
253 i915_request_set_error_once(rq, err);
254 i915_request_add(rq);
255 }
256unpin_hws:
257 i915_vma_unpin(hws);
258unpin_vma:
259 i915_vma_unpin(vma);
260 i915_vm_put(vm);
261 return err ? ERR_PTR(err) : rq;
262}
263
264static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)
265{
266 return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);
267}
268
269static void hang_fini(struct hang *h)
270{
271 *h->batch = MI_BATCH_BUFFER_END;
272 intel_gt_chipset_flush(h->gt);
273
274 i915_gem_object_unpin_map(h->obj);
275 i915_gem_object_put(h->obj);
276
277 i915_gem_object_unpin_map(h->hws);
278 i915_gem_object_put(h->hws);
279
280 kernel_context_close(h->ctx);
281
282 igt_flush_test(h->gt->i915);
283}
284
285static bool wait_until_running(struct hang *h, struct i915_request *rq)
286{
287 return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
288 rq->fence.seqno),
289 10) &&
290 wait_for(i915_seqno_passed(hws_seqno(h, rq),
291 rq->fence.seqno),
292 1000));
293}
294
295static int igt_hang_sanitycheck(void *arg)
296{
297 struct intel_gt *gt = arg;
298 struct i915_request *rq;
299 struct intel_engine_cs *engine;
300 enum intel_engine_id id;
301 struct hang h;
302 int err;
303
304 /* Basic check that we can execute our hanging batch */
305
306 err = hang_init(&h, gt);
307 if (err)
308 return err;
309
310 for_each_engine(engine, gt, id) {
311 struct intel_wedge_me w;
312 long timeout;
313
314 if (!intel_engine_can_store_dword(engine))
315 continue;
316
317 rq = hang_create_request(&h, engine);
318 if (IS_ERR(rq)) {
319 err = PTR_ERR(rq);
320 pr_err("Failed to create request for %s, err=%d\n",
321 engine->name, err);
322 goto fini;
323 }
324
325 i915_request_get(rq);
326
327 *h.batch = MI_BATCH_BUFFER_END;
328 intel_gt_chipset_flush(engine->gt);
329
330 i915_request_add(rq);
331
332 timeout = 0;
333 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
334 timeout = i915_request_wait(rq, 0,
335 MAX_SCHEDULE_TIMEOUT);
336 if (intel_gt_is_wedged(gt))
337 timeout = -EIO;
338
339 i915_request_put(rq);
340
341 if (timeout < 0) {
342 err = timeout;
343 pr_err("Wait for request failed on %s, err=%d\n",
344 engine->name, err);
345 goto fini;
346 }
347 }
348
349fini:
350 hang_fini(&h);
351 return err;
352}
353
354static bool wait_for_idle(struct intel_engine_cs *engine)
355{
356 return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0;
357}
358
359static int igt_reset_nop(void *arg)
360{
361 struct intel_gt *gt = arg;
362 struct i915_gpu_error *global = >->i915->gpu_error;
363 struct intel_engine_cs *engine;
364 unsigned int reset_count, count;
365 enum intel_engine_id id;
366 IGT_TIMEOUT(end_time);
367 int err = 0;
368
369 /* Check that we can reset during non-user portions of requests */
370
371 reset_count = i915_reset_count(global);
372 count = 0;
373 do {
374 for_each_engine(engine, gt, id) {
375 struct intel_context *ce;
376 int i;
377
378 ce = intel_context_create(engine);
379 if (IS_ERR(ce)) {
380 err = PTR_ERR(ce);
381 break;
382 }
383
384 for (i = 0; i < 16; i++) {
385 struct i915_request *rq;
386
387 rq = intel_context_create_request(ce);
388 if (IS_ERR(rq)) {
389 err = PTR_ERR(rq);
390 break;
391 }
392
393 i915_request_add(rq);
394 }
395
396 intel_context_put(ce);
397 }
398
399 igt_global_reset_lock(gt);
400 intel_gt_reset(gt, ALL_ENGINES, NULL);
401 igt_global_reset_unlock(gt);
402
403 if (intel_gt_is_wedged(gt)) {
404 err = -EIO;
405 break;
406 }
407
408 if (i915_reset_count(global) != reset_count + ++count) {
409 pr_err("Full GPU reset not recorded!\n");
410 err = -EINVAL;
411 break;
412 }
413
414 err = igt_flush_test(gt->i915);
415 if (err)
416 break;
417 } while (time_before(jiffies, end_time));
418 pr_info("%s: %d resets\n", __func__, count);
419
420 if (igt_flush_test(gt->i915))
421 err = -EIO;
422 return err;
423}
424
425static int igt_reset_nop_engine(void *arg)
426{
427 struct intel_gt *gt = arg;
428 struct i915_gpu_error *global = >->i915->gpu_error;
429 struct intel_engine_cs *engine;
430 enum intel_engine_id id;
431
432 /* Check that we can engine-reset during non-user portions */
433
434 if (!intel_has_reset_engine(gt))
435 return 0;
436
437 for_each_engine(engine, gt, id) {
438 unsigned int reset_count, reset_engine_count, count;
439 struct intel_context *ce;
440 IGT_TIMEOUT(end_time);
441 int err;
442
443 ce = intel_context_create(engine);
444 if (IS_ERR(ce))
445 return PTR_ERR(ce);
446
447 reset_count = i915_reset_count(global);
448 reset_engine_count = i915_reset_engine_count(global, engine);
449 count = 0;
450
451 st_engine_heartbeat_disable(engine);
452 set_bit(I915_RESET_ENGINE + id, >->reset.flags);
453 do {
454 int i;
455
456 if (!wait_for_idle(engine)) {
457 pr_err("%s failed to idle before reset\n",
458 engine->name);
459 err = -EIO;
460 break;
461 }
462
463 for (i = 0; i < 16; i++) {
464 struct i915_request *rq;
465
466 rq = intel_context_create_request(ce);
467 if (IS_ERR(rq)) {
468 struct drm_printer p =
469 drm_info_printer(gt->i915->drm.dev);
470 intel_engine_dump(engine, &p,
471 "%s(%s): failed to submit request\n",
472 __func__,
473 engine->name);
474
475 GEM_TRACE("%s(%s): failed to submit request\n",
476 __func__,
477 engine->name);
478 GEM_TRACE_DUMP();
479
480 intel_gt_set_wedged(gt);
481
482 err = PTR_ERR(rq);
483 break;
484 }
485
486 i915_request_add(rq);
487 }
488 err = intel_engine_reset(engine, NULL);
489 if (err) {
490 pr_err("intel_engine_reset(%s) failed, err:%d\n",
491 engine->name, err);
492 break;
493 }
494
495 if (i915_reset_count(global) != reset_count) {
496 pr_err("Full GPU reset recorded! (engine reset expected)\n");
497 err = -EINVAL;
498 break;
499 }
500
501 if (i915_reset_engine_count(global, engine) !=
502 reset_engine_count + ++count) {
503 pr_err("%s engine reset not recorded!\n",
504 engine->name);
505 err = -EINVAL;
506 break;
507 }
508 } while (time_before(jiffies, end_time));
509 clear_bit(I915_RESET_ENGINE + id, >->reset.flags);
510 st_engine_heartbeat_enable(engine);
511
512 pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
513
514 intel_context_put(ce);
515 if (igt_flush_test(gt->i915))
516 err = -EIO;
517 if (err)
518 return err;
519 }
520
521 return 0;
522}
523
524static void force_reset_timeout(struct intel_engine_cs *engine)
525{
526 engine->reset_timeout.probability = 999;
527 atomic_set(&engine->reset_timeout.times, -1);
528}
529
530static void cancel_reset_timeout(struct intel_engine_cs *engine)
531{
532 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
533}
534
535static int igt_reset_fail_engine(void *arg)
536{
537 struct intel_gt *gt = arg;
538 struct intel_engine_cs *engine;
539 enum intel_engine_id id;
540
541 /* Check that we can recover from engine-reset failues */
542
543 if (!intel_has_reset_engine(gt))
544 return 0;
545
546 for_each_engine(engine, gt, id) {
547 unsigned int count;
548 struct intel_context *ce;
549 IGT_TIMEOUT(end_time);
550 int err;
551
552 ce = intel_context_create(engine);
553 if (IS_ERR(ce))
554 return PTR_ERR(ce);
555
556 st_engine_heartbeat_disable(engine);
557 set_bit(I915_RESET_ENGINE + id, >->reset.flags);
558
559 force_reset_timeout(engine);
560 err = intel_engine_reset(engine, NULL);
561 cancel_reset_timeout(engine);
562 if (err == 0) /* timeouts only generated on gen8+ */
563 goto skip;
564
565 count = 0;
566 do {
567 struct i915_request *last = NULL;
568 int i;
569
570 if (!wait_for_idle(engine)) {
571 pr_err("%s failed to idle before reset\n",
572 engine->name);
573 err = -EIO;
574 break;
575 }
576
577 for (i = 0; i < count % 15; i++) {
578 struct i915_request *rq;
579
580 rq = intel_context_create_request(ce);
581 if (IS_ERR(rq)) {
582 struct drm_printer p =
583 drm_info_printer(gt->i915->drm.dev);
584 intel_engine_dump(engine, &p,
585 "%s(%s): failed to submit request\n",
586 __func__,
587 engine->name);
588
589 GEM_TRACE("%s(%s): failed to submit request\n",
590 __func__,
591 engine->name);
592 GEM_TRACE_DUMP();
593
594 intel_gt_set_wedged(gt);
595 if (last)
596 i915_request_put(last);
597
598 err = PTR_ERR(rq);
599 goto out;
600 }
601
602 if (last)
603 i915_request_put(last);
604 last = i915_request_get(rq);
605 i915_request_add(rq);
606 }
607
608 if (count & 1) {
609 err = intel_engine_reset(engine, NULL);
610 if (err) {
611 GEM_TRACE_ERR("intel_engine_reset(%s) failed, err:%d\n",
612 engine->name, err);
613 GEM_TRACE_DUMP();
614 i915_request_put(last);
615 break;
616 }
617 } else {
618 force_reset_timeout(engine);
619 err = intel_engine_reset(engine, NULL);
620 cancel_reset_timeout(engine);
621 if (err != -ETIMEDOUT) {
622 pr_err("intel_engine_reset(%s) did not fail, err:%d\n",
623 engine->name, err);
624 i915_request_put(last);
625 break;
626 }
627 }
628
629 err = 0;
630 if (last) {
631 if (i915_request_wait(last, 0, HZ / 2) < 0) {
632 struct drm_printer p =
633 drm_info_printer(gt->i915->drm.dev);
634
635 intel_engine_dump(engine, &p,
636 "%s(%s): failed to complete request\n",
637 __func__,
638 engine->name);
639
640 GEM_TRACE("%s(%s): failed to complete request\n",
641 __func__,
642 engine->name);
643 GEM_TRACE_DUMP();
644
645 err = -EIO;
646 }
647 i915_request_put(last);
648 }
649 count++;
650 } while (err == 0 && time_before(jiffies, end_time));
651out:
652 pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
653skip:
654 clear_bit(I915_RESET_ENGINE + id, >->reset.flags);
655 st_engine_heartbeat_enable(engine);
656 intel_context_put(ce);
657
658 if (igt_flush_test(gt->i915))
659 err = -EIO;
660 if (err)
661 return err;
662 }
663
664 return 0;
665}
666
667static int __igt_reset_engine(struct intel_gt *gt, bool active)
668{
669 struct i915_gpu_error *global = >->i915->gpu_error;
670 struct intel_engine_cs *engine;
671 enum intel_engine_id id;
672 struct hang h;
673 int err = 0;
674
675 /* Check that we can issue an engine reset on an idle engine (no-op) */
676
677 if (!intel_has_reset_engine(gt))
678 return 0;
679
680 if (active) {
681 err = hang_init(&h, gt);
682 if (err)
683 return err;
684 }
685
686 for_each_engine(engine, gt, id) {
687 unsigned int reset_count, reset_engine_count;
688 unsigned long count;
689 IGT_TIMEOUT(end_time);
690
691 if (active && !intel_engine_can_store_dword(engine))
692 continue;
693
694 if (!wait_for_idle(engine)) {
695 pr_err("%s failed to idle before reset\n",
696 engine->name);
697 err = -EIO;
698 break;
699 }
700
701 reset_count = i915_reset_count(global);
702 reset_engine_count = i915_reset_engine_count(global, engine);
703
704 st_engine_heartbeat_disable(engine);
705 set_bit(I915_RESET_ENGINE + id, >->reset.flags);
706 count = 0;
707 do {
708 if (active) {
709 struct i915_request *rq;
710
711 rq = hang_create_request(&h, engine);
712 if (IS_ERR(rq)) {
713 err = PTR_ERR(rq);
714 break;
715 }
716
717 i915_request_get(rq);
718 i915_request_add(rq);
719
720 if (!wait_until_running(&h, rq)) {
721 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
722
723 pr_err("%s: Failed to start request %llx, at %x\n",
724 __func__, rq->fence.seqno, hws_seqno(&h, rq));
725 intel_engine_dump(engine, &p,
726 "%s\n", engine->name);
727
728 i915_request_put(rq);
729 err = -EIO;
730 break;
731 }
732
733 i915_request_put(rq);
734 }
735
736 err = intel_engine_reset(engine, NULL);
737 if (err) {
738 pr_err("intel_engine_reset(%s) failed, err:%d\n",
739 engine->name, err);
740 break;
741 }
742
743 if (i915_reset_count(global) != reset_count) {
744 pr_err("Full GPU reset recorded! (engine reset expected)\n");
745 err = -EINVAL;
746 break;
747 }
748
749 if (i915_reset_engine_count(global, engine) !=
750 ++reset_engine_count) {
751 pr_err("%s engine reset not recorded!\n",
752 engine->name);
753 err = -EINVAL;
754 break;
755 }
756
757 count++;
758 } while (time_before(jiffies, end_time));
759 clear_bit(I915_RESET_ENGINE + id, >->reset.flags);
760 st_engine_heartbeat_enable(engine);
761 pr_info("%s: Completed %lu %s resets\n",
762 engine->name, count, active ? "active" : "idle");
763
764 if (err)
765 break;
766
767 err = igt_flush_test(gt->i915);
768 if (err)
769 break;
770 }
771
772 if (intel_gt_is_wedged(gt))
773 err = -EIO;
774
775 if (active)
776 hang_fini(&h);
777
778 return err;
779}
780
781static int igt_reset_idle_engine(void *arg)
782{
783 return __igt_reset_engine(arg, false);
784}
785
786static int igt_reset_active_engine(void *arg)
787{
788 return __igt_reset_engine(arg, true);
789}
790
791struct active_engine {
792 struct task_struct *task;
793 struct intel_engine_cs *engine;
794 unsigned long resets;
795 unsigned int flags;
796};
797
798#define TEST_ACTIVE BIT(0)
799#define TEST_OTHERS BIT(1)
800#define TEST_SELF BIT(2)
801#define TEST_PRIORITY BIT(3)
802
803static int active_request_put(struct i915_request *rq)
804{
805 int err = 0;
806
807 if (!rq)
808 return 0;
809
810 if (i915_request_wait(rq, 0, 5 * HZ) < 0) {
811 GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n",
812 rq->engine->name,
813 rq->fence.context,
814 rq->fence.seqno);
815 GEM_TRACE_DUMP();
816
817 intel_gt_set_wedged(rq->engine->gt);
818 err = -EIO;
819 }
820
821 i915_request_put(rq);
822
823 return err;
824}
825
826static int active_engine(void *data)
827{
828 I915_RND_STATE(prng);
829 struct active_engine *arg = data;
830 struct intel_engine_cs *engine = arg->engine;
831 struct i915_request *rq[8] = {};
832 struct intel_context *ce[ARRAY_SIZE(rq)];
833 unsigned long count;
834 int err = 0;
835
836 for (count = 0; count < ARRAY_SIZE(ce); count++) {
837 ce[count] = intel_context_create(engine);
838 if (IS_ERR(ce[count])) {
839 err = PTR_ERR(ce[count]);
840 while (--count)
841 intel_context_put(ce[count]);
842 return err;
843 }
844 }
845
846 count = 0;
847 while (!kthread_should_stop()) {
848 unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1);
849 struct i915_request *old = rq[idx];
850 struct i915_request *new;
851
852 new = intel_context_create_request(ce[idx]);
853 if (IS_ERR(new)) {
854 err = PTR_ERR(new);
855 break;
856 }
857
858 rq[idx] = i915_request_get(new);
859 i915_request_add(new);
860
861 if (engine->schedule && arg->flags & TEST_PRIORITY) {
862 struct i915_sched_attr attr = {
863 .priority =
864 i915_prandom_u32_max_state(512, &prng),
865 };
866 engine->schedule(rq[idx], &attr);
867 }
868
869 err = active_request_put(old);
870 if (err)
871 break;
872
873 cond_resched();
874 }
875
876 for (count = 0; count < ARRAY_SIZE(rq); count++) {
877 int err__ = active_request_put(rq[count]);
878
879 /* Keep the first error */
880 if (!err)
881 err = err__;
882
883 intel_context_put(ce[count]);
884 }
885
886 return err;
887}
888
889static int __igt_reset_engines(struct intel_gt *gt,
890 const char *test_name,
891 unsigned int flags)
892{
893 struct i915_gpu_error *global = >->i915->gpu_error;
894 struct intel_engine_cs *engine, *other;
895 enum intel_engine_id id, tmp;
896 struct hang h;
897 int err = 0;
898
899 /* Check that issuing a reset on one engine does not interfere
900 * with any other engine.
901 */
902
903 if (!intel_has_reset_engine(gt))
904 return 0;
905
906 if (flags & TEST_ACTIVE) {
907 err = hang_init(&h, gt);
908 if (err)
909 return err;
910
911 if (flags & TEST_PRIORITY)
912 h.ctx->sched.priority = 1024;
913 }
914
915 for_each_engine(engine, gt, id) {
916 struct active_engine threads[I915_NUM_ENGINES] = {};
917 unsigned long device = i915_reset_count(global);
918 unsigned long count = 0, reported;
919 IGT_TIMEOUT(end_time);
920
921 if (flags & TEST_ACTIVE &&
922 !intel_engine_can_store_dword(engine))
923 continue;
924
925 if (!wait_for_idle(engine)) {
926 pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n",
927 engine->name, test_name);
928 err = -EIO;
929 break;
930 }
931
932 memset(threads, 0, sizeof(threads));
933 for_each_engine(other, gt, tmp) {
934 struct task_struct *tsk;
935
936 threads[tmp].resets =
937 i915_reset_engine_count(global, other);
938
939 if (other == engine && !(flags & TEST_SELF))
940 continue;
941
942 if (other != engine && !(flags & TEST_OTHERS))
943 continue;
944
945 threads[tmp].engine = other;
946 threads[tmp].flags = flags;
947
948 tsk = kthread_run(active_engine, &threads[tmp],
949 "igt/%s", other->name);
950 if (IS_ERR(tsk)) {
951 err = PTR_ERR(tsk);
952 goto unwind;
953 }
954
955 threads[tmp].task = tsk;
956 get_task_struct(tsk);
957 }
958
959 yield(); /* start all threads before we begin */
960
961 st_engine_heartbeat_disable(engine);
962 set_bit(I915_RESET_ENGINE + id, >->reset.flags);
963 do {
964 struct i915_request *rq = NULL;
965
966 if (flags & TEST_ACTIVE) {
967 rq = hang_create_request(&h, engine);
968 if (IS_ERR(rq)) {
969 err = PTR_ERR(rq);
970 break;
971 }
972
973 i915_request_get(rq);
974 i915_request_add(rq);
975
976 if (!wait_until_running(&h, rq)) {
977 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
978
979 pr_err("%s: Failed to start request %llx, at %x\n",
980 __func__, rq->fence.seqno, hws_seqno(&h, rq));
981 intel_engine_dump(engine, &p,
982 "%s\n", engine->name);
983
984 i915_request_put(rq);
985 err = -EIO;
986 break;
987 }
988 }
989
990 err = intel_engine_reset(engine, NULL);
991 if (err) {
992 pr_err("i915_reset_engine(%s:%s): failed, err=%d\n",
993 engine->name, test_name, err);
994 break;
995 }
996
997 count++;
998
999 if (rq) {
1000 if (rq->fence.error != -EIO) {
1001 pr_err("i915_reset_engine(%s:%s):"
1002 " failed to reset request %llx:%lld\n",
1003 engine->name, test_name,
1004 rq->fence.context,
1005 rq->fence.seqno);
1006 i915_request_put(rq);
1007
1008 GEM_TRACE_DUMP();
1009 intel_gt_set_wedged(gt);
1010 err = -EIO;
1011 break;
1012 }
1013
1014 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1015 struct drm_printer p =
1016 drm_info_printer(gt->i915->drm.dev);
1017
1018 pr_err("i915_reset_engine(%s:%s):"
1019 " failed to complete request %llx:%lld after reset\n",
1020 engine->name, test_name,
1021 rq->fence.context,
1022 rq->fence.seqno);
1023 intel_engine_dump(engine, &p,
1024 "%s\n", engine->name);
1025 i915_request_put(rq);
1026
1027 GEM_TRACE_DUMP();
1028 intel_gt_set_wedged(gt);
1029 err = -EIO;
1030 break;
1031 }
1032
1033 i915_request_put(rq);
1034 }
1035
1036 if (!(flags & TEST_SELF) && !wait_for_idle(engine)) {
1037 struct drm_printer p =
1038 drm_info_printer(gt->i915->drm.dev);
1039
1040 pr_err("i915_reset_engine(%s:%s):"
1041 " failed to idle after reset\n",
1042 engine->name, test_name);
1043 intel_engine_dump(engine, &p,
1044 "%s\n", engine->name);
1045
1046 err = -EIO;
1047 break;
1048 }
1049 } while (time_before(jiffies, end_time));
1050 clear_bit(I915_RESET_ENGINE + id, >->reset.flags);
1051 st_engine_heartbeat_enable(engine);
1052
1053 pr_info("i915_reset_engine(%s:%s): %lu resets\n",
1054 engine->name, test_name, count);
1055
1056 reported = i915_reset_engine_count(global, engine);
1057 reported -= threads[engine->id].resets;
1058 if (reported != count) {
1059 pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n",
1060 engine->name, test_name, count, reported);
1061 if (!err)
1062 err = -EINVAL;
1063 }
1064
1065unwind:
1066 for_each_engine(other, gt, tmp) {
1067 int ret;
1068
1069 if (!threads[tmp].task)
1070 continue;
1071
1072 ret = kthread_stop(threads[tmp].task);
1073 if (ret) {
1074 pr_err("kthread for other engine %s failed, err=%d\n",
1075 other->name, ret);
1076 if (!err)
1077 err = ret;
1078 }
1079 put_task_struct(threads[tmp].task);
1080
1081 if (other->uabi_class != engine->uabi_class &&
1082 threads[tmp].resets !=
1083 i915_reset_engine_count(global, other)) {
1084 pr_err("Innocent engine %s was reset (count=%ld)\n",
1085 other->name,
1086 i915_reset_engine_count(global, other) -
1087 threads[tmp].resets);
1088 if (!err)
1089 err = -EINVAL;
1090 }
1091 }
1092
1093 if (device != i915_reset_count(global)) {
1094 pr_err("Global reset (count=%ld)!\n",
1095 i915_reset_count(global) - device);
1096 if (!err)
1097 err = -EINVAL;
1098 }
1099
1100 if (err)
1101 break;
1102
1103 err = igt_flush_test(gt->i915);
1104 if (err)
1105 break;
1106 }
1107
1108 if (intel_gt_is_wedged(gt))
1109 err = -EIO;
1110
1111 if (flags & TEST_ACTIVE)
1112 hang_fini(&h);
1113
1114 return err;
1115}
1116
1117static int igt_reset_engines(void *arg)
1118{
1119 static const struct {
1120 const char *name;
1121 unsigned int flags;
1122 } phases[] = {
1123 { "idle", 0 },
1124 { "active", TEST_ACTIVE },
1125 { "others-idle", TEST_OTHERS },
1126 { "others-active", TEST_OTHERS | TEST_ACTIVE },
1127 {
1128 "others-priority",
1129 TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY
1130 },
1131 {
1132 "self-priority",
1133 TEST_ACTIVE | TEST_PRIORITY | TEST_SELF,
1134 },
1135 { }
1136 };
1137 struct intel_gt *gt = arg;
1138 typeof(*phases) *p;
1139 int err;
1140
1141 for (p = phases; p->name; p++) {
1142 if (p->flags & TEST_PRIORITY) {
1143 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
1144 continue;
1145 }
1146
1147 err = __igt_reset_engines(arg, p->name, p->flags);
1148 if (err)
1149 return err;
1150 }
1151
1152 return 0;
1153}
1154
1155static u32 fake_hangcheck(struct intel_gt *gt, intel_engine_mask_t mask)
1156{
1157 u32 count = i915_reset_count(>->i915->gpu_error);
1158
1159 intel_gt_reset(gt, mask, NULL);
1160
1161 return count;
1162}
1163
1164static int igt_reset_wait(void *arg)
1165{
1166 struct intel_gt *gt = arg;
1167 struct i915_gpu_error *global = >->i915->gpu_error;
1168 struct intel_engine_cs *engine = gt->engine[RCS0];
1169 struct i915_request *rq;
1170 unsigned int reset_count;
1171 struct hang h;
1172 long timeout;
1173 int err;
1174
1175 if (!engine || !intel_engine_can_store_dword(engine))
1176 return 0;
1177
1178 /* Check that we detect a stuck waiter and issue a reset */
1179
1180 igt_global_reset_lock(gt);
1181
1182 err = hang_init(&h, gt);
1183 if (err)
1184 goto unlock;
1185
1186 rq = hang_create_request(&h, engine);
1187 if (IS_ERR(rq)) {
1188 err = PTR_ERR(rq);
1189 goto fini;
1190 }
1191
1192 i915_request_get(rq);
1193 i915_request_add(rq);
1194
1195 if (!wait_until_running(&h, rq)) {
1196 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1197
1198 pr_err("%s: Failed to start request %llx, at %x\n",
1199 __func__, rq->fence.seqno, hws_seqno(&h, rq));
1200 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1201
1202 intel_gt_set_wedged(gt);
1203
1204 err = -EIO;
1205 goto out_rq;
1206 }
1207
1208 reset_count = fake_hangcheck(gt, ALL_ENGINES);
1209
1210 timeout = i915_request_wait(rq, 0, 10);
1211 if (timeout < 0) {
1212 pr_err("i915_request_wait failed on a stuck request: err=%ld\n",
1213 timeout);
1214 err = timeout;
1215 goto out_rq;
1216 }
1217
1218 if (i915_reset_count(global) == reset_count) {
1219 pr_err("No GPU reset recorded!\n");
1220 err = -EINVAL;
1221 goto out_rq;
1222 }
1223
1224out_rq:
1225 i915_request_put(rq);
1226fini:
1227 hang_fini(&h);
1228unlock:
1229 igt_global_reset_unlock(gt);
1230
1231 if (intel_gt_is_wedged(gt))
1232 return -EIO;
1233
1234 return err;
1235}
1236
1237struct evict_vma {
1238 struct completion completion;
1239 struct i915_vma *vma;
1240};
1241
1242static int evict_vma(void *data)
1243{
1244 struct evict_vma *arg = data;
1245 struct i915_address_space *vm = arg->vma->vm;
1246 struct drm_mm_node evict = arg->vma->node;
1247 int err;
1248
1249 complete(&arg->completion);
1250
1251 mutex_lock(&vm->mutex);
1252 err = i915_gem_evict_for_node(vm, &evict, 0);
1253 mutex_unlock(&vm->mutex);
1254
1255 return err;
1256}
1257
1258static int evict_fence(void *data)
1259{
1260 struct evict_vma *arg = data;
1261 int err;
1262
1263 complete(&arg->completion);
1264
1265 /* Mark the fence register as dirty to force the mmio update. */
1266 err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512);
1267 if (err) {
1268 pr_err("Invalid Y-tiling settings; err:%d\n", err);
1269 return err;
1270 }
1271
1272 err = i915_vma_pin(arg->vma, 0, 0, PIN_GLOBAL | PIN_MAPPABLE);
1273 if (err) {
1274 pr_err("Unable to pin vma for Y-tiled fence; err:%d\n", err);
1275 return err;
1276 }
1277
1278 err = i915_vma_pin_fence(arg->vma);
1279 i915_vma_unpin(arg->vma);
1280 if (err) {
1281 pr_err("Unable to pin Y-tiled fence; err:%d\n", err);
1282 return err;
1283 }
1284
1285 i915_vma_unpin_fence(arg->vma);
1286
1287 return 0;
1288}
1289
1290static int __igt_reset_evict_vma(struct intel_gt *gt,
1291 struct i915_address_space *vm,
1292 int (*fn)(void *),
1293 unsigned int flags)
1294{
1295 struct intel_engine_cs *engine = gt->engine[RCS0];
1296 struct drm_i915_gem_object *obj;
1297 struct task_struct *tsk = NULL;
1298 struct i915_request *rq;
1299 struct evict_vma arg;
1300 struct hang h;
1301 unsigned int pin_flags;
1302 int err;
1303
1304 if (!gt->ggtt->num_fences && flags & EXEC_OBJECT_NEEDS_FENCE)
1305 return 0;
1306
1307 if (!engine || !intel_engine_can_store_dword(engine))
1308 return 0;
1309
1310 /* Check that we can recover an unbind stuck on a hanging request */
1311
1312 err = hang_init(&h, gt);
1313 if (err)
1314 return err;
1315
1316 obj = i915_gem_object_create_internal(gt->i915, SZ_1M);
1317 if (IS_ERR(obj)) {
1318 err = PTR_ERR(obj);
1319 goto fini;
1320 }
1321
1322 if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1323 err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512);
1324 if (err) {
1325 pr_err("Invalid X-tiling settings; err:%d\n", err);
1326 goto out_obj;
1327 }
1328 }
1329
1330 arg.vma = i915_vma_instance(obj, vm, NULL);
1331 if (IS_ERR(arg.vma)) {
1332 err = PTR_ERR(arg.vma);
1333 goto out_obj;
1334 }
1335
1336 rq = hang_create_request(&h, engine);
1337 if (IS_ERR(rq)) {
1338 err = PTR_ERR(rq);
1339 goto out_obj;
1340 }
1341
1342 pin_flags = i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER;
1343
1344 if (flags & EXEC_OBJECT_NEEDS_FENCE)
1345 pin_flags |= PIN_MAPPABLE;
1346
1347 err = i915_vma_pin(arg.vma, 0, 0, pin_flags);
1348 if (err) {
1349 i915_request_add(rq);
1350 goto out_obj;
1351 }
1352
1353 if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1354 err = i915_vma_pin_fence(arg.vma);
1355 if (err) {
1356 pr_err("Unable to pin X-tiled fence; err:%d\n", err);
1357 i915_vma_unpin(arg.vma);
1358 i915_request_add(rq);
1359 goto out_obj;
1360 }
1361 }
1362
1363 i915_vma_lock(arg.vma);
1364 err = i915_request_await_object(rq, arg.vma->obj,
1365 flags & EXEC_OBJECT_WRITE);
1366 if (err == 0)
1367 err = i915_vma_move_to_active(arg.vma, rq, flags);
1368 i915_vma_unlock(arg.vma);
1369
1370 if (flags & EXEC_OBJECT_NEEDS_FENCE)
1371 i915_vma_unpin_fence(arg.vma);
1372 i915_vma_unpin(arg.vma);
1373
1374 i915_request_get(rq);
1375 i915_request_add(rq);
1376 if (err)
1377 goto out_rq;
1378
1379 if (!wait_until_running(&h, rq)) {
1380 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1381
1382 pr_err("%s: Failed to start request %llx, at %x\n",
1383 __func__, rq->fence.seqno, hws_seqno(&h, rq));
1384 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1385
1386 intel_gt_set_wedged(gt);
1387 goto out_reset;
1388 }
1389
1390 init_completion(&arg.completion);
1391
1392 tsk = kthread_run(fn, &arg, "igt/evict_vma");
1393 if (IS_ERR(tsk)) {
1394 err = PTR_ERR(tsk);
1395 tsk = NULL;
1396 goto out_reset;
1397 }
1398 get_task_struct(tsk);
1399
1400 wait_for_completion(&arg.completion);
1401
1402 if (wait_for(!list_empty(&rq->fence.cb_list), 10)) {
1403 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1404
1405 pr_err("igt/evict_vma kthread did not wait\n");
1406 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1407
1408 intel_gt_set_wedged(gt);
1409 goto out_reset;
1410 }
1411
1412out_reset:
1413 igt_global_reset_lock(gt);
1414 fake_hangcheck(gt, rq->engine->mask);
1415 igt_global_reset_unlock(gt);
1416
1417 if (tsk) {
1418 struct intel_wedge_me w;
1419
1420 /* The reset, even indirectly, should take less than 10ms. */
1421 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
1422 err = kthread_stop(tsk);
1423
1424 put_task_struct(tsk);
1425 }
1426
1427out_rq:
1428 i915_request_put(rq);
1429out_obj:
1430 i915_gem_object_put(obj);
1431fini:
1432 hang_fini(&h);
1433 if (intel_gt_is_wedged(gt))
1434 return -EIO;
1435
1436 return err;
1437}
1438
1439static int igt_reset_evict_ggtt(void *arg)
1440{
1441 struct intel_gt *gt = arg;
1442
1443 return __igt_reset_evict_vma(gt, >->ggtt->vm,
1444 evict_vma, EXEC_OBJECT_WRITE);
1445}
1446
1447static int igt_reset_evict_ppgtt(void *arg)
1448{
1449 struct intel_gt *gt = arg;
1450 struct i915_ppgtt *ppgtt;
1451 int err;
1452
1453 /* aliasing == global gtt locking, covered above */
1454 if (INTEL_PPGTT(gt->i915) < INTEL_PPGTT_FULL)
1455 return 0;
1456
1457 ppgtt = i915_ppgtt_create(gt);
1458 if (IS_ERR(ppgtt))
1459 return PTR_ERR(ppgtt);
1460
1461 err = __igt_reset_evict_vma(gt, &ppgtt->vm,
1462 evict_vma, EXEC_OBJECT_WRITE);
1463 i915_vm_put(&ppgtt->vm);
1464
1465 return err;
1466}
1467
1468static int igt_reset_evict_fence(void *arg)
1469{
1470 struct intel_gt *gt = arg;
1471
1472 return __igt_reset_evict_vma(gt, >->ggtt->vm,
1473 evict_fence, EXEC_OBJECT_NEEDS_FENCE);
1474}
1475
1476static int wait_for_others(struct intel_gt *gt,
1477 struct intel_engine_cs *exclude)
1478{
1479 struct intel_engine_cs *engine;
1480 enum intel_engine_id id;
1481
1482 for_each_engine(engine, gt, id) {
1483 if (engine == exclude)
1484 continue;
1485
1486 if (!wait_for_idle(engine))
1487 return -EIO;
1488 }
1489
1490 return 0;
1491}
1492
1493static int igt_reset_queue(void *arg)
1494{
1495 struct intel_gt *gt = arg;
1496 struct i915_gpu_error *global = >->i915->gpu_error;
1497 struct intel_engine_cs *engine;
1498 enum intel_engine_id id;
1499 struct hang h;
1500 int err;
1501
1502 /* Check that we replay pending requests following a hang */
1503
1504 igt_global_reset_lock(gt);
1505
1506 err = hang_init(&h, gt);
1507 if (err)
1508 goto unlock;
1509
1510 for_each_engine(engine, gt, id) {
1511 struct i915_request *prev;
1512 IGT_TIMEOUT(end_time);
1513 unsigned int count;
1514
1515 if (!intel_engine_can_store_dword(engine))
1516 continue;
1517
1518 prev = hang_create_request(&h, engine);
1519 if (IS_ERR(prev)) {
1520 err = PTR_ERR(prev);
1521 goto fini;
1522 }
1523
1524 i915_request_get(prev);
1525 i915_request_add(prev);
1526
1527 count = 0;
1528 do {
1529 struct i915_request *rq;
1530 unsigned int reset_count;
1531
1532 rq = hang_create_request(&h, engine);
1533 if (IS_ERR(rq)) {
1534 err = PTR_ERR(rq);
1535 goto fini;
1536 }
1537
1538 i915_request_get(rq);
1539 i915_request_add(rq);
1540
1541 /*
1542 * XXX We don't handle resetting the kernel context
1543 * very well. If we trigger a device reset twice in
1544 * quick succession while the kernel context is
1545 * executing, we may end up skipping the breadcrumb.
1546 * This is really only a problem for the selftest as
1547 * normally there is a large interlude between resets
1548 * (hangcheck), or we focus on resetting just one
1549 * engine and so avoid repeatedly resetting innocents.
1550 */
1551 err = wait_for_others(gt, engine);
1552 if (err) {
1553 pr_err("%s(%s): Failed to idle other inactive engines after device reset\n",
1554 __func__, engine->name);
1555 i915_request_put(rq);
1556 i915_request_put(prev);
1557
1558 GEM_TRACE_DUMP();
1559 intel_gt_set_wedged(gt);
1560 goto fini;
1561 }
1562
1563 if (!wait_until_running(&h, prev)) {
1564 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1565
1566 pr_err("%s(%s): Failed to start request %llx, at %x\n",
1567 __func__, engine->name,
1568 prev->fence.seqno, hws_seqno(&h, prev));
1569 intel_engine_dump(engine, &p,
1570 "%s\n", engine->name);
1571
1572 i915_request_put(rq);
1573 i915_request_put(prev);
1574
1575 intel_gt_set_wedged(gt);
1576
1577 err = -EIO;
1578 goto fini;
1579 }
1580
1581 reset_count = fake_hangcheck(gt, BIT(id));
1582
1583 if (prev->fence.error != -EIO) {
1584 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n",
1585 prev->fence.error);
1586 i915_request_put(rq);
1587 i915_request_put(prev);
1588 err = -EINVAL;
1589 goto fini;
1590 }
1591
1592 if (rq->fence.error) {
1593 pr_err("Fence error status not zero [%d] after unrelated reset\n",
1594 rq->fence.error);
1595 i915_request_put(rq);
1596 i915_request_put(prev);
1597 err = -EINVAL;
1598 goto fini;
1599 }
1600
1601 if (i915_reset_count(global) == reset_count) {
1602 pr_err("No GPU reset recorded!\n");
1603 i915_request_put(rq);
1604 i915_request_put(prev);
1605 err = -EINVAL;
1606 goto fini;
1607 }
1608
1609 i915_request_put(prev);
1610 prev = rq;
1611 count++;
1612 } while (time_before(jiffies, end_time));
1613 pr_info("%s: Completed %d queued resets\n",
1614 engine->name, count);
1615
1616 *h.batch = MI_BATCH_BUFFER_END;
1617 intel_gt_chipset_flush(engine->gt);
1618
1619 i915_request_put(prev);
1620
1621 err = igt_flush_test(gt->i915);
1622 if (err)
1623 break;
1624 }
1625
1626fini:
1627 hang_fini(&h);
1628unlock:
1629 igt_global_reset_unlock(gt);
1630
1631 if (intel_gt_is_wedged(gt))
1632 return -EIO;
1633
1634 return err;
1635}
1636
1637static int igt_handle_error(void *arg)
1638{
1639 struct intel_gt *gt = arg;
1640 struct i915_gpu_error *global = >->i915->gpu_error;
1641 struct intel_engine_cs *engine = gt->engine[RCS0];
1642 struct hang h;
1643 struct i915_request *rq;
1644 struct i915_gpu_coredump *error;
1645 int err;
1646
1647 /* Check that we can issue a global GPU and engine reset */
1648
1649 if (!intel_has_reset_engine(gt))
1650 return 0;
1651
1652 if (!engine || !intel_engine_can_store_dword(engine))
1653 return 0;
1654
1655 err = hang_init(&h, gt);
1656 if (err)
1657 return err;
1658
1659 rq = hang_create_request(&h, engine);
1660 if (IS_ERR(rq)) {
1661 err = PTR_ERR(rq);
1662 goto err_fini;
1663 }
1664
1665 i915_request_get(rq);
1666 i915_request_add(rq);
1667
1668 if (!wait_until_running(&h, rq)) {
1669 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1670
1671 pr_err("%s: Failed to start request %llx, at %x\n",
1672 __func__, rq->fence.seqno, hws_seqno(&h, rq));
1673 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1674
1675 intel_gt_set_wedged(gt);
1676
1677 err = -EIO;
1678 goto err_request;
1679 }
1680
1681 /* Temporarily disable error capture */
1682 error = xchg(&global->first_error, (void *)-1);
1683
1684 intel_gt_handle_error(gt, engine->mask, 0, NULL);
1685
1686 xchg(&global->first_error, error);
1687
1688 if (rq->fence.error != -EIO) {
1689 pr_err("Guilty request not identified!\n");
1690 err = -EINVAL;
1691 goto err_request;
1692 }
1693
1694err_request:
1695 i915_request_put(rq);
1696err_fini:
1697 hang_fini(&h);
1698 return err;
1699}
1700
1701static int __igt_atomic_reset_engine(struct intel_engine_cs *engine,
1702 const struct igt_atomic_section *p,
1703 const char *mode)
1704{
1705 struct tasklet_struct * const t = &engine->execlists.tasklet;
1706 int err;
1707
1708 GEM_TRACE("i915_reset_engine(%s:%s) under %s\n",
1709 engine->name, mode, p->name);
1710
1711 if (t->func)
1712 tasklet_disable(t);
1713 if (strcmp(p->name, "softirq"))
1714 local_bh_disable();
1715 p->critical_section_begin();
1716
1717 err = __intel_engine_reset_bh(engine, NULL);
1718
1719 p->critical_section_end();
1720 if (strcmp(p->name, "softirq"))
1721 local_bh_enable();
1722 if (t->func) {
1723 tasklet_enable(t);
1724 tasklet_hi_schedule(t);
1725 }
1726
1727 if (err)
1728 pr_err("i915_reset_engine(%s:%s) failed under %s\n",
1729 engine->name, mode, p->name);
1730
1731 return err;
1732}
1733
1734static int igt_atomic_reset_engine(struct intel_engine_cs *engine,
1735 const struct igt_atomic_section *p)
1736{
1737 struct i915_request *rq;
1738 struct hang h;
1739 int err;
1740
1741 err = __igt_atomic_reset_engine(engine, p, "idle");
1742 if (err)
1743 return err;
1744
1745 err = hang_init(&h, engine->gt);
1746 if (err)
1747 return err;
1748
1749 rq = hang_create_request(&h, engine);
1750 if (IS_ERR(rq)) {
1751 err = PTR_ERR(rq);
1752 goto out;
1753 }
1754
1755 i915_request_get(rq);
1756 i915_request_add(rq);
1757
1758 if (wait_until_running(&h, rq)) {
1759 err = __igt_atomic_reset_engine(engine, p, "active");
1760 } else {
1761 pr_err("%s(%s): Failed to start request %llx, at %x\n",
1762 __func__, engine->name,
1763 rq->fence.seqno, hws_seqno(&h, rq));
1764 intel_gt_set_wedged(engine->gt);
1765 err = -EIO;
1766 }
1767
1768 if (err == 0) {
1769 struct intel_wedge_me w;
1770
1771 intel_wedge_on_timeout(&w, engine->gt, HZ / 20 /* 50ms */)
1772 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
1773 if (intel_gt_is_wedged(engine->gt))
1774 err = -EIO;
1775 }
1776
1777 i915_request_put(rq);
1778out:
1779 hang_fini(&h);
1780 return err;
1781}
1782
1783static int igt_reset_engines_atomic(void *arg)
1784{
1785 struct intel_gt *gt = arg;
1786 const typeof(*igt_atomic_phases) *p;
1787 int err = 0;
1788
1789 /* Check that the engines resets are usable from atomic context */
1790
1791 if (!intel_has_reset_engine(gt))
1792 return 0;
1793
1794 if (intel_uc_uses_guc_submission(>->uc))
1795 return 0;
1796
1797 igt_global_reset_lock(gt);
1798
1799 /* Flush any requests before we get started and check basics */
1800 if (!igt_force_reset(gt))
1801 goto unlock;
1802
1803 for (p = igt_atomic_phases; p->name; p++) {
1804 struct intel_engine_cs *engine;
1805 enum intel_engine_id id;
1806
1807 for_each_engine(engine, gt, id) {
1808 err = igt_atomic_reset_engine(engine, p);
1809 if (err)
1810 goto out;
1811 }
1812 }
1813
1814out:
1815 /* As we poke around the guts, do a full reset before continuing. */
1816 igt_force_reset(gt);
1817unlock:
1818 igt_global_reset_unlock(gt);
1819
1820 return err;
1821}
1822
1823int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
1824{
1825 static const struct i915_subtest tests[] = {
1826 SUBTEST(igt_hang_sanitycheck),
1827 SUBTEST(igt_reset_nop),
1828 SUBTEST(igt_reset_nop_engine),
1829 SUBTEST(igt_reset_idle_engine),
1830 SUBTEST(igt_reset_active_engine),
1831 SUBTEST(igt_reset_fail_engine),
1832 SUBTEST(igt_reset_engines),
1833 SUBTEST(igt_reset_engines_atomic),
1834 SUBTEST(igt_reset_queue),
1835 SUBTEST(igt_reset_wait),
1836 SUBTEST(igt_reset_evict_ggtt),
1837 SUBTEST(igt_reset_evict_ppgtt),
1838 SUBTEST(igt_reset_evict_fence),
1839 SUBTEST(igt_handle_error),
1840 };
1841 struct intel_gt *gt = &i915->gt;
1842 intel_wakeref_t wakeref;
1843 int err;
1844
1845 if (!intel_has_gpu_reset(gt))
1846 return 0;
1847
1848 if (intel_gt_is_wedged(gt))
1849 return -EIO; /* we're long past hope of a successful reset */
1850
1851 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1852
1853 err = intel_gt_live_subtests(tests, gt);
1854
1855 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1856
1857 return err;
1858}